From 5d497e8821dafd24c2c7db103305ad507c99e219 Mon Sep 17 00:00:00 2001 From: Denis Khalikov Date: Wed, 10 Mar 2021 17:23:13 +0300 Subject: [PATCH] MCOL-4566: Add rebuildEM tool support to work with compressed files. * This patch adds rebuildEM tool support to work with compressed files. * This patch increases a version of the file header. Note: Default version of the `rebuildEM` tool was using very old API, those functions are not present currently. So `rebuildEM` will not work with files created without compression, because we cannot deduce some info which are needed to create column extent. --- tests/rebuild-em-tests.cpp | 4 + tools/CMakeLists.txt | 2 +- .../BRM_saves_em_system_tables_blob.h | 340 ++++++++++++ tools/rebuildEM/CMakeLists.txt | 6 + tools/rebuildEM/main.cpp | 401 ++++---------- tools/rebuildEM/rebuildEM.cpp | 497 ++++++++++++++++++ tools/rebuildEM/rebuildEM.h | 230 ++++++++ utils/compress/idbcompress.cpp | 98 +++- utils/compress/idbcompress.h | 65 ++- versioning/BRM/extentmap.cpp | 92 +++- versioning/BRM/extentmap.h | 13 +- writeengine/bulk/we_colbufcompressed.cpp | 12 + writeengine/bulk/we_columninfo.cpp | 8 + writeengine/bulk/we_columninfo.h | 10 + writeengine/dictionary/we_dctnry.cpp | 42 +- writeengine/dictionary/we_dctnry.h | 3 +- writeengine/shared/we_chunkmanager.cpp | 38 +- writeengine/shared/we_chunkmanager.h | 17 +- writeengine/shared/we_fileop.cpp | 41 +- writeengine/shared/we_fileop.h | 17 +- writeengine/wrapper/we_colop.cpp | 2 +- writeengine/wrapper/we_colopcompress.cpp | 4 +- writeengine/wrapper/we_colopcompress.h | 2 +- writeengine/wrapper/we_dctnrycompress.cpp | 16 +- writeengine/wrapper/we_dctnrycompress.h | 6 +- 25 files changed, 1560 insertions(+), 406 deletions(-) create mode 100644 tools/rebuildEM/BRM_saves_em_system_tables_blob.h create mode 100644 tools/rebuildEM/CMakeLists.txt create mode 100644 tools/rebuildEM/rebuildEM.cpp create mode 100644 tools/rebuildEM/rebuildEM.h diff --git a/tests/rebuild-em-tests.cpp b/tests/rebuild-em-tests.cpp index 8875be460..d9c4b6a1e 100644 --- a/tests/rebuild-em-tests.cpp +++ b/tests/rebuild-em-tests.cpp @@ -20,6 +20,10 @@ #include #include "we_convertor.h" +#include "we_fileop.h" + +using namespace idbdatafile; +using namespace WriteEngine; class RebuildEMTest : public ::testing::Test { diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 4df1fb652..d5b4d95af 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -10,4 +10,4 @@ add_subdirectory(viewtablelock) add_subdirectory(cleartablelock) add_subdirectory(ddlcleanup) add_subdirectory(idbmeminfo) - +add_subdirectory(rebuildEM) diff --git a/tools/rebuildEM/BRM_saves_em_system_tables_blob.h b/tools/rebuildEM/BRM_saves_em_system_tables_blob.h new file mode 100644 index 000000000..d25eaa222 --- /dev/null +++ b/tools/rebuildEM/BRM_saves_em_system_tables_blob.h @@ -0,0 +1,340 @@ +#ifndef BRM_SAVES_EM_SYSTEM_TABLES_BLOB +#define BRM_SAVES_EM_SYSTEM_TABLES_BLOB + +// Generated by hexdump -C BRM_saves_em for initial system catalog. +// TODO: This could be compressed by simple `run-length encoding` +// algo. +static const uint8_t BRM_saves_em_system_tables_blob[] = { + 0x20, 0x8b, 0xf7, 0x76, 0x29, 0x00, 0x00, 0x00, 0x32, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xe9, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xd1, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xea, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x60, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xd4, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xeb, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x90, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xec, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xed, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xb0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xee, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xef, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xd0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xf0, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xf1, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xf2, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xf3, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x10, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xfd, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x30, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x0d, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x50, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xfe, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x70, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x10, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x90, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xff, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xb0, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x13, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, + 0x00, 0xd0, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xe0, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xf0, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x02, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x03, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x10, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x20, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x05, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x30, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x06, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x40, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x07, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x50, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x08, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x70, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x16, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x90, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x09, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xa0, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x0a, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xb0, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x0b, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xc0, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x0c, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xc4, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x0d, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xd4, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x0e, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xe4, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x0f, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x04, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x19, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x24, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x10, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x44, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x1c, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x64, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x11, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x74, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x12, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x94, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1b, 0xff, 0xff, 0x03, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; +#endif diff --git a/tools/rebuildEM/CMakeLists.txt b/tools/rebuildEM/CMakeLists.txt new file mode 100644 index 000000000..3e5feeb9a --- /dev/null +++ b/tools/rebuildEM/CMakeLists.txt @@ -0,0 +1,6 @@ +include_directories(${ENGINE_COMMON_INCLUDES}) + +set(rebuildEM_SRCS main.cpp rebuildEM.cpp) +add_executable(rebuildEM ${rebuildEM_SRCS}) +target_link_libraries(rebuildEM ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${MARIADB_CLIENT_LIBS} boost_filesystem) +install(TARGETS rebuildEM DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine) diff --git a/tools/rebuildEM/main.cpp b/tools/rebuildEM/main.cpp index 819d37ead..a3d876705 100644 --- a/tools/rebuildEM/main.cpp +++ b/tools/rebuildEM/main.cpp @@ -1,329 +1,128 @@ -/***************************************************************************** -* $Id: main.cpp 1739 2012-03-22 12:57:59Z pleblanc $ -* -*****************************************************************************/ +/* Copyright (C) 2021 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + #include -#include -//#define _XOPEN_SOURCE 500 +#include #include -#include -#include -#include -#include -#include -using namespace std; -using namespace boost; +#include #include "configcpp.h" -using namespace config; +#include "rebuildEM.h" +#include "IDBPolicy.h" +#include "IDBFileSystem.h" -#include "extentmap.h" -using namespace BRM; +using namespace idbdatafile; +using namespace RebuildExtentMap; -#include "calpontsystemcatalog.h" -#include "dm.h" - -namespace +static void usage(const string& pname) { - -unsigned vflg = 0; -bool dflg = false; -string DBRoot; -string pattern; -unsigned extentSize; -ExtentMap em; - -void usage(const string& pname) -{ - cout << "usage: " << pname << " [-vdh]" << endl << - " rebuilds the extent map from the contents of the database file system." << endl << - " -v display verbose progress information. More v's, more debug" << endl << - " -d display what would be done--don't do it" << endl << - " -h display this help text" << endl; -} - -OID_t pname2OID(const string& fpath) -{ - dmFilePathArgs_t args; - char aBuff[10]; - char bBuff[10]; - char cBuff[10]; - char fnBuff[16]; - int rc; - OID_t oid; - - args.pDirA = aBuff; - args.pDirB = bBuff; - args.pDirC = cBuff; - args.pFName = fnBuff; - args.ALen = sizeof(aBuff); - args.BLen = sizeof(bBuff); - args.CLen = sizeof(cBuff); - args.FNLen = sizeof(fnBuff); - args.Arc = 0; - args.Brc = 0; - args.Crc = 0; - args.FNrc = 0; - - typedef tokenizer > tokenizer; - char_separator sep("/"); - tokenizer tokens(fpath, sep); - tokenizer::iterator tok_iter = tokens.begin(); - tokenizer::iterator end = tokens.end(); - typedef stack pcomps_t; - pcomps_t pcomps; - - while (tok_iter != end) - { - pcomps.push(*tok_iter); - ++tok_iter; - } - - idbassert(pcomps.size() >= 4); - - string pcomp; - pcomp = pcomps.top(); - pcomps.pop(); - idbassert(pcomp.size() < 16); - strcpy(args.pFName, pcomp.c_str()); - pcomp = pcomps.top(); - pcomps.pop(); - idbassert(pcomp.size() < 10); - strcpy(args.pDirC, pcomp.c_str()); - pcomp = pcomps.top(); - pcomps.pop(); - idbassert(pcomp.size() < 10); - strcpy(args.pDirB, pcomp.c_str()); - pcomp = pcomps.top(); - pcomps.pop(); - idbassert(pcomp.size() < 10); - strcpy(args.pDirA, pcomp.c_str()); - - rc = dmFPath2Oid(&args, (UINT32*)&oid); - - if (rc != 0) - { - oid = 0; - } - - return oid; -} - -bool isCalpontDBFile(const string& fpath) -{ - return (fnmatch(pattern.c_str(), fpath.c_str(), 0) == 0); -} - -int walkDB(const char* fp, const struct stat* sb, int typeflag, struct FTW* ftwbuf) -{ - string fpath(fp); - unsigned numExtents; - unsigned numBlocks; - - if (typeflag != FTW_F) - { - return FTW_CONTINUE; - } - - if (!isCalpontDBFile(fpath)) - { - if (vflg > 2) - { - cout << "Skipping non-Calpont DB file " << fpath << endl; - } - - return FTW_CONTINUE; - } - - if (vflg) - { - cout << "Processing file " << fpath << endl; - numBlocks = (sb->st_size + (BLOCK_SIZE - 1)) / BLOCK_SIZE; - numExtents = (numBlocks + (extentSize - 1)) / extentSize; - - if (vflg > 1) - { - cout << "File is " << numExtents << " extent" << (numExtents > 1 ? "s" : "") << - ", " << numBlocks << " block" << (numBlocks > 1 ? "s" : "") << - ", " << sb->st_size << " bytes, "; - } - } - - OID_t oid; - oid = pname2OID(fpath); - - if (vflg > 1) - { - cout << "OID is " << oid << ", "; - } - - if (oid <= 10) - { - if (vflg) - { - cout << endl << "OID " << oid << " is probably a VBBF and is being skipped" << endl; - } - - return FTW_CONTINUE; - } - - HWM_t hwm; -#ifdef DELETE_FIRST - - try - { - hwm = em.getHWM(oid); - } - catch (exception& ex) - { - if (vflg) - { - cout << endl << "There was no HWM for OID " << oid << - ", it is probably a VBBF and is being skipped: " << ex.what() << endl; - } - - return FTW_CONTINUE; - } - catch (...) - { - if (vflg) - { - cout << endl << "There was no HWM for OID " << oid << - ", it is probably a VBBF and is being skipped" << endl; - } - - return FTW_CONTINUE; - } - -#else - hwm = numBlocks - 1; -#endif - - if (vflg > 1) - { - cout << "HWM is " << hwm << endl; - } - -#ifdef DELETE_FIRST - - if (vflg > 1) - { - cout << "Deleting OID " << oid << " from Extent Map" << endl; - } - - if (!dflg) - { - em.deleteOID(oid); - em.confirmChanges(); - } - -#endif - - vector lbids; - int allocdsize = 0; - - if (vflg > 1) - { - cout << "Creating extent for " << numBlocks << " blocks for OID " << oid << endl; - } - - if (!dflg) - { - em.createExtent(numBlocks, oid, lbids, allocdsize); - em.confirmChanges(); - } - - if (vflg > 1) - { - cout << "Created " << allocdsize << " LBIDs for OID " << oid << endl; - - if (vflg > 2) - { - vector::iterator iter = lbids.begin(); - vector::iterator end = lbids.end(); - cout << "First LBIDs from each extent are "; - - while (iter != end) - { - cout << *iter << ", "; - ++iter; - } - - cout << endl; - } - } - - if (vflg > 1) - { - cout << "Setting OID " << oid << " HWM to " << hwm << endl; - } - - if (!dflg) - { - em.setHWM(oid, hwm); - em.confirmChanges(); - } - - return FTW_CONTINUE; -} - + std::cout << "usage: " << pname << " [-vdhs]" << std::endl; + std::cout + << "rebuilds the extent map from the contents of the database file " + "system." + << std::endl; + std::cout + << " -v display verbose progress information. More v's, more debug" + << std::endl; + std::cout << " -d display what would be done--don't do it" << std::endl; + std::cout << " -h display this help text" << std::endl; + std::cout << " -s show extent map and quit" << std::endl; } int main(int argc, char** argv) { - int c; - string pname(argv[0]); + int32_t option; + std::string pname(argv[0]); + bool showExtentMap = false; + bool verbose = false; + bool display = false; - opterr = 0; - - while ((c = getopt(argc, argv, "vdh")) != EOF) - switch (c) + while ((option = getopt(argc, argv, "vdhs")) != EOF) + { + switch (option) { - case 'v': - vflg++; - break; + case 'v': + verbose = true; + break; - case 'd': - dflg = true; - break; + case 'd': + display = true; + break; - case 'h': - case '?': - default: - usage(pname); - return (c == 'h' ? 0 : 1); - break; + case 's': + showExtentMap = true; + break; + + case 'h': + case '?': + default: + usage(pname); + return (option == 'h' ? 0 : 1); + break; } - - const Config* cf = Config::makeConfig(); - DBRoot = cf->getConfig("SystemConfig", "DBRoot"); - pattern = DBRoot + "/[0-9][0-9][0-9].dir/[0-9][0-9][0-9].dir/[0-9][0-9][0-9].dir/FILE[0-9][0-9][0-9].cdf"; - - if (vflg) - { - cout << "Using DBRoot " << DBRoot << endl; } - if (access(DBRoot.c_str(), X_OK) != 0) + EMReBuilder emReBuilder(verbose, display); + // Just show EM and quit. + if (showExtentMap) + { + emReBuilder.showExtentMap(); + return 0; + } + + auto* config = config::Config::makeConfig(); + const auto BRMSavesEM = + config->getConfig("SystemConfig", "DBRMRoot") + "_em"; + // Check for `BRM_saves_em` file presents. + if (boost::filesystem::exists(BRMSavesEM)) + { + std::cout << BRMSavesEM << " file exists. " << std::endl; + std::cout << "Please note: this tool is only suitable in situations " + "where there is no `BRM_saves_em` file. " + << std::endl; + std::cout << "If `BRM_saves_em` " + "exists extent map will be restored from it. " + << std::endl; + return 0; + } + + // Initialize system extents from the binary blob. + auto rc = emReBuilder.initializeSystemExtents(); + if (rc == -1) { - cerr << "Could not scan DBRoot " << DBRoot << '!' << endl; return 1; } - ExtentMap em; - extentSize = em.getExtentSize(); + // Make config from default path. + std::string count = config->getConfig("SystemConfig", "DBRootCount"); + // Read the number of DBRoots. + uint32_t dbRootCount = config->uFromText(count); - if (vflg) + // Iterate over DBRoots starting from the first one. + for (uint32_t dbRootNumber = 1; dbRootNumber <= dbRootCount; + ++dbRootNumber) { - cout << "System extent size is " << extentSize << " blocks" << endl; - } - - if (nftw(DBRoot.c_str(), walkDB, 64, FTW_PHYS | FTW_ACTIONRETVAL) != 0) - { - cerr << "Error processing files in DBRoot " << DBRoot << '!' << endl; - return 1; + std::string dbRootName = "DBRoot" + std::to_string(dbRootNumber); + auto dbRootPath = config->getConfig("SystemConfig", dbRootName); + emReBuilder.setDBRoot(dbRootNumber); + emReBuilder.collectExtents(dbRootPath.c_str()); + emReBuilder.rebuildExtentMap(); + emReBuilder.clear(); } return 0; } - diff --git a/tools/rebuildEM/rebuildEM.cpp b/tools/rebuildEM/rebuildEM.cpp new file mode 100644 index 000000000..7ba7247d5 --- /dev/null +++ b/tools/rebuildEM/rebuildEM.cpp @@ -0,0 +1,497 @@ +/* Copyright (C) 2021 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#include +#include +#include + +#include "rebuildEM.h" +#include "calpontsystemcatalog.h" +#include "idbcompress.h" +#include "blocksize.h" +#include "we_convertor.h" +#include "we_fileop.h" +#include "IDBPolicy.h" +#include "IDBFileSystem.h" +#include "we_chunkmanager.h" +#include "BRM_saves_em_system_tables_blob.h" +#include "we_dctnrycompress.h" +#include "we_colopcompress.h" + +using namespace idbdatafile; + +namespace RebuildExtentMap +{ + +int32_t EMReBuilder::collectExtents(const string& dbRootPath) +{ + if (doVerbose()) + { + std::cout << "Collect extents for the DBRoot " << dbRootPath + << std::endl; + } + + for (boost::filesystem::recursive_directory_iterator dirIt(dbRootPath), + dirEnd; + dirIt != dirEnd; ++dirIt) + { + (void) collectExtent(dirIt->path().string()); + } + + return 0; +} + +int32_t EMReBuilder::collectExtent(const std::string& fullFileName) +{ + WriteEngine::FileOp fileOp; + uint32_t oid; + uint32_t partition; + uint32_t segment; + // Initialize oid, partition and segment from the given `fullFileName`. + auto rc = WriteEngine::Convertor::fileName2Oid(fullFileName, oid, + partition, segment); + if (rc != 0) + { + return rc; + } + + // Open the given file. + std::unique_ptr dbFile(IDBDataFile::open( + IDBPolicy::getType(fullFileName, IDBPolicy::WRITEENG), + fullFileName.c_str(), "rb", 1)); + + if (!dbFile) + { + std::cerr << "Cannot open file " << fullFileName << std::endl; + return -1; + } + + rc = dbFile->seek(0, 0); + if (rc != 0) + { + std::cerr << "IDBDataFile::seek failed for the file " << fullFileName + << std::endl; + return rc; + } + + // Read and verify header. + char fileHeader[compress::IDBCompressInterface::HDR_BUF_LEN * 2]; + rc = fileOp.readHeaders(dbFile.get(), fileHeader); + if (rc != 0) + { + // FIXME: If the file was created without a compression, it does not + // have a header block, so header verification fails in this case, + // currently we skip it, because we cannot deduce needed data to create + // a column extent from the blob file. + if (doVerbose()) + { + std::cerr + << "Cannot read file header from the file " << fullFileName + << ", probably this file was created without compression. " + << std::endl; + } + return rc; + } + + if (doVerbose()) + { + std::cout << "Processing file: " << fullFileName << std::endl; + std::cout << "fileName2Oid: [OID: " << oid + << ", partition: " << partition << ", segment: " << segment + << "] " << std::endl; + } + + // Read the `colDataType` and `colWidth` from the given header. + compress::IDBCompressInterface compressor; + const auto versionNumber = compressor.getVersionNumber(fileHeader); + // Verify header number. + if (versionNumber < 3) + { + if (doVerbose()) + { + std::cerr << "File version " << versionNumber + << " is not supported. " << std::endl; + } + return -1; + } + + auto colDataType = compressor.getColDataType(fileHeader); + auto colWidth = compressor.getColumnWidth(fileHeader); + auto blockCount = compressor.getBlockCount(fileHeader); + auto lbid = compressor.getLBID0(fileHeader); + + if (colDataType == execplan::CalpontSystemCatalog::UNDEFINED) + { + if (doVerbose()) + { + std::cout << "File header has invalid data. " << std::endl; + } + return -1; + } + + auto isDict = isDictFile(colDataType, colWidth); + if (isDict) + colWidth = 8; + uint64_t hwm = 0; + + if (doVerbose()) + { + std::cout << "Searching for HWM... " << std::endl; + std::cout << "Block count: " << blockCount << std::endl; + } + + rc = + searchHWMInSegmentFile(oid, getDBRoot(), partition, segment, + colDataType, colWidth, blockCount, isDict, hwm); + if (rc != 0) + { + return rc; + } + + const uint32_t extentMaxBlockCount = + getEM().getExtentRows() * colWidth / BLOCK_SIZE; + + // We found multiple extents per one segment file. + if (hwm >= extentMaxBlockCount) + { + auto lbid = compressor.getLBID1(fileHeader); + FileId fileId(oid, partition, segment, colWidth, colDataType, lbid, + hwm, isDict); + extentMap.push_back(fileId); + + // Update HWM. + hwm = extentMaxBlockCount - 1; + if (doVerbose()) + { + std::cout << "Found multiple extents per segment file " + << std::endl; + std::cout << "FileId is collected " << fileId << std::endl; + } + } + + if (doVerbose()) + { + std::cout << "HWM is: " << hwm << std::endl; + } + + FileId fileId(oid, partition, segment, colWidth, colDataType, lbid, hwm, + isDict); + extentMap.push_back(fileId); + + if (doVerbose()) + { + std::cout << "FileId is collected " << fileId << std::endl; + } + return 0; +} + +int32_t EMReBuilder::rebuildExtentMap() +{ + if (doVerbose()) + { + std::cout << "Build extent map with size " << extentMap.size() + << std::endl; + } + + // We have to restore extent map by restoring individual extent in order + // they were created. This is important part, otherwise we will get invalid + // extents for dictionary segment files and will not be able to access + // columns through dictionay segment files. + sort(extentMap.begin(), extentMap.end(), + [](const FileId& lhs, const FileId& rhs) { + return lhs.lbid < rhs.lbid; + }); + + for (const auto& fileId : extentMap) + { + uint32_t startBlockOffset; + int32_t allocdSize; + BRM::LBID_t lbid; + + if (!doDisplay()) + { + try + { + if (fileId.isDict) + { + // Create a dictionary extent for the given oid, partition, + // segment, dbroot. + getEM().createDictStoreExtent( + fileId.oid, getDBRoot(), fileId.partition, + fileId.segment, lbid, allocdSize); + } + else + { + // Create a column extent for the given oid, partition, + // segment, dbroot and column width. + getEM().createColumnExtentExactFile( + fileId.oid, fileId.colWidth, getDBRoot(), + fileId.partition, fileId.segment, fileId.colDataType, + lbid, allocdSize, startBlockOffset); + } + } + // Could throw an logic_error exception. + catch (std::exception& e) + { + getEM().undoChanges(); + std::cerr << "Cannot create column extent: " << e.what() + << std::endl; + return -1; + } + + getEM().confirmChanges(); + if (doVerbose()) + { + std::cout << "Extent is created, allocated size " << allocdSize + << " actual LBID " << lbid << std::endl; + std::cout << "For " << fileId << std::endl; + } + + // This is important part, it sets a status for specific extent + // as 'available' that means we can use it. + if (doVerbose()) + { + std::cout << "Setting a HWM for " << fileId << std::endl; + } + try + { + getEM().setLocalHWM(fileId.oid, fileId.partition, + fileId.segment, fileId.hwm, false, true); + } + catch (std::exception& e) + { + getEM().undoChanges(); + std::cerr << "Cannot set local HWM: " << e.what() << std::endl; + return -1; + } + getEM().confirmChanges(); + } + } + return 0; +} + +int32_t EMReBuilder::searchHWMInSegmentFile( + uint32_t oid, uint32_t dbRoot, uint32_t partition, uint32_t segment, + execplan::CalpontSystemCatalog::ColDataType colDataType, uint32_t colWidth, + uint64_t blockCount, bool isDict, uint64_t& hwm) +{ + std::unique_ptr chunkManagerWrapper; + try + { + if (isDict) + { + chunkManagerWrapper = std::unique_ptr( + new ChunkManagerWrapperDict(oid, dbRoot, partition, segment, + colDataType, colWidth)); + } + else + { + chunkManagerWrapper = std::unique_ptr( + new ChunkManagerWrapperColumn(oid, dbRoot, partition, segment, + colDataType, colWidth)); + } + } + catch (...) + { + std::cerr << "Cannot create ChunkManagerWrapper" << std::endl; + return -1; + } + + hwm = 0; + // Starting from the end. + // Note: This solves problem related to `bulk` insertion. + // Currently it could start to insert values from any block into empty + // column. + for (int32_t currentBlock = blockCount - 1; currentBlock >= 0; + --currentBlock) + { + // Read the block associated to HWM. + // The uncompressed chunk size is 512 * 1024 * 8, so for `abbreviated` + // extent one chunk will hold all blocks, therefore we need to + // decompress chunk only once, the cached chunk will be used later. + chunkManagerWrapper->readBlock(currentBlock); + if (!chunkManagerWrapper->isEmptyBlock()) + { + hwm = currentBlock; + break; + } + } + + return 0; +} + +void EMReBuilder::showExtentMap() +{ + std::cout << "range.start|range.size|fileId|blockOffset|HWM|partition|" + "segment|dbroot|width|status|hiVal|loVal|seqNum|isValid|" + << std::endl; + getEM().dumpTo(std::cout); +} + +// Check for hard-coded values which define dictionary file. +bool EMReBuilder::isDictFile( + execplan::CalpontSystemCatalog::ColDataType colDataType, uint64_t width) +{ + // Dictionary store extent has a width == 0. See more detais in + // `createDictStoreExtent` function. + return (colDataType == execplan::CalpontSystemCatalog::VARCHAR) && + (width == 0); +} + +int32_t EMReBuilder::initializeSystemExtents() +{ + if (!doDisplay()) + { + if (doVerbose()) + { + std::cout << "Initialize system extents from the initial state" + << std::endl; + } + try + { + getEM().loadFromBinaryBlob(reinterpret_cast( + BRM_saves_em_system_tables_blob)); + } + catch (...) + { + std::cerr << "Cannot initialize system extents from binary blob " + << std::endl; + return -1; + } + } + return 0; +} + +ChunkManagerWrapper::ChunkManagerWrapper( + uint32_t oid, uint32_t dbRoot, uint32_t partition, uint32_t segment, + execplan::CalpontSystemCatalog::ColDataType colDataType, uint32_t colWidth) + : oid(oid), dbRoot(dbRoot), partition(partition), segment(segment), + colDataType(colDataType), colWidth(colWidth), size(colWidth), + pFileOp(nullptr) +{ +} + +int32_t ChunkManagerWrapper::readBlock(uint32_t blockNumber) +{ + auto rc = chunkManager.readBlock(pFile, blockData, blockNumber); + if (rc != 0) + return rc; + return 0; +} + +ChunkManagerWrapperColumn::ChunkManagerWrapperColumn( + uint32_t oid, uint32_t dbRoot, uint32_t partition, uint32_t segment, + execplan::CalpontSystemCatalog::ColDataType colDataType, uint32_t colWidth) + : ChunkManagerWrapper(oid, dbRoot, partition, segment, colDataType, + colWidth) +{ + pFileOp = std::unique_ptr( + new WriteEngine::ColumnOpCompress1()); + chunkManager.fileOp(pFileOp.get()); + // Open compressed column segment file. We will read block by block + // from the compressed chunks. + pFile = chunkManager.getSegmentFilePtr(oid, dbRoot, partition, segment, + colDataType, colWidth, fileName, + "rb", size, false, false); + if (!pFile) + { + throw std::bad_alloc(); + } + + emptyValue = pFileOp->getEmptyRowValue(colDataType, colWidth); + midOffset = (WriteEngine::BYTE_PER_BLOCK / 2); + endOffset = (WriteEngine::BYTE_PER_BLOCK - colWidth); +} + +bool ChunkManagerWrapperColumn::isEmptyBlock() +{ + const uint8_t* beginValue = blockData; + const uint8_t* midValue = blockData + midOffset; + const uint8_t* endValue = blockData + endOffset; + + return isEmptyValue(beginValue) && isEmptyValue(midValue) && + isEmptyValue(endValue); +} + +// This function is copy pasted from `ColumnOp` file, unfortunately it's not +// possible to reuse it directly form `ColumnOp` without a creating a some kind +// of object which inherits from `ColumnOp`, probably this should be moved to +// utils and make a static as well, to be able to use it without creating an +// object. +bool ChunkManagerWrapperColumn::isEmptyValue(const uint8_t* value) const +{ + switch (colWidth) + { + case 1: + return *(uint8_t*) value == *(uint8_t*) emptyValue; + + case 2: + return *(uint16_t*) value == *(uint16_t*) emptyValue; + + case 4: + return *(uint32_t*) value == *(uint32_t*) emptyValue; + + case 8: + return *(uint64_t*) value == *(uint64_t*) emptyValue; + + case 16: + return *(uint128_t*) value == *(uint128_t*) emptyValue; + } + + return false; +} + +ChunkManagerWrapperDict::ChunkManagerWrapperDict( + uint32_t oid, uint32_t dbRoot, uint32_t partition, uint32_t segment, + execplan::CalpontSystemCatalog::ColDataType colDataType, uint32_t colWidth) + : ChunkManagerWrapper(oid, dbRoot, partition, segment, colDataType, + colWidth) +{ + pFileOp = std::unique_ptr( + new WriteEngine::DctnryCompress1()); + chunkManager.fileOp(pFileOp.get()); + // Open compressed dict segment file. + pFile = chunkManager.getSegmentFilePtr(oid, dbRoot, partition, segment, + colDataType, colWidth, fileName, + "rb", size, false, true); + if (!pFile) + { + throw std::bad_alloc(); + } + + auto dictBlockHeaderSize = + WriteEngine::HDR_UNIT_SIZE + WriteEngine::NEXT_PTR_BYTES + + WriteEngine::HDR_UNIT_SIZE + WriteEngine::HDR_UNIT_SIZE; + + emptyBlock = WriteEngine::BYTE_PER_BLOCK - dictBlockHeaderSize; +} + +bool ChunkManagerWrapperDict::isEmptyBlock() +{ + // Check that block is completely empty. + return (*(uint16_t*) blockData) == emptyBlock; +} + +std::ostream& operator<<(std::ostream& os, const FileId& fileID) +{ + os << "[OID: " << fileID.oid << ", partition: " << fileID.partition + << ", segment: " << fileID.segment << ", col width: " << fileID.colWidth + << ", lbid:" << fileID.lbid << ", hwm: " << fileID.hwm + << ", isDict: " << fileID.isDict << "]"; + return os; +} +} // namespace RebuildExtentMap diff --git a/tools/rebuildEM/rebuildEM.h b/tools/rebuildEM/rebuildEM.h new file mode 100644 index 000000000..03db4896d --- /dev/null +++ b/tools/rebuildEM/rebuildEM.h @@ -0,0 +1,230 @@ +/* Copyright (C) 2021 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#ifndef REBUILD_EM_H +#define REBUILD_EM_H + +#include +#include +#include + +#include "calpontsystemcatalog.h" +#include "extentmap.h" +#include "IDBPolicy.h" +#include "IDBFileSystem.h" +#include "idbcompress.h" +#include "blocksize.h" +#include "we_convertor.h" +#include "we_fileop.h" +#include "IDBPolicy.h" +#include "we_chunkmanager.h" + +using namespace idbdatafile; + +namespace RebuildExtentMap +{ +// This struct represents a FileId. For internal purpose only. +struct FileId +{ + FileId(uint32_t oid, uint32_t partition, uint32_t segment, + uint32_t colWidth, + execplan::CalpontSystemCatalog::ColDataType colDataType, + int64_t lbid, uint64_t hwm, bool isDict) + : oid(oid), partition(partition), segment(segment), colWidth(colWidth), + colDataType(colDataType), lbid(lbid), hwm(hwm), isDict(isDict) + { + } + + uint32_t oid; + uint32_t partition; + uint32_t segment; + uint32_t colWidth; + execplan::CalpontSystemCatalog::ColDataType colDataType; + int64_t lbid; + uint64_t hwm; + bool isDict; +}; +std::ostream& operator<<(std::ostream& os, const FileId& fileID); + +// This class represents extent map rebuilder. +class EMReBuilder +{ + public: + EMReBuilder(bool verbose, bool display) + : verbose(verbose), display(display) + { + // Initalize plugins. + IDBPolicy::init(true, false, "", 0); + } + ~EMReBuilder() = default; + + // Collects extents from the given DBRoot path. + int32_t collectExtents(const std::string& dbRootPath); + + // Clears collected extents. + void clear() { extentMap.clear(); } + + // Specifies whether we need verbose to output. + bool doVerbose() const { return verbose; } + + // Specifies whether we need just display a pipeline, but not actually run + // it. + bool doDisplay() const { return display; } + + // Returns the number of current DBRoot. + uint32_t getDBRoot() const { return dbRoot; } + + // Retunrs a reference to `ExtentMap` object. + BRM::ExtentMap& getEM() { return em; } + + // Checks if the given data specifies a dictionary file. + static bool + isDictFile(execplan::CalpontSystemCatalog::ColDataType colDataType, + uint64_t width); + + // Initializes system extents from the binary blob. + // This function solves the problem related to system segment files. + // Currently those files do not have file header, so we cannot + // get the data (like width, colType, lbid) to restore an extent for this + // particular segment file. The current approach is to keep a binary blob + // of initial state of the system extents. + // Returns -1 on error. + int32_t initializeSystemExtents(); + + // Rebuilds extent map from the collected map. + int32_t rebuildExtentMap(); + + // Search HWM in the given segment file. + int32_t searchHWMInSegmentFile( + uint32_t oid, uint32_t dbRoot, uint32_t partition, uint32_t segment, + execplan::CalpontSystemCatalog::ColDataType colDataType, + uint32_t width, uint64_t blocksCount, bool isDict, uint64_t& hwm); + + // Sets the dbroot to the given `number`. + void setDBRoot(uint32_t number) { dbRoot = number; } + + // Shows the extent map. + void showExtentMap(); + + private: + EMReBuilder(const EMReBuilder&) = delete; + EMReBuilder(EMReBuilder&&) = delete; + EMReBuilder& operator=(const EMReBuilder&) = delete; + EMReBuilder& operator=(EMReBuilder&&) = delete; + + // Collects the information for extent from the given file and stores + // it in `extentMap` set. + int32_t collectExtent(const std::string& fullFileName); + bool verbose; + bool display; + uint32_t dbRoot; + BRM::ExtentMap em; + std::vector extentMap; +}; + +// The base class aroud `ChunkManager` to read and write decompressed blocks +// from segment file. +class ChunkManagerWrapper +{ + public: + ChunkManagerWrapper( + uint32_t oid, uint32_t dbRoot, uint32_t partition, uint32_t segment, + execplan::CalpontSystemCatalog::ColDataType colDataType, + uint32_t colWidth); + + virtual ~ChunkManagerWrapper() = default; + ChunkManagerWrapper(const ChunkManagerWrapper& other) = delete; + ChunkManagerWrapper& operator=(const ChunkManagerWrapper& other) = delete; + ChunkManagerWrapper(ChunkManagerWrapper&& other) = delete; + ChunkManagerWrapper& operator=(ChunkManagerWrapper&& other) = delete; + + // Reads block, by given `blockNumber` from associated segment file and + // populates internal block buffer. + int32_t readBlock(uint32_t blockNumber); + + // Checks that last read block is empty. + virtual bool isEmptyBlock() = 0; + + protected: + uint32_t oid; + uint32_t dbRoot; + uint32_t partition; + uint32_t segment; + execplan::CalpontSystemCatalog::ColDataType colDataType; + uint32_t colWidth; + int32_t size; + std::string fileName; + std::unique_ptr pFileOp; + // Note: We cannot clear this pointer directly, because + // `ChunkManager` closes this file for us, otherwise we will get double + // free error. + IDBDataFile* pFile; + WriteEngine::ChunkManager chunkManager; + uint8_t blockData[WriteEngine::BYTE_PER_BLOCK]; +}; + +// Class to read decompressed blocks from column segment files. +class ChunkManagerWrapperColumn : public ChunkManagerWrapper +{ + public: + ChunkManagerWrapperColumn( + uint32_t oid, uint32_t dbRoot, uint32_t partition, uint32_t segment, + execplan::CalpontSystemCatalog::ColDataType colDataType, + uint32_t colWidth); + + ~ChunkManagerWrapperColumn() = default; + ChunkManagerWrapperColumn(const ChunkManagerWrapperColumn& other) = delete; + ChunkManagerWrapperColumn& + operator=(const ChunkManagerWrapperColumn& other) = delete; + ChunkManagerWrapperColumn(ChunkManagerWrapperColumn&& other) = delete; + ChunkManagerWrapperColumn& + operator=(ChunkManagerWrapperColumn&& other) = delete; + + bool isEmptyBlock() override; + bool isEmptyValue(const uint8_t* value) const; + + private: + const uint8_t* emptyValue; + uint32_t midOffset; + uint32_t endOffset; +}; + +// Class to read decompressed blocks from dict segment files. +class ChunkManagerWrapperDict : public ChunkManagerWrapper +{ + public: + ChunkManagerWrapperDict( + uint32_t oid, uint32_t dbRoot, uint32_t partition, uint32_t segment, + execplan::CalpontSystemCatalog::ColDataType colDataType, + uint32_t colWidth); + + ~ChunkManagerWrapperDict() = default; + ChunkManagerWrapperDict(const ChunkManagerWrapperDict& other) = delete; + ChunkManagerWrapperDict& + operator=(const ChunkManagerWrapperDict& other) = delete; + ChunkManagerWrapperDict(ChunkManagerWrapperDict&& other) = delete; + ChunkManagerWrapperDict& + operator=(ChunkManagerWrapperDict&& other) = delete; + + bool isEmptyBlock() override; + + private: + uint32_t emptyBlock; +}; + +} // namespace RebuildExtentMap +#endif diff --git a/utils/compress/idbcompress.cpp b/utils/compress/idbcompress.cpp index ae8e04b4d..993a0c224 100644 --- a/utils/compress/idbcompress.cpp +++ b/utils/compress/idbcompress.cpp @@ -38,6 +38,7 @@ namespace const uint64_t MAGIC_NUMBER = 0xfdc119a384d0778eULL; const uint64_t VERSION_NUM1 = 1; const uint64_t VERSION_NUM2 = 2; +const uint64_t VERSION_NUM3 = 3; const int COMPRESSED_CHUNK_INCREMENT_SIZE = 8192; const int PTR_SECTION_OFFSET = compress::IDBCompressInterface::HDR_BUF_LEN; @@ -71,6 +72,8 @@ struct CompressedDBFileHeader uint64_t fBlockCount; uint64_t fColumnWidth; execplan::CalpontSystemCatalog::ColDataType fColDataType; + uint64_t fLBID0; + uint64_t fLBID1; }; // Make the header to be 4K, regardless number of fields being defined/used in header. @@ -80,18 +83,6 @@ union CompressedDBFileHeaderBlock char fDummy[compress::IDBCompressInterface::HDR_BUF_LEN]; }; -void initCompressedDBFileHeader(void* hdrBuf, int compressionType, int hdrSize) -{ - CompressedDBFileHeaderBlock* hdr = reinterpret_cast(hdrBuf); - hdr->fHeader.fMagicNumber = MAGIC_NUMBER; - hdr->fHeader.fVersionNum = VERSION_NUM2; - hdr->fHeader.fCompressionType = compressionType; - hdr->fHeader.fBlockCount = 0; - hdr->fHeader.fHeaderSize = hdrSize; - hdr->fHeader.fColumnWidth = 0; - hdr->fHeader.fColDataType = execplan::CalpontSystemCatalog::ColDataType::UNDEFINED; -} - void initCompressedDBFileHeader( void* hdrBuf, uint32_t columnWidth, execplan::CalpontSystemCatalog::ColDataType colDataType, @@ -99,12 +90,14 @@ void initCompressedDBFileHeader( { CompressedDBFileHeaderBlock* hdr = reinterpret_cast(hdrBuf); hdr->fHeader.fMagicNumber = MAGIC_NUMBER; - hdr->fHeader.fVersionNum = VERSION_NUM2; + hdr->fHeader.fVersionNum = VERSION_NUM3; hdr->fHeader.fCompressionType = compressionType; hdr->fHeader.fBlockCount = 0; hdr->fHeader.fHeaderSize = hdrSize; hdr->fHeader.fColumnWidth = columnWidth; hdr->fHeader.fColDataType = colDataType; + hdr->fHeader.fLBID0 = 0; + hdr->fHeader.fLBID1 = 0; } } // namespace @@ -351,23 +344,18 @@ void IDBCompressInterface::storePtrs(const std::vector& ptrs, void* pt storePtrs(ptrs, reinterpret_cast(ptrBuf) + HDR_BUF_LEN, HDR_BUF_LEN); } -//------------------------------------------------------------------------------ -// Initialize the header blocks to be written at the start of a column file. -//------------------------------------------------------------------------------ -void IDBCompressInterface::initHdr(void* hdrBuf, int compressionType) const -{ - memset(hdrBuf, 0, HDR_BUF_LEN * 2); - initCompressedDBFileHeader(hdrBuf, compressionType, HDR_BUF_LEN * 2); -} - //------------------------------------------------------------------------------ // Initialize the header blocks to be written at the start of a dictionary file. //------------------------------------------------------------------------------ -void IDBCompressInterface::initHdr(void* hdrBuf, void* ptrBuf, int compressionType, int hdrSize) const +void IDBCompressInterface::initHdr( + void* hdrBuf, void* ptrBuf, uint32_t colWidth, + execplan::CalpontSystemCatalog::ColDataType columnType, + int compressionType, int hdrSize) const { memset(hdrBuf, 0, HDR_BUF_LEN); memset(ptrBuf, 0, hdrSize - HDR_BUF_LEN); - initCompressedDBFileHeader(hdrBuf, compressionType, hdrSize); + initCompressedDBFileHeader(hdrBuf, colWidth, columnType, compressionType, + hdrSize); } //------------------------------------------------------------------------------ @@ -383,6 +371,15 @@ void IDBCompressInterface::initHdr( compressionType, HDR_BUF_LEN * 2); } +//------------------------------------------------------------------------------ +// Get the header's version number +//------------------------------------------------------------------------------ +uint64_t IDBCompressInterface::getVersionNumber(const void* hdrBuf) const +{ + return ( + reinterpret_cast(hdrBuf)->fVersionNum); +} + //------------------------------------------------------------------------------ // Set the file's block count //------------------------------------------------------------------------------ @@ -415,6 +412,59 @@ uint64_t IDBCompressInterface::getHdrSize(const void* hdrBuf) const return (reinterpret_cast(hdrBuf)->fHeaderSize); } +//------------------------------------------------------------------------------ +// Get column type +//----------------------------------------------------------------------------- +execplan::CalpontSystemCatalog::ColDataType +IDBCompressInterface::getColDataType(const void* hdrBuf) const +{ + return ( + reinterpret_cast(hdrBuf)->fColDataType); +} + +//------------------------------------------------------------------------------ +// Get column width +//------------------------------------------------------------------------------ +uint64_t IDBCompressInterface::getColumnWidth(const void* hdrBuf) const +{ + return ( + reinterpret_cast(hdrBuf)->fColumnWidth); +} + +//------------------------------------------------------------------------------ +// Get start LBID +//------------------------------------------------------------------------------ +uint64_t IDBCompressInterface::getLBID0(const void* hdrBuf) const +{ + return (reinterpret_cast(hdrBuf)->fLBID0); +} + +//------------------------------------------------------------------------------ +// Set start LBID +//------------------------------------------------------------------------------ +void IDBCompressInterface::setLBID0(void* hdrBuf, uint64_t lbid) const +{ + if (lbid) + reinterpret_cast(hdrBuf)->fLBID0 = lbid; +} + +//------------------------------------------------------------------------------ +// Get start LBID +//------------------------------------------------------------------------------ +uint64_t IDBCompressInterface::getLBID1(const void* hdrBuf) const +{ + return (reinterpret_cast(hdrBuf)->fLBID1); +} + +//------------------------------------------------------------------------------ +// Set start LBID +//------------------------------------------------------------------------------ +void IDBCompressInterface::setLBID1(void* hdrBuf, uint64_t lbid) const +{ + if (lbid) + reinterpret_cast(hdrBuf)->fLBID1 = lbid; +} + //------------------------------------------------------------------------------ // Calculates the chunk and block offset within the chunk for the specified // block number. diff --git a/utils/compress/idbcompress.h b/utils/compress/idbcompress.h index 523ca0a68..196254ffe 100644 --- a/utils/compress/idbcompress.h +++ b/utils/compress/idbcompress.h @@ -101,20 +101,15 @@ public: */ EXPORT int uncompress(const char* in, size_t inLen, char* out) const; - /** - * Initialize header buffer at start of compressed db file. - * - * @warning hdrBuf must be at least HDR_BUF_LEN*2 bytes - */ - EXPORT void initHdr(void* hdrBuf, int compressionType) const; - /** * Initialize header buffer at start of compressed db file. * * @warning hdrBuf must be at least HDR_BUF_LEN bytes * @warning ptrBuf must be at least (hdrSize-HDR_BUF_LEN) bytes */ - EXPORT void initHdr(void* hdrBuf, void* ptrBuf, int compressionType, int hdrSize) const; + EXPORT void initHdr(void* hdrBuf, void* ptrBuf, uint32_t columnWidht, + execplan::CalpontSystemCatalog::ColDataType columnType, + int compressionType, int hdrSize) const; /** * Initialize header buffer at start of compressed db file. @@ -199,6 +194,11 @@ public: /* * Mutator methods for the block count in the file */ + /** + * getVersionNumber + */ + EXPORT uint64_t getVersionNumber(const void* hdrBuf) const; + /** * setBlockCount */ @@ -222,6 +222,37 @@ public: */ EXPORT uint64_t getHdrSize(const void* hdrBuf) const; + /** + * getColumnType + */ + EXPORT execplan::CalpontSystemCatalog::ColDataType + getColDataType(const void* hdrBuf) const; + + /** + * getColumnWidth + */ + EXPORT uint64_t getColumnWidth(const void* hdrBuf) const; + + /** + * getLBID + */ + EXPORT uint64_t getLBID0(const void* hdrBuf) const; + + /** + * setBID + */ + EXPORT void setLBID0(void* hdrBuf, uint64_t lbid) const; + + /** + * getLBID + */ + EXPORT uint64_t getLBID1(const void* hdrBuf) const; + + /** + * setBID + */ + EXPORT void setLBID1(void* hdrBuf, uint64_t lbid) const; + /** * Mutator methods for the user padding bytes */ @@ -286,8 +317,7 @@ inline int IDBCompressInterface::uncompress(const char* in, size_t inLen, char* { return 0; } -inline void IDBCompressInterface::initHdr(void*, int) const {} -inline void IDBCompressInterface::initHdr(void*, void*, int, int) const {} +inline void IDBCompressInterface::initHdr(void*, void*, uint32_t, execplan::CalpontSystemCatalog::ColDataType, int, int) const {} inline void initHdr(void*, uint32_t, execplan::CalpontSystemCatalog::ColDataType, int) const {} inline int IDBCompressInterface::verifyHdr(const void*) const { @@ -317,6 +347,11 @@ inline int IDBCompressInterface::padCompressedChunks(unsigned char* buf, unsigne { return -1; } +inline uint64_t +IDBCompressInterface::getVersionNumber(const void* hdrBuf) const +{ + return 0; +} inline void IDBCompressInterface::setBlockCount(void* hdrBuf, uint64_t count) const {} inline uint64_t IDBCompressInterface::getBlockCount(const void* hdrBuf) const { @@ -327,10 +362,20 @@ inline uint64_t IDBCompressInterface::getHdrSize(const void*) const { return 0; } +inline execplan::CalpontSystemCatalog::ColDataType +IDBCompressInterface::getColDataType(const void* hdrBuf) const +{ + return execplan::CalpontSystemCatalog::ColDataType::UNDEFINED; +} +inline uint64_t getColumnWidth(const void* hdrBuf) const { return 0; } inline uint64_t IDBCompressInterface::maxCompressedSize(uint64_t uncompSize) { return uncompSize; } +inline uint64_t getLBID0(const void* hdrBuf) const { return 0; } +void setLBID0(void* hdrBuf, uint64_t lbid) const {} +inline uint64_t getLBID1(const void* hdrBuf) const { return 0; } +void setLBID1(void* hdrBuf, uint64_t lbid) const {} inline bool IDBCompressInterface::getUncompressedSize(char* in, size_t inLen, size_t* outLen) { return false; diff --git a/versioning/BRM/extentmap.cpp b/versioning/BRM/extentmap.cpp index bb48844b8..dec8c1c02 100644 --- a/versioning/BRM/extentmap.cpp +++ b/versioning/BRM/extentmap.cpp @@ -1271,7 +1271,8 @@ void ExtentMap::reserveLBIDRange(LBID_t start, uint8_t size) */ -void ExtentMap::loadVersion4or5(IDBDataFile* in, bool upgradeV4ToV5) +template +void ExtentMap::loadVersion4or5(T* in, bool upgradeV4ToV5) { int emNumElements = 0, flNumElements = 0; @@ -1456,20 +1457,9 @@ void ExtentMap::load(const string& filename, bool fixFL) try { - int emVersion = 0; - int bytes = in->read((char*) &emVersion, sizeof(int)); - - if (bytes == (int) sizeof(int) && - (emVersion == EM_MAGIC_V4 || emVersion == EM_MAGIC_V5)) - { - loadVersion4or5(in.get(), emVersion == EM_MAGIC_V4); - } - else - { - log("ExtentMap::load(): That file is not a valid ExtentMap image"); - throw runtime_error("ExtentMap::load(): That file is not a valid ExtentMap image"); - } + load(in.get()); } + catch (...) { releaseFreeList(WRITE); @@ -1482,6 +1472,80 @@ void ExtentMap::load(const string& filename, bool fixFL) // checkConsistency(); } +// This is a quick workaround, to be able to initialize initial system tables +// from binary blob. +// This should be updated, probably we need inherit from `IDBDataFile`. +struct EMBinaryReader +{ + EMBinaryReader(const char* data) : src(data) {} + + ssize_t read(char* dst, size_t size) + { + memcpy(dst, src, size); + src += size; + return size; + } + + const char* src; +}; + +void ExtentMap::loadFromBinaryBlob(const char* blob) +{ + grabEMEntryTable(WRITE); + + try + { + grabFreeList(WRITE); + } + catch (...) + { + releaseEMEntryTable(WRITE); + throw; + } + + try + { + EMBinaryReader emBinReader(blob); + load(&emBinReader); + } + catch (...) + { + releaseFreeList(WRITE); + releaseEMEntryTable(WRITE); + throw; + } + + releaseFreeList(WRITE); + releaseEMEntryTable(WRITE); +} + +template void ExtentMap::load(T* in) +{ + if (!in) + return; + + try + { + int emVersion = 0; + int bytes = in->read((char*) &emVersion, sizeof(int)); + + if (bytes == (int) sizeof(int) && + (emVersion == EM_MAGIC_V4 || emVersion == EM_MAGIC_V5)) + { + loadVersion4or5(in, emVersion == EM_MAGIC_V4); + } + else + { + log("ExtentMap::load(): That file is not a valid ExtentMap image"); + throw runtime_error("ExtentMap::load(): That file is not a valid ExtentMap image"); + } + } + catch (...) + { + throw; + } +} + void ExtentMap::save(const string& filename) { #ifdef BRM_INFO diff --git a/versioning/BRM/extentmap.h b/versioning/BRM/extentmap.h index 8f16c3cee..541709319 100644 --- a/versioning/BRM/extentmap.h +++ b/versioning/BRM/extentmap.h @@ -344,6 +344,16 @@ public: */ EXPORT void load(const std::string& filename, bool fixFL = false); + /** @brief Loads the ExtentMap entries from a binayr blob. + * + * Loads the ExtentMap entries from a file. This will + * clear out any existing entries. The intention is that before + * the system starts, an external tool instantiates a single Extent + * Map and loads the stored entries. + * @param pointer to a binary blob. + */ + EXPORT void loadFromBinaryBlob(const char* blob); + /** @brief Saves the ExtentMap entries to a file * * Saves the ExtentMap entries to a file. @@ -1002,13 +1012,14 @@ private: int _markInvalid(const LBID_t lbid, const execplan::CalpontSystemCatalog::ColDataType colDataType); + template void load(T* in); /** @brief Loads the extent map from a file into memory. * * @param in (in) the file to load the extent map from. * @param upgradeV4ToV5 (in) flag indicating whether we are upgrading * extent map from v4 to v5. */ - void loadVersion4or5(idbdatafile::IDBDataFile* in, bool upgradeV4ToV5); + template void loadVersion4or5(T* in, bool upgradeV4ToV5); ExtentMapImpl* fPExtMapImpl; FreeListImpl* fPFreeListImpl; diff --git a/writeengine/bulk/we_colbufcompressed.cpp b/writeengine/bulk/we_colbufcompressed.cpp index 9e1a4f949..ef1c66ed9 100644 --- a/writeengine/bulk/we_colbufcompressed.cpp +++ b/writeengine/bulk/we_colbufcompressed.cpp @@ -582,11 +582,23 @@ int ColumnBufferCompressed::saveCompressionHeaders( ) { // Construct the header records char hdrBuf[IDBCompressInterface::HDR_BUF_LEN * 2]; + RETURN_ON_ERROR(fColInfo->colOp->readHeaders(fFile, hdrBuf)); + + auto lbid = fCompressor->getLBID0(hdrBuf); fCompressor->initHdr(hdrBuf, fColInfo->column.width, fColInfo->column.dataType, fColInfo->column.compressionType); fCompressor->setBlockCount(hdrBuf, (fColInfo->getFileSize() / BYTE_PER_BLOCK) ); + if (lbid) + { + fCompressor->setLBID0(hdrBuf, lbid); + fCompressor->setLBID1(hdrBuf, fColInfo->getLastUpdatedLBID()); + } + else + { + fCompressor->setLBID0(hdrBuf, fColInfo->getLastUpdatedLBID()); + } std::vector ptrs; diff --git a/writeengine/bulk/we_columninfo.cpp b/writeengine/bulk/we_columninfo.cpp index f4a157a76..69bf37fef 100644 --- a/writeengine/bulk/we_columninfo.cpp +++ b/writeengine/bulk/we_columninfo.cpp @@ -146,6 +146,7 @@ ColumnInfo::ColumnInfo(Log* logger, fLog(logger), fDelayedFileStartBlksSkipped(0), fSavedLbid(0), + fLastUpdatedLbid(0), fSizeWrittenStart(0), fSizeWritten(0), fLastInputRowInCurrentExtent(0), @@ -749,6 +750,9 @@ int ColumnInfo::extendColumnNewExtent( "; file-" << segFileNew; fLog->logMsg( oss.str(), MSGLVL_INFO2 ); + // Update lbid. + fLastUpdatedLbid = startLbid; + // Save the LBID with our CP extent info, so that we can update extent map if (saveLBIDForCP) { @@ -1397,6 +1401,9 @@ int ColumnInfo::getHWMInfoForBRM( BRMReporter& brmReporter ) return entriesAdded; } +// Returns last updated LBID. +BRM::LBID_t ColumnInfo::getLastUpdatedLBID() const { return fLastUpdatedLbid; } + //------------------------------------------------------------------------------ // Setup initial extent we will begin loading at start of import. // DBRoot, partition, segment, etc for the starting extent are specified. @@ -1484,6 +1491,7 @@ int ColumnInfo::setupInitialColumnExtent( } fSavedLbid = lbid; + fLastUpdatedLbid = lbid; if (bSkippedToNewExtent) oldHwm = hwm; diff --git a/writeengine/bulk/we_columninfo.h b/writeengine/bulk/we_columninfo.h index f3bb8f23e..dbae719b5 100644 --- a/writeengine/bulk/we_columninfo.h +++ b/writeengine/bulk/we_columninfo.h @@ -257,6 +257,10 @@ public: */ void getSegFileInfo( DBRootExtentInfo& fileInfo ); + /** @brief Get last updated saved LBID. + */ + BRM::LBID_t getLastUpdatedLBID() const; + /** @brief Initialize autoincrement value from the current "next" value * taken from the system catalog. */ @@ -463,6 +467,10 @@ protected: // have written additional rows to that extent as part of an import. BRM::LBID_t fSavedLbid; + // The last updated LBID. In case `bulk` creates a new extent with new LBID + // we have to initialize file header with created LBID. + BRM::LBID_t fLastUpdatedLbid; + // Size of a segment file (in bytes) when the file is opened // to add the next extent. // For compressed data files, this is the "raw" data byte count, @@ -487,6 +495,8 @@ protected: long long fMaxNumRowsPerSegFile; // Max num rows per segment file Dctnry* fStore; // Corresponding dctnry store file + + // For autoincrement column only... Tracks latest autoincrement value used long long fAutoIncLastValue; diff --git a/writeengine/dictionary/we_dctnry.cpp b/writeengine/dictionary/we_dctnry.cpp index ac54bba2c..7bcfc48c2 100644 --- a/writeengine/dictionary/we_dctnry.cpp +++ b/writeengine/dictionary/we_dctnry.cpp @@ -206,16 +206,31 @@ int Dctnry::createDctnry( const OID& dctnryOID, int colWidth, if (flag) { + // Allocate extent before file creation. + // If we got an error while allocating dictionary store extent, + // we do not need to create/close the file, because it was not created + // yet. This logic is the same as column segment file creation - at + // first we allocate an extent, then we create a segment file. + rc = BRMWrapper::getInstance()->allocateDictStoreExtent( + (OID) dctnryOID, dbRoot, partition, segment, startLbid, allocSize); + + if (rc != NO_ERROR) + { + return rc; + } + m_dctnryOID = dctnryOID; m_partition = partition; m_segment = segment; m_dbRoot = dbRoot; + RETURN_ON_ERROR( ( rc = oid2FileName( m_dctnryOID, fileName, true, m_dbRoot, m_partition, m_segment ) ) ); m_segFileName = fileName; // if obsolete file exists, "w+b" will truncate and write over - m_dFile = createDctnryFile(fileName, colWidth, "w+b", DEFAULT_BUFSIZ); + m_dFile = createDctnryFile(fileName, colWidth, "w+b", DEFAULT_BUFSIZ, + startLbid); { // We presume the path will contain / @@ -228,21 +243,17 @@ int Dctnry::createDctnry( const OID& dctnryOID, int colWidth, } else { - RETURN_ON_ERROR( setFileOffset(m_dFile, 0, SEEK_END) ); - } + rc = BRMWrapper::getInstance()->allocateDictStoreExtent( + (OID) m_dctnryOID, m_dbRoot, m_partition, m_segment, startLbid, + allocSize); - rc = BRMWrapper::getInstance()->allocateDictStoreExtent( - (OID)m_dctnryOID, m_dbRoot, m_partition, m_segment, - startLbid, allocSize); - - if (rc != NO_ERROR) - { - if (flag) + if (rc != NO_ERROR) { - closeDctnryFile(false, oids); + return rc; } - return rc; + RETURN_ON_ERROR(setFileOffset(m_dFile, 0, SEEK_END)); + } // We allocate a full extent from BRM, but only write an abbreviated 256K @@ -278,8 +289,8 @@ int Dctnry::createDctnry( const OID& dctnryOID, int colWidth, m_dctnryHeader2, m_totalHdrBytes, false, - true ); // explicitly optimize - + true, // explicitly optimize + startLbid ); if (rc != NO_ERROR) { if (flag) @@ -1488,8 +1499,9 @@ int Dctnry::updateDctnry(unsigned char* sigValue, int& sigSize, * open dictionary file ******************************************************************************/ IDBDataFile* Dctnry::createDctnryFile( - const char* name, int, const char* mode, int ioBuffSize) + const char* name, int, const char* mode, int ioBuffSize, LBID_t lbid) { + (void) lbid; return openFile(name, mode, ioBuffSize, false); } diff --git a/writeengine/dictionary/we_dctnry.h b/writeengine/dictionary/we_dctnry.h index 536022bf7..fd9d1bf3a 100644 --- a/writeengine/dictionary/we_dctnry.h +++ b/writeengine/dictionary/we_dctnry.h @@ -305,7 +305,8 @@ protected: // methods to be overriden by compression classes // (width argument in createDctnryFile() is string width, not token width) virtual IDBDataFile* createDctnryFile(const char* name, int width, - const char* mode, int ioBuffSize); + const char* mode, int ioBuffSize, + BRM::LBID_t lbid = -1); virtual IDBDataFile* openDctnryFile(bool useTmpSuffix); virtual void closeDctnryFile(bool doFlush, std::map& oids); virtual int numOfBlocksInFile(); diff --git a/writeengine/shared/we_chunkmanager.cpp b/writeengine/shared/we_chunkmanager.cpp index 7dd7f1027..bdcdd7f3e 100644 --- a/writeengine/shared/we_chunkmanager.cpp +++ b/writeengine/shared/we_chunkmanager.cpp @@ -290,6 +290,25 @@ IDBDataFile* ChunkManager::getFilePtr(const FID& fid, return (fileData ? fileData->fFilePtr : NULL); } +//------------------------------------------------------------------------------ +// Get/Return IDBDataFile* for specified OID, root, partition, and segment. +// Function is to be used to open column/dict segment file. +// If the IDBDataFile* is not found, then a segment file will be opened using +// the mode (mode) and I/O buffer size (size) that is given. Name of the +// resulting file is returned in filename. +//------------------------------------------------------------------------------ +IDBDataFile* ChunkManager::getSegmentFilePtr( + FID& fid, uint16_t root, uint32_t partition, uint16_t segment, + execplan::CalpontSystemCatalog::ColDataType colDataType, uint32_t colWidth, + std::string& filename, const char* mode, int32_t size, bool useTmpSuffix, + bool isDict) const +{ + CompFileData* fileData = + getFileData(fid, root, partition, segment, filename, mode, size, + colDataType, colWidth, useTmpSuffix, isDict); + return (fileData ? fileData->fFilePtr : NULL); +} + //------------------------------------------------------------------------------ // Get/Return CompFileData* for specified column OID, root, partition, and // segment. If the IDBDataFile* is not found, then a segment file will be opened @@ -411,7 +430,8 @@ IDBDataFile* ChunkManager::createDctnryFile(const FID& fid, uint16_t segment, const char* filename, const char* mode, - int size) + int size, + BRM::LBID_t lbid) { FileID fileID(fid, root, partition, segment); CompFileData* fileData = new CompFileData(fileID, fid, CalpontSystemCatalog::VARCHAR, width); @@ -440,8 +460,13 @@ IDBDataFile* ChunkManager::createDctnryFile(const FID& fid, fileData->fFileHeader.fLongPtrSectData.reset(fileData->fFileHeader.fPtrSection); } - fCompressor.initHdr(fileData->fFileHeader.fControlData, fileData->fFileHeader.fPtrSection, + // Dictionary store extent width == 0. See more details in function + // `createDictStoreExtent`. + fCompressor.initHdr(fileData->fFileHeader.fControlData, + fileData->fFileHeader.fPtrSection, + /*colWidth=*/0, fileData->fColDataType, fFileOp->compressionType(), hdrSize); + fCompressor.setLBID0(fileData->fFileHeader.fControlData, lbid); if (writeHeader(fileData, __LINE__) != NO_ERROR) { @@ -1376,7 +1401,8 @@ int ChunkManager::expandAbbrevColumnExtent(IDBDataFile* pFile, const uint8_t* em // Increment the block count stored in the chunk header used to track how many // blocks are allocated to the corresponding segment file. //------------------------------------------------------------------------------ -int ChunkManager::updateColumnExtent(IDBDataFile* pFile, int addBlockCount) +// same here as for dict. +int ChunkManager::updateColumnExtent(IDBDataFile* pFile, int addBlockCount, int64_t lbid) { map::iterator i = fFilePtrMap.find(pFile); @@ -1397,6 +1423,7 @@ int ChunkManager::updateColumnExtent(IDBDataFile* pFile, int addBlockCount) int rc = NO_ERROR; char* hdr = pFileData->fFileHeader.fControlData; fCompressor.setBlockCount(hdr, fCompressor.getBlockCount(hdr) + addBlockCount); + fCompressor.setLBID1(hdr, lbid); ChunkData* chunkData = (pFileData)->findChunk(0); if (chunkData != NULL) @@ -1428,7 +1455,8 @@ int ChunkManager::updateColumnExtent(IDBDataFile* pFile, int addBlockCount) // Increment the block count stored in the chunk header used to track how many // blocks are allocated to the corresponding segment file. //------------------------------------------------------------------------------ -int ChunkManager::updateDctnryExtent(IDBDataFile* pFile, int addBlockCount) +int ChunkManager::updateDctnryExtent(IDBDataFile* pFile, int addBlockCount, + BRM::LBID_t lbid) { map::iterator i = fFilePtrMap.find(pFile); @@ -1485,6 +1513,8 @@ int ChunkManager::updateDctnryExtent(IDBDataFile* pFile, int addBlockCount) if (rc == NO_ERROR) fCompressor.setBlockCount(hdr, fCompressor.getBlockCount(hdr) + addBlockCount); + if (currentBlockCount) + fCompressor.setLBID1(hdr, lbid); return rc; } diff --git a/writeengine/shared/we_chunkmanager.h b/writeengine/shared/we_chunkmanager.h index d2eaaafac..b79a9b377 100644 --- a/writeengine/shared/we_chunkmanager.h +++ b/writeengine/shared/we_chunkmanager.h @@ -189,6 +189,14 @@ public: int size, bool useTmpSuffix) const; + // @brief Retrieve a file pointer in the chunk manager. + // for column/dict segment file + IDBDataFile* getSegmentFilePtr( + FID& fid, uint16_t root, uint32_t partition, uint16_t segment, + execplan::CalpontSystemCatalog::ColDataType colDataType, + uint32_t colWidth, std::string& filename, const char* mode, + int32_t size, bool useTmpSuffix, bool isDict) const; + // @brief Create a compressed dictionary file with an appropriate header. IDBDataFile* createDctnryFile(const FID& fid, int64_t width, @@ -197,7 +205,8 @@ public: uint16_t segment, const char* filename, const char* mode, - int size); + int size, + int64_t lbid); // @brief Read a block from pFile at offset fbo. // The data may copied from memory if the chunk it belongs to is already available. @@ -217,10 +226,12 @@ public: int expandAbbrevColumnExtent(IDBDataFile* pFile, const uint8_t* emptyVal, int width); // @brief Update column extent - int updateColumnExtent(IDBDataFile* pFile, int addBlockCount); + int updateColumnExtent(IDBDataFile* pFile, int addBlockCount, + int64_t lbid); // @brief Update dictionary extent - int updateDctnryExtent(IDBDataFile* pFile, int addBlockCount); + int updateDctnryExtent(IDBDataFile* pFile, int addBlockCount, + int64_t lbid); // @brief Read in n continuous blocks to read buffer. // for backing up blocks to version buffer diff --git a/writeengine/shared/we_fileop.cpp b/writeengine/shared/we_fileop.cpp index 26acc7612..02dc6610e 100644 --- a/writeengine/shared/we_fileop.cpp +++ b/writeengine/shared/we_fileop.cpp @@ -163,7 +163,8 @@ int FileOp::createDir( const char* dirName, mode_t mode ) const int FileOp::createFile( const char* fileName, int numOfBlock, const uint8_t* emptyVal, int width, execplan::CalpontSystemCatalog::ColDataType colDataType, - uint16_t dbRoot ) + uint16_t dbRoot, + BRM::LBID_t startLbid ) { IDBDataFile* pFile = IDBDataFile::open( @@ -185,6 +186,7 @@ int FileOp::createFile( const char* fileName, int numOfBlock, numOfBlock, emptyVal, width, + startLbid, colDataType ); } else @@ -284,7 +286,8 @@ int FileOp::createFile(FID fid, //timer.stop( "allocateColExtent" ); - return createFile( fileName, totalSize, emptyVal, width, colDataType, dbRoot ); + return createFile(fileName, totalSize, emptyVal, width, colDataType, + dbRoot, startLbid); } /*********************************************************** @@ -815,6 +818,7 @@ int FileOp::extendFile( { IDBCompressInterface compressor; compressor.initHdr(hdrs, width, colDataType, m_compressionType); + compressor.setLBID0(hdrs, startLbid); } } @@ -849,7 +853,8 @@ int FileOp::extendFile( newFile, // new or existing file false, // don't expand; new extent false, // add full (not abbreviated) extent - true); // try to optimize extent creation + true, // try to optimize extent creation + startLbid ); return rc; } @@ -973,6 +978,7 @@ int FileOp::addExtentExactFile( { IDBCompressInterface compressor; compressor.initHdr(hdrs, width, colDataType, m_compressionType); + compressor.setLBID0(hdrs, startLbid); } } @@ -1007,7 +1013,8 @@ int FileOp::addExtentExactFile( colDataType, newFile, // new or existing file false, // don't expand; new extent - false ); // add full (not abbreviated) extent + false, // add full (not abbreviated) extent + startLbid ); closeFile( pFile ); return rc; @@ -1052,13 +1059,15 @@ int FileOp::initColumnExtent( bool bNewFile, bool bExpandExtent, bool bAbbrevExtent, - bool bOptExtension) + bool bOptExtension, + int64_t lbid) { if ((bNewFile) && (m_compressionType)) { char hdrs[IDBCompressInterface::HDR_BUF_LEN * 2]; IDBCompressInterface compressor; compressor.initHdr(hdrs, width, colDataType, m_compressionType); + compressor.setLBID0(hdrs, lbid); if (bAbbrevExtent) compressor.setBlockCount(hdrs, nBlocks); @@ -1072,7 +1081,7 @@ int FileOp::initColumnExtent( //@Bug 3219. update the compression header after the extent is expanded. if ((!bNewFile) && (m_compressionType) && (bExpandExtent)) { - updateColumnExtent(pFile, nBlocks); + updateColumnExtent(pFile, nBlocks, lbid); } // @bug 2378. Synchronize here to avoid write buffer pile up too much, @@ -1188,7 +1197,7 @@ int FileOp::initColumnExtent( //@Bug 3219. update the compression header after the extent is expanded. if ((!bNewFile) && (m_compressionType) && (bExpandExtent)) { - updateColumnExtent(pFile, nBlocks); + updateColumnExtent(pFile, nBlocks, lbid); } // @bug 2378. Synchronize here to avoid write buffer pile up too much, @@ -1229,6 +1238,7 @@ int FileOp::initAbbrevCompColumnExtent( int nBlocks, const uint8_t* emptyVal, int width, + BRM::LBID_t startLBID, execplan::CalpontSystemCatalog::ColDataType colDataType) { // Reserve disk space for optimized abbreviated extent @@ -1241,7 +1251,8 @@ int FileOp::initAbbrevCompColumnExtent( true, // new file false, // don't expand; add new extent true, // add abbreviated extent - true); // optimize the initial extent + true, // optimize the initial extent + startLBID); if (rc != NO_ERROR) { return rc; @@ -1257,6 +1268,7 @@ int FileOp::initAbbrevCompColumnExtent( INITIAL_EXTENT_ROWS_TO_DISK, emptyVal, width, + startLBID, colDataType, hdrs ); @@ -1292,6 +1304,7 @@ int FileOp::writeInitialCompColumnChunk( int nRows, const uint8_t* emptyVal, int width, + BRM::LBID_t startLBID, execplan::CalpontSystemCatalog::ColDataType colDataType, char* hdrs) { @@ -1336,6 +1349,7 @@ int FileOp::writeInitialCompColumnChunk( compressor.initHdr(hdrs, width, colDataType, m_compressionType); compressor.setBlockCount(hdrs, nBlocksAllocated); + compressor.setLBID0(hdrs, startLBID); // Store compression pointers in the header std::vector ptrs; @@ -1841,13 +1855,14 @@ int FileOp::initDctnryExtent( unsigned char* blockHdrInit, int blockHdrInitSize, bool bExpandExtent, - bool bOptExtension ) + bool bOptExtension, + int64_t lbid) { // @bug5769 Don't initialize extents or truncate db files on HDFS if (idbdatafile::IDBPolicy::useHdfs()) { if (m_compressionType) - updateDctnryExtent(pFile, nBlocks); + updateDctnryExtent(pFile, nBlocks, lbid); // Synchronize to avoid write buffer pile up too much, which could cause // controllernode to timeout later when it needs to save a snapshot. @@ -1972,7 +1987,7 @@ int FileOp::initDctnryExtent( // MCOL-498 CS has to set a number of blocs in the chunk header if ( m_compressionType ) { - updateDctnryExtent(pFile, nBlocks); + updateDctnryExtent(pFile, nBlocks, lbid); } pFile->flush(); } @@ -2897,12 +2912,12 @@ int FileOp::flushFile(int rc, std::map& oids) return NO_ERROR; } -int FileOp::updateColumnExtent(IDBDataFile* pFile, int nBlocks) +int FileOp::updateColumnExtent(IDBDataFile* pFile, int nBlocks, int64_t lbid) { return NO_ERROR; } -int FileOp::updateDctnryExtent(IDBDataFile* pFile, int nBlocks) +int FileOp::updateDctnryExtent(IDBDataFile* pFile, int nBlocks, int64_t lbid) { return NO_ERROR; } diff --git a/writeengine/shared/we_fileop.h b/writeengine/shared/we_fileop.h index 9557a5949..a136d4528 100644 --- a/writeengine/shared/we_fileop.h +++ b/writeengine/shared/we_fileop.h @@ -103,7 +103,8 @@ public: int createFile( const char* fileName, int fileSize, const uint8_t* emptyVal, int width, execplan::CalpontSystemCatalog::ColDataType colDataType, - uint16_t dbRoot ); + uint16_t dbRoot , + BRM::LBID_t lbid = -1 ); /** * @brief Delete a file @@ -358,7 +359,8 @@ public: unsigned char* blockHdrInit, int blockHdrInitSize, bool bExpandExtent, - bool bOptExtension = false ); + bool bOptExtension = false, + int64_t lbid = 0); /** * @brief Check whether it is an directory @@ -510,14 +512,15 @@ public: bool bNewFile, bool bExpandExtent, bool bAbbrevExtent, - bool bOptExtension=false ); + bool bOptExtension=false, + int64_t lbid = 0 ); // Calls a chown and logs an error message bool chownDataPath(const std::string& fileName) const; protected: - EXPORT virtual int updateColumnExtent(IDBDataFile* pFile, int nBlocks); - EXPORT virtual int updateDctnryExtent(IDBDataFile* pFile, int nBlocks); + EXPORT virtual int updateColumnExtent(IDBDataFile* pFile, int nBlocks, int64_t lbid); + EXPORT virtual int updateDctnryExtent(IDBDataFile* pFile, int nBlocks, int64_t lbid); int m_compressionType; // compresssion type @@ -534,7 +537,7 @@ private: int initAbbrevCompColumnExtent( IDBDataFile* pFile, uint16_t dbRoot, int nBlocks, - const uint8_t* emptyVal, int width, + const uint8_t* emptyVal, int width, BRM::LBID_t lbid, execplan::CalpontSystemCatalog::ColDataType colDataType); static void initDbRootExtentMutexes(); @@ -542,7 +545,7 @@ private: int writeInitialCompColumnChunk( IDBDataFile* pFile, int nBlocksAllocated, int nRows, - const uint8_t* emptyVal, int width, + const uint8_t* emptyVal, int width, BRM::LBID_t lbid, execplan::CalpontSystemCatalog::ColDataType colDataType, char* hdrs); TxnID m_transId; diff --git a/writeengine/wrapper/we_colop.cpp b/writeengine/wrapper/we_colop.cpp index 13511582d..96349651e 100644 --- a/writeengine/wrapper/we_colop.cpp +++ b/writeengine/wrapper/we_colop.cpp @@ -483,7 +483,7 @@ int ColumnOp::allocRowId(const TxnID& txnid, bool useStartingExtent, return rc; //@Bug 3164 update compressed extent - updateColumnExtent(newCol.dataFile.pFile, allocSize); + updateColumnExtent(newCol.dataFile.pFile, allocSize, /*lbid=*/0); //..Search first block of new extent for empty rows rc = readBlock(newCol.dataFile.pFile, buf, newHwm); diff --git a/writeengine/wrapper/we_colopcompress.cpp b/writeengine/wrapper/we_colopcompress.cpp index dc4a2d473..ae5659c03 100644 --- a/writeengine/wrapper/we_colopcompress.cpp +++ b/writeengine/wrapper/we_colopcompress.cpp @@ -210,9 +210,9 @@ int ColumnOpCompress1::expandAbbrevColumnExtent( } -int ColumnOpCompress1::updateColumnExtent(IDBDataFile* pFile, int nBlocks) +int ColumnOpCompress1::updateColumnExtent(IDBDataFile* pFile, int nBlocks, int64_t lbid) { - return m_chunkManager->updateColumnExtent(pFile, nBlocks); + return m_chunkManager->updateColumnExtent(pFile, nBlocks, lbid); } diff --git a/writeengine/wrapper/we_colopcompress.h b/writeengine/wrapper/we_colopcompress.h index b611a2f58..681d911f8 100644 --- a/writeengine/wrapper/we_colopcompress.h +++ b/writeengine/wrapper/we_colopcompress.h @@ -160,7 +160,7 @@ protected: /** * @brief virtual method in FileOp */ - int updateColumnExtent(IDBDataFile* pFile, int nBlocks); + int updateColumnExtent(IDBDataFile* pFile, int nBlocks, int64_t lbid); /** * @brief virtual method in ColumnOp diff --git a/writeengine/wrapper/we_dctnrycompress.cpp b/writeengine/wrapper/we_dctnrycompress.cpp index 6367e3c63..22677b491 100644 --- a/writeengine/wrapper/we_dctnrycompress.cpp +++ b/writeengine/wrapper/we_dctnrycompress.cpp @@ -90,16 +90,20 @@ DctnryCompress1::~DctnryCompress1() delete m_chunkManager; } -int DctnryCompress1::updateDctnryExtent(IDBDataFile* pFile, int nBlocks) +int DctnryCompress1::updateDctnryExtent(IDBDataFile* pFile, int nBlocks, + int64_t lbid) { - return m_chunkManager->updateDctnryExtent(pFile, nBlocks); + return m_chunkManager->updateDctnryExtent(pFile, nBlocks, lbid); } - -IDBDataFile* DctnryCompress1::createDctnryFile(const char* name, int width, const char* mode, int ioBuffSize) +IDBDataFile* DctnryCompress1::createDctnryFile(const char* name, int width, + const char* mode, + int ioBuffSize, + BRM::LBID_t lbid) { - return m_chunkManager->createDctnryFile( - m_dctnryOID, width, m_dbRoot, m_partition, m_segment, name, mode, ioBuffSize); + return m_chunkManager->createDctnryFile(m_dctnryOID, width, m_dbRoot, + m_partition, m_segment, name, mode, + ioBuffSize, lbid); } diff --git a/writeengine/wrapper/we_dctnrycompress.h b/writeengine/wrapper/we_dctnrycompress.h index 9e0aa9bcd..968253d45 100644 --- a/writeengine/wrapper/we_dctnrycompress.h +++ b/writeengine/wrapper/we_dctnrycompress.h @@ -97,7 +97,9 @@ public: /** * @brief virtual method in Dctnry */ - IDBDataFile* createDctnryFile(const char* name, int width, const char* mode, int ioBuffSize); + IDBDataFile* createDctnryFile(const char* name, int width, + const char* mode, int ioBuffSize, + int64_t lbid); /** * @brief virtual method in Dctnry @@ -159,7 +161,7 @@ protected: /** * @brief virtual method in FileOp */ - int updateDctnryExtent(IDBDataFile* pFile, int nBlocks); + int updateDctnryExtent(IDBDataFile* pFile, int nBlocks, int64_t lbid); /** * @brief convert lbid to fbo