database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-27 21:01:50 +03:00
Code Activity

MCOL-4566: Add rebuildEM tool support to work with compressed files.

Browse Source 
* This patch adds rebuildEM tool support to work with compressed files.
* This patch increases a version of the file header.

Note: Default version of the `rebuildEM` tool was using very old API,
those functions are not present currently. So `rebuildEM` will not work with
files created without compression, because we cannot deduce some info which are
needed to create column extent.
This commit is contained in:
Denis Khalikov
2021-03-10 17:23:13 +03:00
parent 2eec956977
commit 5d497e8821
25 changed files with 1560 additions and 406 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
tests/rebuild-em-tests.cpp
View File

@ -20,6 +20,10 @@
#include <vector>
#include "we_convertor.h"
#include "we_fileop.h"
using namespace idbdatafile;
using namespace WriteEngine;
class RebuildEMTest : public ::testing::Test
{

2
tools/CMakeLists.txt
View File

@ -10,4 +10,4 @@ add_subdirectory(viewtablelock)
add_subdirectory(cleartablelock)
add_subdirectory(ddlcleanup)
add_subdirectory(idbmeminfo)
add_subdirectory(rebuildEM)

340
tools/rebuildEM/BRM_saves_em_system_tables_blob.h Normal file
View File

@ -0,0 +1,340 @@
#ifndef BRM_SAVES_EM_SYSTEM_TABLES_BLOB
#define BRM_SAVES_EM_SYSTEM_TABLES_BLOB
// Generated by hexdump -C BRM_saves_em for initial system catalog.
// TODO: This could be compressed by simple `run-length encoding`
// algo.
static const uint8_t BRM_saves_em_system_tables_blob[] = {
0x20, 0x8b, 0xf7, 0x76, 0x29, 0x00, 0x00, 0x00, 0x32, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0xe9, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00,
0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0xd1, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80,
0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0xea, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00,
0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x60, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0xd4, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80,
0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0xeb, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00,
0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x90, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0xec, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00,
0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0xed, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00,
0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0xb0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0xee, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00,
0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0xef, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00,
0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0xd0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0xf0, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00,
0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0xf1, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00,
0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0xf2, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00,
0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0xf3, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00,
0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x10, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0xfd, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00,
0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x30, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x0d, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80,
0x00, 0x50, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0xfe, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00,
0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x70, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x10, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80,
0x00, 0x90, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0xff, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00,
0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0xb0, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x13, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80,
0x00, 0xd0, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00,
0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0xe0, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00,
0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0xf0, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x02, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00,
0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x03, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00,
0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x10, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x04, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00,
0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x20, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x05, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00,
0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x30, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x06, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00,
0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x40, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x07, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00,
0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x50, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x08, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00,
0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x70, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x16, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80,
0x00, 0x90, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x09, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00,
0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0xa0, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x0a, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00,
0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0xb0, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x0b, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00,
0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0xc0, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x0c, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00,
0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0xc4, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x0d, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00,
0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0xd4, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x0e, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00,
0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0xe4, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x0f, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00,
0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x04, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x19, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80,
0x00, 0x24, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x10, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00,
0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x44, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x1c, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80,
0x00, 0x64, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x11, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00,
0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x74, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x12, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00,
0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x94, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1b, 0xff, 0xff, 0x03,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
#endif

6
tools/rebuildEM/CMakeLists.txt Normal file
View File

@ -0,0 +1,6 @@
include_directories(${ENGINE_COMMON_INCLUDES})
set(rebuildEM_SRCS main.cpp rebuildEM.cpp)
add_executable(rebuildEM ${rebuildEM_SRCS})
target_link_libraries(rebuildEM ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${MARIADB_CLIENT_LIBS} boost_filesystem)
install(TARGETS rebuildEM DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine)

383
tools/rebuildEM/main.cpp
View File

@ -1,329 +1,128 @@
/*****************************************************************************
* $Id: main.cpp 1739 2012-03-22 12:57:59Z pleblanc $
*
*****************************************************************************/
/* Copyright (C) 2021 MariaDB Corporation
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#include <iostream>
#include <unistd.h>
//#define _XOPEN_SOURCE 500
#include <string>
#include <ftw.h>
#include <fnmatch.h>
#include <cassert>
#include <boost/tokenizer.hpp>
#include <stack>
#include <stdexcept>
using namespace std;
using namespace boost;
#include <boost/filesystem.hpp>
#include "configcpp.h"
using namespace config;
#include "rebuildEM.h"
#include "IDBPolicy.h"
#include "IDBFileSystem.h"
#include "extentmap.h"
using namespace BRM;
using namespace idbdatafile;
using namespace RebuildExtentMap;
#include "calpontsystemcatalog.h"
#include "dm.h"
namespace
static void usage(const string& pname)
{
unsigned vflg = 0;
bool dflg = false;
string DBRoot;
string pattern;
unsigned extentSize;
ExtentMap em;
void usage(const string& pname)
{
cout << "usage: " << pname << " [-vdh]" << endl <<
" rebuilds the extent map from the contents of the database file system." << endl <<
" -v display verbose progress information. More v's, more debug" << endl <<
" -d display what would be done--don't do it" << endl <<
" -h display this help text" << endl;
}
OID_t pname2OID(const string& fpath)
{
dmFilePathArgs_t args;
char aBuff[10];
char bBuff[10];
char cBuff[10];
char fnBuff[16];
int rc;
OID_t oid;
args.pDirA = aBuff;
args.pDirB = bBuff;
args.pDirC = cBuff;
args.pFName = fnBuff;
args.ALen = sizeof(aBuff);
args.BLen = sizeof(bBuff);
args.CLen = sizeof(cBuff);
args.FNLen = sizeof(fnBuff);
args.Arc = 0;
args.Brc = 0;
args.Crc = 0;
args.FNrc = 0;
typedef tokenizer<char_separator<char> > tokenizer;
char_separator<char> sep("/");
tokenizer tokens(fpath, sep);
tokenizer::iterator tok_iter = tokens.begin();
tokenizer::iterator end = tokens.end();
typedef stack<string> pcomps_t;
pcomps_t pcomps;
while (tok_iter != end)
{
pcomps.push(*tok_iter);
++tok_iter;
}
idbassert(pcomps.size() >= 4);
string pcomp;
pcomp = pcomps.top();
pcomps.pop();
idbassert(pcomp.size() < 16);
strcpy(args.pFName, pcomp.c_str());
pcomp = pcomps.top();
pcomps.pop();
idbassert(pcomp.size() < 10);
strcpy(args.pDirC, pcomp.c_str());
pcomp = pcomps.top();
pcomps.pop();
idbassert(pcomp.size() < 10);
strcpy(args.pDirB, pcomp.c_str());
pcomp = pcomps.top();
pcomps.pop();
idbassert(pcomp.size() < 10);
strcpy(args.pDirA, pcomp.c_str());
rc = dmFPath2Oid(&args, (UINT32*)&oid);
if (rc != 0)
{
oid = 0;
}
return oid;
}
bool isCalpontDBFile(const string& fpath)
{
return (fnmatch(pattern.c_str(), fpath.c_str(), 0) == 0);
}
int walkDB(const char* fp, const struct stat* sb, int typeflag, struct FTW* ftwbuf)
{
string fpath(fp);
unsigned numExtents;
unsigned numBlocks;
if (typeflag != FTW_F)
{
return FTW_CONTINUE;
}
if (!isCalpontDBFile(fpath))
{
if (vflg > 2)
{
cout << "Skipping non-Calpont DB file " << fpath << endl;
}
return FTW_CONTINUE;
}
if (vflg)
{
cout << "Processing file " << fpath << endl;
numBlocks = (sb->st_size + (BLOCK_SIZE - 1)) / BLOCK_SIZE;
numExtents = (numBlocks + (extentSize - 1)) / extentSize;
if (vflg > 1)
{
cout << "File is " << numExtents << " extent" << (numExtents > 1 ? "s" : "") <<
", " << numBlocks << " block" << (numBlocks > 1 ? "s" : "") <<
", " << sb->st_size << " bytes, ";
}
}
OID_t oid;
oid = pname2OID(fpath);
if (vflg > 1)
{
cout << "OID is " << oid << ", ";
}
if (oid <= 10)
{
if (vflg)
{
cout << endl << "OID " << oid << " is probably a VBBF and is being skipped" << endl;
}
return FTW_CONTINUE;
}
HWM_t hwm;
#ifdef DELETE_FIRST
try
{
hwm = em.getHWM(oid);
}
catch (exception& ex)
{
if (vflg)
{
cout << endl << "There was no HWM for OID " << oid <<
", it is probably a VBBF and is being skipped: " << ex.what() << endl;
}
return FTW_CONTINUE;
}
catch (...)
{
if (vflg)
{
cout << endl << "There was no HWM for OID " << oid <<
", it is probably a VBBF and is being skipped" << endl;
}
return FTW_CONTINUE;
}
#else
hwm = numBlocks - 1;
#endif
if (vflg > 1)
{
cout << "HWM is " << hwm << endl;
}
#ifdef DELETE_FIRST
if (vflg > 1)
{
cout << "Deleting OID " << oid << " from Extent Map" << endl;
}
if (!dflg)
{
em.deleteOID(oid);
em.confirmChanges();
}
#endif
vector<LBID_t> lbids;
int allocdsize = 0;
if (vflg > 1)
{
cout << "Creating extent for " << numBlocks << " blocks for OID " << oid << endl;
}
if (!dflg)
{
em.createExtent(numBlocks, oid, lbids, allocdsize);
em.confirmChanges();
}
if (vflg > 1)
{
cout << "Created " << allocdsize << " LBIDs for OID " << oid << endl;
if (vflg > 2)
{
vector<LBID_t>::iterator iter = lbids.begin();
vector<LBID_t>::iterator end = lbids.end();
cout << "First LBIDs from each extent are ";
while (iter != end)
{
cout << *iter << ", ";
++iter;
}
cout << endl;
}
}
if (vflg > 1)
{
cout << "Setting OID " << oid << " HWM to " << hwm << endl;
}
if (!dflg)
{
em.setHWM(oid, hwm);
em.confirmChanges();
}
return FTW_CONTINUE;
}
std::cout << "usage: " << pname << " [-vdhs]" << std::endl;
std::cout
<< "rebuilds the extent map from the contents of the database file "
"system."
<< std::endl;
std::cout
<< " -v display verbose progress information. More v's, more debug"
<< std::endl;
std::cout << " -d display what would be done--don't do it" << std::endl;
std::cout << " -h display this help text" << std::endl;
std::cout << " -s show extent map and quit" << std::endl;
}
int main(int argc, char** argv)
{
int c;
string pname(argv[0]);
int32_t option;
std::string pname(argv[0]);
bool showExtentMap = false;
bool verbose = false;
bool display = false;
opterr = 0;
while ((c = getopt(argc, argv, "vdh")) != EOF)
switch (c)
while ((option = getopt(argc, argv, "vdhs")) != EOF)
{
switch (option)
{
case 'v':
vflg++;
verbose = true;
break;
case 'd':
dflg = true;
display = true;
break;
case 's':
showExtentMap = true;
break;
case 'h':
case '?':
default:
usage(pname);
return (c == 'h' ? 0 : 1);
return (option == 'h' ? 0 : 1);
break;
}
const Config* cf = Config::makeConfig();
DBRoot = cf->getConfig("SystemConfig", "DBRoot");
pattern = DBRoot + "/[0-9][0-9][0-9].dir/[0-9][0-9][0-9].dir/[0-9][0-9][0-9].dir/FILE[0-9][0-9][0-9].cdf";
if (vflg)
{
cout << "Using DBRoot " << DBRoot << endl;
}
if (access(DBRoot.c_str(), X_OK) != 0)
EMReBuilder emReBuilder(verbose, display);
// Just show EM and quit.
if (showExtentMap)
{
emReBuilder.showExtentMap();
return 0;
}
auto* config = config::Config::makeConfig();
const auto BRMSavesEM =
config->getConfig("SystemConfig", "DBRMRoot") + "_em";
// Check for `BRM_saves_em` file presents.
if (boost::filesystem::exists(BRMSavesEM))
{
std::cout << BRMSavesEM << " file exists. " << std::endl;
std::cout << "Please note: this tool is only suitable in situations "
"where there is no `BRM_saves_em` file. "
<< std::endl;
std::cout << "If `BRM_saves_em` "
"exists extent map will be restored from it. "
<< std::endl;
return 0;
}
// Initialize system extents from the binary blob.
auto rc = emReBuilder.initializeSystemExtents();
if (rc == -1)
{
cerr << "Could not scan DBRoot " << DBRoot << '!' << endl;
return 1;
}
ExtentMap em;
extentSize = em.getExtentSize();
// Make config from default path.
std::string count = config->getConfig("SystemConfig", "DBRootCount");
// Read the number of DBRoots.
uint32_t dbRootCount = config->uFromText(count);
if (vflg)
// Iterate over DBRoots starting from the first one.
for (uint32_t dbRootNumber = 1; dbRootNumber <= dbRootCount;
++dbRootNumber)
{
cout << "System extent size is " << extentSize << " blocks" << endl;
}
if (nftw(DBRoot.c_str(), walkDB, 64, FTW_PHYS | FTW_ACTIONRETVAL) != 0)
{
cerr << "Error processing files in DBRoot " << DBRoot << '!' << endl;
return 1;
std::string dbRootName = "DBRoot" + std::to_string(dbRootNumber);
auto dbRootPath = config->getConfig("SystemConfig", dbRootName);
emReBuilder.setDBRoot(dbRootNumber);
emReBuilder.collectExtents(dbRootPath.c_str());
emReBuilder.rebuildExtentMap();
emReBuilder.clear();
}
return 0;
}

497
tools/rebuildEM/rebuildEM.cpp Normal file
View File

@ -0,0 +1,497 @@
/* Copyright (C) 2021 MariaDB Corporation
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#include <iostream>
#include <boost/filesystem.hpp>
#include <stdint.h>
#include "rebuildEM.h"
#include "calpontsystemcatalog.h"
#include "idbcompress.h"
#include "blocksize.h"
#include "we_convertor.h"
#include "we_fileop.h"
#include "IDBPolicy.h"
#include "IDBFileSystem.h"
#include "we_chunkmanager.h"
#include "BRM_saves_em_system_tables_blob.h"
#include "we_dctnrycompress.h"
#include "we_colopcompress.h"
using namespace idbdatafile;
namespace RebuildExtentMap
{
int32_t EMReBuilder::collectExtents(const string& dbRootPath)
{
if (doVerbose())
{
std::cout << "Collect extents for the DBRoot " << dbRootPath
<< std::endl;
}
for (boost::filesystem::recursive_directory_iterator dirIt(dbRootPath),
dirEnd;
dirIt != dirEnd; ++dirIt)
{
(void) collectExtent(dirIt->path().string());
}
return 0;
}
int32_t EMReBuilder::collectExtent(const std::string& fullFileName)
{
WriteEngine::FileOp fileOp;
uint32_t oid;
uint32_t partition;
uint32_t segment;
// Initialize oid, partition and segment from the given `fullFileName`.
auto rc = WriteEngine::Convertor::fileName2Oid(fullFileName, oid,
partition, segment);
if (rc != 0)
{
return rc;
}
// Open the given file.
std::unique_ptr<IDBDataFile> dbFile(IDBDataFile::open(
IDBPolicy::getType(fullFileName, IDBPolicy::WRITEENG),
fullFileName.c_str(), "rb", 1));
if (!dbFile)
{
std::cerr << "Cannot open file " << fullFileName << std::endl;
return -1;
}
rc = dbFile->seek(0, 0);
if (rc != 0)
{
std::cerr << "IDBDataFile::seek failed for the file " << fullFileName
<< std::endl;
return rc;
}
// Read and verify header.
char fileHeader[compress::IDBCompressInterface::HDR_BUF_LEN * 2];
rc = fileOp.readHeaders(dbFile.get(), fileHeader);
if (rc != 0)
{
// FIXME: If the file was created without a compression, it does not
// have a header block, so header verification fails in this case,
// currently we skip it, because we cannot deduce needed data to create
// a column extent from the blob file.
if (doVerbose())
{
std::cerr
<< "Cannot read file header from the file " << fullFileName
<< ", probably this file was created without compression. "
<< std::endl;
}
return rc;
}
if (doVerbose())
{
std::cout << "Processing file: " << fullFileName << std::endl;
std::cout << "fileName2Oid: [OID: " << oid
<< ", partition: " << partition << ", segment: " << segment
<< "] " << std::endl;
}
// Read the `colDataType` and `colWidth` from the given header.
compress::IDBCompressInterface compressor;
const auto versionNumber = compressor.getVersionNumber(fileHeader);
// Verify header number.
if (versionNumber < 3)
{
if (doVerbose())
{
std::cerr << "File version " << versionNumber
<< " is not supported. " << std::endl;
}
return -1;
}
auto colDataType = compressor.getColDataType(fileHeader);
auto colWidth = compressor.getColumnWidth(fileHeader);
auto blockCount = compressor.getBlockCount(fileHeader);
auto lbid = compressor.getLBID0(fileHeader);
if (colDataType == execplan::CalpontSystemCatalog::UNDEFINED)
{
if (doVerbose())
{
std::cout << "File header has invalid data. " << std::endl;
}
return -1;
}
auto isDict = isDictFile(colDataType, colWidth);
if (isDict)
colWidth = 8;
uint64_t hwm = 0;
if (doVerbose())
{
std::cout << "Searching for HWM... " << std::endl;
std::cout << "Block count: " << blockCount << std::endl;
}
rc =
searchHWMInSegmentFile(oid, getDBRoot(), partition, segment,
colDataType, colWidth, blockCount, isDict, hwm);
if (rc != 0)
{
return rc;
}
const uint32_t extentMaxBlockCount =
getEM().getExtentRows() * colWidth / BLOCK_SIZE;
// We found multiple extents per one segment file.
if (hwm >= extentMaxBlockCount)
{
auto lbid = compressor.getLBID1(fileHeader);
FileId fileId(oid, partition, segment, colWidth, colDataType, lbid,
hwm, isDict);
extentMap.push_back(fileId);
// Update HWM.
hwm = extentMaxBlockCount - 1;
if (doVerbose())
{
std::cout << "Found multiple extents per segment file "
<< std::endl;
std::cout << "FileId is collected " << fileId << std::endl;
}
}
if (doVerbose())
{
std::cout << "HWM is: " << hwm << std::endl;
}
FileId fileId(oid, partition, segment, colWidth, colDataType, lbid, hwm,
isDict);
extentMap.push_back(fileId);
if (doVerbose())
{
std::cout << "FileId is collected " << fileId << std::endl;
}
return 0;
}
int32_t EMReBuilder::rebuildExtentMap()
{
if (doVerbose())
{
std::cout << "Build extent map with size " << extentMap.size()
<< std::endl;
}
// We have to restore extent map by restoring individual extent in order
// they were created. This is important part, otherwise we will get invalid
// extents for dictionary segment files and will not be able to access
// columns through dictionay segment files.
sort(extentMap.begin(), extentMap.end(),
[](const FileId& lhs, const FileId& rhs) {
return lhs.lbid < rhs.lbid;
});
for (const auto& fileId : extentMap)
{
uint32_t startBlockOffset;
int32_t allocdSize;
BRM::LBID_t lbid;
if (!doDisplay())
{
try
{
if (fileId.isDict)
{
// Create a dictionary extent for the given oid, partition,
// segment, dbroot.
getEM().createDictStoreExtent(
fileId.oid, getDBRoot(), fileId.partition,
fileId.segment, lbid, allocdSize);
}
else
{
// Create a column extent for the given oid, partition,
// segment, dbroot and column width.
getEM().createColumnExtentExactFile(
fileId.oid, fileId.colWidth, getDBRoot(),
fileId.partition, fileId.segment, fileId.colDataType,
lbid, allocdSize, startBlockOffset);
}
}
// Could throw an logic_error exception.
catch (std::exception& e)
{
getEM().undoChanges();
std::cerr << "Cannot create column extent: " << e.what()
<< std::endl;
return -1;
}
getEM().confirmChanges();
if (doVerbose())
{
std::cout << "Extent is created, allocated size " << allocdSize
<< " actual LBID " << lbid << std::endl;
std::cout << "For " << fileId << std::endl;
}
// This is important part, it sets a status for specific extent
// as 'available' that means we can use it.
if (doVerbose())
{
std::cout << "Setting a HWM for " << fileId << std::endl;
}
try
{
getEM().setLocalHWM(fileId.oid, fileId.partition,
fileId.segment, fileId.hwm, false, true);
}
catch (std::exception& e)
{
getEM().undoChanges();
std::cerr << "Cannot set local HWM: " << e.what() << std::endl;
return -1;
}
getEM().confirmChanges();
}
}
return 0;
}
int32_t EMReBuilder::searchHWMInSegmentFile(
uint32_t oid, uint32_t dbRoot, uint32_t partition, uint32_t segment,
execplan::CalpontSystemCatalog::ColDataType colDataType, uint32_t colWidth,
uint64_t blockCount, bool isDict, uint64_t& hwm)
{
std::unique_ptr<ChunkManagerWrapper> chunkManagerWrapper;
try
{
if (isDict)
{
chunkManagerWrapper = std::unique_ptr<ChunkManagerWrapperDict>(
new ChunkManagerWrapperDict(oid, dbRoot, partition, segment,
colDataType, colWidth));
}
else
{
chunkManagerWrapper = std::unique_ptr<ChunkManagerWrapperColumn>(
new ChunkManagerWrapperColumn(oid, dbRoot, partition, segment,
colDataType, colWidth));
}
}
catch (...)
{
std::cerr << "Cannot create ChunkManagerWrapper" << std::endl;
return -1;
}
hwm = 0;
// Starting from the end.
// Note: This solves problem related to `bulk` insertion.
// Currently it could start to insert values from any block into empty
// column.
for (int32_t currentBlock = blockCount - 1; currentBlock >= 0;
--currentBlock)
{
// Read the block associated to HWM.
// The uncompressed chunk size is 512 * 1024 * 8, so for `abbreviated`
// extent one chunk will hold all blocks, therefore we need to
// decompress chunk only once, the cached chunk will be used later.
chunkManagerWrapper->readBlock(currentBlock);
if (!chunkManagerWrapper->isEmptyBlock())
{
hwm = currentBlock;
break;
}
}
return 0;
}
void EMReBuilder::showExtentMap()
{
std::cout << "range.start|range.size|fileId|blockOffset|HWM|partition|"
"segment|dbroot|width|status|hiVal|loVal|seqNum|isValid|"
<< std::endl;
getEM().dumpTo(std::cout);
}
// Check for hard-coded values which define dictionary file.
bool EMReBuilder::isDictFile(
execplan::CalpontSystemCatalog::ColDataType colDataType, uint64_t width)
{
// Dictionary store extent has a width == 0. See more detais in
// `createDictStoreExtent` function.
return (colDataType == execplan::CalpontSystemCatalog::VARCHAR) &&
(width == 0);
}
int32_t EMReBuilder::initializeSystemExtents()
{
if (!doDisplay())
{
if (doVerbose())
{
std::cout << "Initialize system extents from the initial state"
<< std::endl;
}
try
{
getEM().loadFromBinaryBlob(reinterpret_cast<const char*>(
BRM_saves_em_system_tables_blob));
}
catch (...)
{
std::cerr << "Cannot initialize system extents from binary blob "
<< std::endl;
return -1;
}
}
return 0;
}
ChunkManagerWrapper::ChunkManagerWrapper(
uint32_t oid, uint32_t dbRoot, uint32_t partition, uint32_t segment,
execplan::CalpontSystemCatalog::ColDataType colDataType, uint32_t colWidth)
: oid(oid), dbRoot(dbRoot), partition(partition), segment(segment),
colDataType(colDataType), colWidth(colWidth), size(colWidth),
pFileOp(nullptr)
{
}
int32_t ChunkManagerWrapper::readBlock(uint32_t blockNumber)
{
auto rc = chunkManager.readBlock(pFile, blockData, blockNumber);
if (rc != 0)
return rc;
return 0;
}
ChunkManagerWrapperColumn::ChunkManagerWrapperColumn(
uint32_t oid, uint32_t dbRoot, uint32_t partition, uint32_t segment,
execplan::CalpontSystemCatalog::ColDataType colDataType, uint32_t colWidth)
: ChunkManagerWrapper(oid, dbRoot, partition, segment, colDataType,
colWidth)
{
pFileOp = std::unique_ptr<WriteEngine::ColumnOpCompress1>(
new WriteEngine::ColumnOpCompress1());
chunkManager.fileOp(pFileOp.get());
// Open compressed column segment file. We will read block by block
// from the compressed chunks.
pFile = chunkManager.getSegmentFilePtr(oid, dbRoot, partition, segment,
colDataType, colWidth, fileName,
"rb", size, false, false);
if (!pFile)
{
throw std::bad_alloc();
}
emptyValue = pFileOp->getEmptyRowValue(colDataType, colWidth);
midOffset = (WriteEngine::BYTE_PER_BLOCK / 2);
endOffset = (WriteEngine::BYTE_PER_BLOCK - colWidth);
}
bool ChunkManagerWrapperColumn::isEmptyBlock()
{
const uint8_t* beginValue = blockData;
const uint8_t* midValue = blockData + midOffset;
const uint8_t* endValue = blockData + endOffset;
return isEmptyValue(beginValue) && isEmptyValue(midValue) &&
isEmptyValue(endValue);
}
// This function is copy pasted from `ColumnOp` file, unfortunately it's not
// possible to reuse it directly form `ColumnOp` without a creating a some kind
// of object which inherits from `ColumnOp`, probably this should be moved to
// utils and make a static as well, to be able to use it without creating an
// object.
bool ChunkManagerWrapperColumn::isEmptyValue(const uint8_t* value) const
{
switch (colWidth)
{
case 1:
return *(uint8_t*) value == *(uint8_t*) emptyValue;
case 2:
return *(uint16_t*) value == *(uint16_t*) emptyValue;
case 4:
return *(uint32_t*) value == *(uint32_t*) emptyValue;
case 8:
return *(uint64_t*) value == *(uint64_t*) emptyValue;
case 16:
return *(uint128_t*) value == *(uint128_t*) emptyValue;
}
return false;
}
ChunkManagerWrapperDict::ChunkManagerWrapperDict(
uint32_t oid, uint32_t dbRoot, uint32_t partition, uint32_t segment,
execplan::CalpontSystemCatalog::ColDataType colDataType, uint32_t colWidth)
: ChunkManagerWrapper(oid, dbRoot, partition, segment, colDataType,
colWidth)
{
pFileOp = std::unique_ptr<WriteEngine::DctnryCompress1>(
new WriteEngine::DctnryCompress1());
chunkManager.fileOp(pFileOp.get());
// Open compressed dict segment file.
pFile = chunkManager.getSegmentFilePtr(oid, dbRoot, partition, segment,
colDataType, colWidth, fileName,
"rb", size, false, true);
if (!pFile)
{
throw std::bad_alloc();
}
auto dictBlockHeaderSize =
WriteEngine::HDR_UNIT_SIZE + WriteEngine::NEXT_PTR_BYTES +
WriteEngine::HDR_UNIT_SIZE + WriteEngine::HDR_UNIT_SIZE;
emptyBlock = WriteEngine::BYTE_PER_BLOCK - dictBlockHeaderSize;
}
bool ChunkManagerWrapperDict::isEmptyBlock()
{
// Check that block is completely empty.
return (*(uint16_t*) blockData) == emptyBlock;
}
std::ostream& operator<<(std::ostream& os, const FileId& fileID)
{
os << "[OID: " << fileID.oid << ", partition: " << fileID.partition
<< ", segment: " << fileID.segment << ", col width: " << fileID.colWidth
<< ", lbid:" << fileID.lbid << ", hwm: " << fileID.hwm
<< ", isDict: " << fileID.isDict << "]";
return os;
}
} // namespace RebuildExtentMap

230
tools/rebuildEM/rebuildEM.h Normal file
View File

@ -0,0 +1,230 @@
/* Copyright (C) 2021 MariaDB Corporation
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#ifndef REBUILD_EM_H
#define REBUILD_EM_H
#include <string>
#include <map>
#include <ftw.h>
#include "calpontsystemcatalog.h"
#include "extentmap.h"
#include "IDBPolicy.h"
#include "IDBFileSystem.h"
#include "idbcompress.h"
#include "blocksize.h"
#include "we_convertor.h"
#include "we_fileop.h"
#include "IDBPolicy.h"
#include "we_chunkmanager.h"
using namespace idbdatafile;
namespace RebuildExtentMap
{
// This struct represents a FileId. For internal purpose only.
struct FileId
{
FileId(uint32_t oid, uint32_t partition, uint32_t segment,
uint32_t colWidth,
execplan::CalpontSystemCatalog::ColDataType colDataType,
int64_t lbid, uint64_t hwm, bool isDict)
: oid(oid), partition(partition), segment(segment), colWidth(colWidth),
colDataType(colDataType), lbid(lbid), hwm(hwm), isDict(isDict)
{
}
uint32_t oid;
uint32_t partition;
uint32_t segment;
uint32_t colWidth;
execplan::CalpontSystemCatalog::ColDataType colDataType;
int64_t lbid;
uint64_t hwm;
bool isDict;
};
std::ostream& operator<<(std::ostream& os, const FileId& fileID);
// This class represents extent map rebuilder.
class EMReBuilder
{
public:
EMReBuilder(bool verbose, bool display)
: verbose(verbose), display(display)
{
// Initalize plugins.
IDBPolicy::init(true, false, "", 0);
}
~EMReBuilder() = default;
// Collects extents from the given DBRoot path.
int32_t collectExtents(const std::string& dbRootPath);
// Clears collected extents.
void clear() { extentMap.clear(); }
// Specifies whether we need verbose to output.
bool doVerbose() const { return verbose; }
// Specifies whether we need just display a pipeline, but not actually run
// it.
bool doDisplay() const { return display; }
// Returns the number of current DBRoot.
uint32_t getDBRoot() const { return dbRoot; }
// Retunrs a reference to `ExtentMap` object.
BRM::ExtentMap& getEM() { return em; }
// Checks if the given data specifies a dictionary file.
static bool
isDictFile(execplan::CalpontSystemCatalog::ColDataType colDataType,
uint64_t width);
// Initializes system extents from the binary blob.
// This function solves the problem related to system segment files.
// Currently those files do not have file header, so we cannot
// get the data (like width, colType, lbid) to restore an extent for this
// particular segment file. The current approach is to keep a binary blob
// of initial state of the system extents.
// Returns -1 on error.
int32_t initializeSystemExtents();
// Rebuilds extent map from the collected map.
int32_t rebuildExtentMap();
// Search HWM in the given segment file.
int32_t searchHWMInSegmentFile(
uint32_t oid, uint32_t dbRoot, uint32_t partition, uint32_t segment,
execplan::CalpontSystemCatalog::ColDataType colDataType,
uint32_t width, uint64_t blocksCount, bool isDict, uint64_t& hwm);
// Sets the dbroot to the given `number`.
void setDBRoot(uint32_t number) { dbRoot = number; }
// Shows the extent map.
void showExtentMap();
private:
EMReBuilder(const EMReBuilder&) = delete;
EMReBuilder(EMReBuilder&&) = delete;
EMReBuilder& operator=(const EMReBuilder&) = delete;
EMReBuilder& operator=(EMReBuilder&&) = delete;
// Collects the information for extent from the given file and stores
// it in `extentMap` set.
int32_t collectExtent(const std::string& fullFileName);
bool verbose;
bool display;
uint32_t dbRoot;
BRM::ExtentMap em;
std::vector<FileId> extentMap;
};
// The base class aroud `ChunkManager` to read and write decompressed blocks
// from segment file.
class ChunkManagerWrapper
{
public:
ChunkManagerWrapper(
uint32_t oid, uint32_t dbRoot, uint32_t partition, uint32_t segment,
execplan::CalpontSystemCatalog::ColDataType colDataType,
uint32_t colWidth);
virtual ~ChunkManagerWrapper() = default;
ChunkManagerWrapper(const ChunkManagerWrapper& other) = delete;
ChunkManagerWrapper& operator=(const ChunkManagerWrapper& other) = delete;
ChunkManagerWrapper(ChunkManagerWrapper&& other) = delete;
ChunkManagerWrapper& operator=(ChunkManagerWrapper&& other) = delete;
// Reads block, by given `blockNumber` from associated segment file and
// populates internal block buffer.
int32_t readBlock(uint32_t blockNumber);
// Checks that last read block is empty.
virtual bool isEmptyBlock() = 0;
protected:
uint32_t oid;
uint32_t dbRoot;
uint32_t partition;
uint32_t segment;
execplan::CalpontSystemCatalog::ColDataType colDataType;
uint32_t colWidth;
int32_t size;
std::string fileName;
std::unique_ptr<WriteEngine::FileOp> pFileOp;
// Note: We cannot clear this pointer directly, because
// `ChunkManager` closes this file for us, otherwise we will get double
// free error.
IDBDataFile* pFile;
WriteEngine::ChunkManager chunkManager;
uint8_t blockData[WriteEngine::BYTE_PER_BLOCK];
};
// Class to read decompressed blocks from column segment files.
class ChunkManagerWrapperColumn : public ChunkManagerWrapper
{
public:
ChunkManagerWrapperColumn(
uint32_t oid, uint32_t dbRoot, uint32_t partition, uint32_t segment,
execplan::CalpontSystemCatalog::ColDataType colDataType,
uint32_t colWidth);
~ChunkManagerWrapperColumn() = default;
ChunkManagerWrapperColumn(const ChunkManagerWrapperColumn& other) = delete;
ChunkManagerWrapperColumn&
operator=(const ChunkManagerWrapperColumn& other) = delete;
ChunkManagerWrapperColumn(ChunkManagerWrapperColumn&& other) = delete;
ChunkManagerWrapperColumn&
operator=(ChunkManagerWrapperColumn&& other) = delete;
bool isEmptyBlock() override;
bool isEmptyValue(const uint8_t* value) const;
private:
const uint8_t* emptyValue;
uint32_t midOffset;
uint32_t endOffset;
};
// Class to read decompressed blocks from dict segment files.
class ChunkManagerWrapperDict : public ChunkManagerWrapper
{
public:
ChunkManagerWrapperDict(
uint32_t oid, uint32_t dbRoot, uint32_t partition, uint32_t segment,
execplan::CalpontSystemCatalog::ColDataType colDataType,
uint32_t colWidth);
~ChunkManagerWrapperDict() = default;
ChunkManagerWrapperDict(const ChunkManagerWrapperDict& other) = delete;
ChunkManagerWrapperDict&
operator=(const ChunkManagerWrapperDict& other) = delete;
ChunkManagerWrapperDict(ChunkManagerWrapperDict&& other) = delete;
ChunkManagerWrapperDict&
operator=(ChunkManagerWrapperDict&& other) = delete;
bool isEmptyBlock() override;
private:
uint32_t emptyBlock;
};
} // namespace RebuildExtentMap
#endif

98
utils/compress/idbcompress.cpp
View File

@ -38,6 +38,7 @@ namespace
const uint64_t MAGIC_NUMBER = 0xfdc119a384d0778eULL;
const uint64_t VERSION_NUM1 = 1;
const uint64_t VERSION_NUM2 = 2;
const uint64_t VERSION_NUM3 = 3;
const int COMPRESSED_CHUNK_INCREMENT_SIZE = 8192;
const int PTR_SECTION_OFFSET = compress::IDBCompressInterface::HDR_BUF_LEN;
@ -71,6 +72,8 @@ struct CompressedDBFileHeader
uint64_t fBlockCount;
uint64_t fColumnWidth;
execplan::CalpontSystemCatalog::ColDataType fColDataType;
uint64_t fLBID0;
uint64_t fLBID1;
};
// Make the header to be 4K, regardless number of fields being defined/used in header.
@ -80,18 +83,6 @@ union CompressedDBFileHeaderBlock
char fDummy[compress::IDBCompressInterface::HDR_BUF_LEN];
};
void initCompressedDBFileHeader(void* hdrBuf, int compressionType, int hdrSize)
{
CompressedDBFileHeaderBlock* hdr = reinterpret_cast<CompressedDBFileHeaderBlock*>(hdrBuf);
hdr->fHeader.fMagicNumber = MAGIC_NUMBER;
hdr->fHeader.fVersionNum = VERSION_NUM2;
hdr->fHeader.fCompressionType = compressionType;
hdr->fHeader.fBlockCount = 0;
hdr->fHeader.fHeaderSize = hdrSize;
hdr->fHeader.fColumnWidth = 0;
hdr->fHeader.fColDataType = execplan::CalpontSystemCatalog::ColDataType::UNDEFINED;
}
void initCompressedDBFileHeader(
void* hdrBuf, uint32_t columnWidth,
execplan::CalpontSystemCatalog::ColDataType colDataType,
@ -99,12 +90,14 @@ void initCompressedDBFileHeader(
{
CompressedDBFileHeaderBlock* hdr = reinterpret_cast<CompressedDBFileHeaderBlock*>(hdrBuf);
hdr->fHeader.fMagicNumber = MAGIC_NUMBER;
hdr->fHeader.fVersionNum = VERSION_NUM2;
hdr->fHeader.fVersionNum = VERSION_NUM3;
hdr->fHeader.fCompressionType = compressionType;
hdr->fHeader.fBlockCount = 0;
hdr->fHeader.fHeaderSize = hdrSize;
hdr->fHeader.fColumnWidth = columnWidth;
hdr->fHeader.fColDataType = colDataType;
hdr->fHeader.fLBID0 = 0;
hdr->fHeader.fLBID1 = 0;
}
} // namespace
@ -351,23 +344,18 @@ void IDBCompressInterface::storePtrs(const std::vector<uint64_t>& ptrs, void* pt
storePtrs(ptrs, reinterpret_cast<char*>(ptrBuf) + HDR_BUF_LEN, HDR_BUF_LEN);
}
//------------------------------------------------------------------------------
// Initialize the header blocks to be written at the start of a column file.
//------------------------------------------------------------------------------
void IDBCompressInterface::initHdr(void* hdrBuf, int compressionType) const
{
memset(hdrBuf, 0, HDR_BUF_LEN * 2);
initCompressedDBFileHeader(hdrBuf, compressionType, HDR_BUF_LEN * 2);
}
//------------------------------------------------------------------------------
// Initialize the header blocks to be written at the start of a dictionary file.
//------------------------------------------------------------------------------
void IDBCompressInterface::initHdr(void* hdrBuf, void* ptrBuf, int compressionType, int hdrSize) const
void IDBCompressInterface::initHdr(
void* hdrBuf, void* ptrBuf, uint32_t colWidth,
execplan::CalpontSystemCatalog::ColDataType columnType,
int compressionType, int hdrSize) const
{
memset(hdrBuf, 0, HDR_BUF_LEN);
memset(ptrBuf, 0, hdrSize - HDR_BUF_LEN);
initCompressedDBFileHeader(hdrBuf, compressionType, hdrSize);
initCompressedDBFileHeader(hdrBuf, colWidth, columnType, compressionType,
hdrSize);
}
//------------------------------------------------------------------------------
@ -383,6 +371,15 @@ void IDBCompressInterface::initHdr(
compressionType, HDR_BUF_LEN * 2);
}
//------------------------------------------------------------------------------
// Get the header's version number
//------------------------------------------------------------------------------
uint64_t IDBCompressInterface::getVersionNumber(const void* hdrBuf) const
{
return (
reinterpret_cast<const CompressedDBFileHeader*>(hdrBuf)->fVersionNum);
}
//------------------------------------------------------------------------------
// Set the file's block count
//------------------------------------------------------------------------------
@ -415,6 +412,59 @@ uint64_t IDBCompressInterface::getHdrSize(const void* hdrBuf) const
return (reinterpret_cast<const CompressedDBFileHeader*>(hdrBuf)->fHeaderSize);
}
//------------------------------------------------------------------------------
// Get column type
//-----------------------------------------------------------------------------
execplan::CalpontSystemCatalog::ColDataType
IDBCompressInterface::getColDataType(const void* hdrBuf) const
{
return (
reinterpret_cast<const CompressedDBFileHeader*>(hdrBuf)->fColDataType);
}
//------------------------------------------------------------------------------
// Get column width
//------------------------------------------------------------------------------
uint64_t IDBCompressInterface::getColumnWidth(const void* hdrBuf) const
{
return (
reinterpret_cast<const CompressedDBFileHeader*>(hdrBuf)->fColumnWidth);
}
//------------------------------------------------------------------------------
// Get start LBID
//------------------------------------------------------------------------------
uint64_t IDBCompressInterface::getLBID0(const void* hdrBuf) const
{
return (reinterpret_cast<const CompressedDBFileHeader*>(hdrBuf)->fLBID0);
}
//------------------------------------------------------------------------------
// Set start LBID
//------------------------------------------------------------------------------
void IDBCompressInterface::setLBID0(void* hdrBuf, uint64_t lbid) const
{
if (lbid)
reinterpret_cast<CompressedDBFileHeader*>(hdrBuf)->fLBID0 = lbid;
}
//------------------------------------------------------------------------------
// Get start LBID
//------------------------------------------------------------------------------
uint64_t IDBCompressInterface::getLBID1(const void* hdrBuf) const
{
return (reinterpret_cast<const CompressedDBFileHeader*>(hdrBuf)->fLBID1);
}
//------------------------------------------------------------------------------
// Set start LBID
//------------------------------------------------------------------------------
void IDBCompressInterface::setLBID1(void* hdrBuf, uint64_t lbid) const
{
if (lbid)
reinterpret_cast<CompressedDBFileHeader*>(hdrBuf)->fLBID1 = lbid;
}
//------------------------------------------------------------------------------
// Calculates the chunk and block offset within the chunk for the specified
// block number.

65
utils/compress/idbcompress.h
View File

@ -101,20 +101,15 @@ public:
*/
EXPORT int uncompress(const char* in, size_t inLen, char* out) const;
/**
* Initialize header buffer at start of compressed db file.
*
* @warning hdrBuf must be at least HDR_BUF_LEN*2 bytes
*/
EXPORT void initHdr(void* hdrBuf, int compressionType) const;
/**
* Initialize header buffer at start of compressed db file.
*
* @warning hdrBuf must be at least HDR_BUF_LEN bytes
* @warning ptrBuf must be at least (hdrSize-HDR_BUF_LEN) bytes
*/
EXPORT void initHdr(void* hdrBuf, void* ptrBuf, int compressionType, int hdrSize) const;
EXPORT void initHdr(void* hdrBuf, void* ptrBuf, uint32_t columnWidht,
execplan::CalpontSystemCatalog::ColDataType columnType,
int compressionType, int hdrSize) const;
/**
* Initialize header buffer at start of compressed db file.
@ -199,6 +194,11 @@ public:
/*
* Mutator methods for the block count in the file
*/
/**
* getVersionNumber
*/
EXPORT uint64_t getVersionNumber(const void* hdrBuf) const;
/**
* setBlockCount
*/
@ -222,6 +222,37 @@ public:
*/
EXPORT uint64_t getHdrSize(const void* hdrBuf) const;
/**
* getColumnType
*/
EXPORT execplan::CalpontSystemCatalog::ColDataType
getColDataType(const void* hdrBuf) const;
/**
* getColumnWidth
*/
EXPORT uint64_t getColumnWidth(const void* hdrBuf) const;
/**
* getLBID
*/
EXPORT uint64_t getLBID0(const void* hdrBuf) const;
/**
* setBID
*/
EXPORT void setLBID0(void* hdrBuf, uint64_t lbid) const;
/**
* getLBID
*/
EXPORT uint64_t getLBID1(const void* hdrBuf) const;
/**
* setBID
*/
EXPORT void setLBID1(void* hdrBuf, uint64_t lbid) const;
/**
* Mutator methods for the user padding bytes
*/
@ -286,8 +317,7 @@ inline int IDBCompressInterface::uncompress(const char* in, size_t inLen, char*
{
return 0;
}
inline void IDBCompressInterface::initHdr(void*, int) const {}
inline void IDBCompressInterface::initHdr(void*, void*, int, int) const {}
inline void IDBCompressInterface::initHdr(void*, void*, uint32_t, execplan::CalpontSystemCatalog::ColDataType, int, int) const {}
inline void initHdr(void*, uint32_t, execplan::CalpontSystemCatalog::ColDataType, int) const {}
inline int IDBCompressInterface::verifyHdr(const void*) const
{
@ -317,6 +347,11 @@ inline int IDBCompressInterface::padCompressedChunks(unsigned char* buf, unsigne
{
return -1;
}
inline uint64_t
IDBCompressInterface::getVersionNumber(const void* hdrBuf) const
{
return 0;
}
inline void IDBCompressInterface::setBlockCount(void* hdrBuf, uint64_t count) const {}
inline uint64_t IDBCompressInterface::getBlockCount(const void* hdrBuf) const
{
@ -327,10 +362,20 @@ inline uint64_t IDBCompressInterface::getHdrSize(const void*) const
{
return 0;
}
inline execplan::CalpontSystemCatalog::ColDataType
IDBCompressInterface::getColDataType(const void* hdrBuf) const
{
return execplan::CalpontSystemCatalog::ColDataType::UNDEFINED;
}
inline uint64_t getColumnWidth(const void* hdrBuf) const { return 0; }
inline uint64_t IDBCompressInterface::maxCompressedSize(uint64_t uncompSize)
{
return uncompSize;
}
inline uint64_t getLBID0(const void* hdrBuf) const { return 0; }
void setLBID0(void* hdrBuf, uint64_t lbid) const {}
inline uint64_t getLBID1(const void* hdrBuf) const { return 0; }
void setLBID1(void* hdrBuf, uint64_t lbid) const {}
inline bool IDBCompressInterface::getUncompressedSize(char* in, size_t inLen, size_t* outLen)
{
return false;

92
versioning/BRM/extentmap.cpp
View File

@ -1271,7 +1271,8 @@ void ExtentMap::reserveLBIDRange(LBID_t start, uint8_t size)
*/
void ExtentMap::loadVersion4or5(IDBDataFile* in, bool upgradeV4ToV5)
template <class T>
void ExtentMap::loadVersion4or5(T* in, bool upgradeV4ToV5)
{
int emNumElements = 0, flNumElements = 0;
@ -1456,20 +1457,9 @@ void ExtentMap::load(const string& filename, bool fixFL)
try
{
int emVersion = 0;
int bytes = in->read((char*) &emVersion, sizeof(int));
load(in.get());
}
if (bytes == (int) sizeof(int) &&
(emVersion == EM_MAGIC_V4 || emVersion == EM_MAGIC_V5))
{
loadVersion4or5(in.get(), emVersion == EM_MAGIC_V4);
}
else
{
log("ExtentMap::load(): That file is not a valid ExtentMap image");
throw runtime_error("ExtentMap::load(): That file is not a valid ExtentMap image");
}
}
catch (...)
{
releaseFreeList(WRITE);
@ -1482,6 +1472,80 @@ void ExtentMap::load(const string& filename, bool fixFL)
// checkConsistency();
}
// This is a quick workaround, to be able to initialize initial system tables
// from binary blob.
// This should be updated, probably we need inherit from `IDBDataFile`.
struct EMBinaryReader
{
EMBinaryReader(const char* data) : src(data) {}
ssize_t read(char* dst, size_t size)
{
memcpy(dst, src, size);
src += size;
return size;
}
const char* src;
};
void ExtentMap::loadFromBinaryBlob(const char* blob)
{
grabEMEntryTable(WRITE);
try
{
grabFreeList(WRITE);
}
catch (...)
{
releaseEMEntryTable(WRITE);
throw;
}
try
{
EMBinaryReader emBinReader(blob);
load(&emBinReader);
}
catch (...)
{
releaseFreeList(WRITE);
releaseEMEntryTable(WRITE);
throw;
}
releaseFreeList(WRITE);
releaseEMEntryTable(WRITE);
}
template <typename T> void ExtentMap::load(T* in)
{
if (!in)
return;
try
{
int emVersion = 0;
int bytes = in->read((char*) &emVersion, sizeof(int));
if (bytes == (int) sizeof(int) &&
(emVersion == EM_MAGIC_V4 || emVersion == EM_MAGIC_V5))
{
loadVersion4or5(in, emVersion == EM_MAGIC_V4);
}
else
{
log("ExtentMap::load(): That file is not a valid ExtentMap image");
throw runtime_error("ExtentMap::load(): That file is not a valid ExtentMap image");
}
}
catch (...)
{
throw;
}
}
void ExtentMap::save(const string& filename)
{
#ifdef BRM_INFO

13
versioning/BRM/extentmap.h
View File

@ -344,6 +344,16 @@ public:
*/
EXPORT void load(const std::string& filename, bool fixFL = false);
/** @brief Loads the ExtentMap entries from a binayr blob.
*
* Loads the ExtentMap entries from a file. This will
* clear out any existing entries. The intention is that before
* the system starts, an external tool instantiates a single Extent
* Map and loads the stored entries.
* @param pointer to a binary blob.
*/
EXPORT void loadFromBinaryBlob(const char* blob);
/** @brief Saves the ExtentMap entries to a file
*
* Saves the ExtentMap entries to a file.
@ -1002,13 +1012,14 @@ private:
int _markInvalid(const LBID_t lbid, const execplan::CalpontSystemCatalog::ColDataType colDataType);
template <class T> void load(T* in);
/** @brief Loads the extent map from a file into memory.
*
* @param in (in) the file to load the extent map from.
* @param upgradeV4ToV5 (in) flag indicating whether we are upgrading
* extent map from v4 to v5.
*/
void loadVersion4or5(idbdatafile::IDBDataFile* in, bool upgradeV4ToV5);
template <class T> void loadVersion4or5(T* in, bool upgradeV4ToV5);
ExtentMapImpl* fPExtMapImpl;
FreeListImpl* fPFreeListImpl;

12
writeengine/bulk/we_colbufcompressed.cpp
View File

@ -582,11 +582,23 @@ int ColumnBufferCompressed::saveCompressionHeaders( )
{
// Construct the header records
char hdrBuf[IDBCompressInterface::HDR_BUF_LEN * 2];
RETURN_ON_ERROR(fColInfo->colOp->readHeaders(fFile, hdrBuf));
auto lbid = fCompressor->getLBID0(hdrBuf);
fCompressor->initHdr(hdrBuf, fColInfo->column.width,
fColInfo->column.dataType,
fColInfo->column.compressionType);
fCompressor->setBlockCount(hdrBuf,
(fColInfo->getFileSize() / BYTE_PER_BLOCK) );
if (lbid)
{
fCompressor->setLBID0(hdrBuf, lbid);
fCompressor->setLBID1(hdrBuf, fColInfo->getLastUpdatedLBID());
}
else
{
fCompressor->setLBID0(hdrBuf, fColInfo->getLastUpdatedLBID());
}
std::vector<uint64_t> ptrs;

8
writeengine/bulk/we_columninfo.cpp
View File

@ -146,6 +146,7 @@ ColumnInfo::ColumnInfo(Log* logger,
fLog(logger),
fDelayedFileStartBlksSkipped(0),
fSavedLbid(0),
fLastUpdatedLbid(0),
fSizeWrittenStart(0),
fSizeWritten(0),
fLastInputRowInCurrentExtent(0),
@ -749,6 +750,9 @@ int ColumnInfo::extendColumnNewExtent(
"; file-" << segFileNew;
fLog->logMsg( oss.str(), MSGLVL_INFO2 );
// Update lbid.
fLastUpdatedLbid = startLbid;
// Save the LBID with our CP extent info, so that we can update extent map
if (saveLBIDForCP)
{
@ -1397,6 +1401,9 @@ int ColumnInfo::getHWMInfoForBRM( BRMReporter& brmReporter )
return entriesAdded;
}
// Returns last updated LBID.
BRM::LBID_t ColumnInfo::getLastUpdatedLBID() const { return fLastUpdatedLbid; }
//------------------------------------------------------------------------------
// Setup initial extent we will begin loading at start of import.
// DBRoot, partition, segment, etc for the starting extent are specified.
@ -1484,6 +1491,7 @@ int ColumnInfo::setupInitialColumnExtent(
}
fSavedLbid = lbid;
fLastUpdatedLbid = lbid;
if (bSkippedToNewExtent)
oldHwm = hwm;

10
writeengine/bulk/we_columninfo.h
View File

@ -257,6 +257,10 @@ public:
*/
void getSegFileInfo( DBRootExtentInfo& fileInfo );
/** @brief Get last updated saved LBID.
*/
BRM::LBID_t getLastUpdatedLBID() const;
/** @brief Initialize autoincrement value from the current "next" value
* taken from the system catalog.
*/
@ -463,6 +467,10 @@ protected:
// have written additional rows to that extent as part of an import.
BRM::LBID_t fSavedLbid;
// The last updated LBID. In case `bulk` creates a new extent with new LBID
// we have to initialize file header with created LBID.
BRM::LBID_t fLastUpdatedLbid;
// Size of a segment file (in bytes) when the file is opened
// to add the next extent.
// For compressed data files, this is the "raw" data byte count,
@ -487,6 +495,8 @@ protected:
long long fMaxNumRowsPerSegFile; // Max num rows per segment file
Dctnry* fStore; // Corresponding dctnry store file
// For autoincrement column only... Tracks latest autoincrement value used
long long fAutoIncLastValue;

38
writeengine/dictionary/we_dctnry.cpp
View File

@ -206,16 +206,31 @@ int Dctnry::createDctnry( const OID& dctnryOID, int colWidth,
if (flag)
{
// Allocate extent before file creation.
// If we got an error while allocating dictionary store extent,
// we do not need to create/close the file, because it was not created
// yet. This logic is the same as column segment file creation - at
// first we allocate an extent, then we create a segment file.
rc = BRMWrapper::getInstance()->allocateDictStoreExtent(
(OID) dctnryOID, dbRoot, partition, segment, startLbid, allocSize);
if (rc != NO_ERROR)
{
return rc;
}
m_dctnryOID = dctnryOID;
m_partition = partition;
m_segment = segment;
m_dbRoot = dbRoot;
RETURN_ON_ERROR( ( rc = oid2FileName( m_dctnryOID, fileName, true,
m_dbRoot, m_partition, m_segment ) ) );
m_segFileName = fileName;
// if obsolete file exists, "w+b" will truncate and write over
m_dFile = createDctnryFile(fileName, colWidth, "w+b", DEFAULT_BUFSIZ);
m_dFile = createDctnryFile(fileName, colWidth, "w+b", DEFAULT_BUFSIZ,
startLbid);
{
// We presume the path will contain /
@ -228,21 +243,17 @@ int Dctnry::createDctnry( const OID& dctnryOID, int colWidth,
}
else
{
RETURN_ON_ERROR( setFileOffset(m_dFile, 0, SEEK_END) );
}
rc = BRMWrapper::getInstance()->allocateDictStoreExtent(
(OID)m_dctnryOID, m_dbRoot, m_partition, m_segment,
startLbid, allocSize);
(OID) m_dctnryOID, m_dbRoot, m_partition, m_segment, startLbid,
allocSize);
if (rc != NO_ERROR)
{
if (flag)
{
closeDctnryFile(false, oids);
return rc;
}
return rc;
RETURN_ON_ERROR(setFileOffset(m_dFile, 0, SEEK_END));
}
// We allocate a full extent from BRM, but only write an abbreviated 256K
@ -278,8 +289,8 @@ int Dctnry::createDctnry( const OID& dctnryOID, int colWidth,
m_dctnryHeader2,
m_totalHdrBytes,
false,
true ); // explicitly optimize
true, // explicitly optimize
startLbid );
if (rc != NO_ERROR)
{
if (flag)
@ -1488,8 +1499,9 @@ int Dctnry::updateDctnry(unsigned char* sigValue, int& sigSize,
* open dictionary file
******************************************************************************/
IDBDataFile* Dctnry::createDctnryFile(
const char* name, int, const char* mode, int ioBuffSize)
const char* name, int, const char* mode, int ioBuffSize, LBID_t lbid)
{
(void) lbid;
return openFile(name, mode, ioBuffSize, false);
}

3
writeengine/dictionary/we_dctnry.h
View File

@ -305,7 +305,8 @@ protected:
// methods to be overriden by compression classes
// (width argument in createDctnryFile() is string width, not token width)
virtual IDBDataFile* createDctnryFile(const char* name, int width,
const char* mode, int ioBuffSize);
const char* mode, int ioBuffSize,
BRM::LBID_t lbid = -1);
virtual IDBDataFile* openDctnryFile(bool useTmpSuffix);
virtual void closeDctnryFile(bool doFlush, std::map<FID, FID>& oids);
virtual int numOfBlocksInFile();

38
writeengine/shared/we_chunkmanager.cpp
View File

@ -290,6 +290,25 @@ IDBDataFile* ChunkManager::getFilePtr(const FID& fid,
return (fileData ? fileData->fFilePtr : NULL);
}
//------------------------------------------------------------------------------
// Get/Return IDBDataFile* for specified OID, root, partition, and segment.
// Function is to be used to open column/dict segment file.
// If the IDBDataFile* is not found, then a segment file will be opened using
// the mode (mode) and I/O buffer size (size) that is given. Name of the
// resulting file is returned in filename.
//------------------------------------------------------------------------------
IDBDataFile* ChunkManager::getSegmentFilePtr(
FID& fid, uint16_t root, uint32_t partition, uint16_t segment,
execplan::CalpontSystemCatalog::ColDataType colDataType, uint32_t colWidth,
std::string& filename, const char* mode, int32_t size, bool useTmpSuffix,
bool isDict) const
{
CompFileData* fileData =
getFileData(fid, root, partition, segment, filename, mode, size,
colDataType, colWidth, useTmpSuffix, isDict);
return (fileData ? fileData->fFilePtr : NULL);
}
//------------------------------------------------------------------------------
// Get/Return CompFileData* for specified column OID, root, partition, and
// segment. If the IDBDataFile* is not found, then a segment file will be opened
@ -411,7 +430,8 @@ IDBDataFile* ChunkManager::createDctnryFile(const FID& fid,
uint16_t segment,
const char* filename,
const char* mode,
int size)
int size,
BRM::LBID_t lbid)
{
FileID fileID(fid, root, partition, segment);
CompFileData* fileData = new CompFileData(fileID, fid, CalpontSystemCatalog::VARCHAR, width);
@ -440,8 +460,13 @@ IDBDataFile* ChunkManager::createDctnryFile(const FID& fid,
fileData->fFileHeader.fLongPtrSectData.reset(fileData->fFileHeader.fPtrSection);
}
fCompressor.initHdr(fileData->fFileHeader.fControlData, fileData->fFileHeader.fPtrSection,
// Dictionary store extent width == 0. See more details in function
// `createDictStoreExtent`.
fCompressor.initHdr(fileData->fFileHeader.fControlData,
fileData->fFileHeader.fPtrSection,
/*colWidth=*/0, fileData->fColDataType,
fFileOp->compressionType(), hdrSize);
fCompressor.setLBID0(fileData->fFileHeader.fControlData, lbid);
if (writeHeader(fileData, __LINE__) != NO_ERROR)
{
@ -1376,7 +1401,8 @@ int ChunkManager::expandAbbrevColumnExtent(IDBDataFile* pFile, const uint8_t* em
// Increment the block count stored in the chunk header used to track how many
// blocks are allocated to the corresponding segment file.
//------------------------------------------------------------------------------
int ChunkManager::updateColumnExtent(IDBDataFile* pFile, int addBlockCount)
// same here as for dict.
int ChunkManager::updateColumnExtent(IDBDataFile* pFile, int addBlockCount, int64_t lbid)
{
map<IDBDataFile*, CompFileData*>::iterator i = fFilePtrMap.find(pFile);
@ -1397,6 +1423,7 @@ int ChunkManager::updateColumnExtent(IDBDataFile* pFile, int addBlockCount)
int rc = NO_ERROR;
char* hdr = pFileData->fFileHeader.fControlData;
fCompressor.setBlockCount(hdr, fCompressor.getBlockCount(hdr) + addBlockCount);
fCompressor.setLBID1(hdr, lbid);
ChunkData* chunkData = (pFileData)->findChunk(0);
if (chunkData != NULL)
@ -1428,7 +1455,8 @@ int ChunkManager::updateColumnExtent(IDBDataFile* pFile, int addBlockCount)
// Increment the block count stored in the chunk header used to track how many
// blocks are allocated to the corresponding segment file.
//------------------------------------------------------------------------------
int ChunkManager::updateDctnryExtent(IDBDataFile* pFile, int addBlockCount)
int ChunkManager::updateDctnryExtent(IDBDataFile* pFile, int addBlockCount,
BRM::LBID_t lbid)
{
map<IDBDataFile*, CompFileData*>::iterator i = fFilePtrMap.find(pFile);
@ -1485,6 +1513,8 @@ int ChunkManager::updateDctnryExtent(IDBDataFile* pFile, int addBlockCount)
if (rc == NO_ERROR)
fCompressor.setBlockCount(hdr, fCompressor.getBlockCount(hdr) + addBlockCount);
if (currentBlockCount)
fCompressor.setLBID1(hdr, lbid);
return rc;
}

17
writeengine/shared/we_chunkmanager.h
View File

@ -189,6 +189,14 @@ public:
int size,
bool useTmpSuffix) const;
// @brief Retrieve a file pointer in the chunk manager.
// for column/dict segment file
IDBDataFile* getSegmentFilePtr(
FID& fid, uint16_t root, uint32_t partition, uint16_t segment,
execplan::CalpontSystemCatalog::ColDataType colDataType,
uint32_t colWidth, std::string& filename, const char* mode,
int32_t size, bool useTmpSuffix, bool isDict) const;
// @brief Create a compressed dictionary file with an appropriate header.
IDBDataFile* createDctnryFile(const FID& fid,
int64_t width,
@ -197,7 +205,8 @@ public:
uint16_t segment,
const char* filename,
const char* mode,
int size);
int size,
int64_t lbid);
// @brief Read a block from pFile at offset fbo.
// The data may copied from memory if the chunk it belongs to is already available.
@ -217,10 +226,12 @@ public:
int expandAbbrevColumnExtent(IDBDataFile* pFile, const uint8_t* emptyVal, int width);
// @brief Update column extent
int updateColumnExtent(IDBDataFile* pFile, int addBlockCount);
int updateColumnExtent(IDBDataFile* pFile, int addBlockCount,
int64_t lbid);
// @brief Update dictionary extent
int updateDctnryExtent(IDBDataFile* pFile, int addBlockCount);
int updateDctnryExtent(IDBDataFile* pFile, int addBlockCount,
int64_t lbid);
// @brief Read in n continuous blocks to read buffer.
// for backing up blocks to version buffer

41
writeengine/shared/we_fileop.cpp
View File

@ -163,7 +163,8 @@ int FileOp::createDir( const char* dirName, mode_t mode ) const
int FileOp::createFile( const char* fileName, int numOfBlock,
const uint8_t* emptyVal, int width,
execplan::CalpontSystemCatalog::ColDataType colDataType,
uint16_t dbRoot )
uint16_t dbRoot,
BRM::LBID_t startLbid )
{
IDBDataFile* pFile =
IDBDataFile::open(
@ -185,6 +186,7 @@ int FileOp::createFile( const char* fileName, int numOfBlock,
numOfBlock,
emptyVal,
width,
startLbid,
colDataType );
}
else
@ -284,7 +286,8 @@ int FileOp::createFile(FID fid,
//timer.stop( "allocateColExtent" );
return createFile( fileName, totalSize, emptyVal, width, colDataType, dbRoot );
return createFile(fileName, totalSize, emptyVal, width, colDataType,
dbRoot, startLbid);
}
/***********************************************************
@ -815,6 +818,7 @@ int FileOp::extendFile(
{
IDBCompressInterface compressor;
compressor.initHdr(hdrs, width, colDataType, m_compressionType);
compressor.setLBID0(hdrs, startLbid);
}
}
@ -849,7 +853,8 @@ int FileOp::extendFile(
newFile, // new or existing file
false, // don't expand; new extent
false, // add full (not abbreviated) extent
true); // try to optimize extent creation
true, // try to optimize extent creation
startLbid );
return rc;
}
@ -973,6 +978,7 @@ int FileOp::addExtentExactFile(
{
IDBCompressInterface compressor;
compressor.initHdr(hdrs, width, colDataType, m_compressionType);
compressor.setLBID0(hdrs, startLbid);
}
}
@ -1007,7 +1013,8 @@ int FileOp::addExtentExactFile(
colDataType,
newFile, // new or existing file
false, // don't expand; new extent
false ); // add full (not abbreviated) extent
false, // add full (not abbreviated) extent
startLbid );
closeFile( pFile );
return rc;
@ -1052,13 +1059,15 @@ int FileOp::initColumnExtent(
bool bNewFile,
bool bExpandExtent,
bool bAbbrevExtent,
bool bOptExtension)
bool bOptExtension,
int64_t lbid)
{
if ((bNewFile) && (m_compressionType))
{
char hdrs[IDBCompressInterface::HDR_BUF_LEN * 2];
IDBCompressInterface compressor;
compressor.initHdr(hdrs, width, colDataType, m_compressionType);
compressor.setLBID0(hdrs, lbid);
if (bAbbrevExtent)
compressor.setBlockCount(hdrs, nBlocks);
@ -1072,7 +1081,7 @@ int FileOp::initColumnExtent(
//@Bug 3219. update the compression header after the extent is expanded.
if ((!bNewFile) && (m_compressionType) && (bExpandExtent))
{
updateColumnExtent(pFile, nBlocks);
updateColumnExtent(pFile, nBlocks, lbid);
}
// @bug 2378. Synchronize here to avoid write buffer pile up too much,
@ -1188,7 +1197,7 @@ int FileOp::initColumnExtent(
//@Bug 3219. update the compression header after the extent is expanded.
if ((!bNewFile) && (m_compressionType) && (bExpandExtent))
{
updateColumnExtent(pFile, nBlocks);
updateColumnExtent(pFile, nBlocks, lbid);
}
// @bug 2378. Synchronize here to avoid write buffer pile up too much,
@ -1229,6 +1238,7 @@ int FileOp::initAbbrevCompColumnExtent(
int nBlocks,
const uint8_t* emptyVal,
int width,
BRM::LBID_t startLBID,
execplan::CalpontSystemCatalog::ColDataType colDataType)
{
// Reserve disk space for optimized abbreviated extent
@ -1241,7 +1251,8 @@ int FileOp::initAbbrevCompColumnExtent(
true, // new file
false, // don't expand; add new extent
true, // add abbreviated extent
true); // optimize the initial extent
true, // optimize the initial extent
startLBID);
if (rc != NO_ERROR)
{
return rc;
@ -1257,6 +1268,7 @@ int FileOp::initAbbrevCompColumnExtent(
INITIAL_EXTENT_ROWS_TO_DISK,
emptyVal,
width,
startLBID,
colDataType,
hdrs );
@ -1292,6 +1304,7 @@ int FileOp::writeInitialCompColumnChunk(
int nRows,
const uint8_t* emptyVal,
int width,
BRM::LBID_t startLBID,
execplan::CalpontSystemCatalog::ColDataType colDataType,
char* hdrs)
{
@ -1336,6 +1349,7 @@ int FileOp::writeInitialCompColumnChunk(
compressor.initHdr(hdrs, width, colDataType, m_compressionType);
compressor.setBlockCount(hdrs, nBlocksAllocated);
compressor.setLBID0(hdrs, startLBID);
// Store compression pointers in the header
std::vector<uint64_t> ptrs;
@ -1841,13 +1855,14 @@ int FileOp::initDctnryExtent(
unsigned char* blockHdrInit,
int blockHdrInitSize,
bool bExpandExtent,
bool bOptExtension )
bool bOptExtension,
int64_t lbid)
{
// @bug5769 Don't initialize extents or truncate db files on HDFS
if (idbdatafile::IDBPolicy::useHdfs())
{
if (m_compressionType)
updateDctnryExtent(pFile, nBlocks);
updateDctnryExtent(pFile, nBlocks, lbid);
// Synchronize to avoid write buffer pile up too much, which could cause
// controllernode to timeout later when it needs to save a snapshot.
@ -1972,7 +1987,7 @@ int FileOp::initDctnryExtent(
// MCOL-498 CS has to set a number of blocs in the chunk header
if ( m_compressionType )
{
updateDctnryExtent(pFile, nBlocks);
updateDctnryExtent(pFile, nBlocks, lbid);
}
pFile->flush();
}
@ -2897,12 +2912,12 @@ int FileOp::flushFile(int rc, std::map<FID, FID>& oids)
return NO_ERROR;
}
int FileOp::updateColumnExtent(IDBDataFile* pFile, int nBlocks)
int FileOp::updateColumnExtent(IDBDataFile* pFile, int nBlocks, int64_t lbid)
{
return NO_ERROR;
}
int FileOp::updateDctnryExtent(IDBDataFile* pFile, int nBlocks)
int FileOp::updateDctnryExtent(IDBDataFile* pFile, int nBlocks, int64_t lbid)
{
return NO_ERROR;
}

17
writeengine/shared/we_fileop.h
View File

@ -103,7 +103,8 @@ public:
int createFile( const char* fileName, int fileSize,
const uint8_t* emptyVal, int width,
execplan::CalpontSystemCatalog::ColDataType colDataType,
uint16_t dbRoot );
uint16_t dbRoot ,
BRM::LBID_t lbid = -1 );
/**
* @brief Delete a file
@ -358,7 +359,8 @@ public:
unsigned char* blockHdrInit,
int blockHdrInitSize,
bool bExpandExtent,
bool bOptExtension = false );
bool bOptExtension = false,
int64_t lbid = 0);
/**
* @brief Check whether it is an directory
@ -510,14 +512,15 @@ public:
bool bNewFile,
bool bExpandExtent,
bool bAbbrevExtent,
bool bOptExtension=false );
bool bOptExtension=false,
int64_t lbid = 0 );
// Calls a chown and logs an error message
bool chownDataPath(const std::string& fileName) const;
protected:
EXPORT virtual int updateColumnExtent(IDBDataFile* pFile, int nBlocks);
EXPORT virtual int updateDctnryExtent(IDBDataFile* pFile, int nBlocks);
EXPORT virtual int updateColumnExtent(IDBDataFile* pFile, int nBlocks, int64_t lbid);
EXPORT virtual int updateDctnryExtent(IDBDataFile* pFile, int nBlocks, int64_t lbid);
int m_compressionType; // compresssion type
@ -534,7 +537,7 @@ private:
int initAbbrevCompColumnExtent(
IDBDataFile* pFile, uint16_t dbRoot, int nBlocks,
const uint8_t* emptyVal, int width,
const uint8_t* emptyVal, int width, BRM::LBID_t lbid,
execplan::CalpontSystemCatalog::ColDataType colDataType);
static void initDbRootExtentMutexes();
@ -542,7 +545,7 @@ private:
int writeInitialCompColumnChunk(
IDBDataFile* pFile, int nBlocksAllocated, int nRows,
const uint8_t* emptyVal, int width,
const uint8_t* emptyVal, int width, BRM::LBID_t lbid,
execplan::CalpontSystemCatalog::ColDataType colDataType, char* hdrs);
TxnID m_transId;

2
writeengine/wrapper/we_colop.cpp
View File

@ -483,7 +483,7 @@ int ColumnOp::allocRowId(const TxnID& txnid, bool useStartingExtent,
return rc;
//@Bug 3164 update compressed extent
updateColumnExtent(newCol.dataFile.pFile, allocSize);
updateColumnExtent(newCol.dataFile.pFile, allocSize, /*lbid=*/0);
//..Search first block of new extent for empty rows
rc = readBlock(newCol.dataFile.pFile, buf, newHwm);

4
writeengine/wrapper/we_colopcompress.cpp
View File

@ -210,9 +210,9 @@ int ColumnOpCompress1::expandAbbrevColumnExtent(
}
int ColumnOpCompress1::updateColumnExtent(IDBDataFile* pFile, int nBlocks)
int ColumnOpCompress1::updateColumnExtent(IDBDataFile* pFile, int nBlocks, int64_t lbid)
{
return m_chunkManager->updateColumnExtent(pFile, nBlocks);
return m_chunkManager->updateColumnExtent(pFile, nBlocks, lbid);
}

2
writeengine/wrapper/we_colopcompress.h
View File

@ -160,7 +160,7 @@ protected:
/**
* @brief virtual method in FileOp
*/
int updateColumnExtent(IDBDataFile* pFile, int nBlocks);
int updateColumnExtent(IDBDataFile* pFile, int nBlocks, int64_t lbid);
/**
* @brief virtual method in ColumnOp

16
writeengine/wrapper/we_dctnrycompress.cpp
View File

@ -90,16 +90,20 @@ DctnryCompress1::~DctnryCompress1()
delete m_chunkManager;
}
int DctnryCompress1::updateDctnryExtent(IDBDataFile* pFile, int nBlocks)
int DctnryCompress1::updateDctnryExtent(IDBDataFile* pFile, int nBlocks,
int64_t lbid)
{
return m_chunkManager->updateDctnryExtent(pFile, nBlocks);
return m_chunkManager->updateDctnryExtent(pFile, nBlocks, lbid);
}
IDBDataFile* DctnryCompress1::createDctnryFile(const char* name, int width, const char* mode, int ioBuffSize)
IDBDataFile* DctnryCompress1::createDctnryFile(const char* name, int width,
const char* mode,
int ioBuffSize,
BRM::LBID_t lbid)
{
return m_chunkManager->createDctnryFile(
m_dctnryOID, width, m_dbRoot, m_partition, m_segment, name, mode, ioBuffSize);
return m_chunkManager->createDctnryFile(m_dctnryOID, width, m_dbRoot,
m_partition, m_segment, name, mode,
ioBuffSize, lbid);
}

6
writeengine/wrapper/we_dctnrycompress.h
View File

@ -97,7 +97,9 @@ public:
/**
* @brief virtual method in Dctnry
*/
IDBDataFile* createDctnryFile(const char* name, int width, const char* mode, int ioBuffSize);
IDBDataFile* createDctnryFile(const char* name, int width,
const char* mode, int ioBuffSize,
int64_t lbid);
/**
* @brief virtual method in Dctnry
@ -159,7 +161,7 @@ protected:
/**
* @brief virtual method in FileOp
*/
int updateDctnryExtent(IDBDataFile* pFile, int nBlocks);
int updateDctnryExtent(IDBDataFile* pFile, int nBlocks, int64_t lbid);
/**
* @brief convert lbid to fbo