database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-29 08:21:15 +03:00
Code Activity

MCOL-498 Revived unit tests for writeengine/shared and add new tests

Browse Source 
for extent extention.
Added a getter, moved some methods from protected into public to use
with unit tests, e.g createFile, setPreallocSpace. Added code stub in
FileOp::oid2FileName to use with UT.
This commit is contained in:
Roman Nozdrin
2019-02-20 19:56:28 +03:00
parent ecbf6b7606
commit bc3c780e35
11 changed files with 417 additions and 278 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
CMakeLists.txt
View File

@ -276,6 +276,16 @@ ADD_SUBDIRECTORY(writeengine/server)
ADD_SUBDIRECTORY(writeengine/bulk) ADD_SUBDIRECTORY(writeengine/bulk)
ADD_SUBDIRECTORY(writeengine/splitter) ADD_SUBDIRECTORY(writeengine/splitter)
# WriteEngine component tests
IF( WITH_SHARED_COMP_TESTS )
# search for cppunit
INCLUDE (findcppunit.cmake)
if (NOT CPPUNIT_FOUND)
MESSAGE(FATAL_ERROR "CPPUnit not found please install cppunit-devel for CentOS/RedHat or libcppunit-dev for Ubuntu/Debian")
endif()
ADD_SUBDIRECTORY(writeengine/shared)
ENDIF( WITH_SHARED_COMP_TESTS )
INCLUDE(cpackEngineRPM.cmake) INCLUDE(cpackEngineRPM.cmake)
INCLUDE(cpackEngineDEB.cmake) INCLUDE(cpackEngineDEB.cmake)

5
utils/idbdatafile/IDBPolicy.cpp
View File

@ -297,4 +297,9 @@ void IDBPolicy::configIDBPolicy()
} }
} }
void IDBPolicy::setPreallocSpace(uint16_t dbRoot)
{
s_PreallocSpace.push_back(dbRoot);
}
} }

4
utils/idbdatafile/IDBPolicy.h
View File

@ -129,6 +129,10 @@ public:
static int listDirectory(const char* pathname, std::list<std::string>& contents); static int listDirectory(const char* pathname, std::list<std::string>& contents);
static bool isDir(const char* pathname); static bool isDir(const char* pathname);
static int copyFile(const char* srcPath, const char* destPath); static int copyFile(const char* srcPath, const char* destPath);
/**
* This is used in WE shared components Unit Tests
*/
static void setPreallocSpace(uint16_t dbRoot);
private: private:
/** /**

41
writeengine/dictionary/CMakeLists.txt
View File

@ -1,44 +1,3 @@
include_directories(${KDE4_INCLUDES} ${KDE4_INCLUDE_DIR} ${QT_INCLUDES} )
########### install files ############### ########### install files ###############
install(FILES we_dctnry.h DESTINATION include) install(FILES we_dctnry.h DESTINATION include)
#original Makefile.am contents follow:
## Copyright (C) 2014 InfiniDB, Inc.
##
## This program is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License
## as published by the Free Software Foundation; version 2 of
## the License.
##
## This program is distributed in the hope that it will be useful,
## but WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
## GNU General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with this program; if not, write to the Free Software
## Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
## MA 02110-1301, USA.
#
## $Id: Makefile.am 864 2009-04-02 19:22:49Z rdempsey $
### Process this file with automake to produce Makefile.in
#
#include_HEADERS = we_dctnry.h
#
#test:
#
#coverage:
#
#leakcheck:
#
#docs:
#
#bootstrap: install-data-am
#

15
writeengine/dictionary/we_dctnry.h
View File

@ -132,6 +132,10 @@ public:
{ {
return m_curLbid; return m_curLbid;
} }
const unsigned char* getDctnryHeader2() const
{
return m_dctnryHeader2;
}
/** /**
* @brief Insert a signature value to a file block and return token/pointer. * @brief Insert a signature value to a file block and return token/pointer.
@ -222,6 +226,17 @@ public:
{ {
return NO_ERROR; return NO_ERROR;
} }
/**
* @brief Use this only in Unit Tests and not in prod
*/
virtual IDBDataFile* createDctnryFileUnit(const char* name,
int width,
const char* mode,
int ioBuffSize)
{
return createDctnryFile(name, width, mode, ioBuffSize);
}
//------------------------------------------------------------------------------ //------------------------------------------------------------------------------
// Protected members // Protected members

43
writeengine/shared/CMakeLists.txt
View File

@ -1,44 +1,11 @@
include_directories(${KDE4_INCLUDES} ${KDE4_INCLUDE_DIR} ${QT_INCLUDES} ) include_directories(${ENGINE_COMMON_INCLUDES} ../dictionary)
add_executable(we_shared_components_tests ./shared_components_tests.cpp)
target_link_libraries(we_shared_components_tests ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES})
install(TARGETS we_shared_components_tests DESTINATION ${ENGINE_BINDIR} COMPONENT platform)
########### install files ############### ########### install files ###############
install(FILES we_index.h we_define.h we_type.h we_fileop.h we_blockop.h we_dbfileop.h we_obj.h we_log.h we_simplesyslog.h we_convertor.h we_brm.h we_macro.h we_config.h we_cache.h we_stats.h we_bulkrollbackmgr.h we_typeext.h we_chunkmanager.h we_bulkrollbackfilecompressed.h we_bulkrollbackfilecompressedhdfs.h we_bulkrollbackfile.h we_rbmetawriter.h we_dbrootextenttracker.h we_confirmhdfsdbfile.h DESTINATION include) install(FILES we_index.h we_define.h we_type.h we_fileop.h we_blockop.h we_dbfileop.h we_obj.h we_log.h we_simplesyslog.h we_convertor.h we_brm.h we_macro.h we_config.h we_cache.h we_stats.h we_bulkrollbackmgr.h we_typeext.h we_chunkmanager.h we_bulkrollbackfilecompressed.h we_bulkrollbackfilecompressedhdfs.h we_bulkrollbackfile.h we_rbmetawriter.h we_dbrootextenttracker.h we_confirmhdfsdbfile.h DESTINATION include)
#original Makefile.am contents follow:
## Copyright (C) 2014 InfiniDB, Inc.
##
## This program is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License
## as published by the Free Software Foundation; version 2 of
## the License.
##
## This program is distributed in the hope that it will be useful,
## but WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
## GNU General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with this program; if not, write to the Free Software
## Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
## MA 02110-1301, USA.
#
## $Id: Makefile.am 3720 2012-04-04 18:18:49Z rdempsey $
### Process this file with automake to produce Makefile.in
#
#include_HEADERS = we_index.h we_define.h we_type.h we_fileop.h we_blockop.h we_dbfileop.h we_obj.h we_log.h we_simplesyslog.h we_convertor.h we_brm.h we_macro.h we_config.h we_cache.h we_stats.h we_bulkrollbackmgr.h we_typeext.h we_chunkmanager.h we_bulkrollbackfilecompressed.h we_bulkrollbackfilecompressedhdfs.h we_bulkrollbackfile.h we_rbmetawriter.h we_dbrootextenttracker.h we_confirmhdfsdbfile.h
#
#test:
#
#coverage:
#
#leakcheck:
#
#docs:
#
#bootstrap: install-data-am
#

493
writeengine/shared/tshared.cpp → writeengine/shared/shared_components_tests.cpp
View File

@ -29,18 +29,28 @@ using namespace boost;
#include <cppunit/extensions/HelperMacros.h> #include <cppunit/extensions/HelperMacros.h>
#include <we_dbfileop.h> #include "we_dbfileop.h"
#include <we_type.h> #include "we_type.h"
#include <we_semop.h> #include "we_log.h"
#include <we_log.h> #include "we_convertor.h"
#include <we_convertor.h> #include "we_brm.h"
#include <we_brm.h> #include "we_cache.h"
#include <we_cache.h> #include "we_colop.h"
#include "IDBDataFile.h"
#include "BufferedFile.h"
#include "IDBPolicy.h"
#include "IDBFileSystem.h"
#include "idbcompress.h"
#include "calpontsystemcatalog.h"
#include "we_colopcompress.h"
#include "we_dctnrycompress.h"
using namespace compress;
using namespace idbdatafile;
using namespace WriteEngine; using namespace WriteEngine;
using namespace BRM; using namespace BRM;
int compare (const void* a, const void* b) /*int compare (const void* a, const void* b)
{ {
return ( *(uint32_t*)a - * (uint32_t*)b ); return ( *(uint32_t*)a - * (uint32_t*)b );
} }
@ -49,51 +59,55 @@ int compare1(const void* a, const void* b)
{ {
return ( (*(SortTuple*)a).key - (*(SortTuple*)b).key ); return ( (*(SortTuple*)a).key - (*(SortTuple*)b).key );
} }
*/
class SharedTest : public CppUnit::TestFixture class SharedTest : public CppUnit::TestFixture
{ {
CPPUNIT_TEST_SUITE( SharedTest ); CPPUNIT_TEST_SUITE( SharedTest );
//CPPUNIT_TEST(setUp); CPPUNIT_TEST(setUp);
//CPPUNIT_TEST( test1 ); //CPPUNIT_TEST( test1 );
// File operation testing // File operation testing
CPPUNIT_TEST( testFileNameOp ); // CPPUNIT_TEST( testFileNameOp );
CPPUNIT_TEST( testFileHandleOp ); // CPPUNIT_TEST( testFileHandleOp );
CPPUNIT_TEST( testDirBasic ); CPPUNIT_TEST( testDirBasic );
CPPUNIT_TEST( testCreateDeleteFile );
// Data block related testing // Data block related testing
CPPUNIT_TEST( testCalculateRowIdBitmap ); // CPPUNIT_TEST( testCalculateRowIdBitmap );
CPPUNIT_TEST( testBlockBuffer ); // CPPUNIT_TEST( testBlockBuffer );
CPPUNIT_TEST( testBitBasic ); // CPPUNIT_TEST( testBitBasic );
CPPUNIT_TEST( testBufferBit ); // CPPUNIT_TEST( testBufferBit );
CPPUNIT_TEST( testBitShift ); // CPPUNIT_TEST( testBitShift );
CPPUNIT_TEST( testEmptyRowValue ); // CPPUNIT_TEST( testEmptyRowValue );
CPPUNIT_TEST( testCorrectRowWidth ); // CPPUNIT_TEST( testCorrectRowWidth );
// DB File Block related testing // DB File Block related testing
CPPUNIT_TEST( testDbBlock ); // CPPUNIT_TEST( testDbBlock );
CPPUNIT_TEST( testCopyDbFile ); // CPPUNIT_TEST( testCopyDbFile );
// Extent & dict related testing
CPPUNIT_TEST( testExtensionWOPrealloc );
CPPUNIT_TEST( testDictExtensionWOPrealloc );
// Semaphore related testing // Semaphore related testing
CPPUNIT_TEST( testSem ); // CPPUNIT_TEST( testSem );
// Log related testing // Log related testing
CPPUNIT_TEST( testLog ); CPPUNIT_TEST( testLog );
// Version Buffer related testing // Version Buffer related testing
CPPUNIT_TEST( testHWM ); // CPPUNIT_TEST( testHWM );
CPPUNIT_TEST( testVB ); // CPPUNIT_TEST( testVB );
// Disk manager related testing // Disk manager related testing
CPPUNIT_TEST( testDM ); // CPPUNIT_TEST( testDM );
CPPUNIT_TEST( tearDown ); // CPPUNIT_TEST( tearDown );
// Cache related testing // Cache related testing
CPPUNIT_TEST( testCacheBasic ); // CPPUNIT_TEST( testCacheBasic );
CPPUNIT_TEST( testCacheReadWrite ); // CPPUNIT_TEST( testCacheReadWrite );
CPPUNIT_TEST( testCleanup ); // NEVER COMMENT OUT THIS LINE CPPUNIT_TEST( testCleanup ); // NEVER COMMENT OUT THIS LINE
CPPUNIT_TEST_SUITE_END(); CPPUNIT_TEST_SUITE_END();
@ -113,8 +127,6 @@ public:
void test1() void test1()
{ {
//m_wrapper.test();
// int numOfBlock = 10240;
int numOfBlock = 1024; int numOfBlock = 1024;
FILE* pFile; FILE* pFile;
unsigned char writeBuf[BYTE_PER_BLOCK * 10]; unsigned char writeBuf[BYTE_PER_BLOCK * 10];
@ -139,7 +151,7 @@ public:
} }
} }
} }
/*
void testFileNameOp() void testFileNameOp()
{ {
FileOp fileOp; FileOp fileOp;
@ -342,23 +354,74 @@ public:
} }
*/
void testDirBasic() void testDirBasic()
{ {
FileOp fileOp; FileOp fileOp;
char dirName[30]; char dirName[30];
int rc; int rc;
strcpy( dirName, "testdir" ); printf("\nRunning testDirBasic \n");
fileOp.removeDir( dirName ); idbdatafile::IDBPolicy::init(true, false, "", 0);
IDBFileSystem& fs = IDBPolicy::getFs( "/tmp" );
strcpy( dirName, "/tmp/testdir42" );
fs.remove( dirName );
CPPUNIT_ASSERT( fileOp.isDir( dirName ) == false ); CPPUNIT_ASSERT( fileOp.isDir( dirName ) == false );
rc = fileOp.createDir( dirName ); rc = fileOp.createDir( dirName );
CPPUNIT_ASSERT( rc == NO_ERROR ); CPPUNIT_ASSERT( rc == NO_ERROR );
CPPUNIT_ASSERT( fileOp.isDir( dirName ) == true ); CPPUNIT_ASSERT( fileOp.isDir( dirName ) == true );
fileOp.removeDir( dirName ); fs.remove( dirName );
} }
void testCreateDeleteFile()
{
IDBDataFile* pFile = NULL;
FileOp fileOp;
BlockOp blockOp;
char fileName[20];
int rc;
char hdrs[ IDBCompressInterface::HDR_BUF_LEN * 2 ];
printf("\nRunning testCreateDeleteFile \n");
idbdatafile::IDBPolicy::init(true, false, "", 0);
// Set to versionbuffer to satisfy IDBPolicy::getType
strcpy( fileName, "versionbuffer" );
fileOp.compressionType(1);
fileOp.deleteFile( fileName );
CPPUNIT_ASSERT( fileOp.exists( fileName ) == false );
int width = blockOp.getCorrectRowWidth( execplan::CalpontSystemCatalog::BIGINT, 8 );
int nBlocks = INITIAL_EXTENT_ROWS_TO_DISK / BYTE_PER_BLOCK * width;
uint64_t emptyVal = blockOp.getEmptyRowValue( execplan::CalpontSystemCatalog::BIGINT, 8 );
// createFile runs IDBDataFile::open + initAbrevCompColumnExtent
// under the hood
// bigint column file
rc = fileOp.createFile( fileName,
nBlocks, // number of blocks
emptyVal, // NULL value
width, // width
1 ); // dbroot
CPPUNIT_ASSERT( rc == NO_ERROR );
fileOp.closeFile(pFile);
pFile = IDBDataFile::open(IDBPolicy::getType(fileName,
IDBPolicy::WRITEENG), fileName, "rb", 1);
rc = pFile->seek(0, 0);
CPPUNIT_ASSERT(rc == NO_ERROR);
rc = fileOp.readHeaders(pFile, hdrs);
CPPUNIT_ASSERT( rc == NO_ERROR );
// Couldn't use IDBDataFile->close() here w/o excplicit cast
fileOp.closeFile(pFile);
fileOp.deleteFile( fileName );
CPPUNIT_ASSERT( fileOp.exists( fileName ) == false );
}
/*
void testCalculateRowIdBitmap() void testCalculateRowIdBitmap()
{ {
BlockOp blockOp; BlockOp blockOp;
@ -374,23 +437,23 @@ public:
CPPUNIT_ASSERT( bio == 16 ); CPPUNIT_ASSERT( bio == 16 );
// Assuming 2048 per data block, 4 byte width // Assuming 2048 per data block, 4 byte width
/* rowId = 2049; // rowId = 2049;
CPPUNIT_ASSERT( blockOp.calculateRowId( rowId, 2048, 4, fbo, bio ) == true ); // CPPUNIT_ASSERT( blockOp.calculateRowId( rowId, 2048, 4, fbo, bio ) == true );
CPPUNIT_ASSERT( fbo == 1 ); // CPPUNIT_ASSERT( fbo == 1 );
CPPUNIT_ASSERT( bio == 16 ); // CPPUNIT_ASSERT( bio == 16 );
//
// Assuming 4096 per data block, 2 byte width // // Assuming 4096 per data block, 2 byte width
rowId = 2049; // rowId = 2049;
CPPUNIT_ASSERT( blockOp.calculateRowId( rowId, 4096, 2, fbo, bio ) == true ); // CPPUNIT_ASSERT( blockOp.calculateRowId( rowId, 4096, 2, fbo, bio ) == true );
CPPUNIT_ASSERT( fbo == 1 ); // CPPUNIT_ASSERT( fbo == 1 );
CPPUNIT_ASSERT( bio == 16 ); // CPPUNIT_ASSERT( bio == 16 );
//
// Assuming 8192 per data block, 1 byte width // // Assuming 8192 per data block, 1 byte width
rowId = 2049; // rowId = 2049;
CPPUNIT_ASSERT( blockOp.calculateRowId( rowId, 8192, 1, fbo, bio ) == true ); // CPPUNIT_ASSERT( blockOp.calculateRowId( rowId, 8192, 1, fbo, bio ) == true );
CPPUNIT_ASSERT( fbo == 1 ); // CPPUNIT_ASSERT( fbo == 1 );
CPPUNIT_ASSERT( bio == 16 ); // CPPUNIT_ASSERT( bio == 16 );
*/ //
rowId = 65546; rowId = 65546;
CPPUNIT_ASSERT( blockOp.calculateRowBitmap( rowId, BYTE_PER_BLOCK * 8, fbo, bio, bbo ) == true ); CPPUNIT_ASSERT( blockOp.calculateRowBitmap( rowId, BYTE_PER_BLOCK * 8, fbo, bio, bbo ) == true );
CPPUNIT_ASSERT( fbo == 1 ); CPPUNIT_ASSERT( fbo == 1 );
@ -550,19 +613,19 @@ public:
curVal = blockOp.getEmptyRowValue( WriteEngine::DECIMAL, 8 ); curVal = blockOp.getEmptyRowValue( WriteEngine::DECIMAL, 8 );
CPPUNIT_ASSERT( curVal == 0x8000000000000001LL ); CPPUNIT_ASSERT( curVal == 0x8000000000000001LL );
/*
curVal = blockOp.getEmptyRowValue( WriteEngine::DECIMAL, 9 ); // curVal = blockOp.getEmptyRowValue( WriteEngine::DECIMAL, 9 );
CPPUNIT_ASSERT( curVal == 0x80000001 ); // CPPUNIT_ASSERT( curVal == 0x80000001 );
//
curVal = blockOp.getEmptyRowValue( WriteEngine::DECIMAL, 10 ); // curVal = blockOp.getEmptyRowValue( WriteEngine::DECIMAL, 10 );
CPPUNIT_ASSERT( curVal == 0x8000000000000001LL ); // CPPUNIT_ASSERT( curVal == 0x8000000000000001LL );
//
curVal = blockOp.getEmptyRowValue( WriteEngine::DECIMAL, 12 ); // curVal = blockOp.getEmptyRowValue( WriteEngine::DECIMAL, 12 );
CPPUNIT_ASSERT( curVal == 0x8000000000000001LL ); // CPPUNIT_ASSERT( curVal == 0x8000000000000001LL );
//
curVal = blockOp.getEmptyRowValue( WriteEngine::DECIMAL, 19 ); // curVal = blockOp.getEmptyRowValue( WriteEngine::DECIMAL, 19 );
CPPUNIT_ASSERT( curVal == 0xFFFFFFFFFFFFFFFFLL ); // CPPUNIT_ASSERT( curVal == 0xFFFFFFFFFFFFFFFFLL );
*/ //
curVal = blockOp.getEmptyRowValue( WriteEngine::DATE, 4 ); curVal = blockOp.getEmptyRowValue( WriteEngine::DATE, 4 );
CPPUNIT_ASSERT( curVal == 0xFFFFFFFF ); CPPUNIT_ASSERT( curVal == 0xFFFFFFFF );
@ -645,18 +708,18 @@ public:
curVal = blockOp.getCorrectRowWidth( WriteEngine::DECIMAL, 8 ); curVal = blockOp.getCorrectRowWidth( WriteEngine::DECIMAL, 8 );
CPPUNIT_ASSERT( curVal == 8 ); CPPUNIT_ASSERT( curVal == 8 );
/* curVal = blockOp.getCorrectRowWidth( WriteEngine::DECIMAL, 9 ); // curVal = blockOp.getCorrectRowWidth( WriteEngine::DECIMAL, 9 );
CPPUNIT_ASSERT( curVal == 4 ); // CPPUNIT_ASSERT( curVal == 4 );
//
curVal = blockOp.getCorrectRowWidth( WriteEngine::DECIMAL, 10 ); // curVal = blockOp.getCorrectRowWidth( WriteEngine::DECIMAL, 10 );
CPPUNIT_ASSERT( curVal == 8 ); // CPPUNIT_ASSERT( curVal == 8 );
//
curVal = blockOp.getCorrectRowWidth( WriteEngine::DECIMAL, 12 ); // curVal = blockOp.getCorrectRowWidth( WriteEngine::DECIMAL, 12 );
CPPUNIT_ASSERT( curVal == 8 ); // CPPUNIT_ASSERT( curVal == 8 );
//
curVal = blockOp.getCorrectRowWidth( WriteEngine::DECIMAL, 19 ); // curVal = blockOp.getCorrectRowWidth( WriteEngine::DECIMAL, 19 );
CPPUNIT_ASSERT( curVal == 8 ); // CPPUNIT_ASSERT( curVal == 8 );
*/ //
curVal = blockOp.getCorrectRowWidth( WriteEngine::DATE, 8 ); curVal = blockOp.getCorrectRowWidth( WriteEngine::DATE, 8 );
CPPUNIT_ASSERT( curVal == 4 ); CPPUNIT_ASSERT( curVal == 4 );
@ -892,126 +955,229 @@ public:
dbFileOp.closeFile( pTargetFile ); dbFileOp.closeFile( pTargetFile );
} }
*/
void testSem() void testExtensionWOPrealloc()
{ {
SemOp semOp; IDBDataFile* pFile = NULL;
FileOp fileOp; FileOp fileOp;
BlockOp blockOp;
char fileName[20];
int rc; int rc;
bool bSuccess; char hdrs[ IDBCompressInterface::HDR_BUF_LEN * 2 ];
key_t key; int dbRoot = 1;
int sid, totalNum = 5;
char fileName[100];
semOp.setMaxSemVal( 3 ); printf("\nRunning testExtensionWOPrealloc \n");
idbdatafile::IDBPolicy::init(true, false, "", 0);
// Set to versionbuffer to satisfy IDBPolicy::getType
strcpy( fileName, "versionbuffer" );
fileOp.compressionType(1);
bSuccess = semOp.getKey( NULL, key ); fileOp.deleteFile( fileName );
CPPUNIT_ASSERT( bSuccess == false ); CPPUNIT_ASSERT( fileOp.exists( fileName ) == false );
rc = fileOp.getFileName( 9991, fileName ); int width = blockOp.getCorrectRowWidth( execplan::CalpontSystemCatalog::BIGINT, 8 );
int nBlocks = INITIAL_EXTENT_ROWS_TO_DISK / BYTE_PER_BLOCK * width;
uint64_t emptyVal = blockOp.getEmptyRowValue( execplan::CalpontSystemCatalog::BIGINT, 8 );
// createFile runs IDBDataFile::open + initAbrevCompColumnExtent
// under the hood
// bigint column file
rc = fileOp.createFile( fileName,
nBlocks, // number of blocks
emptyVal, // NULL value
width, // width
dbRoot ); // dbroot
CPPUNIT_ASSERT( rc == NO_ERROR ); CPPUNIT_ASSERT( rc == NO_ERROR );
bSuccess = semOp.getKey( fileName, key ); // open created compressed file and check its header
CPPUNIT_ASSERT( bSuccess == false ); pFile = IDBDataFile::open(IDBPolicy::getType(fileName,
IDBPolicy::WRITEENG), fileName, "rb", dbRoot);
rc = fileOp.getFileName( 999, fileName ); rc = pFile->seek(0, 0);
CPPUNIT_ASSERT(rc == NO_ERROR);
rc = fileOp.readHeaders(pFile, hdrs);
CPPUNIT_ASSERT( rc == NO_ERROR ); CPPUNIT_ASSERT( rc == NO_ERROR );
bSuccess = semOp.getKey( fileName, key ); // Couldn't use IDBDataFile->close() here w/o excplicit cast
printf( "\nkey=%d", key ); fileOp.closeFile(pFile);
CPPUNIT_ASSERT( bSuccess == true );
if ( semOp.existSem( sid, key ) ) // Extend the extent up to 64MB
semOp.deleteSem( sid ); // first run w preallocation
idbdatafile::BufferedFile* bFile = new idbdatafile::BufferedFile(fileName, "r+b", 0);
pFile = dynamic_cast<IDBDataFile*>(bFile);
rc = fileOp.initColumnExtent(pFile,
dbRoot,
BYTE_PER_BLOCK, // number of blocks
emptyVal,
width,
false, // use existing file
true, // expand the extent
false, // add full (not abbreviated) extent
false); // don't optimize extention
rc = semOp.createSem( sid, key, 1000 );
CPPUNIT_ASSERT( rc == ERR_MAX_SEM );
rc = semOp.createSem( sid, key, totalNum ); CPPUNIT_ASSERT(rc == NO_ERROR);
CPPUNIT_ASSERT(bFile->size() == 67108864);
fileOp.closeFile(pFile);
// file has been extended delete the file before
// the second run
fileOp.deleteFile( fileName );
CPPUNIT_ASSERT(fileOp.exists( fileName ) == false);
// second run with disabled preallocation
rc = fileOp.createFile( fileName,
nBlocks, // number of blocks
emptyVal, // NULL value
width, // width
dbRoot ); // dbroot
CPPUNIT_ASSERT( rc == NO_ERROR ); CPPUNIT_ASSERT( rc == NO_ERROR );
rc = semOp.createSem( sid, key, totalNum ); // open created compressed file and check its header
CPPUNIT_ASSERT( rc == ERR_SEM_EXIST ); pFile = IDBDataFile::open(IDBPolicy::getType(fileName,
IDBPolicy::WRITEENG), fileName, "rb", dbRoot);
rc = semOp.openSem( sid, key ); rc = pFile->seek(0, 0);
CPPUNIT_ASSERT(rc == NO_ERROR);
rc = fileOp.readHeaders(pFile, hdrs);
CPPUNIT_ASSERT( rc == NO_ERROR ); CPPUNIT_ASSERT( rc == NO_ERROR );
semOp.printAllVal( sid ); fileOp.closeFile(pFile);
// lock bFile = new idbdatafile::BufferedFile(fileName, "r+b", 0);
printf( "\nlock one in 2" ); pFile = dynamic_cast<IDBDataFile*>(bFile);
rc = semOp.lockSem( sid, 2 );
CPPUNIT_ASSERT( rc == NO_ERROR ); // disable disk space preallocation and extend
CPPUNIT_ASSERT( semOp.getVal( sid, 2 ) == 2 ); idbdatafile::IDBPolicy::setPreallocSpace(dbRoot);
semOp.printAllVal( sid ); rc = fileOp.initColumnExtent(pFile,
dbRoot,
printf( "\nlock one in 2" ); BYTE_PER_BLOCK, // number of blocks
rc = semOp.lockSem( sid, 2 ); emptyVal,
CPPUNIT_ASSERT( rc == NO_ERROR ); width,
CPPUNIT_ASSERT( semOp.getVal( sid, 2 ) == 1 ); false, // use existing file
semOp.printAllVal( sid ); true, // expand the extent
false, // add full (not abbreviated) extent
printf( "\nlock one in 2" ); true); // optimize extention
rc = semOp.lockSem( sid, 2 );
CPPUNIT_ASSERT( rc == NO_ERROR );
CPPUNIT_ASSERT( semOp.getVal( sid, 2 ) == 0 );
semOp.printAllVal( sid );
rc = semOp.lockSem( sid, 2 );
CPPUNIT_ASSERT( rc == ERR_NO_SEM_RESOURCE );
CPPUNIT_ASSERT( semOp.getVal( sid, 2 ) == 0 );
rc = semOp.lockSem( sid, -2 );
CPPUNIT_ASSERT( rc == ERR_VALUE_OUTOFRANGE );
rc = semOp.lockSem( sid + 1, 1 );
CPPUNIT_ASSERT( rc == ERR_LOCK_FAIL );
// unlock
rc = semOp.unlockSem( sid, -2 );
CPPUNIT_ASSERT( rc == ERR_VALUE_OUTOFRANGE );
rc = semOp.unlockSem( sid, 1 );
CPPUNIT_ASSERT( rc == ERR_NO_SEM_LOCK );
rc = semOp.unlockSem( sid + 1, 2 );
CPPUNIT_ASSERT( rc == ERR_UNLOCK_FAIL );
printf( "\nunlock one in 2" );
rc = semOp.unlockSem( sid, 2 );
CPPUNIT_ASSERT( rc == NO_ERROR );
CPPUNIT_ASSERT( semOp.getVal( sid, 2 ) == 1 );
semOp.printAllVal( sid );
semOp.deleteSem( sid );
CPPUNIT_ASSERT( semOp.existSem( sid, key ) == false );
CPPUNIT_ASSERT( semOp.getSemCount( sid + 1 ) == 0 );
CPPUNIT_ASSERT(rc == NO_ERROR);
CPPUNIT_ASSERT(bFile->size() == 2105344);
fileOp.closeFile(pFile);
// file has been extended
fileOp.deleteFile( fileName );
CPPUNIT_ASSERT(fileOp.exists( fileName ) == false);
} }
// Create a dict file. Extend it w and w/o preallocation.
// Check the file sizes.
void testDictExtensionWOPrealloc()
{
FileOp fileOp;
BlockOp blockOp;
char fileName[20];
int rc;
int dbRoot = 1;
int colWidth = 65535;
DctnryCompress1 m_Dctnry;
// This is the magic for the stub in FileOp::oid2FileName
int oId = 42;
printf("\nRunning testDictExtensionWOPrealloc ");
printf("There could be InetStreamSocket::connect errors \n");
m_Dctnry.setDebugLevel( DEBUG_3 );
idbdatafile::IDBPolicy::init(true, false, "", 0);
// Set to versionbuffer to satisfy IDBPolicy::getType
strcpy( fileName, "versionbuffer" );
rc = m_Dctnry.dropDctnry(oId);
// FileOp::oid2FileName is called under the hood
// Dctnry::createDctnry could be used with running CS
// createDctnryFile also uses DBRM under the hood it works though.
IDBDataFile* m_dFile = m_Dctnry.createDctnryFileUnit(fileName,
colWidth,
"w+b",
DEFAULT_BUFSIZ);
idbdatafile::BufferedFile* bFile = (idbdatafile::BufferedFile*)m_dFile;
CPPUNIT_ASSERT(m_dFile != NULL);
const int m_totalHdrBytes = HDR_UNIT_SIZE + NEXT_PTR_BYTES + HDR_UNIT_SIZE + HDR_UNIT_SIZE;
m_Dctnry.compressionType(1);
rc = m_Dctnry.initDctnryExtent( m_dFile,
dbRoot,
BYTE_PER_BLOCK, // 8192
const_cast<unsigned char*>(m_Dctnry.getDctnryHeader2()),
m_totalHdrBytes,
false,
false ); //enable preallocation
// Check the file size and remove the file
CPPUNIT_ASSERT(bFile->size() == 67379200);
CPPUNIT_ASSERT(rc == NO_ERROR);
fileOp.deleteFile( fileName );
CPPUNIT_ASSERT(fileOp.exists( fileName ) == false);
// Create a Dictionary for the second time
m_dFile = m_Dctnry.createDctnryFileUnit(fileName,
colWidth,
"w+b",
DEFAULT_BUFSIZ);
// Get the file size later
bFile = (idbdatafile::BufferedFile*)m_dFile;
CPPUNIT_ASSERT(m_dFile != NULL);
// disable preallocation and create a Dictionary
idbdatafile::IDBPolicy::setPreallocSpace(dbRoot);
m_Dctnry.compressionType(1);
rc = m_Dctnry.initDctnryExtent( m_dFile,
dbRoot,
BYTE_PER_BLOCK,
const_cast<unsigned char*>(m_Dctnry.getDctnryHeader2()),
m_totalHdrBytes,
false,
true ); //skip preallocation
// Check the size and remove the file.
CPPUNIT_ASSERT(bFile->size() == 483328);
CPPUNIT_ASSERT(rc == NO_ERROR);
fileOp.deleteFile(fileName);
CPPUNIT_ASSERT(fileOp.exists( fileName ) == false);
}
void testLog() void testLog()
{ {
Log log; Log log;
FileOp fileOp;
string msg; string msg;
int iVal = 3; int iVal = 3;
float fVal = 2.0; char logFile[] = "test1.log";
char logErrFile[] = "test1err.log";
log.setLogFileName( "test1.log", "test1err.log" ); log.setLogFileName( logFile, logErrFile );
msg = Convertor::int2Str( iVal ); msg = Convertor::int2Str( iVal );
log.logMsg( msg + " this is a info message", INFO ); log.logMsg( msg + " this is a info message", MSGLVL_INFO1 );
msg = Convertor::getTimeStr(); msg = Convertor::getTimeStr();
log.logMsg( Convertor::float2Str( fVal ) + " this is a warning message", WARNING ); log.logMsg( " this is a warning message", MSGLVL_WARNING );
log.logMsg( "this is an error message ", 1011, ERROR ); log.logMsg( "this is an error message ", 1011, MSGLVL_ERROR );
log.logMsg( "this is a critical message", 1211, CRITICAL ); log.logMsg( "this is a critical message", 1211, MSGLVL_CRITICAL );
CPPUNIT_ASSERT( fileOp.exists( logFile ) == true );
//...Test formatting an unsigned 64 bit integer. CPPUNIT_ASSERT( fileOp.exists( logErrFile ) == true );
uint64_t i64Value(UINT64_MAX); fileOp.deleteFile( logFile );
msg = Convertor::i64ToStr( i64Value ); fileOp.deleteFile( logErrFile );
CPPUNIT_ASSERT( (msg == "18446744073709551615") ); CPPUNIT_ASSERT( fileOp.exists( logFile ) == false );
log.logMsg( msg + " this is an info message with the max uint64_t integer value", INFO ); CPPUNIT_ASSERT( fileOp.exists( logErrFile ) == false );
} }
/*
void testHWM() void testHWM()
{ {
int rc ; int rc ;
@ -1375,6 +1541,7 @@ public:
} }
} }
*/
void testCleanup() void testCleanup()
{ {

20
writeengine/shared/we_fileop.cpp
View File

@ -985,6 +985,8 @@ int FileOp::addExtentExactFile(
return rc; return rc;
// Initialize the contents of the extent. // Initialize the contents of the extent.
// CS doesn't optimize file operations to have a valid
// segment files with empty magics
rc = initColumnExtent( pFile, rc = initColumnExtent( pFile,
dbRoot, dbRoot,
allocSize, allocSize,
@ -1118,6 +1120,8 @@ int FileOp::initColumnExtent(
int savedErrno = 0; int savedErrno = 0;
// MCOL-498 fallocate the abbreviated extent, // MCOL-498 fallocate the abbreviated extent,
// fallback to sequential write if fallocate failed // fallback to sequential write if fallocate failed
// Couldn't use fallocate for full extents, e.g. ADD COLUMN DDL
// b/c CS has to fill the file with empty magics.
if ( !bOptExtension || ( nBlocks <= MAX_INITIAL_EXTENT_BLOCKS_TO_DISK if ( !bOptExtension || ( nBlocks <= MAX_INITIAL_EXTENT_BLOCKS_TO_DISK
&& pFile->fallocate(0, currFileSize, writeSize) ) && pFile->fallocate(0, currFileSize, writeSize) )
) )
@ -1944,12 +1948,7 @@ int FileOp::initDctnryExtent(
Stats::startParseEvent(WE_STATS_CREATE_DCT_EXTENT); Stats::startParseEvent(WE_STATS_CREATE_DCT_EXTENT);
#endif #endif
//std::ostringstream oss; if (remWriteSize > 0)
//oss << "initDctnryExtent: width-8(assumed)" <<
//"; loopCount-" << loopCount <<
//"; writeSize-" << writeSize;
//std::cout << oss.str() << std::endl;
if (remWriteSize > 0)
{ {
if (pFile->write( writeBuf, remWriteSize ) != remWriteSize) if (pFile->write( writeBuf, remWriteSize ) != remWriteSize)
{ {
@ -2302,6 +2301,15 @@ int FileOp::oid2FileName( FID fid,
#endif #endif
// Need this stub to use ColumnOp::writeRow in the unit tests
#ifdef WITH_UNIT_TESTS
if (fid == 42)
{
sprintf(fullFileName, "./versionbuffer");
return NO_ERROR;
}
#endif
/* If is a version buffer file, the format is different. */ /* If is a version buffer file, the format is different. */
if (fid < 1000) if (fid < 1000)
{ {

60
writeengine/shared/we_fileop.h
View File

@ -92,6 +92,15 @@ public:
execplan::CalpontSystemCatalog::ColDataType colDataType, execplan::CalpontSystemCatalog::ColDataType colDataType,
uint64_t emptyVal = 0, int width = 1 ) ; uint64_t emptyVal = 0, int width = 1 ) ;
/**
* @brief Create a file with a fixed file size by its name.
* Changed to public for UT.
*/
int createFile( const char* fileName, int fileSize,
uint64_t emptyVal, int width,
uint16_t dbRoot );
/** /**
* @brief Delete a file * @brief Delete a file
*/ */
@ -467,34 +476,6 @@ public:
int compressionType() const; int compressionType() const;
EXPORT virtual int flushFile(int rc, std::map<FID, FID>& oids); EXPORT virtual int flushFile(int rc, std::map<FID, FID>& oids);
protected:
EXPORT virtual int updateColumnExtent(IDBDataFile* pFile, int nBlocks);
EXPORT virtual int updateDctnryExtent(IDBDataFile* pFile, int nBlocks);
int m_compressionType; // compresssion type
private:
//not copyable
FileOp(const FileOp& rhs);
FileOp& operator=(const FileOp& rhs);
int createFile( const char* fileName, int fileSize,
uint64_t emptyVal, int width,
uint16_t dbRoot );
int expandAbbrevColumnChunk( IDBDataFile* pFile,
uint64_t emptyVal,
int colWidth,
const compress::CompChunkPtr& chunkInPtr,
compress::CompChunkPtr& chunkOutPt);
int initAbbrevCompColumnExtent( IDBDataFile* pFile,
uint16_t dbRoot,
int nBlocks,
uint64_t emptyVal,
int width);
// Initialize an extent in a column segment file // Initialize an extent in a column segment file
// pFile (in) IDBDataFile* of column segment file to be written to // pFile (in) IDBDataFile* of column segment file to be written to
// dbRoot (in) - DBRoot of pFile // dbRoot (in) - DBRoot of pFile
@ -515,6 +496,29 @@ private:
bool bAbbrevExtent, bool bAbbrevExtent,
bool bOptExtension=false ); bool bOptExtension=false );
protected:
EXPORT virtual int updateColumnExtent(IDBDataFile* pFile, int nBlocks);
EXPORT virtual int updateDctnryExtent(IDBDataFile* pFile, int nBlocks);
int m_compressionType; // compresssion type
private:
//not copyable
FileOp(const FileOp& rhs);
FileOp& operator=(const FileOp& rhs);
int expandAbbrevColumnChunk( IDBDataFile* pFile,
uint64_t emptyVal,
int colWidth,
const compress::CompChunkPtr& chunkInPtr,
compress::CompChunkPtr& chunkOutPt);
int initAbbrevCompColumnExtent( IDBDataFile* pFile,
uint16_t dbRoot,
int nBlocks,
uint64_t emptyVal,
int width);
static void initDbRootExtentMutexes(); static void initDbRootExtentMutexes();
static void removeDbRootExtentMutexes(); static void removeDbRootExtentMutexes();

1
writeengine/wrapper/CMakeLists.txt
View File

@ -44,4 +44,3 @@ target_link_libraries(writeengine ${NETSNMP_LIBRARIES})
set_target_properties(writeengine PROPERTIES VERSION 1.0.0 SOVERSION 1) set_target_properties(writeengine PROPERTIES VERSION 1.0.0 SOVERSION 1)
install(TARGETS writeengine DESTINATION ${ENGINE_LIBDIR} COMPONENT libs) install(TARGETS writeengine DESTINATION ${ENGINE_LIBDIR} COMPONENT libs)

3
writeengine/wrapper/we_colop.cpp
View File

@ -1529,7 +1529,8 @@ void ColumnOp::setColParam(Column& column,
* rowIdArray - the array of row id, for performance purpose, I am assuming the rowIdArray is sorted * rowIdArray - the array of row id, for performance purpose, I am assuming the rowIdArray is sorted
* valArray - the array of row values * valArray - the array of row values
* oldValArray - the array of old value * oldValArray - the array of old value
* bDelete - yet. The flag must be useless. * bDelete - yet. The flag must be useless b/c writeRows
* is used for deletion.
* RETURN: * RETURN:
* NO_ERROR if success, other number otherwise * NO_ERROR if success, other number otherwise
***********************************************************/ ***********************************************************/