database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2026-01-06 08:21:10 +03:00
Code Activity

MCOL-4328 There is a new option in both cpimport and cpimport.bin to asign

Browse Source 
an owner for all data files created by cpimport

The patch consists of two parts: cpimport.bin changes, cpimport splitter
changes

cpimport.bin computes uid_t and gid_t early and propagates it down the stack
where MCS creates data files
This commit is contained in:
Roman Nozdrin
2020-10-01 12:19:32 +00:00
parent f584bab846
commit 328ae25650
19 changed files with 200 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
writeengine/bulk/cpimport.cpp
View File

@@ -105,7 +105,8 @@ void printUsage()
" [-c readBufSize] [-e maxErrs] [-B libBufSize] [-n NullOption] " << endl <<
" [-E encloseChar] [-C escapeChar] [-I binaryOpt] [-S] "
"[-d debugLevel] [-i] " << endl <<
" [-D] [-N] [-L rejectDir] [-T timeZone]" << endl;
" [-D] [-N] [-L rejectDir] [-T timeZone]" << endl <<
" [-U username]" << endl << endl;
cout << endl << "Traditional usage without positional parameters "
"(XML job file required):" << endl <<
@@ -115,7 +116,8 @@ void printUsage()
" [-E encloseChar] [-C escapeChar] [-I binaryOpt] [-S] "
"[-d debugLevel] [-i] " << endl <<
" [-p path] [-l loadFile]" << endl <<
" [-D] [-N] [-L rejectDir] [-T timeZone]" << endl << endl;
" [-D] [-N] [-L rejectDir] [-T timeZone]" << endl <<
" [-U username]" << endl << endl;
cout << " Positional parameters:" << endl <<
" dbName Name of database to load" << endl <<
@@ -171,7 +173,8 @@ void printUsage()
" -K S3 Authentication Secret (for S3 imports)" << endl <<
" -t S3 Bucket (for S3 imports)" << endl <<
" -H S3 Hostname (for S3 imports, Amazon's S3 default)" << endl <<
" -g S3 Regions (for S3 imports)" << endl;
" -g S3 Regions (for S3 imports)" << endl <<
" -U username of new data files owner. Default is mysql" << endl;
cout << " Example1:" << endl <<
" cpimport.bin -j 1234" << endl <<
@@ -322,7 +325,7 @@ void parseCmdLineArgs(
std::string jobUUID;
while ( (option = getopt(
argc, argv, "b:c:d:e:f:hij:kl:m:n:p:r:s:u:w:B:C:DE:I:P:R:ST:X:NL:y:K:t:H:g:")) != EOF )
argc, argv, "b:c:d:e:f:hij:kl:m:n:p:r:s:u:w:B:C:DE:I:P:R:ST:X:NL:y:K:t:H:g:U:")) != EOF )
{
switch (option)
{
@@ -743,6 +746,11 @@ void parseCmdLineArgs(
break;
}
case 'U':
{
curJob.setUsername(optarg);
break;
}
default :
{
@@ -1083,12 +1091,6 @@ int main(int argc, char** argv)
{
#ifdef _MSC_VER
_setmaxstdio(2048);
#else
// set effective ID to root
if( setuid( 0 ) < 0 )
{
std::cerr << " cpimport: couldn't set uid " << std::endl;
}
#endif
setupSignalHandlers();

26
writeengine/bulk/we_bulkload.cpp
View File

@@ -38,6 +38,7 @@
#include <boost/filesystem/convenience.hpp>
#include <boost/uuid/uuid.hpp>
#include <boost/uuid/uuid_generators.hpp>
#include <pwd.h>
#include "we_bulkstatus.h"
#include "we_rbmetawriter.h"
@@ -157,7 +158,8 @@ BulkLoad::BulkLoad() :
fbContinue(false),
fDisableTimeOut(false),
fUUID(boost::uuids::nil_generator()()),
fTimeZone("SYSTEM")
fTimeZone("SYSTEM"),
fUsername("mysql") // MCOL-4328 default file owner
{
fTableInfo.clear();
setDebugLevel( DEBUG_0 );
@@ -484,6 +486,23 @@ int BulkLoad::preProcess( Job& job, int tableNo,
tableInfo->setTimeZone(fTimeZone);
tableInfo->setJobUUID(fUUID);
// MCOL-4328 Get username gid and uid if they are set
// We inject uid and gid into TableInfo and All ColumnInfo-s later.
struct passwd* pwd = nullptr;
errno = 0;
if (fUsername.length() && (pwd = getpwnam(fUsername.c_str())) == nullptr)
{
std::ostringstream oss;
oss << "Error getting pwd for " << fUsername
<< " with errno "
<< errno;
fLog.logMsg( oss.str(), MSGLVL_ERROR );
return ERR_FILE_CHOWN;
}
if (pwd)
tableInfo->setUIDGID(pwd->pw_uid, pwd->pw_gid);
if (fMaxErrors != -1)
tableInfo->setMaxErrorRows(fMaxErrors);
else
@@ -685,6 +704,9 @@ int BulkLoad::preProcess( Job& job, int tableNo,
pDBRootExtentTracker,
tableInfo);
if (pwd)
info->setUIDGID(pwd->pw_uid, pwd->pw_gid);
// For auto increment column, we need to get the starting value
if (info->column.autoIncFlag)
{
@@ -921,7 +943,7 @@ int BulkLoad::preProcessHwmLbid(
return rc;
}
//------------------------------------------------------------------------------
// DESCRIPTION:
// NO_ERROR if success

7
writeengine/bulk/we_bulkload.h
View File

@@ -160,6 +160,7 @@ public:
void setS3Bucket ( const std::string& bucket );
void setS3Host ( const std::string& host );
void setS3Region ( const std::string& region );
void setUsername ( const std::string& username );
// Timer functions
void startTimer ( );
void stopTimer ( );
@@ -244,6 +245,7 @@ private:
std::string fS3Host; // S3 Host
std::string fS3Bucket; // S3 Bucket
std::string fS3Region; // S3 Region
std::string fUsername; // data files owner name mysql by default
//--------------------------------------------------------------------------
// Private Functions
@@ -536,6 +538,11 @@ inline void BulkLoad::setS3Region( const std::string& region )
fS3Region = region;
}
inline void BulkLoad::setUsername( const std::string& username )
{
fUsername = username;
}
inline void BulkLoad::startTimer( )
{
gettimeofday( &fStartTime, 0 );

4
writeengine/bulk/we_columninfo.cpp
View File

@@ -458,6 +458,9 @@ int ColumnInfo::createDelayedFileIfNeeded( const std::string& tableName )
}
boost::scoped_ptr<Dctnry> refDctnry(tempD);
// MCOL-4328 Define a file owner uid and gid
refDctnry->setUIDGID(this);
rc = tempD->createDctnry(
column.dctnry.dctnryOid,
column.dctnryWidth,
@@ -1681,6 +1684,7 @@ int ColumnInfo::openDctnryStore( bool bMustExist )
fStore->setLogger(fLog);
fStore->setColWidth( column.dctnryWidth );
fStore->setUIDGID(this);
if (column.fWithDefault)
fStore->setDefault( column.fDefaultChr );

12
writeengine/bulk/we_columninfo.h
View File

@@ -123,8 +123,9 @@ struct LockInfo
/** @brief Maintains information about a DB column.
*/
struct ColumnInfo
class ColumnInfo: public WeUIDGID
{
public:
//--------------------------------------------------------------------------
// Public Data Members
//--------------------------------------------------------------------------
@@ -397,6 +398,8 @@ struct ColumnInfo
*/
unsigned rowsPerExtent( );
void setUIDGID(const uid_t uid, const gid_t gid) override;
protected:
//--------------------------------------------------------------------------
@@ -507,6 +510,13 @@ protected:
//------------------------------------------------------------------------------
// Inline functions
//------------------------------------------------------------------------------
inline void ColumnInfo::setUIDGID(const uid_t p_uid, const gid_t p_gid)
{
WeUIDGID::setUIDGID(p_uid, p_gid);
if (colOp)
colOp->setUIDGID(this);
}
inline boost::mutex& ColumnInfo::colMutex()
{
return fColMutex;

2
writeengine/bulk/we_tableinfo.cpp
View File

@@ -2428,6 +2428,8 @@ int TableInfo::saveBulkRollbackMetaData( Job& job,
} // end of loop through columns
fRBMetaWriter.setUIDGID(this);
try
{
fRBMetaWriter.saveBulkRollbackMetaData(

2
writeengine/bulk/we_tableinfo.h
View File

@@ -55,7 +55,7 @@ namespace WriteEngine
/* @brief Class which maintains the information for a table.
*/
class TableInfo
class TableInfo : public WeUIDGID
{
private: