database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-08-08 14:22:09 +03:00
Code Activity

MCOL-4328 MCS avoids chown() calls for files that are on S3

Browse Source 
MCS now chowns created directories hierarchy not only files and
immediate parent directories

Minor changes to cpimport's help printout

cpimport's -f option is now mandatory with mode 2
This commit is contained in:
Roman Nozdrin
2020-10-08 13:21:22 +00:00
parent d85fc579ba
commit 6f120d2637
9 changed files with 97 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files

81
writeengine/shared/we_fileop.cpp
View File

@@ -791,8 +791,7 @@ int FileOp::extendFile(
{
// We presume the path will contain /
std::string filePath(fileName);
std::ostringstream ossChown;
if (chownDataFileDir(ossChown, filePath))
if (chownDataPath(filePath))
return ERR_FILE_CHOWN;
}
@@ -2365,57 +2364,27 @@ int FileOp::oid2FileName( FID fid,
return ERR_FILE_NOT_EXIST;
}
/*
char dirName[FILE_NAME_SIZE];
sprintf( dirName, "%s/%s", Config::getDBRootByNum(dbRoot).c_str(),
dbDir[0] );
if( !isDir( dirName ) )
RETURN_ON_ERROR( createDir( dirName ));
sprintf( dirName, "%s/%s", dirName, dbDir[1] );
if( !isDir( dirName ) )
RETURN_ON_ERROR( createDir( dirName ));
sprintf( dirName, "%s/%s", dirName, dbDir[2] );
if( !isDir( dirName ) )
RETURN_ON_ERROR( createDir( dirName ));
sprintf( dirName, "%s/%s", dirName, dbDir[3] );
if( !isDir( dirName ) )
RETURN_ON_ERROR( createDir( dirName ));
sprintf( dirName, "%s/%s", dirName, dbDir[4] );
if( !isDir( dirName ) )
RETURN_ON_ERROR( createDir( dirName ));
*/
std::stringstream aDirName;
for (size_t i = 0; i < MaxDirLevels; i++)
{
if (i == 0)
{
aDirName << Config::getDBRootByNum(dbRoot).c_str()
<< "/" << dbDir[i];
}
else
{
aDirName << "/" << dbDir[i];
}
if (!isDir(aDirName.str().c_str()))
RETURN_ON_ERROR( createDir(aDirName.str().c_str()) );
aDirName << Config::getDBRootByNum(dbRoot).c_str() << "/" << dbDir[0];
if (!isDir((aDirName.str()).c_str()))
RETURN_ON_ERROR( createDir((aDirName.str()).c_str()) );
aDirName << "/" << dbDir[1];
if (!isDir(aDirName.str().c_str()))
RETURN_ON_ERROR( createDir(aDirName.str().c_str()) );
aDirName << "/" << dbDir[2];
if (!isDir(aDirName.str().c_str()))
RETURN_ON_ERROR( createDir(aDirName.str().c_str()) );
aDirName << "/" << dbDir[3];
if (!isDir(aDirName.str().c_str()))
RETURN_ON_ERROR( createDir(aDirName.str().c_str()) );
aDirName << "/" << dbDir[4];
if (!isDir(aDirName.str().c_str()))
RETURN_ON_ERROR( createDir(aDirName.str().c_str()) );
{
std::ostringstream ossChown;
if (chownDataPath(aDirName.str()))
return ERR_FILE_CHOWN;
}
}
return NO_ERROR;
}
@@ -2932,11 +2901,13 @@ void FileOp::setFixFlag(bool isFix)
m_isFix = isFix;
}
bool FileOp::chownDataFileDir(std::ostringstream& error,
const std::string& fileName)
// Small note. We call chownFileDir in couple places to chown of the
// target file and call in oid2Filename() chowns directories created
bool FileOp::chownDataPath(const std::string& fileName) const
{
std::string dirName = fileName.substr(0, fileName.find_last_of('/'));
if (chownFileDir(error, fileName, dirName))
std::ostringstream error;
idbdatafile::IDBFileSystem& fs = IDBPolicy::getFs(fileName);
if (chownPath(error, fileName, fs))
{
logging::Message::Args args;
logging::Message message(1);

5
writeengine/shared/we_fileop.h
View File

@@ -58,6 +58,8 @@
namespace WriteEngine
{
constexpr size_t MaxDirLevels = 5;
/** Class FileOp */
class FileOp : public BlockOp, public WeUIDGID
{
@@ -503,8 +505,7 @@ public:
bool bOptExtension=false );
// Calls a chown and logs an error message
bool chownDataFileDir(std::ostringstream& error,
const std::string& fileName);
bool chownDataPath(const std::string& fileName) const;
protected:
EXPORT virtual int updateColumnExtent(IDBDataFile* pFile, int nBlocks);

12
writeengine/shared/we_rbmetawriter.cpp
View File

@@ -454,8 +454,12 @@ std::string RBMetaWriter::openMetaFile ( uint16_t dbRoot )
{
std::ostringstream ossChown;
if (chownFileDir(ossChown, tmpMetaFileName, bulkRollbackPath))
idbdatafile::IDBFileSystem& fs = IDBPolicy::getFs(tmpMetaFileName.c_str());
if (chownPath(ossChown, tmpMetaFileName, fs)
|| chownPath(ossChown, bulkRollbackPath, fs))
{
throw WeException(ossChown.str(), ERR_FILE_CHOWN);
}
}
fMetaDataStream <<
@@ -1336,8 +1340,12 @@ int RBMetaWriter::writeHWMChunk(
{
std::ostringstream ossChown;
if (chownFileDir(ossChown, fileName, dirPath))
idbdatafile::IDBFileSystem& fs = IDBPolicy::getFs(fileName.c_str());
if (chownPath(ossChown, fileName, fs)
|| chownPath(ossChown, dirPath, fs))
{
throw WeException(ossChown.str(), ERR_FILE_CHOWN);
}
}
return NO_ERROR;

19
writeengine/shared/we_typeext.h
View File

@@ -32,6 +32,8 @@
#include <sys/types.h>
#include <pwd.h>
#include <sstream>
#include "IDBFileSystem.h"
/** Namespace WriteEngine */
namespace WriteEngine
@@ -67,8 +69,9 @@ public:
virtual ~WeUIDGID() {};
virtual void setUIDGID(const uid_t uid, const gid_t gid);
void setUIDGID(const WeUIDGID* id);
bool chownFileDir(std::ostringstream& error,
const std::string& fileName, const std::string& dirName) const;
bool chownPath(std::ostringstream& error,
const std::string& fileName,
const idbdatafile::IDBFileSystem& fs) const;
;
private:
@@ -87,18 +90,18 @@ inline void WeUIDGID::setUIDGID(const WeUIDGID* id)
*this = *id;
}
inline bool WeUIDGID::chownFileDir(std::ostringstream& error,
const std::string& fileName, const std::string& dirName) const
inline bool WeUIDGID::chownPath(std::ostringstream& error,
const std::string& fileName,
const idbdatafile::IDBFileSystem& fs) const
{
if (uid != UID_NONE)
{
errno = 0;
if (chown(fileName.c_str(), uid, gid) == -1 ||
chown(dirName.c_str(), uid, gid) == -1)
int funcErrno = 0;
if (fs.chown(fileName.c_str(), uid, gid, funcErrno) == -1)
{
error << "Error calling chown() with uid " << uid
<< " and gid " << gid << " with the file "
<< fileName << " with errno " << errno;
<< fileName << " with errno " << funcErrno;
return true;
}
}