database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-08-08 14:22:09 +03:00
Code Activity

Reformat all code to coding standard

Browse Source
This commit is contained in:
Andrew Hutchings
2017-10-26 17:18:17 +01:00
parent 4985f3456e
commit 01446d1e22
1296 changed files with 403852 additions and 353747 deletions
Download Patch File Download Diff File Expand all files Collapse all files

462
utils/compress/idbcompress.cpp
View File

@@ -65,39 +65,39 @@ const uint8_t CHUNK_MAGIC3 = 0xfd;
struct CompressedDBFileHeader
{
uint64_t fMagicNumber;
uint64_t fVersionNum;
uint64_t fCompressionType;
uint64_t fHeaderSize;
uint64_t fBlockCount;
uint64_t fMagicNumber;
uint64_t fVersionNum;
uint64_t fCompressionType;
uint64_t fHeaderSize;
uint64_t fBlockCount;
};
// Make the header to be 4K, regardless number of fields being defined/used in header.
union CompressedDBFileHeaderBlock
{
CompressedDBFileHeader fHeader;
char fDummy[compress::IDBCompressInterface::HDR_BUF_LEN];
CompressedDBFileHeader fHeader;
char fDummy[compress::IDBCompressInterface::HDR_BUF_LEN];
};
void initCompressedDBFileHeader(void* hdrBuf, int compressionType, int hdrSize)
{
CompressedDBFileHeaderBlock* hdr = reinterpret_cast<CompressedDBFileHeaderBlock*>(hdrBuf);
hdr->fHeader.fMagicNumber = MAGIC_NUMBER;
hdr->fHeader.fVersionNum = VERSION_NUM2;
hdr->fHeader.fCompressionType = compressionType;
hdr->fHeader.fBlockCount = 0;
hdr->fHeader.fHeaderSize = hdrSize;
CompressedDBFileHeaderBlock* hdr = reinterpret_cast<CompressedDBFileHeaderBlock*>(hdrBuf);
hdr->fHeader.fMagicNumber = MAGIC_NUMBER;
hdr->fHeader.fVersionNum = VERSION_NUM2;
hdr->fHeader.fCompressionType = compressionType;
hdr->fHeader.fBlockCount = 0;
hdr->fHeader.fHeaderSize = hdrSize;
}
void log(const string &s)
void log(const string& s)
{
logging::MessageLog logger((logging::LoggingID()));
logging::Message message;
logging::Message::Args args;
logging::MessageLog logger((logging::LoggingID()));
logging::Message message;
logging::Message::Args args;
args.add(s);
message.format(args);
logger.logErrorMessage(message);
args.add(s);
message.format(args);
logger.logErrorMessage(message);
}
} // namespace
@@ -108,7 +108,7 @@ namespace compress
#ifndef SKIP_IDB_COMPRESSION
IDBCompressInterface::IDBCompressInterface(unsigned int numUserPaddingBytes) :
fNumUserPaddingBytes(numUserPaddingBytes)
fNumUserPaddingBytes(numUserPaddingBytes)
{ }
IDBCompressInterface::~IDBCompressInterface()
@@ -120,144 +120,169 @@ IDBCompressInterface::~IDBCompressInterface()
*/
bool IDBCompressInterface::isCompressionAvail(int compressionType) const
{
if ( (compressionType == 0) ||
(compressionType == 1) ||
(compressionType == 2) )
return true;
return false;
if ( (compressionType == 0) ||
(compressionType == 1) ||
(compressionType == 2) )
return true;
return false;
}
//------------------------------------------------------------------------------
// Compress a block of data
//------------------------------------------------------------------------------
int IDBCompressInterface::compressBlock(const char* in,
const size_t inLen,
unsigned char* out,
unsigned int& outLen) const
const size_t inLen,
unsigned char* out,
unsigned int& outLen) const
{
size_t snaplen = 0;
utils::Hasher128 hasher;
size_t snaplen = 0;
utils::Hasher128 hasher;
// loose input checking.
if (outLen < snappy::MaxCompressedLength(inLen) + HEADER_SIZE)
{
cerr << "got outLen = " << outLen << " for inLen = " << inLen << ", needed " <<
(snappy::MaxCompressedLength(inLen) + HEADER_SIZE) << endl;
return ERR_BADOUTSIZE;
}
// loose input checking.
if (outLen < snappy::MaxCompressedLength(inLen) + HEADER_SIZE)
{
cerr << "got outLen = " << outLen << " for inLen = " << inLen << ", needed " <<
(snappy::MaxCompressedLength(inLen) + HEADER_SIZE) << endl;
return ERR_BADOUTSIZE;
}
//apparently this never fails?
snappy::RawCompress(in, inLen, reinterpret_cast<char*>(&out[HEADER_SIZE]), &snaplen);
//apparently this never fails?
snappy::RawCompress(in, inLen, reinterpret_cast<char*>(&out[HEADER_SIZE]), &snaplen);
uint8_t *signature = (uint8_t *) &out[SIG_OFFSET];
uint32_t *checksum = (uint32_t *) &out[CHECKSUM_OFFSET];
uint32_t *len = (uint32_t *) &out[LEN_OFFSET];
*signature = CHUNK_MAGIC3;
*checksum = hasher((char *) &out[HEADER_SIZE], snaplen);
*len = snaplen;
uint8_t* signature = (uint8_t*) &out[SIG_OFFSET];
uint32_t* checksum = (uint32_t*) &out[CHECKSUM_OFFSET];
uint32_t* len = (uint32_t*) &out[LEN_OFFSET];
*signature = CHUNK_MAGIC3;
*checksum = hasher((char*) &out[HEADER_SIZE], snaplen);
*len = snaplen;
//cerr << "cb: " << inLen << '/' << outLen << '/' << (snappy::MaxCompressedLength(inLen) + HEADER_SIZE) <<
// " : " << (snaplen + HEADER_SIZE) << endl;
//cerr << "cb: " << inLen << '/' << outLen << '/' << (snappy::MaxCompressedLength(inLen) + HEADER_SIZE) <<
// " : " << (snaplen + HEADER_SIZE) << endl;
outLen = snaplen + HEADER_SIZE;
outLen = snaplen + HEADER_SIZE;
return ERR_OK;
return ERR_OK;
}
//------------------------------------------------------------------------------
// Decompress a block of data
//------------------------------------------------------------------------------
int IDBCompressInterface::uncompressBlock(const char* in, const size_t inLen, unsigned char* out,
unsigned int& outLen) const
unsigned int& outLen) const
{
bool comprc = false;
size_t ol = 0;
bool comprc = false;
size_t ol = 0;
uint32_t realChecksum;
uint32_t storedChecksum;
uint32_t storedLen;
uint8_t storedMagic;
utils::Hasher128 hasher;
uint32_t realChecksum;
uint32_t storedChecksum;
uint32_t storedLen;
uint8_t storedMagic;
utils::Hasher128 hasher;
outLen = 0;
if (inLen < 1) {
return ERR_BADINPUT;
}
storedMagic = *((uint8_t *) &in[SIG_OFFSET]);
outLen = 0;
if (storedMagic == CHUNK_MAGIC3)
{
if (inLen < HEADER_SIZE) {
return ERR_BADINPUT;
}
storedChecksum = *((uint32_t *) &in[CHECKSUM_OFFSET]);
storedLen = *((uint32_t *) (&in[LEN_OFFSET]));
if (inLen < storedLen + HEADER_SIZE) {
return ERR_BADINPUT;
}
if (inLen < 1)
{
return ERR_BADINPUT;
}
realChecksum = hasher(&in[HEADER_SIZE], storedLen);
if (storedChecksum != realChecksum) {
return ERR_CHECKSUM;
}
storedMagic = *((uint8_t*) &in[SIG_OFFSET]);
comprc = snappy::GetUncompressedLength(&in[HEADER_SIZE], storedLen, &ol) &&
snappy::RawUncompress(&in[HEADER_SIZE], storedLen, reinterpret_cast<char*>(out));
}
else if (storedMagic == CHUNK_MAGIC1 || storedMagic == CHUNK_MAGIC2)
{
if (inLen < HEADER_SIZE) {
return ERR_BADINPUT;
}
storedChecksum = *((uint32_t *) &in[CHECKSUM_OFFSET]);
storedLen = *((uint32_t *) (&in[LEN_OFFSET]));
if (inLen < storedLen + HEADER_SIZE) {
return ERR_BADINPUT;
}
/* We can no longer verify the checksum on ver 1.1 */
if (storedMagic == CHUNK_MAGIC2) {
realChecksum = hasher(&in[HEADER_SIZE], storedLen);
if (storedChecksum != realChecksum) {
return ERR_CHECKSUM;
}
}
if (storedMagic == CHUNK_MAGIC3)
{
if (inLen < HEADER_SIZE)
{
return ERR_BADINPUT;
}
try {
comprc = v1::decompress(&in[HEADER_SIZE], storedLen, out, &ol);
} catch (runtime_error& rex) {
//cerr << "decomp caught exception: " << rex.what() << endl;
ostringstream os;
os << "decomp caught exception: " << rex.what();
log(os.str());
comprc = false;
} catch (exception& ex) {
ostringstream os;
os << "decomp caught exception: " << ex.what();
log(os.str());
comprc = false;
} catch (...) {
comprc = false;
}
}
else if ((storedMagic & 0x80) != 0)
{
return ERR_BADINPUT;
}
else
{
comprc = v1::decompress(in, inLen, out, &ol);
}
storedChecksum = *((uint32_t*) &in[CHECKSUM_OFFSET]);
storedLen = *((uint32_t*) (&in[LEN_OFFSET]));
if (!comprc)
{
cerr << "decomp failed!" << endl;
return ERR_DECOMPRESS;
}
if (inLen < storedLen + HEADER_SIZE)
{
return ERR_BADINPUT;
}
outLen = ol;
//cerr << "ub: " << inLen << " : " << outLen << endl;
realChecksum = hasher(&in[HEADER_SIZE], storedLen);
return ERR_OK;
if (storedChecksum != realChecksum)
{
return ERR_CHECKSUM;
}
comprc = snappy::GetUncompressedLength(&in[HEADER_SIZE], storedLen, &ol) &&
snappy::RawUncompress(&in[HEADER_SIZE], storedLen, reinterpret_cast<char*>(out));
}
else if (storedMagic == CHUNK_MAGIC1 || storedMagic == CHUNK_MAGIC2)
{
if (inLen < HEADER_SIZE)
{
return ERR_BADINPUT;
}
storedChecksum = *((uint32_t*) &in[CHECKSUM_OFFSET]);
storedLen = *((uint32_t*) (&in[LEN_OFFSET]));
if (inLen < storedLen + HEADER_SIZE)
{
return ERR_BADINPUT;
}
/* We can no longer verify the checksum on ver 1.1 */
if (storedMagic == CHUNK_MAGIC2)
{
realChecksum = hasher(&in[HEADER_SIZE], storedLen);
if (storedChecksum != realChecksum)
{
return ERR_CHECKSUM;
}
}
try
{
comprc = v1::decompress(&in[HEADER_SIZE], storedLen, out, &ol);
}
catch (runtime_error& rex)
{
//cerr << "decomp caught exception: " << rex.what() << endl;
ostringstream os;
os << "decomp caught exception: " << rex.what();
log(os.str());
comprc = false;
}
catch (exception& ex)
{
ostringstream os;
os << "decomp caught exception: " << ex.what();
log(os.str());
comprc = false;
}
catch (...)
{
comprc = false;
}
}
else if ((storedMagic & 0x80) != 0)
{
return ERR_BADINPUT;
}
else
{
comprc = v1::decompress(in, inLen, out, &ol);
}
if (!comprc)
{
cerr << "decomp failed!" << endl;
return ERR_DECOMPRESS;
}
outLen = ol;
//cerr << "ub: " << inLen << " : " << outLen << endl;
return ERR_OK;
}
//------------------------------------------------------------------------------
@@ -265,13 +290,15 @@ int IDBCompressInterface::uncompressBlock(const char* in, const size_t inLen, un
//------------------------------------------------------------------------------
int IDBCompressInterface::verifyHdr(const void* hdrBuf) const
{
const CompressedDBFileHeader* hdr = reinterpret_cast<const CompressedDBFileHeader*>(hdrBuf);
if (hdr->fMagicNumber != MAGIC_NUMBER)
return -1;
if (!isCompressionAvail(hdr->fCompressionType))
return -2;
const CompressedDBFileHeader* hdr = reinterpret_cast<const CompressedDBFileHeader*>(hdrBuf);
return 0;
if (hdr->fMagicNumber != MAGIC_NUMBER)
return -1;
if (!isCompressionAvail(hdr->fCompressionType))
return -2;
return 0;
}
//------------------------------------------------------------------------------
@@ -279,26 +306,27 @@ int IDBCompressInterface::verifyHdr(const void* hdrBuf) const
// passed in. ptrBuf points to the pointer section of the compression hdr.
//------------------------------------------------------------------------------
int IDBCompressInterface::getPtrList(const char* ptrBuf,
const int ptrBufSize,
CompChunkPtrList& chunkPtrs ) const
const int ptrBufSize,
CompChunkPtrList& chunkPtrs ) const
{
int rc = 0;
chunkPtrs.clear();
int rc = 0;
chunkPtrs.clear();
const uint64_t* ptrs = reinterpret_cast<const uint64_t*>(ptrBuf);
const unsigned int NUM_PTRS = ptrBufSize / sizeof(uint64_t);
for (unsigned int i = 0; (i < NUM_PTRS) && (rc == 0); i++)
{
if (ptrs[i+1] == 0) // 0 offset means end of data
break;
const uint64_t* ptrs = reinterpret_cast<const uint64_t*>(ptrBuf);
const unsigned int NUM_PTRS = ptrBufSize / sizeof(uint64_t);
if (ptrs[i+1] > ptrs[i])
chunkPtrs.push_back(make_pair( ptrs[i], (ptrs[i+1]-ptrs[i])));
else
rc = -1;
}
for (unsigned int i = 0; (i < NUM_PTRS) && (rc == 0); i++)
{
if (ptrs[i + 1] == 0) // 0 offset means end of data
break;
return rc;
if (ptrs[i + 1] > ptrs[i])
chunkPtrs.push_back(make_pair( ptrs[i], (ptrs[i + 1] - ptrs[i])));
else
rc = -1;
}
return rc;
}
//------------------------------------------------------------------------------
@@ -309,28 +337,29 @@ int IDBCompressInterface::getPtrList(const char* ptrBuf,
//------------------------------------------------------------------------------
int IDBCompressInterface::getPtrList(const char* hdrBuf, CompChunkPtrList& chunkPtrs ) const
{
return getPtrList(hdrBuf+HDR_BUF_LEN, HDR_BUF_LEN, chunkPtrs);
return getPtrList(hdrBuf + HDR_BUF_LEN, HDR_BUF_LEN, chunkPtrs);
}
//------------------------------------------------------------------------------
// Count the number of chunk pointers in the pointer header(s)
//------------------------------------------------------------------------------
unsigned int IDBCompressInterface::getPtrCount(const char* ptrBuf,
const int ptrBufSize) const
const int ptrBufSize) const
{
unsigned int chunkCount = 0;
unsigned int chunkCount = 0;
const uint64_t* ptrs = reinterpret_cast<const uint64_t*>(ptrBuf);
const unsigned int NUM_PTRS = ptrBufSize / sizeof(uint64_t);
for (unsigned int i = 0; i < NUM_PTRS; i++)
{
if (ptrs[i+1] == 0) // 0 offset means end of data
break;
const uint64_t* ptrs = reinterpret_cast<const uint64_t*>(ptrBuf);
const unsigned int NUM_PTRS = ptrBufSize / sizeof(uint64_t);
chunkCount++;
}
for (unsigned int i = 0; i < NUM_PTRS; i++)
{
if (ptrs[i + 1] == 0) // 0 offset means end of data
break;
return chunkCount;
chunkCount++;
}
return chunkCount;
}
//------------------------------------------------------------------------------
@@ -341,23 +370,23 @@ unsigned int IDBCompressInterface::getPtrCount(const char* ptrBuf,
//------------------------------------------------------------------------------
unsigned int IDBCompressInterface::getPtrCount(const char* hdrBuf) const
{
return getPtrCount(hdrBuf+HDR_BUF_LEN, HDR_BUF_LEN);
return getPtrCount(hdrBuf + HDR_BUF_LEN, HDR_BUF_LEN);
}
//------------------------------------------------------------------------------
// Store list of compression pointers into the specified header.
//------------------------------------------------------------------------------
void IDBCompressInterface::storePtrs(const std::vector<uint64_t>& ptrs,
void* ptrBuf,
int ptrSectionSize) const
void* ptrBuf,
int ptrSectionSize) const
{
memset((ptrBuf), 0, ptrSectionSize); // reset the pointer section to 0
uint64_t* hdrPtrs = reinterpret_cast<uint64_t*>(ptrBuf);
memset((ptrBuf), 0, ptrSectionSize); // reset the pointer section to 0
uint64_t* hdrPtrs = reinterpret_cast<uint64_t*>(ptrBuf);
for (unsigned i=0; i<ptrs.size(); i++)
{
hdrPtrs[i] = ptrs[i];
}
for (unsigned i = 0; i < ptrs.size(); i++)
{
hdrPtrs[i] = ptrs[i];
}
}
//------------------------------------------------------------------------------
@@ -365,7 +394,7 @@ void IDBCompressInterface::storePtrs(const std::vector<uint64_t>& ptrs,
//------------------------------------------------------------------------------
void IDBCompressInterface::storePtrs(const std::vector<uint64_t>& ptrs, void* ptrBuf) const
{
storePtrs(ptrs, reinterpret_cast<char*>(ptrBuf) + HDR_BUF_LEN, HDR_BUF_LEN);
storePtrs(ptrs, reinterpret_cast<char*>(ptrBuf) + HDR_BUF_LEN, HDR_BUF_LEN);
}
//------------------------------------------------------------------------------
@@ -373,16 +402,16 @@ void IDBCompressInterface::storePtrs(const std::vector<uint64_t>& ptrs, void* pt
//------------------------------------------------------------------------------
void IDBCompressInterface::initHdr(void* hdrBuf, int compressionType) const
{
memset(hdrBuf, 0, HDR_BUF_LEN*2);
initCompressedDBFileHeader(hdrBuf, compressionType, HDR_BUF_LEN*2);
memset(hdrBuf, 0, HDR_BUF_LEN * 2);
initCompressedDBFileHeader(hdrBuf, compressionType, HDR_BUF_LEN * 2);
}
//------------------------------------------------------------------------------
// Initialize the header blocks to be written at the start of a dictionary file.
//------------------------------------------------------------------------------
void IDBCompressInterface::initHdr(void* hdrBuf,void* ptrBuf,int compressionType,int hdrSize) const
void IDBCompressInterface::initHdr(void* hdrBuf, void* ptrBuf, int compressionType, int hdrSize) const
{
memset(hdrBuf, 0, HDR_BUF_LEN);
memset(hdrBuf, 0, HDR_BUF_LEN);
memset(ptrBuf, 0, hdrSize - HDR_BUF_LEN);
initCompressedDBFileHeader(hdrBuf, compressionType, hdrSize);
}
@@ -392,7 +421,7 @@ void IDBCompressInterface::initHdr(void* hdrBuf,void* ptrBuf,int compressionType
//------------------------------------------------------------------------------
void IDBCompressInterface::setBlockCount(void* hdrBuf, uint64_t count) const
{
reinterpret_cast<CompressedDBFileHeader*>(hdrBuf)->fBlockCount = count;
reinterpret_cast<CompressedDBFileHeader*>(hdrBuf)->fBlockCount = count;
}
//------------------------------------------------------------------------------
@@ -400,7 +429,7 @@ void IDBCompressInterface::setBlockCount(void* hdrBuf, uint64_t count) const
//------------------------------------------------------------------------------
uint64_t IDBCompressInterface::getBlockCount(const void* hdrBuf) const
{
return (reinterpret_cast<const CompressedDBFileHeader*>(hdrBuf)->fBlockCount);
return (reinterpret_cast<const CompressedDBFileHeader*>(hdrBuf)->fBlockCount);
}
//------------------------------------------------------------------------------
@@ -408,7 +437,7 @@ uint64_t IDBCompressInterface::getBlockCount(const void* hdrBuf) const
//------------------------------------------------------------------------------
void IDBCompressInterface::setHdrSize(void* hdrBuf, uint64_t size) const
{
reinterpret_cast<CompressedDBFileHeader*>(hdrBuf)->fHeaderSize = size;
reinterpret_cast<CompressedDBFileHeader*>(hdrBuf)->fHeaderSize = size;
}
//------------------------------------------------------------------------------
@@ -416,7 +445,7 @@ void IDBCompressInterface::setHdrSize(void* hdrBuf, uint64_t size) const
//------------------------------------------------------------------------------
uint64_t IDBCompressInterface::getHdrSize(const void* hdrBuf) const
{
return (reinterpret_cast<const CompressedDBFileHeader*>(hdrBuf)->fHeaderSize);
return (reinterpret_cast<const CompressedDBFileHeader*>(hdrBuf)->fHeaderSize);
}
//------------------------------------------------------------------------------
@@ -424,17 +453,17 @@ uint64_t IDBCompressInterface::getHdrSize(const void* hdrBuf) const
// block number.
//------------------------------------------------------------------------------
void IDBCompressInterface::locateBlock(unsigned int block,
unsigned int& chunkIndex,
unsigned int& blockOffsetWithinChunk) const
unsigned int& chunkIndex,
unsigned int& blockOffsetWithinChunk) const
{
const uint64_t BUFLEN = UNCOMPRESSED_INBUF_LEN;
const uint64_t BUFLEN = UNCOMPRESSED_INBUF_LEN;
uint64_t byteOffset = (uint64_t)block * BLOCK_SIZE;
uint64_t chunk = byteOffset / BUFLEN;
uint64_t blockInChunk = (byteOffset % BUFLEN) / BLOCK_SIZE;
uint64_t byteOffset = (uint64_t)block * BLOCK_SIZE;
uint64_t chunk = byteOffset / BUFLEN;
uint64_t blockInChunk = (byteOffset % BUFLEN) / BLOCK_SIZE;
chunkIndex = chunk;
blockOffsetWithinChunk = blockInChunk;
chunkIndex = chunk;
blockOffsetWithinChunk = blockInChunk;
}
//------------------------------------------------------------------------------
@@ -443,52 +472,53 @@ void IDBCompressInterface::locateBlock(unsigned int block,
// bytes to 0.
//------------------------------------------------------------------------------
int IDBCompressInterface::padCompressedChunks(unsigned char* buf,
unsigned int& len,
unsigned int maxLen) const
unsigned int& len,
unsigned int maxLen) const
{
int nPaddingBytes = 0;
int nRem = len % COMPRESSED_CHUNK_INCREMENT_SIZE;
if (nRem != 0)
{
nPaddingBytes = COMPRESSED_CHUNK_INCREMENT_SIZE - nRem;
}
int nPaddingBytes = 0;
int nRem = len % COMPRESSED_CHUNK_INCREMENT_SIZE;
nPaddingBytes = nPaddingBytes + fNumUserPaddingBytes;
if (nRem != 0)
{
nPaddingBytes = COMPRESSED_CHUNK_INCREMENT_SIZE - nRem;
}
if (nPaddingBytes > 0)
{
if ((len + nPaddingBytes) > maxLen)
return -1;
nPaddingBytes = nPaddingBytes + fNumUserPaddingBytes;
memset(buf+len, 0, nPaddingBytes);
len = len + nPaddingBytes;
}
if (nPaddingBytes > 0)
{
if ((len + nPaddingBytes) > maxLen)
return -1;
return 0;
memset(buf + len, 0, nPaddingBytes);
len = len + nPaddingBytes;
}
return 0;
}
/* static */
uint64_t IDBCompressInterface::maxCompressedSize(uint64_t uncompSize)
{
return (snappy::MaxCompressedLength(uncompSize) + HEADER_SIZE);
return (snappy::MaxCompressedLength(uncompSize) + HEADER_SIZE);
}
int IDBCompressInterface::compress(const char *in, size_t inLen, char *out,
size_t *outLen) const
int IDBCompressInterface::compress(const char* in, size_t inLen, char* out,
size_t* outLen) const
{
snappy::RawCompress(in, inLen, out, outLen);
return 0;
snappy::RawCompress(in, inLen, out, outLen);
return 0;
}
int IDBCompressInterface::uncompress(const char *in, size_t inLen, char *out) const
int IDBCompressInterface::uncompress(const char* in, size_t inLen, char* out) const
{
return !(snappy::RawUncompress(in, inLen, out));
return !(snappy::RawUncompress(in, inLen, out));
}
/* static */
bool IDBCompressInterface::getUncompressedSize(char *in, size_t inLen, size_t *outLen)
bool IDBCompressInterface::getUncompressedSize(char* in, size_t inLen, size_t* outLen)
{
return snappy::GetUncompressedLength(in, inLen, outLen);
return snappy::GetUncompressedLength(in, inLen, outLen);
}
#endif

431
utils/compress/idbcompress.h
View File

@@ -36,242 +36,293 @@
namespace compress
{
typedef std::pair<uint64_t,uint64_t> CompChunkPtr;
typedef std::pair<uint64_t, uint64_t> CompChunkPtr;
typedef std::vector<CompChunkPtr> CompChunkPtrList;
class IDBCompressInterface
{
public:
static const unsigned int HDR_BUF_LEN = 4096;
static const unsigned int UNCOMPRESSED_INBUF_LEN = 512 * 1024 * 8;
static const unsigned int HDR_BUF_LEN = 4096;
static const unsigned int UNCOMPRESSED_INBUF_LEN = 512 * 1024 * 8;
// error codes from uncompressBlock()
static const int ERR_OK = 0;
static const int ERR_CHECKSUM = -1;
static const int ERR_DECOMPRESS = -2;
static const int ERR_BADINPUT = -3;
static const int ERR_BADOUTSIZE = -4;
// error codes from uncompressBlock()
static const int ERR_OK = 0;
static const int ERR_CHECKSUM = -1;
static const int ERR_DECOMPRESS = -2;
static const int ERR_BADINPUT = -3;
static const int ERR_BADOUTSIZE = -4;
/**
* When IDBCompressInterface object is being used to compress a chunk, this
* construct can be used to specify the padding added by padCompressedChunks
*/
EXPORT explicit IDBCompressInterface(unsigned int numUserPaddingBytes=0);
/**
* When IDBCompressInterface object is being used to compress a chunk, this
* construct can be used to specify the padding added by padCompressedChunks
*/
EXPORT explicit IDBCompressInterface(unsigned int numUserPaddingBytes = 0);
/**
* dtor
*/
EXPORT virtual ~IDBCompressInterface();
/**
* dtor
*/
EXPORT virtual ~IDBCompressInterface();
/**
* see if the algo is available in this lib
*/
EXPORT bool isCompressionAvail(int compressionType = 0) const;
/**
* see if the algo is available in this lib
*/
EXPORT bool isCompressionAvail(int compressionType = 0) const;
/**
* Compresses specified "in" buffer of length "inLen" bytes.
* Compressed data and size are returned in "out" and "outLen".
* "out" should be sized using maxCompressedSize() to allow for incompressible data.
* Returns 0 if success.
*/
EXPORT int compressBlock(const char* in,
const size_t inLen,
unsigned char* out,
unsigned int& outLen) const;
/**
* Compresses specified "in" buffer of length "inLen" bytes.
* Compressed data and size are returned in "out" and "outLen".
* "out" should be sized using maxCompressedSize() to allow for incompressible data.
* Returns 0 if success.
*/
EXPORT int compressBlock(const char* in,
const size_t inLen,
unsigned char* out,
unsigned int& outLen) const;
/**
* outLen must be initialized with the size of the out buffer before calling uncompressBlock.
* On return, outLen will have the number of bytes used in out.
*/
EXPORT int uncompressBlock(const char* in, const size_t inLen, unsigned char* out,
unsigned int& outLen) const;
/**
* outLen must be initialized with the size of the out buffer before calling uncompressBlock.
* On return, outLen will have the number of bytes used in out.
*/
EXPORT int uncompressBlock(const char* in, const size_t inLen, unsigned char* out,
unsigned int& outLen) const;
/**
* This fcn wraps whatever compression algorithm we're using at the time, and
* is not specific to blocks on disk.
*/
EXPORT int compress(const char *in, size_t inLen, char *out, size_t *outLen) const;
/**
* This fcn wraps whatever compression algorithm we're using at the time, and
* is not specific to blocks on disk.
*/
EXPORT int compress(const char* in, size_t inLen, char* out, size_t* outLen) const;
/**
* This fcn wraps whatever compression algorithm we're using at the time, and
* is not specific to blocks on disk. The caller needs to make sure out is big
* enough to contain the output by using getUncompressedSize().
*/
EXPORT int uncompress(const char *in, size_t inLen, char *out) const;
/**
* This fcn wraps whatever compression algorithm we're using at the time, and
* is not specific to blocks on disk. The caller needs to make sure out is big
* enough to contain the output by using getUncompressedSize().
*/
EXPORT int uncompress(const char* in, size_t inLen, char* out) const;
/**
* Initialize header buffer at start of compressed db file.
*
* @warning hdrBuf must be at least HDR_BUF_LEN*2 bytes
*/
EXPORT void initHdr(void* hdrBuf, int compressionType) const;
/**
* Initialize header buffer at start of compressed db file.
*
* @warning hdrBuf must be at least HDR_BUF_LEN*2 bytes
*/
EXPORT void initHdr(void* hdrBuf, int compressionType) const;
/**
* Initialize header buffer at start of compressed db file.
*
* @warning hdrBuf must be at least HDR_BUF_LEN bytes
* @warning ptrBuf must be at least (hdrSize-HDR_BUF_LEN) bytes
*/
EXPORT void initHdr(void* hdrBuf, void* ptrBuf, int compressionType, int hdrSize) const;
/**
* Initialize header buffer at start of compressed db file.
*
* @warning hdrBuf must be at least HDR_BUF_LEN bytes
* @warning ptrBuf must be at least (hdrSize-HDR_BUF_LEN) bytes
*/
EXPORT void initHdr(void* hdrBuf, void* ptrBuf, int compressionType, int hdrSize) const;
/**
* Verify the passed in buffer contains a compressed db file header.
*/
EXPORT int verifyHdr(const void* hdrBuf) const;
/**
* Verify the passed in buffer contains a compressed db file header.
*/
EXPORT int verifyHdr(const void* hdrBuf) const;
/**
* Extracts list of compression pointers from the specified ptr buffer.
* ptrBuf points to the pointer section taken from the headers.
* chunkPtrs is a vector of offset, size pairs for the compressed chunks.
* Returns 0 if success.
*/
EXPORT int getPtrList(const char* ptrBuf,
const int ptrBufSize,
CompChunkPtrList& chunkPtrs) const;
/**
* Extracts list of compression pointers from the specified ptr buffer.
* ptrBuf points to the pointer section taken from the headers.
* chunkPtrs is a vector of offset, size pairs for the compressed chunks.
* Returns 0 if success.
*/
EXPORT int getPtrList(const char* ptrBuf,
const int ptrBufSize,
CompChunkPtrList& chunkPtrs) const;
/**
* Extracts list of compression pointers from the specified header.
* hdrBuf points to start of 2 buffer headers from compressed db file.
* Overloaded for backward compatibility. For none dictionary columns.
* Note: the pointer passed in is the beginning of the header,
* not the pointer section as above.
*/
EXPORT int getPtrList(const char* hdrBuf, CompChunkPtrList& chunkPtrs) const;
/**
* Extracts list of compression pointers from the specified header.
* hdrBuf points to start of 2 buffer headers from compressed db file.
* Overloaded for backward compatibility. For none dictionary columns.
* Note: the pointer passed in is the beginning of the header,
* not the pointer section as above.
*/
EXPORT int getPtrList(const char* hdrBuf, CompChunkPtrList& chunkPtrs) const;
/**
* Return the number of chunk pointers contained in the specified ptr buffer.
* ptrBuf points to the pointer section taken from the headers.
*/
EXPORT unsigned int getPtrCount(const char* ptrBuf,
const int ptrBufSize) const;
/**
* Return the number of chunk pointers contained in the specified ptr buffer.
* ptrBuf points to the pointer section taken from the headers.
*/
EXPORT unsigned int getPtrCount(const char* ptrBuf,
const int ptrBufSize) const;
/**
* Return the number of chunk pointers contained in the specified header.
* hdrBuf points to start of 2 buffer headers from compressed db file.
* For non-dictionary columns.
*/
EXPORT unsigned int getPtrCount(const char* hdrBuf) const;
/**
* Return the number of chunk pointers contained in the specified header.
* hdrBuf points to start of 2 buffer headers from compressed db file.
* For non-dictionary columns.
*/
EXPORT unsigned int getPtrCount(const char* hdrBuf) const;
/**
* Store vector of pointers into the specified buffer header's pointer section.
*/
EXPORT void storePtrs(const std::vector<uint64_t>& ptrs,
void* hdrBuf,
int ptrSectionSize) const;
/**
* Store vector of pointers into the specified buffer header's pointer section.
*/
EXPORT void storePtrs(const std::vector<uint64_t>& ptrs,
void* hdrBuf,
int ptrSectionSize) const;
/**
* Store vector of pointers into the specified buffer header.
* Overloaded for backward compatibility. For none dictionary columns.
* Note: the pointer passed in is the beginning of the header,
* not the pointer section as above.
*/
EXPORT void storePtrs(const std::vector<uint64_t>& ptrs, void* hdrBuf) const;
/**
* Store vector of pointers into the specified buffer header.
* Overloaded for backward compatibility. For none dictionary columns.
* Note: the pointer passed in is the beginning of the header,
* not the pointer section as above.
*/
EXPORT void storePtrs(const std::vector<uint64_t>& ptrs, void* hdrBuf) const;
/**
* Calculates the chunk, and the block offset within the chunk, for the
* specified block number.
*/
EXPORT void locateBlock(unsigned int block,
unsigned int& chunkIndex,
unsigned int& blockOffsetWithinChunk) const;
/**
* Calculates the chunk, and the block offset within the chunk, for the
* specified block number.
*/
EXPORT void locateBlock(unsigned int block,
unsigned int& chunkIndex,
unsigned int& blockOffsetWithinChunk) const;
/**
* Pads the specified compressed chunk to the nearest compressed chunk
* increment, by padding buf with 0's, and updating len accordingly.
* maxLen is the maximum size for buf. nonzero return code means the
* result output buffer length is > than maxLen.
*/
EXPORT int padCompressedChunks(unsigned char* buf,
unsigned int& len,
unsigned int maxLen ) const;
/**
* Pads the specified compressed chunk to the nearest compressed chunk
* increment, by padding buf with 0's, and updating len accordingly.
* maxLen is the maximum size for buf. nonzero return code means the
* result output buffer length is > than maxLen.
*/
EXPORT int padCompressedChunks(unsigned char* buf,
unsigned int& len,
unsigned int maxLen ) const;
/*
* Mutator methods for the block count in the file
*/
/**
* setBlockCount
*/
EXPORT void setBlockCount(void* hdrBuf, uint64_t count) const;
/*
* Mutator methods for the block count in the file
*/
/**
* setBlockCount
*/
EXPORT void setBlockCount(void* hdrBuf, uint64_t count) const;
/**
* getBlockCount
*/
EXPORT uint64_t getBlockCount(const void* hdrBuf) const;
/**
* getBlockCount
*/
EXPORT uint64_t getBlockCount(const void* hdrBuf) const;
/*
* Mutator methods for the overall header size
*/
/**
* setHdrSize
*/
EXPORT void setHdrSize(void* hdrBuf, uint64_t size) const;
/*
* Mutator methods for the overall header size
*/
/**
* setHdrSize
*/
EXPORT void setHdrSize(void* hdrBuf, uint64_t size) const;
/**
* getHdrSize
*/
EXPORT uint64_t getHdrSize(const void* hdrBuf) const;
/**
* getHdrSize
*/
EXPORT uint64_t getHdrSize(const void* hdrBuf) const;
/**
* Mutator methods for the user padding bytes
*/
/**
* set numUserPaddingBytes
*/
EXPORT void numUserPaddingBytes(uint64_t num) { fNumUserPaddingBytes = num; }
/**
* Mutator methods for the user padding bytes
*/
/**
* set numUserPaddingBytes
*/
EXPORT void numUserPaddingBytes(uint64_t num)
{
fNumUserPaddingBytes = num;
}
/**
* get numUserPaddingBytes
*/
EXPORT uint64_t numUserPaddingBytes() const { return fNumUserPaddingBytes; }
/**
* get numUserPaddingBytes
*/
EXPORT uint64_t numUserPaddingBytes() const
{
return fNumUserPaddingBytes;
}
/**
* Given an input, uncompressed block, what's the maximum possible output,
* compressed size?
*/
EXPORT static uint64_t maxCompressedSize(uint64_t uncompSize);
/**
* Given an input, uncompressed block, what's the maximum possible output,
* compressed size?
*/
EXPORT static uint64_t maxCompressedSize(uint64_t uncompSize);
/**
* Given a compressed block, returns the uncompressed size in outLen.
* Returns false on error, true on success.
*/
EXPORT static bool getUncompressedSize(char *in, size_t inLen, size_t *outLen);
/**
* Given a compressed block, returns the uncompressed size in outLen.
* Returns false on error, true on success.
*/
EXPORT static bool getUncompressedSize(char* in, size_t inLen, size_t* outLen);
protected:
private:
//defaults okay
//IDBCompressInterface(const IDBCompressInterface& rhs);
//IDBCompressInterface& operator=(const IDBCompressInterface& rhs);
//defaults okay
//IDBCompressInterface(const IDBCompressInterface& rhs);
//IDBCompressInterface& operator=(const IDBCompressInterface& rhs);
unsigned int fNumUserPaddingBytes; // Num bytes to pad compressed chunks
unsigned int fNumUserPaddingBytes; // Num bytes to pad compressed chunks
};
#ifdef SKIP_IDB_COMPRESSION
inline IDBCompressInterface::IDBCompressInterface(unsigned int /*numUserPaddingBytes*/) {}
inline IDBCompressInterface::~IDBCompressInterface() {}
inline bool IDBCompressInterface::isCompressionAvail(int c) const { return (c == 0); }
inline int IDBCompressInterface::compressBlock(const char*,const size_t,unsigned char*,unsigned int&) const { return -1; }
inline int IDBCompressInterface::uncompressBlock(const char* in, const size_t inLen, unsigned char* out, unsigned int& outLen) const { return -1; }
inline int IDBCompressInterface::compress(const char* in, size_t inLen, char* out, size_t* outLen) const { return -1; }
inline int IDBCompressInterface::uncompress(const char* in, size_t inLen, char* out) const { return 0; }
inline void IDBCompressInterface::initHdr(void*,int) const {}
inline void IDBCompressInterface::initHdr(void*, void*, int,int) const {}
inline int IDBCompressInterface::verifyHdr(const void*) const { return -1; }
inline int IDBCompressInterface::getPtrList(const char*, const int, CompChunkPtrList&) const { return -1; }
inline int IDBCompressInterface::getPtrList(const char*, CompChunkPtrList&) const { return -1; }
inline unsigned int IDBCompressInterface::getPtrCount(const char*, const int) const { return 0; }
inline unsigned int IDBCompressInterface::getPtrCount(const char*) const { return 0; }
inline void IDBCompressInterface::storePtrs(const std::vector<uint64_t>&,void*,int) const {}
inline bool IDBCompressInterface::isCompressionAvail(int c) const
{
return (c == 0);
}
inline int IDBCompressInterface::compressBlock(const char*, const size_t, unsigned char*, unsigned int&) const
{
return -1;
}
inline int IDBCompressInterface::uncompressBlock(const char* in, const size_t inLen, unsigned char* out, unsigned int& outLen) const
{
return -1;
}
inline int IDBCompressInterface::compress(const char* in, size_t inLen, char* out, size_t* outLen) const
{
return -1;
}
inline int IDBCompressInterface::uncompress(const char* in, size_t inLen, char* out) const
{
return 0;
}
inline void IDBCompressInterface::initHdr(void*, int) const {}
inline void IDBCompressInterface::initHdr(void*, void*, int, int) const {}
inline int IDBCompressInterface::verifyHdr(const void*) const
{
return -1;
}
inline int IDBCompressInterface::getPtrList(const char*, const int, CompChunkPtrList&) const
{
return -1;
}
inline int IDBCompressInterface::getPtrList(const char*, CompChunkPtrList&) const
{
return -1;
}
inline unsigned int IDBCompressInterface::getPtrCount(const char*, const int) const
{
return 0;
}
inline unsigned int IDBCompressInterface::getPtrCount(const char*) const
{
return 0;
}
inline void IDBCompressInterface::storePtrs(const std::vector<uint64_t>&, void*, int) const {}
inline void IDBCompressInterface::storePtrs(const std::vector<uint64_t>&, void*) const {}
inline void IDBCompressInterface::locateBlock(unsigned int block,
unsigned int& chunkIndex, unsigned int& blockOffsetWithinChunk) const {}
inline int IDBCompressInterface::padCompressedChunks(unsigned char* buf, unsigned int& len, unsigned int maxLen) const { return -1; }
unsigned int& chunkIndex, unsigned int& blockOffsetWithinChunk) const {}
inline int IDBCompressInterface::padCompressedChunks(unsigned char* buf, unsigned int& len, unsigned int maxLen) const
{
return -1;
}
inline void IDBCompressInterface::setBlockCount(void* hdrBuf, uint64_t count) const {}
inline uint64_t IDBCompressInterface::getBlockCount(const void* hdrBuf) const { return 0; }
inline uint64_t IDBCompressInterface::getBlockCount(const void* hdrBuf) const
{
return 0;
}
inline void IDBCompressInterface::setHdrSize(void*, uint64_t) const {}
inline uint64_t IDBCompressInterface::getHdrSize(const void*) const { return 0; }
inline uint64_t IDBCompressInterface::maxCompressedSize(uint64_t uncompSize) { return uncompSize; }
inline bool IDBCompressInterface::getUncompressedSize(char* in, size_t inLen, size_t* outLen) { return false; }
inline uint64_t IDBCompressInterface::getHdrSize(const void*) const
{
return 0;
}
inline uint64_t IDBCompressInterface::maxCompressedSize(uint64_t uncompSize)
{
return uncompSize;
}
inline bool IDBCompressInterface::getUncompressedSize(char* in, size_t inLen, size_t* outLen)
{
return false;
}
#endif
}

789
utils/compress/version1.cpp
View File

@@ -47,7 +47,7 @@ using namespace std;
#include <boost/interprocess/mapped_region.hpp>
#include <boost/interprocess/sync/scoped_lock.hpp>
#include <boost/interprocess/sync/interprocess_mutex.hpp>
namespace bi=boost::interprocess;
namespace bi = boost::interprocess;
#include <boost/thread.hpp>
@@ -90,62 +90,69 @@ namespace
{
short DSPort = 9199;
void log(const string &s)
void log(const string& s)
{
logging::MessageLog logger((logging::LoggingID()));
logging::Message message;
logging::Message::Args args;
logging::MessageLog logger((logging::LoggingID()));
logging::Message message;
logging::Message::Args args;
args.add(s);
message.format(args);
logger.logErrorMessage(message);
args.add(s);
message.format(args);
logger.logErrorMessage(message);
}
struct ScopedCleaner
{
#ifdef _MSC_VER
ScopedCleaner() : ctlsock(INVALID_SOCKET),
cpipeh(INVALID_HANDLE_VALUE),
upipeh(INVALID_HANDLE_VALUE)
{ }
ScopedCleaner() : ctlsock(INVALID_SOCKET),
cpipeh(INVALID_HANDLE_VALUE),
upipeh(INVALID_HANDLE_VALUE)
{ }
~ScopedCleaner()
{
if (cpipeh != INVALID_HANDLE_VALUE)
CloseHandle(cpipeh);
if (upipeh != INVALID_HANDLE_VALUE)
CloseHandle(upipeh);
if (ctlsock != INVALID_SOCKET) {
shutdown(ctlsock, SHUT_RDWR);
closesocket(ctlsock);
}
}
~ScopedCleaner()
{
if (cpipeh != INVALID_HANDLE_VALUE)
CloseHandle(cpipeh);
SOCKET ctlsock;
HANDLE cpipeh;
HANDLE upipeh;
if (upipeh != INVALID_HANDLE_VALUE)
CloseHandle(upipeh);
if (ctlsock != INVALID_SOCKET)
{
shutdown(ctlsock, SHUT_RDWR);
closesocket(ctlsock);
}
}
SOCKET ctlsock;
HANDLE cpipeh;
HANDLE upipeh;
#else
ScopedCleaner() : fd(-1), ctlsock(-1)
{ }
ScopedCleaner() : fd(-1), ctlsock(-1)
{ }
~ScopedCleaner()
{
if (fd >= 0)
close(fd);
if (!cpipename.empty())
unlink(cpipename.c_str());
if (!upipename.empty())
unlink(upipename.c_str());
if (ctlsock >= 0) {
shutdown(ctlsock, SHUT_RDWR);
close(ctlsock);
}
}
~ScopedCleaner()
{
if (fd >= 0)
close(fd);
int fd;
int ctlsock;
string cpipename;
string upipename;
if (!cpipename.empty())
unlink(cpipename.c_str());
if (!upipename.empty())
unlink(upipename.c_str());
if (ctlsock >= 0)
{
shutdown(ctlsock, SHUT_RDWR);
close(ctlsock);
}
}
int fd;
int ctlsock;
string cpipename;
string upipename;
#endif
};
@@ -153,7 +160,7 @@ boost::mutex CtlShmMutex;
struct CtlShmImage
{
bi::interprocess_mutex controlFifoMutex;
bi::interprocess_mutex controlFifoMutex;
};
CtlShmImage* Ctlshmptr = 0;
bi::shared_memory_object Ctlshmobj;
@@ -161,117 +168,129 @@ bi::mapped_region Ctlshmregion;
void initCtlShm()
{
BRM::ShmKeys shmKeys;
string DecomShmName(BRM::ShmKeys::keyToName(shmKeys.DECOMSVRMUTEX_SYSVKEY));
BRM::ShmKeys shmKeys;
string DecomShmName(BRM::ShmKeys::keyToName(shmKeys.DECOMSVRMUTEX_SYSVKEY));
boost::mutex::scoped_lock Ctlshmlk(CtlShmMutex);
if (Ctlshmptr)
return;
boost::mutex::scoped_lock Ctlshmlk(CtlShmMutex);
CtlShmImage* tmpptr = 0;
if (Ctlshmptr)
return;
try {
bi::shared_memory_object shm(bi::open_only, DecomShmName.c_str(), bi::read_write);
bi::mapped_region region(shm, bi::read_write);
tmpptr = reinterpret_cast<CtlShmImage*>(region.get_address());
Ctlshmobj.swap(shm);
Ctlshmregion.swap(region);
} catch (bi::interprocess_exception&) {
CtlShmImage* tmpptr = 0;
try
{
bi::shared_memory_object shm(bi::open_only, DecomShmName.c_str(), bi::read_write);
bi::mapped_region region(shm, bi::read_write);
tmpptr = reinterpret_cast<CtlShmImage*>(region.get_address());
Ctlshmobj.swap(shm);
Ctlshmregion.swap(region);
}
catch (bi::interprocess_exception&)
{
#if BOOST_VERSION < 104500
bi::shared_memory_object shm(bi::create_only, DecomShmName.c_str(), bi::read_write);
bi::shared_memory_object shm(bi::create_only, DecomShmName.c_str(), bi::read_write);
#else
bi::permissions perms;
perms.set_unrestricted();
bi::shared_memory_object shm(bi::create_only, DecomShmName.c_str(), bi::read_write, perms);
bi::permissions perms;
perms.set_unrestricted();
bi::shared_memory_object shm(bi::create_only, DecomShmName.c_str(), bi::read_write, perms);
#endif
shm.truncate(sizeof(CtlShmImage));
bi::mapped_region region(shm, bi::read_write);
tmpptr = new (region.get_address()) CtlShmImage;
Ctlshmobj.swap(shm);
Ctlshmregion.swap(region);
}
const string pname("DecomSvr");
string srvrpath(startup::StartUp::installDir());
srvrpath += "/bin/" + pname;
shm.truncate(sizeof(CtlShmImage));
bi::mapped_region region(shm, bi::read_write);
tmpptr = new (region.get_address()) CtlShmImage;
Ctlshmobj.swap(shm);
Ctlshmregion.swap(region);
}
const string pname("DecomSvr");
string srvrpath(startup::StartUp::installDir());
srvrpath += "/bin/" + pname;
#ifndef OAM_FORKS_DECOMSVR
#ifndef _MSC_VER
int rc;
int rc;
rc = fork();
idbassert_s(rc >= 0, "couldn't fork DecomSvr");
rc = fork();
idbassert_s(rc >= 0, "couldn't fork DecomSvr");
// if (rc < 0)
// throw runtime_error("couldn't fork DecomSvr");
// else if (rc == 0)
if (rc == 0)
{
for (int fd = 0; fd < sysconf(_SC_OPEN_MAX); fd++)
close(fd);
open("/dev/null", O_RDONLY);
open("/dev/null", O_WRONLY);
open("/dev/null", O_WRONLY);
execl(srvrpath.c_str(), pname.c_str(), (char*)NULL);
idbassert_s(0, "couldn't exec DecomSvr");
//throw runtime_error("couldn't exec DecomSvr");
}
if (rc == 0)
{
for (int fd = 0; fd < sysconf(_SC_OPEN_MAX); fd++)
close(fd);
open("/dev/null", O_RDONLY);
open("/dev/null", O_WRONLY);
open("/dev/null", O_WRONLY);
execl(srvrpath.c_str(), pname.c_str(), (char*)NULL);
idbassert_s(0, "couldn't exec DecomSvr");
//throw runtime_error("couldn't exec DecomSvr");
}
#else
srvrpath += ".exe";
PROCESS_INFORMATION pInfo;
ZeroMemory(&pInfo, sizeof(pInfo));
STARTUPINFO sInfo;
ZeroMemory(&sInfo, sizeof(sInfo));
srvrpath += ".exe";
PROCESS_INFORMATION pInfo;
ZeroMemory(&pInfo, sizeof(pInfo));
STARTUPINFO sInfo;
ZeroMemory(&sInfo, sizeof(sInfo));
idbassert_s(CreateProcess(0, (LPSTR)srvrpath.c_str(), 0, 0, false, DETACHED_PROCESS, 0, 0, &sInfo, &pInfo) != 0,
"couldn't exec DecomSvr");
//if (CreateProcess(0, (LPSTR)srvrpath.c_str(), 0, 0, false, 0, 0, 0, &sInfo, &pInfo) == 0)
// throw runtime_error("couldn't exec DecomSvr");
CloseHandle(pInfo.hProcess);
idbassert_s(CreateProcess(0, (LPSTR)srvrpath.c_str(), 0, 0, false, DETACHED_PROCESS, 0, 0, &sInfo, &pInfo) != 0,
"couldn't exec DecomSvr");
//if (CreateProcess(0, (LPSTR)srvrpath.c_str(), 0, 0, false, 0, 0, 0, &sInfo, &pInfo) == 0)
// throw runtime_error("couldn't exec DecomSvr");
CloseHandle(pInfo.hProcess);
sleep(5);
sleep(5);
#endif
#endif
char* p = getenv("IDB_DECOMSVR_PORT");
if (p && *p)
{
DSPort = atoi(p);
if (DSPort <= 0)
DSPort = 9199;
}
char* p = getenv("IDB_DECOMSVR_PORT");
Ctlshmptr = tmpptr;
if (p && *p)
{
DSPort = atoi(p);
if (DSPort <= 0)
DSPort = 9199;
}
Ctlshmptr = tmpptr;
}
void sendn(int fd, const char* p, size_t wanted)
{
size_t needed = wanted;
size_t sofar = 0;
ssize_t rrc = -1;
pollfd fds[1];
int en = 0;
size_t needed = wanted;
size_t sofar = 0;
ssize_t rrc = -1;
pollfd fds[1];
int en = 0;
fds[0].fd = fd;
fds[0].events = POLLOUT;
fds[0].fd = fd;
fds[0].events = POLLOUT;
while (wanted > sofar)
{
fds[0].revents = 0;
poll(fds, 1, -1);
errno = 0;
rrc = send(fd, (p + sofar), needed, 0);
en = errno;
if (rrc < 0)
{
if (en == EAGAIN || en == EINTR || en == 512)
continue;
ostringstream oss;
oss << "send() returned " << rrc << " (" << strerror(en) << ")";
idbassert_s(0, oss.str());
//throw runtime_error(oss.str());
}
needed -= rrc;
sofar += rrc;
}
while (wanted > sofar)
{
fds[0].revents = 0;
poll(fds, 1, -1);
errno = 0;
rrc = send(fd, (p + sofar), needed, 0);
en = errno;
if (rrc < 0)
{
if (en == EAGAIN || en == EINTR || en == 512)
continue;
ostringstream oss;
oss << "send() returned " << rrc << " (" << strerror(en) << ")";
idbassert_s(0, oss.str());
//throw runtime_error(oss.str());
}
needed -= rrc;
sofar += rrc;
}
}
}
@@ -284,272 +303,292 @@ namespace v1
#ifndef _MSC_VER
void readn(int fd, void* buf, const size_t wanted)
{
size_t needed = wanted;
size_t sofar = 0;
char* p = reinterpret_cast<char*>(buf);
ssize_t rrc = -1;
pollfd fds[1];
int en = 0;
int prc = 0;
ostringstream oss;
unsigned zerocount=0;
size_t needed = wanted;
size_t sofar = 0;
char* p = reinterpret_cast<char*>(buf);
ssize_t rrc = -1;
pollfd fds[1];
int en = 0;
int prc = 0;
ostringstream oss;
unsigned zerocount = 0;
fds[0].fd = fd;
fds[0].events = POLLIN;
fds[0].fd = fd;
fds[0].events = POLLIN;
while (wanted > sofar)
{
fds[0].revents = 0;
errno = 0;
prc = poll(fds, 1, -1);
en = errno;
if (prc <= 0)
{
if (en == EAGAIN || en == EINTR || en == 512)
continue;
oss << "compress::v1::readn: poll() returned " << prc << " (" << strerror(en) << ")";
idbassert_s(0, oss.str());
}
//Check if there's data to be read
if ((fds[0].revents & POLLIN) == 0)
{
oss << "compress::v1::readn: revents for fd " << fds[0].fd << " was " << fds[0].revents;
idbassert_s(0, oss.str());
}
errno = 0;
rrc = read(fd, (p + sofar), needed);
en = errno;
if (rrc < 0)
{
if (en == EAGAIN || en == EINTR || en == 512)
continue;
oss << "compress::v1::readn(): read() returned " << rrc << " (" << strerror(en) << ")";
//this throws logging::IDBExcept()
idbassert_s(0, oss.str());
}
if (rrc == 0)
{
ostringstream os;
zerocount++;
if (zerocount >= 10)
{
os << "compress::v1::readn(): too many zero-length reads!";
idbassert_s(0, oss.str());
}
logging::MessageLog logger((logging::LoggingID()));
logging::Message message;
logging::Message::Args args;
os << "compress::v1::readn(): zero-length read on fd " << fd;
args.add(os.str());
message.format(args);
logger.logWarningMessage(message);
sleep(1);
}
else
zerocount = 0;
needed -= rrc;
sofar += rrc;
}
while (wanted > sofar)
{
fds[0].revents = 0;
errno = 0;
prc = poll(fds, 1, -1);
en = errno;
if (prc <= 0)
{
if (en == EAGAIN || en == EINTR || en == 512)
continue;
oss << "compress::v1::readn: poll() returned " << prc << " (" << strerror(en) << ")";
idbassert_s(0, oss.str());
}
//Check if there's data to be read
if ((fds[0].revents & POLLIN) == 0)
{
oss << "compress::v1::readn: revents for fd " << fds[0].fd << " was " << fds[0].revents;
idbassert_s(0, oss.str());
}
errno = 0;
rrc = read(fd, (p + sofar), needed);
en = errno;
if (rrc < 0)
{
if (en == EAGAIN || en == EINTR || en == 512)
continue;
oss << "compress::v1::readn(): read() returned " << rrc << " (" << strerror(en) << ")";
//this throws logging::IDBExcept()
idbassert_s(0, oss.str());
}
if (rrc == 0)
{
ostringstream os;
zerocount++;
if (zerocount >= 10)
{
os << "compress::v1::readn(): too many zero-length reads!";
idbassert_s(0, oss.str());
}
logging::MessageLog logger((logging::LoggingID()));
logging::Message message;
logging::Message::Args args;
os << "compress::v1::readn(): zero-length read on fd " << fd;
args.add(os.str());
message.format(args);
logger.logWarningMessage(message);
sleep(1);
}
else
zerocount = 0;
needed -= rrc;
sofar += rrc;
}
}
size_t writen(int fd, const void *data, size_t nbytes)
size_t writen(int fd, const void* data, size_t nbytes)
{
size_t nleft;
ssize_t nwritten;
const char *bufp = (const char *) data;
nleft = nbytes;
size_t nleft;
ssize_t nwritten;
const char* bufp = (const char*) data;
nleft = nbytes;
while (nleft > 0)
{
// the O_NONBLOCK flag is not set, this is a blocking I/O.
if ((nwritten = ::write(fd, bufp, nleft)) < 0)
{
if (errno == EINTR)
nwritten = 0;
else {
// save the error no first
int e = errno;
string errorMsg = "v1::writen() error: ";
boost::scoped_array<char> buf(new char[80]);
while (nleft > 0)
{
// the O_NONBLOCK flag is not set, this is a blocking I/O.
if ((nwritten = ::write(fd, bufp, nleft)) < 0)
{
if (errno == EINTR)
nwritten = 0;
else
{
// save the error no first
int e = errno;
string errorMsg = "v1::writen() error: ";
boost::scoped_array<char> buf(new char[80]);
#if STRERROR_R_CHAR_P
const char* p;
if ((p = strerror_r(e, buf.get(), 80)) != 0)
errorMsg += p;
const char* p;
if ((p = strerror_r(e, buf.get(), 80)) != 0)
errorMsg += p;
#else
int p;
if ((p = strerror_r(e, buf.get(), 80)) == 0)
errorMsg += buf.get();
int p;
if ((p = strerror_r(e, buf.get(), 80)) == 0)
errorMsg += buf.get();
#endif
idbassert_s(0, errorMsg);
//throw runtime_error(errorMsg);
}
}
nleft -= nwritten;
bufp += nwritten;
}
return nbytes;
idbassert_s(0, errorMsg);
//throw runtime_error(errorMsg);
}
}
nleft -= nwritten;
bufp += nwritten;
}
return nbytes;
}
#endif
bool decompress(const char* in, const uint32_t inLen, unsigned char* out, size_t* ol)
{
uint32_t u32;
uint64_t u64;
ostringstream oss;
string s;
string cpipe;
string upipe;
ScopedCleaner cleaner;
int fd = -1;
uint32_t u32;
uint64_t u64;
ostringstream oss;
string s;
string cpipe;
string upipe;
ScopedCleaner cleaner;
int fd = -1;
if (!Ctlshmptr)
initCtlShm();
if (!Ctlshmptr)
initCtlShm();
bi::scoped_lock<bi::interprocess_mutex> cfLock(Ctlshmptr->controlFifoMutex, bi::defer_lock);
bi::scoped_lock<bi::interprocess_mutex> cfLock(Ctlshmptr->controlFifoMutex, bi::defer_lock);
#ifndef _MSC_VER
pthread_t thdid = pthread_self();
pthread_t thdid = pthread_self();
#else
DWORD thdid = GetCurrentThreadId();
DWORD thdid = GetCurrentThreadId();
#endif
#ifdef _MSC_VER
oss << "\\\\.\\pipe\\cdatafifo" << thdid;
s = oss.str();
cpipe = s + ".c";
upipe = s + ".u";
oss << "\\\\.\\pipe\\cdatafifo" << thdid;
s = oss.str();
cpipe = s + ".c";
upipe = s + ".u";
HANDLE cpipeh;
cpipeh = CreateNamedPipe(cpipe.c_str(), PIPE_ACCESS_OUTBOUND, PIPE_TYPE_BYTE|PIPE_READMODE_BYTE|PIPE_WAIT,
1, 0, 8192, 0, 0);
idbassert_s(cpipeh != INVALID_HANDLE_VALUE, "while creating cdata fifo");
//if (cpipeh == INVALID_HANDLE_VALUE)
// throw runtime_error("while creating cdata fifo");
cleaner.cpipeh = cpipeh;
HANDLE cpipeh;
cpipeh = CreateNamedPipe(cpipe.c_str(), PIPE_ACCESS_OUTBOUND, PIPE_TYPE_BYTE | PIPE_READMODE_BYTE | PIPE_WAIT,
1, 0, 8192, 0, 0);
idbassert_s(cpipeh != INVALID_HANDLE_VALUE, "while creating cdata fifo");
//if (cpipeh == INVALID_HANDLE_VALUE)
// throw runtime_error("while creating cdata fifo");
cleaner.cpipeh = cpipeh;
HANDLE upipeh;
upipeh = CreateNamedPipe(upipe.c_str(), PIPE_ACCESS_INBOUND, PIPE_TYPE_BYTE|PIPE_READMODE_BYTE|PIPE_WAIT,
1, 8192, 0, 0, 0);
idbassert_s(upipeh != INVALID_HANDLE_VALUE, "while creating udata fifo");
//if (upipeh == INVALID_HANDLE_VALUE)
// throw runtime_error("while creating udata fifo");
cleaner.upipeh = upipeh;
HANDLE upipeh;
upipeh = CreateNamedPipe(upipe.c_str(), PIPE_ACCESS_INBOUND, PIPE_TYPE_BYTE | PIPE_READMODE_BYTE | PIPE_WAIT,
1, 8192, 0, 0, 0);
idbassert_s(upipeh != INVALID_HANDLE_VALUE, "while creating udata fifo");
//if (upipeh == INVALID_HANDLE_VALUE)
// throw runtime_error("while creating udata fifo");
cleaner.upipeh = upipeh;
#else
oss << "/tmp/cdatafifo" << hex << thdid;
s = oss.str();
cpipe = s + ".c";
upipe = s + ".u";
cleaner.cpipename = cpipe;
cleaner.upipename = upipe;
unlink(cpipe.c_str());
idbassert_s(mknod(cpipe.c_str(), S_IFIFO|0666, 0) == 0, "while creating cdata fifo");
//if (mknod(cpipe.c_str(), S_IFIFO|0666, 0) != 0)
// throw runtime_error("while creating cdata fifo");
unlink(upipe.c_str());
idbassert_s(mknod(upipe.c_str(), S_IFIFO|0666, 0) == 0, "while creating udata fifo");
//if (mknod(upipe.c_str(), S_IFIFO|0666, 0) != 0)
// throw runtime_error("while creating udata fifo");
oss << "/tmp/cdatafifo" << hex << thdid;
s = oss.str();
cpipe = s + ".c";
upipe = s + ".u";
cleaner.cpipename = cpipe;
cleaner.upipename = upipe;
unlink(cpipe.c_str());
idbassert_s(mknod(cpipe.c_str(), S_IFIFO | 0666, 0) == 0, "while creating cdata fifo");
//if (mknod(cpipe.c_str(), S_IFIFO|0666, 0) != 0)
// throw runtime_error("while creating cdata fifo");
unlink(upipe.c_str());
idbassert_s(mknod(upipe.c_str(), S_IFIFO | 0666, 0) == 0, "while creating udata fifo");
//if (mknod(upipe.c_str(), S_IFIFO|0666, 0) != 0)
// throw runtime_error("while creating udata fifo");
#endif
int rc = -1;
fd = ::socket(PF_INET, SOCK_STREAM, IPPROTO_TCP);
int rc = -1;
fd = ::socket(PF_INET, SOCK_STREAM, IPPROTO_TCP);
#ifdef _MSC_VER
idbassert_s(fd != INVALID_SOCKET,
string("socket create error: ") + strerror(errno));
idbassert_s(fd != INVALID_SOCKET,
string("socket create error: ") + strerror(errno));
#else
idbassert_s(fd >= 0,
string("socket create error: ") + strerror(errno));
idbassert_s(fd >= 0,
string("socket create error: ") + strerror(errno));
#endif
cleaner.ctlsock = fd;
struct sockaddr_in serv_addr;
struct in_addr la;
::inet_aton("127.0.0.1", &la);
memset(&serv_addr, 0, sizeof(serv_addr));
serv_addr.sin_family = AF_INET;
serv_addr.sin_addr.s_addr = la.s_addr;
serv_addr.sin_port = htons(DSPort);
const int MaxTries = 30;
int tries = 0;
cleaner.ctlsock = fd;
struct sockaddr_in serv_addr;
struct in_addr la;
::inet_aton("127.0.0.1", &la);
memset(&serv_addr, 0, sizeof(serv_addr));
serv_addr.sin_family = AF_INET;
serv_addr.sin_addr.s_addr = la.s_addr;
serv_addr.sin_port = htons(DSPort);
const int MaxTries = 30;
int tries = 0;
again:
cfLock.lock();
rc = ::connect(fd, (struct sockaddr*)&serv_addr, sizeof(serv_addr));
if (rc < 0)
{
cfLock.lock();
rc = ::connect(fd, (struct sockaddr*)&serv_addr, sizeof(serv_addr));
if (rc < 0)
{
#ifdef _MSC_VER
int x = WSAGetLastError();
if (x == WSAECONNREFUSED)
int x = WSAGetLastError();
if (x == WSAECONNREFUSED)
#else
if (errno == ECONNREFUSED)
if (errno == ECONNREFUSED)
#endif
{
idbassert_s(++tries < MaxTries, string("socket connect error: ") + strerror(errno));
//if (++tries >= MaxTries)
// throw runtime_error(string("socket connect error: ") + strerror(errno));
cfLock.unlock();
sleep(2);
goto again;
}
idbassert_s(0, string("socket connect error: ") + strerror(errno));
//throw runtime_error(string("socket connect error: ") + strerror(errno));
}
{
idbassert_s(++tries < MaxTries, string("socket connect error: ") + strerror(errno));
//if (++tries >= MaxTries)
// throw runtime_error(string("socket connect error: ") + strerror(errno));
cfLock.unlock();
sleep(2);
goto again;
}
u32 = s.length();
idbassert_s(0, string("socket connect error: ") + strerror(errno));
//throw runtime_error(string("socket connect error: ") + strerror(errno));
}
sendn(fd, reinterpret_cast<const char*>(&u32), 4);
u32 = s.length();
sendn(fd, s.c_str(), u32);
sendn(fd, reinterpret_cast<const char*>(&u32), 4);
shutdown(fd, SHUT_RDWR);
sendn(fd, s.c_str(), u32);
shutdown(fd, SHUT_RDWR);
#ifdef _MSC_VER
closesocket(fd);
cleaner.ctlsock = INVALID_SOCKET;
closesocket(fd);
cleaner.ctlsock = INVALID_SOCKET;
#else
close(fd);
cleaner.ctlsock = -1;
close(fd);
cleaner.ctlsock = -1;
#endif
cfLock.unlock();
cfLock.unlock();
#ifdef _MSC_VER
BOOL dwrc;
BOOL dwrc;
dwrc = ConnectNamedPipe(cpipeh, 0);
idbassert_s(!(dwrc == 0 && GetLastError() != ERROR_PIPE_CONNECTED), "connecting to cpipe");
//if (dwrc == 0 && GetLastError() != ERROR_PIPE_CONNECTED)
// throw runtime_error("connecting to cpipe");
dwrc = ConnectNamedPipe(cpipeh, 0);
idbassert_s(!(dwrc == 0 && GetLastError() != ERROR_PIPE_CONNECTED), "connecting to cpipe");
//if (dwrc == 0 && GetLastError() != ERROR_PIPE_CONNECTED)
// throw runtime_error("connecting to cpipe");
u64 = static_cast<uint64_t>(inLen);
idbassert_s(u64 < 8 * 1024 * 1024, "preposterous inLen!");
//if (!(u64 < 8 * 1024 * 1024))
// throw runtime_error("preposterous inLen!");
u64 = static_cast<uint64_t>(inLen);
idbassert_s(u64 < 8 * 1024 * 1024, "preposterous inLen!");
//if (!(u64 < 8 * 1024 * 1024))
// throw runtime_error("preposterous inLen!");
DWORD nwrite;
dwrc = WriteFile(cpipeh, &u64, 8, &nwrite, 0);
idbassert_s(dwrc != 0 && nwrite == 8, "while writing to cpipe");
//if (!(dwrc != 0 && nwrite == 8))
// throw runtime_error("while writing to cpipe");
DWORD nwrite;
dwrc = WriteFile(cpipeh, &u64, 8, &nwrite, 0);
idbassert_s(dwrc != 0 && nwrite == 8, "while writing to cpipe");
//if (!(dwrc != 0 && nwrite == 8))
// throw runtime_error("while writing to cpipe");
dwrc = WriteFile(cpipeh, in, u64, &nwrite, 0);
idbassert_s(dwrc != 0 && nwrite == u64, "while writing to cpipe");
//if (!(dwrc != 0 && nwrite == u64))
// throw runtime_error("while writing to cpipe");
dwrc = WriteFile(cpipeh, in, u64, &nwrite, 0);
idbassert_s(dwrc != 0 && nwrite == u64, "while writing to cpipe");
//if (!(dwrc != 0 && nwrite == u64))
// throw runtime_error("while writing to cpipe");
FlushFileBuffers(cpipeh);
CloseHandle(cpipeh);
cleaner.cpipeh = INVALID_HANDLE_VALUE;
FlushFileBuffers(cpipeh);
CloseHandle(cpipeh);
cleaner.cpipeh = INVALID_HANDLE_VALUE;
#else
ssize_t wrc;
fd = open(cpipe.c_str(), O_WRONLY);
idbassert_s(fd >= 0, "while opening data fifo for write");
//if (fd < 0)
// throw runtime_error("while opening data fifo for write");
ssize_t wrc;
fd = open(cpipe.c_str(), O_WRONLY);
idbassert_s(fd >= 0, "while opening data fifo for write");
//if (fd < 0)
// throw runtime_error("while opening data fifo for write");
cleaner.fd = fd;
cleaner.fd = fd;
u64 = static_cast<uint64_t>(inLen);
errno = 0;
wrc = writen(fd, &u64, 8);
int err = errno;
idbassert_s(wrc == 8, string("while writing compressed len to the DS: ") + strerror(err));
u64 = static_cast<uint64_t>(inLen);
errno = 0;
wrc = writen(fd, &u64, 8);
int err = errno;
idbassert_s(wrc == 8, string("while writing compressed len to the DS: ") + strerror(err));
// if (wrc != 8)
// {
// ostringstream oss;
@@ -557,52 +596,52 @@ again:
// throw runtime_error(oss.str());
// }
wrc = writen(fd, in, u64);
idbassert_s(wrc == static_cast<ssize_t>(u64), "while writing compressed data to the DS");
wrc = writen(fd, in, u64);
idbassert_s(wrc == static_cast<ssize_t>(u64), "while writing compressed data to the DS");
// if (wrc != static_cast<ssize_t>(u64))
// throw runtime_error("while writing compressed data to the DS");
close(fd);
cleaner.fd = -1;
close(fd);
cleaner.fd = -1;
#endif
#ifdef _MSC_VER
dwrc = ConnectNamedPipe(upipeh, 0);
idbassert_s(!(dwrc == 0 && GetLastError() != ERROR_PIPE_CONNECTED), "connecting to upipe");
//if (dwrc == 0 && GetLastError() != ERROR_PIPE_CONNECTED)
// throw runtime_error("connecting to upipe");
dwrc = ConnectNamedPipe(upipeh, 0);
idbassert_s(!(dwrc == 0 && GetLastError() != ERROR_PIPE_CONNECTED), "connecting to upipe");
//if (dwrc == 0 && GetLastError() != ERROR_PIPE_CONNECTED)
// throw runtime_error("connecting to upipe");
DWORD nread;
dwrc = ReadFile(upipeh, &u64, 8, &nread, 0);
idbassert_s(dwrc != 0 && nread == 8, "while reading from upipe");
//if (!(dwrc != 0 && nread == 8))
// throw runtime_error("while reading from upipe");
DWORD nread;
dwrc = ReadFile(upipeh, &u64, 8, &nread, 0);
idbassert_s(dwrc != 0 && nread == 8, "while reading from upipe");
//if (!(dwrc != 0 && nread == 8))
// throw runtime_error("while reading from upipe");
dwrc = ReadFile(upipeh, out, u64, &nread, 0);
idbassert_s(dwrc != 0 && nread == u64, "while reading from upipe");
//if (!(dwrc != 0 && nread == u64))
// throw runtime_error("while reading from upipe");
dwrc = ReadFile(upipeh, out, u64, &nread, 0);
idbassert_s(dwrc != 0 && nread == u64, "while reading from upipe");
//if (!(dwrc != 0 && nread == u64))
// throw runtime_error("while reading from upipe");
CloseHandle(upipeh);
cleaner.upipeh = INVALID_HANDLE_VALUE;
CloseHandle(upipeh);
cleaner.upipeh = INVALID_HANDLE_VALUE;
#else
fd = open(upipe.c_str(), O_RDONLY);
idbassert_s(fd >= 0, "while opening data fifo for read");
fd = open(upipe.c_str(), O_RDONLY);
idbassert_s(fd >= 0, "while opening data fifo for read");
// if (fd < 0)
// throw runtime_error("while opening data fifo for read");
cleaner.fd = fd;
cleaner.fd = fd;
readn(fd, &u64, 8);
readn(fd, out, u64);
readn(fd, &u64, 8);
readn(fd, out, u64);
close(fd);
cleaner.fd = -1;
close(fd);
cleaner.fd = -1;
#endif
*ol = static_cast<size_t>(u64);
*ol = static_cast<size_t>(u64);
return (u64 != 0);
return (u64 != 0);
}
} //namespace v1

8
utils/compress/version1.h
View File

@@ -27,12 +27,12 @@
namespace compress
{
namespace v1
{
namespace v1
{
bool decompress(const char* in, const uint32_t inLen, unsigned char* out, size_t* ol);
bool decompress(const char* in, const uint32_t inLen, unsigned char* out, size_t* ol);
} //namespace v1
} //namespace v1
} // namespace compress
#endif