mariadb-columnstore-engine/utils/compress/idbcompress.h

/* Copyright (C) 2014 InfiniDB, Inc.

   This program is free software; you can redistribute it and/or
   modify it under the terms of the GNU General Public License
   as published by the Free Software Foundation; version 2 of
   the License.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with this program; if not, write to the Free Software
   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
   MA 02110-1301, USA. */

/** @file */

#ifndef IDBCOMPRESS_H__
#define IDBCOMPRESS_H__

#include <unistd.h>
#ifdef __linux__
#include <sys/types.h>
#endif
#include <vector>
#include <utility>

#if defined(_MSC_VER) && defined(xxxIDBCOMP_DLLEXPORT)
#define EXPORT __declspec(dllexport)
#else
#define EXPORT
#endif

namespace compress
{

typedef std::pair<uint64_t,uint64_t> CompChunkPtr;
typedef std::vector<CompChunkPtr> CompChunkPtrList;

class IDBCompressInterface
{
public:
	static const unsigned int HDR_BUF_LEN            = 4096;
	static const unsigned int UNCOMPRESSED_INBUF_LEN = 512 * 1024 * 8;

	// error codes from uncompressBlock()
	static const int ERR_OK = 0;
	static const int ERR_CHECKSUM = -1;
	static const int ERR_DECOMPRESS = -2;
	static const int ERR_BADINPUT = -3;
	static const int ERR_BADOUTSIZE = -4;

	/**
	* When IDBCompressInterface object is being used to compress a chunk, this
	* construct can be used to specify the padding added by padCompressedChunks
	*/
	EXPORT explicit IDBCompressInterface(unsigned int numUserPaddingBytes=0);

	/**
	 * dtor
	 */
	EXPORT virtual ~IDBCompressInterface();

	/**
	 * see if the algo is available in this lib
	 */
	EXPORT bool isCompressionAvail(int compressionType = 0) const;

	/**
	* Compresses specified "in" buffer of length "inLen" bytes.
	* Compressed data and size are returned in "out" and "outLen".
	* "out" should be sized using maxCompressedSize() to allow for incompressible data.
	* Returns 0 if success.
	*/
	EXPORT int compressBlock(const char* in,
		const size_t   inLen,
		unsigned char* out,
		unsigned int&  outLen) const;

	/**
 	* outLen must be initialized with the size of the out buffer before calling uncompressBlock.
 	* On return, outLen will have the number of bytes used in out.
 	*/
	EXPORT int uncompressBlock(const char* in, const size_t inLen, unsigned char* out,
		unsigned int& outLen) const;

	/**
	 * This fcn wraps whatever compression algorithm we're using at the time, and
	 * is not specific to blocks on disk.
	 */
	EXPORT int compress(const char *in, size_t inLen, char *out, size_t *outLen) const;

	/**
	 * This fcn wraps whatever compression algorithm we're using at the time, and
	 * is not specific to blocks on disk.  The caller needs to make sure out is big
	 * enough to contain the output by using getUncompressedSize().
	 */
	EXPORT int uncompress(const char *in, size_t inLen, char *out) const;

	/**
	* Initialize header buffer at start of compressed db file.
	*
	* @warning hdrBuf must be at least HDR_BUF_LEN*2 bytes
	*/
	EXPORT void initHdr(void* hdrBuf, int compressionType) const;

	/**
	* Initialize header buffer at start of compressed db file.
	*
	* @warning hdrBuf must be at least HDR_BUF_LEN bytes
	* @warning ptrBuf must be at least (hdrSize-HDR_BUF_LEN) bytes
	*/
	EXPORT void initHdr(void* hdrBuf, void* ptrBuf, int compressionType, int hdrSize) const;

	/**
	* Verify the passed in buffer contains a compressed db file header.
	*/
	EXPORT int verifyHdr(const void* hdrBuf) const;

	/**
	* Extracts list of compression pointers from the specified ptr buffer.
	* ptrBuf points to the pointer section taken from the headers.
	* chunkPtrs is a vector of offset, size pairs for the compressed chunks.
	* Returns 0 if success.
	*/
	EXPORT int getPtrList(const char* ptrBuf,
		const int ptrBufSize,
		CompChunkPtrList& chunkPtrs) const;

	/**
	* Extracts list of compression pointers from the specified header.
	* hdrBuf points to start of 2 buffer headers from compressed db file.
	* Overloaded for backward compatibility. For none dictionary columns.
	* Note: the pointer passed in is the beginning of the header,
	*       not the pointer section as above.
	*/
	EXPORT int getPtrList(const char* hdrBuf, CompChunkPtrList& chunkPtrs) const;

	/**
	* Return the number of chunk pointers contained in the specified ptr buffer.
	* ptrBuf points to the pointer section taken from the headers.
	*/
	EXPORT unsigned int getPtrCount(const char* ptrBuf,
		const int ptrBufSize) const;

	/**
	* Return the number of chunk pointers contained in the specified header.
	* hdrBuf points to start of 2 buffer headers from compressed db file.
	* For non-dictionary columns.
	*/
	EXPORT unsigned int getPtrCount(const char* hdrBuf) const;

	/**
	* Store vector of pointers into the specified buffer header's pointer section.
	*/
	EXPORT void storePtrs(const std::vector<uint64_t>& ptrs,
		void* hdrBuf,
		int ptrSectionSize) const;

	/**
	* Store vector of pointers into the specified buffer header.
	* Overloaded for backward compatibility. For none dictionary columns.
	* Note: the pointer passed in is the beginning of the header,
	*       not the pointer section as above.
	*/
	EXPORT void storePtrs(const std::vector<uint64_t>& ptrs, void* hdrBuf) const;

	/**
	* Calculates the chunk, and the block offset within the chunk, for the
	* specified block number.
	*/
	EXPORT void locateBlock(unsigned int block,
		unsigned int& chunkIndex,
		unsigned int& blockOffsetWithinChunk) const;

	/**
	 * Pads the specified compressed chunk to the nearest compressed chunk
	 * increment, by padding buf with 0's, and updating len accordingly.
	 * maxLen is the maximum size for buf.  nonzero return code means the
	 * result output buffer length is > than maxLen.
	 */
	EXPORT int padCompressedChunks(unsigned char* buf,
		unsigned int& len,
		unsigned int  maxLen ) const;

	/*
	 * Mutator methods for the block count in the file
	 */
	/**
	 * setBlockCount
	 */
	EXPORT void setBlockCount(void* hdrBuf, uint64_t count) const;

	/**
	 * getBlockCount
	 */
	EXPORT uint64_t getBlockCount(const void* hdrBuf) const;

	/*
	 * Mutator methods for the overall header size
	 */
	/**
	 * setHdrSize
	 */
	EXPORT void setHdrSize(void* hdrBuf, uint64_t size) const;

	/**
	 * getHdrSize
	 */
	EXPORT uint64_t getHdrSize(const void* hdrBuf) const;

	/**
	 * Mutator methods for the user padding bytes
	 */
	/**
	 * set numUserPaddingBytes
	 */
	EXPORT void numUserPaddingBytes(uint64_t num) { fNumUserPaddingBytes = num;  }

	/**
	 * get numUserPaddingBytes
	 */
	EXPORT uint64_t numUserPaddingBytes() const   { return fNumUserPaddingBytes; }

	/**
	 * Given an input, uncompressed block, what's the maximum possible output,
	 * compressed size?
	 */
	EXPORT static uint64_t maxCompressedSize(uint64_t uncompSize);

	/**
	 * Given a compressed block, returns the uncompressed size in outLen.
	 * Returns false on error, true on success.
	 */
	EXPORT static bool getUncompressedSize(char *in, size_t inLen, size_t *outLen);

protected:

private:
	//defaults okay
	//IDBCompressInterface(const IDBCompressInterface& rhs);
	//IDBCompressInterface& operator=(const IDBCompressInterface& rhs);

	unsigned int fNumUserPaddingBytes; // Num bytes to pad compressed chunks
};

#ifdef SKIP_IDB_COMPRESSION
inline IDBCompressInterface::IDBCompressInterface(unsigned int /*numUserPaddingBytes*/) {}
inline IDBCompressInterface::~IDBCompressInterface() {}
inline bool IDBCompressInterface::isCompressionAvail(int c) const { return (c == 0); }
inline int IDBCompressInterface::compressBlock(const char*,const size_t,unsigned char*,unsigned int&) const { return -1; }
inline int IDBCompressInterface::uncompressBlock(const char* in, const size_t inLen, unsigned char* out, unsigned int& outLen) const { return -1; }
inline int IDBCompressInterface::compress(const char* in, size_t inLen, char* out, size_t* outLen) const { return -1; }
inline int IDBCompressInterface::uncompress(const char* in, size_t inLen, char* out) const { return 0; }
inline void IDBCompressInterface::initHdr(void*,int) const {}
inline void IDBCompressInterface::initHdr(void*, void*, int,int) const {}
inline int IDBCompressInterface::verifyHdr(const void*) const { return -1; }
inline int IDBCompressInterface::getPtrList(const char*, const int, CompChunkPtrList&) const { return -1; }
inline int IDBCompressInterface::getPtrList(const char*, CompChunkPtrList&) const { return -1; }
inline unsigned int IDBCompressInterface::getPtrCount(const char*, const int) const { return 0; }
inline unsigned int IDBCompressInterface::getPtrCount(const char*) const { return 0; }
inline void IDBCompressInterface::storePtrs(const std::vector<uint64_t>&,void*,int) const {}
inline void IDBCompressInterface::storePtrs(const std::vector<uint64_t>&, void*) const {}
inline void IDBCompressInterface::locateBlock(unsigned int block,
	unsigned int& chunkIndex, unsigned int& blockOffsetWithinChunk) const {}
inline int IDBCompressInterface::padCompressedChunks(unsigned char* buf, unsigned int& len, unsigned int maxLen) const { return -1; }
inline void IDBCompressInterface::setBlockCount(void* hdrBuf, uint64_t count) const {}
inline uint64_t IDBCompressInterface::getBlockCount(const void* hdrBuf) const { return 0; }
inline void IDBCompressInterface::setHdrSize(void*, uint64_t) const {}
inline uint64_t IDBCompressInterface::getHdrSize(const void*) const { return 0; }
inline uint64_t IDBCompressInterface::maxCompressedSize(uint64_t uncompSize) { return uncompSize; }
inline bool IDBCompressInterface::getUncompressedSize(char* in, size_t inLen, size_t* outLen) { return false; }
#endif

}

#undef EXPORT

#endif