mariadb-columnstore-engine/utils/messageqcpp/bytestream.h

/*
   Copyright (c) 2017, MariaDB
   Copyright (C) 2014 InfiniDB, Inc.

   This program is free software; you can redistribute it and/or
   modify it under the terms of the GNU General Public License
   as published by the Free Software Foundation; version 2 of
   the License.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with this program; if not, write to the Free Software
   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
   MA 02110-1301, USA.
*/

#ifndef MESSAGEQCPP_BYTESTREAM_H
#define MESSAGEQCPP_BYTESTREAM_H

#include <string>
#include <iostream>
#include <sys/types.h>
#include <stdexcept>
#include <vector>
#include <set>
#include <boost/shared_ptr.hpp>
#include <boost/version.hpp>
#include <boost/uuid/uuid.hpp>
#include <stdint.h>
#include <cstring>

#include "exceptclasses.h"
#include "serializeable.h"
#include "any.hpp"

class ByteStreamTestSuite;

#if defined(_MSC_VER) && defined(xxxBYTESTREAM_DLLEXPORT)
#define EXPORT __declspec(dllexport)
#else
#define EXPORT
#endif

namespace messageqcpp
{

typedef boost::shared_ptr<ByteStream> SBS;

/**
 * @brief A class to marshall bytes as a stream
 *
 * The ByteStream class is used to marshall numeric data into and out of a stream of
 * bytes (unsigned chars). It is a FIFO queue that maintains current input and output
 * pointers.
 *
 * @warning Alas, due to recent changes this class no longer implements a strong execption guarantee.
 *
 * @warning the current implementation does not know how to compact memory, so it should be
 * destructed or have reset() called to clear out the current uint8_t array. Also, multi-uint8_t
 * numeric values are pushed and dequeued in the native byte order, so they are not portable
 * across machines with different byte orders.
 *
 */
class ByteStream : public Serializeable
{
public:
    // We now use the standard Linux types of uint8_t, uint16_t, etc.
    // These are kept around for backward compatibility
    typedef uint8_t  byte;
    typedef uint16_t doublebyte;
    typedef uint32_t quadbyte;
    typedef uint64_t octbyte;
    typedef boost::uuids::uuid uuid;

    /**
     *	default ctor
     */
    EXPORT explicit ByteStream(uint32_t initSize = 8192); // multiples of pagesize are best
    /**
     *	ctor with a uint8_t array and len initializer
     */
    inline ByteStream(const uint8_t* bp, const uint32_t len);
    /**
     *	copy ctor
     */
    EXPORT ByteStream(const ByteStream& rhs);
    EXPORT ByteStream(const SBS& rhs);
    /**
     *	assign op
     */
    EXPORT ByteStream& operator=(const ByteStream& rhs);
    /**
     *	assign op
     */
    inline ByteStream& operator=(const SBS& rhs);
    /**
     *	dtor
     */
    inline virtual ~ByteStream();

    /**
     *	push a int8_t onto the end of the stream
     */
    EXPORT ByteStream& operator<<(const int8_t b);
    /**
     *	push a uint8_t onto the end of the stream
     */
    EXPORT ByteStream& operator<<(const uint8_t b);
    /**
     *	push a int16_t onto the end of the stream. The byte order is whatever the native byte order is.
     */
    EXPORT ByteStream& operator<<(const int16_t d);
    /**
     *	push a uint16_t onto the end of the stream. The byte order is whatever the native byte order is.
     */
    EXPORT ByteStream& operator<<(const uint16_t d);
    /**
     *	push a int32_t onto the end of the stream. The byte order is whatever the native byte order is.
     */
    EXPORT ByteStream& operator<<(const int32_t q);
    /**
     *	push a uint32_t onto the end of the stream. The byte order is whatever the native byte order is.
     */
    EXPORT ByteStream& operator<<(const uint32_t q);
#ifdef _MSC_VER
#if BOOST_VERSION < 104500
    //These 2 are to make MS VC++ w/ boost < 1.45 happy
    // TODO: Do we still need these?
    EXPORT ByteStream& operator<<(const uint32_t ui);

    EXPORT ByteStream& operator>>(uint32_t& ui);
#endif
#endif
    /**
     *	push an int64_t onto the end of the stream. The byte order is whatever the native byte order is.
     */
    EXPORT ByteStream& operator<<(const int64_t o);
    /**
     *	push an uint64_t onto the end of the stream. The byte order is whatever the native byte order is.
     */
    EXPORT ByteStream& operator<<(const uint64_t o);
    /**
     *  push a float onto the end of the stream. The byte order is
     *  whatever the native byte order is.
     */
    EXPORT ByteStream& operator<<(const float f);
    /**
     *  push a double onto the end of the stream. The byte order is
     *  whatever the native byte order is.
     */
    EXPORT ByteStream& operator<<(const double d);
    /**
     * push a std::string onto the end of the stream.
     */
    EXPORT ByteStream& operator<<(const std::string& s);
    /**
     * push an arbitrary class onto the end of the stream.
     */
    inline ByteStream& operator<<(const Serializeable& s);
    /**
     * push a ByteStream onto the end of the stream.
     */
    EXPORT ByteStream& operator<<(const ByteStream& bs);
    /**
     * push a UUID onto the end of the stream.
     */
    EXPORT ByteStream& operator<<(const uuid& u);

    /**
     *	extract a int8_t from the front of the stream.
     */
    EXPORT ByteStream& operator>>(int8_t& b);
    /**
     *	extract a uint8_t from the front of the stream.
     */
    EXPORT ByteStream& operator>>(uint8_t& b);
    /**
     *	extract a int16_t from the front of the stream. The byte order is whatever the native byte order is.
     */
    EXPORT ByteStream& operator>>(int16_t& d);
    /**
     *	extract a uint16_t from the front of the stream. The byte order is whatever the native byte order is.
     */
    EXPORT ByteStream& operator>>(uint16_t& d);
    /**
     *	extract a int32_t from the front of the stream. The byte order is whatever the native byte order is.
     */
    EXPORT ByteStream& operator>>(int32_t& q);
    /**
     *	extract a uint32_t from the front of the stream. The byte order is whatever the native byte order is.
     */
    EXPORT ByteStream& operator>>(uint32_t& q);
    /**
     *	extract an int64_t from the front of the stream. The byte order is whatever the native byte order is.
     */
    EXPORT ByteStream& operator>>(int64_t& o);
    /**
     *	extract an uint64_t from the front of the stream. The byte order is whatever the native byte order is.
     */
    EXPORT ByteStream& operator>>(uint64_t& o);
    /**
     *  extract a float from the front of the stream. The byte
     *  order is whatever the native byte order is.
     */
    EXPORT ByteStream& operator>>(float& f);
    /**
     *  extract a double from the front of the stream. The byte
     *  order is whatever the native byte order is.
     */
    EXPORT ByteStream& operator>>(double& d);
    /**
     * extract a std::string from the front of the stream.
     */
    EXPORT ByteStream& operator>>(std::string& s);
    /**
     *	write the current stream into b. The ByteStream will be empty after this operation.
     * @warning the caller is responsible for making sure b is big enough to hold all the data (perhaps by
     * calling length()).
     */
    EXPORT ByteStream& operator>>(uint8_t*& b);
    /**
     * extract an arbitrary object from the front of the stream.
     */
    inline ByteStream& operator>>(Serializeable& s);
    /**
     * extract a ByteStream from the front of the stream.
     */
    EXPORT ByteStream& operator>>(ByteStream& bs);
    /**
     * extract a UUID from the front of the stream.
     */
    EXPORT ByteStream& operator>>(uuid& u);

    /**
     *	Peek at a int8_t from the front of the stream.
     */
    EXPORT void peek(int8_t& b) const;
    /**
     *	Peek at a uint8_t from the front of the stream.
     */
    EXPORT void peek(uint8_t& b) const;
    /**
     *	Peek at a int16_t from the front of the stream. The byte order is whatever the native byte order is.
     */
    EXPORT void peek(int16_t& d) const;
    /**
     *	Peek at a uint16_t from the front of the stream. The byte order is whatever the native byte order is.
     */
    EXPORT void peek(uint16_t& d) const;
    /**
     *	Peek at a int32_t from the front of the stream. The byte order is whatever the native byte order is.
     */
    EXPORT void peek(int32_t& q) const;
    /**
     *	Peek at a uint32_t from the front of the stream. The byte order is whatever the native byte order is.
     */
    EXPORT void peek(uint32_t& q) const;
    /**
     *	Peek at an int64_t from the front of the stream. The byte order is whatever the native byte order is.
     */
    EXPORT void peek(int64_t& o) const;
    /**
     *	Peek at an uint64_t from the front of the stream. The byte order is whatever the native byte order is.
     */
    EXPORT void peek(uint64_t& o) const;
    /**
     *  Peek at a float from the front of the stream. The byte order
     *  is whatever the native byte order is.
     */
    EXPORT void peek(float& f) const;
    /**
     *  Peek at a double from the front of the stream. The byte
     *  order is whatever the native byte order is.
     */
    EXPORT void peek(double& f) const;
    /**
     * Peek at a std::string from the front of the stream.
     */
    EXPORT void peek(std::string& s) const;
    /**
     * Peek at the whole ByteStream buffer.
     * @warning the caller is responsible for making sure b is big enough to hold all the data (perhaps by
     * calling length()).
     */
    inline void peek(uint8_t*& b) const;
    /**
     * Peek at a ByteStream from the front of the stream.
     */
    EXPORT void peek(ByteStream& bs) const;
    /**
     * Peek at a UUID from the front of the stream.
     */
    EXPORT void peek(uuid& u) const;

    /**
     *	load the stream from an array. Clears out any previous data.
     */
    EXPORT void load(const uint8_t* bp, uint32_t len);

    /**
     *	append bytes to the end of the stream.
     */
    EXPORT void append(const uint8_t* bp, uint32_t len);

    /**
     * equality check on buffer contents.
     */
    EXPORT bool operator==(const ByteStream& b) const;

    /**
     * inequality check on buffer contents.
     */
    EXPORT bool operator!=(const ByteStream& b) const;

    /**
     * these fcns are an alternative to code like {build a msg, bs.load or bs.append}.
     * This will let us build the msg directly in the BS buffer.
     */
    EXPORT void needAtLeast(size_t amount);
    inline uint8_t* getInputPtr();
    inline void advanceInputPtr(size_t amount);

    /**
     *	returns a const pointer to the current head of the queue.  If you use it for
     *  raw access, you might want to advance the current head.
     */
    inline const uint8_t* buf() const;

    /**
     *	returns a pointer to the current head of the queue.  If you use it for
     *  raw access, you might want to advance the current head.
     */
    inline uint8_t* buf();

    /**
     *	advance the output ptr without having to extract bytes
     * @warning be careful advancing near 4GB!
     */
    inline void advance(uint32_t amt);

    /**
     *	returns the length of the queue (in bytes)
     * @warning do not attempt to make a ByteStream bigger than 4GB!
     */
    inline uint32_t length() const;
    inline bool empty() const;

    /**
     *	returns the length of the queue, including header overhead (in bytes)
     */
    inline uint32_t lengthWithHdrOverhead() const;

    /**
     *	clears the stream. Releases any current stream and sets all pointers to 0. The state of the object
     * is identical to its state immediately after default construction.
     */
    inline void reset();

    /**
     * operator+=
     */
    inline ByteStream& operator+=(const ByteStream& rhs);

    /**
     * swap this ByteStream with another ByteStream
     */
    EXPORT void swap(ByteStream& rhs);

    /**
     * reset the input & output pointers to the beginning
     */
    inline void restart();

    /**
     * Move the input pointer back to the beginning so the contents
     * can be read again.
     */
    inline void rewind();

    /**
     * Serializeable interface
     */
    EXPORT void serialize(ByteStream& bs) const;

    /**
     * Serializeable interface
     */
    EXPORT void deserialize(ByteStream& bs);

    /**
     *	memory allocation chunk size
     */
    EXPORT static const uint32_t BlockSize = 4096;

    /** size of the space we want in front of the data */
    EXPORT static const uint32_t ISSOverhead = 2 * sizeof(uint32_t); //space for the BS magic & length

    friend class ::ByteStreamTestSuite;

protected:
    /**
     *	pushes one uint8_t onto the end of the stream
     */
    void add(const uint8_t b);
    /**
     *	adds another BlockSize bytes to the internal buffer
     */
    void growBuf(uint32_t toSize = 0);
    /**
     *	handles member copying from one ByteStream to another
     */
    void doCopy(const ByteStream& rhs);

private:

    uint8_t* fBuf; ///the start of the allocated buffer
    uint8_t* fCurInPtr; //the point in fBuf where data is inserted next
    uint8_t* fCurOutPtr; //the point in fBuf where data is extracted from next
    uint32_t fMaxLen; //how big fBuf is currently
};

// type descriptors to let ByteStream point out protocol errors, WIP
static const uint8_t BS_UINT8 = 0;
static const uint8_t BS_UINT16 = 1;
static const uint8_t BS_UINT32 = 2;
static const uint8_t BS_UINT64 = 3;
static const uint8_t BS_INT8 = 4;
static const uint8_t BS_INT16 = 5;
static const uint8_t BS_INT32 = 6;
static const uint8_t BS_INT64 = 7;
static const uint8_t BS_STRING = 8;
static const uint8_t BS_BLOB = 9;
static const uint8_t BS_SERIALIZABLE = 10;
static const uint8_t BS_UUID = 11;

inline ByteStream::ByteStream(const uint8_t* bp, const uint32_t len) : fBuf(0), fMaxLen(0)
{
    load(bp, len);
}
inline ByteStream::~ByteStream()
{
    delete [] fBuf;
}

inline const uint8_t* ByteStream::buf() const
{
    return fCurOutPtr;
}
inline uint8_t* ByteStream::buf()
{
    return fCurOutPtr;
}
inline uint32_t ByteStream::length() const
{
    return (uint32_t)(fCurInPtr - fCurOutPtr);
}
inline bool ByteStream::empty() const
{
    return (length() == 0);
}
inline uint32_t ByteStream::lengthWithHdrOverhead() const
{
    return (length() + ISSOverhead);
}
inline void ByteStream::reset()
{
    delete [] fBuf;
    fMaxLen = 0;
    fCurInPtr = fCurOutPtr = fBuf = 0;
}
inline void ByteStream::restart()
{
    fCurInPtr = fCurOutPtr = fBuf + ISSOverhead;
}
inline void ByteStream::rewind()
{
    fCurOutPtr = fBuf + ISSOverhead;
}
inline void ByteStream::advance(uint32_t adv)
{
    //fCurOutPtr is always >= fBuf, so fCurOutPtr - fBuf is >= 0, and this difference is always <= 32 bits
    //there is an edge condition not detected here: if fCurOutPtr - fBuf is nearly 4GB and you try to
    //advance by a lot, you will wrap over, so be warned!
    if (adv > length())
        throw std::length_error("ByteStream: advanced beyond the end of the buffer");

    fCurOutPtr += adv;
}
inline uint8_t* ByteStream::getInputPtr()
{
    return fCurInPtr;
}
inline void ByteStream::advanceInputPtr(size_t amount)
{
    fCurInPtr += amount;
}
inline void ByteStream::peek(uint8_t*& bpr) const
{
    memcpy(bpr, fCurOutPtr, length());
}

inline ByteStream& ByteStream::operator+=(const ByteStream& rhs)
{
    append(rhs.buf(), rhs.length());
    return *this;
}
inline ByteStream operator+(const ByteStream& lhs, const ByteStream& rhs)
{
    ByteStream temp(lhs);
    return temp += rhs;
}
inline ByteStream& ByteStream::operator>>(Serializeable& s)
{
    s.deserialize(*this);
    return *this;
}
inline ByteStream& ByteStream::operator<<(const Serializeable& s)
{
    s.serialize(*this);
    return *this;
}
inline ByteStream& ByteStream::operator=(const SBS& rhs)
{
    *this = *rhs;
    return *this;
}

/**
 * stream a ByteStream out to any ostream
 */
inline std::ostream& operator<<(std::ostream& os, const ByteStream& bs)
{
    return os.write(reinterpret_cast<const char*>(bs.buf()), bs.length());
}

/**
 * stream a ByteStream in from a file
 */
EXPORT std::ifstream& operator>>(std::ifstream& os, ByteStream& bs);

/// Generic method to export a vector of T's that implement Serializeable
template<typename T>
void serializeVector(ByteStream& bs, const std::vector<T>& v)
{
    typename std::vector<T>::const_iterator it;
    uint64_t size;

    size = v.size();
    bs << size;

    for (it = v.begin(); it != v.end(); it++)
        bs << *it;
}

/// Generic method to deserialize a vector of T's that implement Serializeable
template<typename T>
void deserializeVector(ByteStream& bs, std::vector<T>& v)
{
    uint32_t i;
    T tmp;
    uint64_t size;

    v.clear();
    bs >> size;

    for (i = 0; i < size; i++)
    {
        bs >> tmp;
        v.push_back(tmp);
    }
}

#ifdef _MSC_VER
//Until the API is fixed to be 64-bit clean...
#pragma warning (push)
#pragma warning (disable : 4267)
#endif

template<typename T>
void serializeInlineVector(ByteStream& bs, const std::vector<T>& v)
{
    uint64_t size = v.size();
    bs << size;

    if (size > 0)
        bs.append((const uint8_t*) & (v[0]), sizeof(T) * size);
}

inline void serializeVector(ByteStream& bs, const std::vector<int64_t>& v)
{
    serializeInlineVector<int64_t>(bs, v);
}

template<typename T>
void deserializeInlineVector(ByteStream& bs, std::vector<T>& v)
{
    uint64_t size;
    const uint8_t* buf;

    v.clear();
    bs >> size;

    if (size > 0)
    {
        v.resize(size);
        buf = bs.buf();
        memcpy(&(v[0]), buf, sizeof(T) * size);
        bs.advance(sizeof(T) * size);
    }
}

#ifdef _MSC_VER
#pragma warning (pop)
#endif

inline void deserializeVector(ByteStream& bs, std::vector<int64_t>& v)
{
    deserializeInlineVector<int64_t>(bs, v);
}

/// Generic method to serialize a set of T's that implement Serializeable
template<typename T>
void serializeSet(ByteStream& bs, const std::set<T>& s)
{
    uint64_t size = s.size();
    bs << size;
    typename std::set<T>::const_iterator it;

    for (it = s.begin(); it != s.end(); ++it)
        bs << *it;
}

/// Generic method to deserialize a set of T's that implement Serializeable
template<typename T>
void deserializeSet(ByteStream& bs, std::set<T>& s)
{
    uint32_t i;
    T tmp;
    uint64_t size;

    s.clear();
    bs >> size;

    for (i = 0; i < size; i++)
    {
        bs >> tmp;
        s.insert(tmp);
    }
}

}//namespace messageqcpp

namespace std
{
/** total specialization of std::swap
 *
 */
template<> inline void swap<messageqcpp::ByteStream>(messageqcpp::ByteStream& lhs, messageqcpp::ByteStream& rhs)
{
    lhs.swap(rhs);
}
}//namespace std

#undef EXPORT

#endif //MESSAGEQCPP_BYTESTREAM_H