mariadb-columnstore-engine/primitives/linux-port/primitiveprocessor.h

/* Copyright (C) 2014 InfiniDB, Inc.

   This program is free software; you can redistribute it and/or
   modify it under the terms of the GNU General Public License
   as published by the Free Software Foundation; version 2 of
   the License.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with this program; if not, write to the Free Software
   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
   MA 02110-1301, USA. */

/******************************************************************************
 * $Id: primitiveprocessor.h 2035 2013-01-21 14:12:19Z rdempsey $
 *
 *****************************************************************************/

/** @file */

#ifndef PRIMITIVEPROCESSOR_H_
#define PRIMITIVEPROCESSOR_H_

#include <stdexcept>
#include <vector>
#ifndef _MSC_VER
#include <tr1/unordered_set>
#else
#include <unordered_set>
#endif

#ifdef __linux__
#define POSIX_REGEX
#endif

#ifdef POSIX_REGEX
#include <regex.h>
#else
#include <boost/regex.hpp>
#endif
#include <cstddef>
#include <boost/shared_ptr.hpp>
#include <boost/shared_array.hpp>

#include "primitivemsg.h"
#include "calpontsystemcatalog.h"
#include "stats.h"
#include "primproc.h"
#include "hasher.h"

class PrimTest;

namespace primitives
{

enum ColumnFilterMode
{
    STANDARD,
    TWO_ARRAYS,
    UNORDERED_SET
};

class pcfHasher
{
public:
    inline size_t operator()(const int64_t i) const
    {
        return i;
    }
};

class pcfEqual
{
public:
    inline size_t operator()(const int64_t f1, const int64_t f2) const
    {
        return f1 == f2;
    }
};

typedef std::tr1::unordered_set<int64_t, pcfHasher, pcfEqual> prestored_set_t;
typedef std::tr1::unordered_set<std::string, utils::Hasher> DictEqualityFilter;

struct idb_regex_t
{
#ifdef POSIX_REGEX
    regex_t regex;
#else
    boost::regex regex;
#endif
    bool used;
    idb_regex_t() : used(false) { }
    ~idb_regex_t()
    {
#ifdef POSIX_REGEX

        if (used)
            regfree(&regex);

#endif
    }
};

struct ParsedColumnFilter
{
    ColumnFilterMode columnFilterMode;
    boost::shared_array<int64_t> prestored_argVals;
    boost::shared_array<uint8_t> prestored_cops;
    boost::shared_array<uint8_t> prestored_rfs;
    boost::shared_ptr<prestored_set_t> prestored_set;
    boost::shared_array<idb_regex_t> prestored_regex;
    uint8_t  likeOps;

    ParsedColumnFilter();
    ~ParsedColumnFilter();
};

//@bug 1828 These need to be public so that column operations can use it for 'like'
struct p_DataValue
{
    int len;
    const uint8_t* data;
};

boost::shared_ptr<ParsedColumnFilter> parseColumnFilter(const uint8_t* filterString,
        uint32_t colWidth, uint32_t colType, uint32_t filterCount, uint32_t BOP);

/** @brief This class encapsulates the primitive processing functionality of the system.
 *
 *  This class encapsulates the primitive processing functionality of the system.
 */
class PrimitiveProcessor
{
public:
    PrimitiveProcessor(int debugLevel = 0);
    virtual ~PrimitiveProcessor();

    /** @brief Sets the block to operate on
     *
     * The primitive processing functions operate on one block at a time.  The caller
     * sets which block to operate on next with this function.
     */
    void setBlockPtr(int* data)
    {
        block = data;
    }
    void setPMStatsPtr(dbbc::Stats* p)
    {
        fStatsPtr = p;
    }


    /** @brief The interface to Mark's NIOS primitive processing code.
     *
     * The interface to Mark's NIOS primitive processing code.  Instead of reading
     * and writing to a bus, it will read/write to buffers specified by inBuf
     * and outBuf.  The primitives implemented this way are:
     * - p_Col and p_ColAggregate
     * - p_GetSignature
     *
     * @param inBuf (in) The buffer containing a command to execute
     * @param inLength (in) The size of inBuf in 4-byte words
     * @param outBuf (in) The buffer to store the output in
     * @param outLength (in) The size of outBuf in 4-byte words
     * @param written (out) The number of bytes written to outBuf.
     * @note Throws logic_error if the output buffer is too small for the result.
     */
    void processBuffer(int* inBuf, unsigned inLength, int* outBuf, unsigned outLength,
                       unsigned* written);

    /* Patrick */

    /** @brief The p_TokenByScan primitive processor
     *
     * The p_TokenByScan primitive processor.  It relies on the caller setting
     * the block to operate on with setBlockPtr().  It assumes the continuation
     * pointer is not used.
     * @param t (in) The arguments to the primitive
     * @param out (out) This must point to memory of some currently unknown max size
     * @param outSize (in) The size of the output buffer in bytes.
     * @note Throws logic_error if the output buffer is too small for the result.
     */
    void p_TokenByScan(const TokenByScanRequestHeader* t,
                       TokenByScanResultHeader* out, unsigned outSize, bool utf8,
                       boost::shared_ptr<DictEqualityFilter> eqFilter);

    /** @brief The p_IdxWalk primitive processor
     *
     * The p_IdxWalk primitive processor.  The caller must set the block to operate
     * on with setBlockPtr().  This primitive can return intermediate results.
     * All results returned will have an different LBID than the input.  They can
     * also be in varying states of completion.  A result is final when
     * Shift >= SSlen, otherwise it is intermediate and needs to be reissued with
     * the specified LBID loaded.
     * @note If in->NVALS > 2, new vectors may be returned in the result set, which
     * will have to be deleted by the caller.  The test to use right now is
     * ({element}->NVALS > 2 && {element}->State == 0).  If that condition is true,
     * delete the vector, otherwise don't.  This kludginess is for efficiency's sake
     * and may go away for the sake of sanity later.
     * @note It is safe to delete any vector passed in after the call.
     * @param out The caller should pass in an empty vector.  The results
     * will be returned as elements of this vector.
     */
    void p_IdxWalk(const IndexWalkHeader* in, std::vector<IndexWalkHeader*>* out) throw();

    /** @brief The p_IdxList primitive processor.
     *
     * The p_IdxList primitive processor.  The caller must set the block to operate
     * on with setBlockPtr().  This primitive can return one intermediate result
     * for every call made.  If there is an intermediate result returned, it will
     * be the first element, distinguished by its type field.  If the
     * first element has a type == RID (3) , there is no intermediate result.  If
     * the first element had a type == LLP_SUBBLK (4) or type == LLP_BLK (5),
     * that element is the intermediate result.  Its value field will be a pointer
     * to the next section of the list.
     *
     * @param rqst (in) The request header followed by NVALS IndexWalkParams
     * @param rslt (out) The caller passes in a buffer which will be filled
     * by the primitive on return.  It will consist of an IndexListHeader,
     * followed by NVALS IndexListEntrys.
     * @param mode (optional, in) 0 specifies old behavior (the last entry of a block might
     * be a pointer).  1 specifies new behavior (the last entry should be ignored).
     */
    void p_IdxList(const IndexListHeader* rqst, IndexListHeader* rslt, int mode = 1);

    /** @brief The p_AggregateSignature primitive processor.
     *
     * The p_AggregateSignature primitive processor.  It operates on a dictionary
     * block and assumes the continuation pointer is not used.
     * @param in The input parameters
     * @param out A pointer to a buffer where the result will be written.
     * @param outSize The size of the output buffer in bytes.
     * @param written (out parameter) A pointer to 1 int, which will contain the
     * number of bytes written to out.
     */
    void p_AggregateSignature(const AggregateSignatureRequestHeader* in,
                              AggregateSignatureResultHeader* out, unsigned outSize, unsigned* written, bool utf8);

    /** @brief The p_Col primitive processor.
     *
     * The p_Col primitive processor.  It operates on a column block specified using setBlockPtr().
     * @param in The buffer containing the command parameters.
     * 		The buffer should begin with a NewColRequestHeader structure, followed by
     * 		an array of 'NOPS' defining the filter to apply (optional),
     * 		followed by an array of RIDs to apply the filter to (optional).
     * @param out The buffer that will contain the results.  On return, it will start with
     * a NewColResultHeader, followed by the output type specified by in->OutputType.
     * \li If OT_RID, it will be an array of RIDs
     * \li If OT_DATAVALUE, it will be an array of matching data values stored in the column
     * \li If OT_BOTH, it will be an array of <DataValue, RID> pairs
     * @param outSize The size of the output buffer in bytes.
     * @param written (out parameter) A pointer to 1 int, which will contain the
     * number of bytes written to out.
     * @note See PrimitiveMsg.h for the type definitions.
     */
    void p_Col(NewColRequestHeader* in, NewColResultHeader* out, unsigned outSize,
               unsigned* written);

    boost::shared_ptr<ParsedColumnFilter> parseColumnFilter(const uint8_t* filterString,
            uint32_t colWidth, uint32_t colType, uint32_t filterCount, uint32_t BOP);
    void setParsedColumnFilter(boost::shared_ptr<ParsedColumnFilter>);

    /** @brief The p_ColAggregate primitive processor.
     *
     * The p_ColAggregate primitive processor.  It operates on a column block
     * specified using setBlockPtr().
     * @param in The buffer containing the command parameters.  The buffer should begin
     *		with a NewColAggRequestHeader, followed by an array of RIDs to generate
     * 		the data for (optional).
     * @param out The buffer to put the result in.  On return, it will contain a
     * NewCollAggResultHeader.
     * @note See PrimitiveMsg.h for the type definitions.
     */
//	void p_ColAggregate(const NewColAggRequestHeader *in, NewColAggResultHeader *out);

    void p_Dictionary(const DictInput* in, std::vector<uint8_t>* out, bool utf8,
                      bool skipNulls, boost::shared_ptr<DictEqualityFilter> eqFilter,
                      uint8_t eqOp);

    inline void setLogicalBlockMode(bool b)
    {
        logicalBlockMode = b;
    }



    static int convertToRegexp(idb_regex_t* regex, const p_DataValue* str);
    inline static bool isEscapedChar(char c);
    boost::shared_array<idb_regex_t> makeLikeFilter(const DictFilterElement* inputMsg, uint32_t count);
    void setLikeFilter(boost::shared_array<idb_regex_t> filter)
    {
        parsedLikeFilter = filter;
    }

private:
    PrimitiveProcessor(const PrimitiveProcessor& rhs);
    PrimitiveProcessor& operator=(const PrimitiveProcessor& rhs);

    int* block;

    bool compare(int cmpResult, uint8_t COP, int len1, int len2) throw();
    int compare(int val1, int val2, uint8_t COP, bool lastStage) throw();
    void indexWalk_1(const IndexWalkHeader* in, std::vector<IndexWalkHeader*>* out) throw();
    void indexWalk_2(const IndexWalkHeader* in, std::vector<IndexWalkHeader*>* out) throw();
    void indexWalk_many(const IndexWalkHeader* in, std::vector<IndexWalkHeader*>* out) throw();
    void grabSubTree(const IndexWalkHeader* in, std::vector<IndexWalkHeader*>* out) throw();

    void nextSig(int NVALS, const PrimToken* tokens, p_DataValue* ret,
                 uint8_t outputFlags = 0, bool oldGetSigBehavior = false, bool skipNulls = false) throw();
    bool isLike(const p_DataValue* dict, const idb_regex_t* arg) throw();

//	void do_sum8(NewColAggResultHeader *out, int64_t val);
//    void do_unsignedsum8(NewColAggResultHeader *out, int64_t val);

    uint64_t masks[11];
    int dict_OffsetIndex, currentOffsetIndex;		// used by p_dictionary
    int fDebugLevel;
    dbbc::Stats* fStatsPtr; // pointer for pmstats
    bool logicalBlockMode;

    boost::shared_ptr<ParsedColumnFilter> parsedColumnFilter;
    boost::shared_array<idb_regex_t> parsedLikeFilter;

    friend class ::PrimTest;
};

} //namespace primitives

#endif
// vim:ts=4 sw=4: