mariadb-columnstore-engine/dbcon/dmlpackageproc/dmlpackageprocessor.cpp

/* Copyright (C) 2014 InfiniDB, Inc.

   This program is free software; you can redistribute it and/or
   modify it under the terms of the GNU General Public License
   as published by the Free Software Foundation; version 2 of
   the License.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with this program; if not, write to the Free Software
   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
   MA 02110-1301, USA. */

/***********************************************************************
 *   $Id: dmlpackageprocessor.cpp 9673 2013-07-09 15:59:49Z chao $
 *
 *
 ***********************************************************************/
#include "dmlpackageprocessor.h"

#include <math.h>
using namespace std;

#include <boost/algorithm/string/case_conv.hpp>
using namespace boost::algorithm;
#include <boost/tokenizer.hpp>
#include <boost/shared_ptr.hpp>

#include "we_messages.h"
using namespace WriteEngine;
using namespace dmlpackage;
#include "calpontselectexecutionplan.h"
#include "simplecolumn.h"
#include "constantcolumn.h"
#include "simplefilter.h"
#include "constantfilter.h"
#include "columnresult.h"
using namespace execplan;
using namespace logging;
#include "configcpp.h"
using namespace config;
#include "joblistfactory.h"
#include "joblist.h"
#include "distributedenginecomm.h"
using namespace joblist;
#include "bytestream.h"
#include "messagequeue.h"
using namespace messageqcpp;
#include "tablelockdata.h"
#include "exceptclasses.h"

namespace
{
using namespace execplan;

const SOP opeq(new Operator("="));
const SOP opne(new Operator("<>"));
const SOP opor(new Operator("or"));
const SOP opand(new Operator("and"));
}

namespace dmlpackageprocessor
{

DMLPackageProcessor::~DMLPackageProcessor()
{
    //cout << "In DMLPackageProcessor destructor " << this << endl;
    if (fWEClient)
        delete fWEClient;

    if (fExeMgr)
        delete fExeMgr;
}

//@bug 397
void DMLPackageProcessor::cleanString(string& s)
{
    string::size_type pos = s.find_first_not_of(" ");

    //stripe off space and ' or '' at beginning and end
    if ( pos < s.length() )
    {
        s = s.substr( pos, s.length() - pos );

        if ( (pos = s.find_last_of(" ")) < s.length())
        {
            s = s.substr(0, pos );
        }
    }

    if  ( s[0] == '\'')
    {
        s = s.substr(1, s.length() - 2);

        if  ( s[0] == '\'')
            s = s.substr(1, s.length() - 2);
    }
}
#if 0
boost::any DMLPackageProcessor::tokenizeData( execplan::CalpontSystemCatalog::SCN txnID,
        execplan::CalpontSystemCatalog::ColType colType,
        const std::string& data, DMLResult& result, bool isNULL )
{
    SUMMARY_INFO("DMLPackageProcessor::tokenizeData");

    bool retval = true;
    boost::any value;

    if (isNULL)
    {
        WriteEngine::Token nullToken;
        value = nullToken;
    }
    else
    {
        if ( data.length() > (unsigned int)colType.colWidth )
        {
            retval = false;
            // build the logging message
            logging::Message::Args args;
            logging::Message message(6);
            args.add("Insert value is too large for colum ");
            message.format( args );

            result.result = INSERT_ERROR;
            result.message = message;
        }
        else
        {
            //Tokenize the data value
            WriteEngine::DctnryStruct dictStruct;
            dictStruct.dctnryOid = colType.ddn.dictOID;
            //cout << "Dictionary OIDs: " << colType.ddn.treeOID << " " << colType.ddn.listOID << endl;
            WriteEngine::DctnryTuple  dictTuple;
            dictTuple.sigValue = data.c_str();
            dictTuple.sigSize = data.length();
            int error = NO_ERROR;

            if ( NO_ERROR != (error = fWriteEngine.tokenize( txnID, dictStruct, dictTuple)) )
            {
                retval = false;
                //cout << "Error code from WE: " << error << endl;
                // build the logging message
                logging::Message::Args args;
                logging::Message message(1);
                args.add("Tokenization failed on: ");
                args.add(data);
                args.add("error number: ");
                args.add( error );
                message.format( args );

                result.result = TOKEN_ERROR;
                result.message = message;
            }

            WriteEngine::Token aToken = dictTuple.token;
            value = aToken;
        }
    }

    return value;
}
#endif
void DMLPackageProcessor::getColumnsForTable(uint32_t sessionID, std::string schema,
        std::string table, dmlpackage::ColumnList& colList)
{

    CalpontSystemCatalog::TableName tableName;
    tableName.schema = schema;
    tableName.table = table;

    boost::shared_ptr<CalpontSystemCatalog> systemCatalogPtr = CalpontSystemCatalog::makeCalpontSystemCatalog( sessionID );
    CalpontSystemCatalog::RIDList ridList = systemCatalogPtr->columnRIDs(tableName, true);

    CalpontSystemCatalog::RIDList::const_iterator rid_iterator = ridList.begin();

    while (rid_iterator != ridList.end())
    {
        CalpontSystemCatalog::ROPair roPair = *rid_iterator;
        DMLColumn* columnPtr = new DMLColumn();
        CalpontSystemCatalog::TableColName tblColName = systemCatalogPtr->colName( roPair.objnum );
        columnPtr->set_Name(tblColName.column);

        colList.push_back(columnPtr);

        ++rid_iterator;
    }

}

char* DMLPackageProcessor::strlower(char* in)
{
    char* p = in;

    if (p)
    {
        while (*p)
        {
            *p = tolower(*p);
            p++;
        }
    }

    return in;
}

void DMLPackageProcessor::convertRidToColumn(uint64_t& rid, unsigned& dbRoot, unsigned& partition,
        unsigned& segment, unsigned filesPerColumnPartition, unsigned  extentsPerSegmentFile, unsigned extentRows,
        unsigned startDBRoot, unsigned dbrootCnt, const unsigned startPartitionNum)
{
    partition = rid / (filesPerColumnPartition * extentsPerSegmentFile * extentRows);

    segment = (((rid % ( filesPerColumnPartition * extentsPerSegmentFile * extentRows)) /
                extentRows)) % filesPerColumnPartition;

    dbRoot = ((startDBRoot - 1 + segment) % dbrootCnt) + 1;

    //Calculate the relative rid for this segment file
    uint64_t relRidInPartition = rid - ((uint64_t)partition * (uint64_t)filesPerColumnPartition *
                                        (uint64_t)extentsPerSegmentFile * (uint64_t)extentRows);
    idbassert(relRidInPartition <= (uint64_t)filesPerColumnPartition * (uint64_t)extentsPerSegmentFile *
              (uint64_t)extentRows);
    uint32_t numExtentsInThisPart = relRidInPartition / extentRows;
    unsigned numExtentsInThisSegPart = numExtentsInThisPart / filesPerColumnPartition;
    uint64_t relRidInThisExtent = relRidInPartition - numExtentsInThisPart * extentRows;
    rid = relRidInThisExtent +  numExtentsInThisSegPart * extentRows;
}


string DMLPackageProcessor::projectTableErrCodeToMsg(uint32_t ec)
{
    if (ec < 1000) // pre IDB error code
    {
        ErrorCodes ecObj;
        string errMsg("Statement failed.");
        errMsg += ecObj.errorString(ec).substr(150); // substr removes ErrorCodes::fPreamble
        return errMsg;
    }

    // IDB error
    return IDBErrorInfo::instance()->errorMsg(ec);
}

bool DMLPackageProcessor::validateVarbinaryVal( std::string& inStr)
{
    bool invalid = false;

    for (unsigned i = 0; i < inStr.length(); i++)
    {
        if (!isxdigit(inStr[i]))
        {
            invalid = true;
            break;
        }

    }

    return invalid;
}

int DMLPackageProcessor::commitTransaction(uint64_t uniqueId, BRM::TxnID txnID)
{
    int rc = fDbrm->vbCommit(txnID.id);
    return rc;
}

int DMLPackageProcessor::rollBackTransaction(uint64_t uniqueId, BRM::TxnID txnID, uint32_t sessionID,
        std::string& errorMsg)
{
    std::vector<BRM::LBID_t> lbidList;
    std::vector<BRM::LBIDRange> lbidRangeList;
    BRM::LBIDRange   range;
    int rc = 0;
    //Check BRM status before processing.
    rc = fDbrm->isReadWrite();

    if (rc != 0 )
    {
        std::string brmMsg;
        errorMsg = "Can't read DBRM isReadWrite [ ";
        BRM::errString(rc, brmMsg);
        errorMsg += brmMsg;
        errorMsg += "]";
        return rc;
    }

    ByteStream bytestream;
    fWEClient->addQueue(uniqueId);
    //cout << "adding to queue with id " << uniqueId << endl;
    bytestream << (ByteStream::byte) WE_SVR_ROLLBACK_BLOCKS;
    bytestream << uniqueId;
    bytestream << sessionID;
    bytestream << (uint32_t)txnID.id;
    uint32_t msgRecived = 0;

    try
    {
        fWEClient->write_to_all(bytestream);
        boost::shared_ptr<messageqcpp::ByteStream> bsIn;
        bsIn.reset(new ByteStream());
        ByteStream::byte tmp8;

        while (1)
        {
            if (msgRecived == fWEClient->getPmCount())
                break;

            fWEClient->read(uniqueId, bsIn);

            if ( bsIn->length() == 0 ) //read error
            {
                rc = NETWORK_ERROR;
                errorMsg = "Network error reading WEClient";
                fWEClient->removeQueue(uniqueId);
                //cout << "erroring out remove queue id " << uniqueId << endl;
                break;
            }
            else
            {
                *bsIn >> tmp8;
                rc = tmp8;

                if (rc != 0)
                {
                    char szrc[20];
                    *bsIn >> errorMsg;
                    errorMsg += " (WriteEngine returns error ";
                    sprintf(szrc, "%d", rc);
                    errorMsg += szrc;
                    errorMsg += ")";
                    fWEClient->removeQueue(uniqueId);
                    cout << "erroring out remove queue id " << uniqueId << endl;
                    break;
                }
                else
                    msgRecived++;
            }
        }
    }
    catch (std::exception& e)
    {
        rc = NETWORK_ERROR;
        errorMsg = "Network error occured when rolling back blocks";
        errorMsg += e.what();
        fWEClient->removeQueue(uniqueId);
        cout << "erroring out remove queue id " << uniqueId << endl;
        //delete fWEClient;
        return rc;
    }
    catch ( ... )
    {
        rc = NETWORK_ERROR;
        errorMsg = "Unknown exception caught while rolling back transaction.";
        fWEClient->removeQueue(uniqueId);
        cout << "erroring out remove queue id " << uniqueId << endl;
        //delete fWEClient;
        return rc;
    }

    if (rc != 0)
    {
        //delete fWEClient;
        return rc;
    }

    fWEClient->removeQueue(uniqueId);
    //delete fWEClient;
//	cout << "success. remove queue id " << uniqueId << endl;
    rc = fDbrm->getUncommittedLBIDs(txnID.id, lbidList);

    if (rc != 0 )
    {
        std::string brmMsg;
        errorMsg = "DBRM getUncommittedLBIDs [ ";
        BRM::errString(rc, brmMsg);
        errorMsg += brmMsg;
        errorMsg += "]";
        return rc;
    }

    for (size_t i = 0; i < lbidList.size(); i++)
    {
        range.start = lbidList[i];
        range.size = 1;
        lbidRangeList.push_back(range);
    }

    rc =  fDbrm->vbRollback(txnID.id, lbidRangeList);

    if (rc != 0 )
    {
        std::string brmMsg;
        errorMsg = "DBRM vbRollback [ ";
        BRM::errString(rc, brmMsg);
        errorMsg += brmMsg;
        errorMsg += "]";
        return rc;
    }

    return rc;
}

int DMLPackageProcessor::commitBatchAutoOnTransaction(uint64_t uniqueId, BRM::TxnID txnID, const uint32_t tableOid, std::string& errorMsg)
{
    //collect hwm info from all pms and set them here. remove table metadata if all successful
    ByteStream bytestream;
    fWEClient->addQueue(uniqueId);
    bytestream << (ByteStream::byte)WE_SVR_COMMIT_BATCH_AUTO_ON;
    bytestream << uniqueId;
    bytestream << (uint32_t) txnID.id;
    bytestream << tableOid;
    bytestream << fSessionID;

    uint32_t msgRecived = 0;
    fWEClient->write_to_all(bytestream);
    boost::shared_ptr<messageqcpp::ByteStream> bsIn;
    bsIn.reset(new ByteStream());
    int rc = 0;
    ByteStream::byte tmp8;
    typedef std::vector<BRM::BulkSetHWMArg> BulkSetHWMArgs;
    std::vector<BulkSetHWMArgs> hwmArgsAllPms;

    while (1)
    {
        if (msgRecived == fWEClient->getPmCount())
            break;

        fWEClient->read(uniqueId, bsIn);

        if ( bsIn->length() == 0 ) //read error
        {
            rc = NETWORK_ERROR;
            fWEClient->removeQueue(uniqueId);
            break;
        }
        else
        {
            *bsIn >> tmp8;
            rc = tmp8;

            if (rc != 0)
            {
                *bsIn >> errorMsg;
                fWEClient->removeQueue(uniqueId);
                break;
            }
            else
            {
                //get hwm info
                *bsIn >> errorMsg;
                BulkSetHWMArgs setHWMArgs;
                //cout << "received from WES bytestream length = " <<  bsIn->length() << endl;
                deserializeInlineVector(*(bsIn.get()), setHWMArgs);
                //cout << "get hwm info from WES size " << setHWMArgs.size() << endl;
                hwmArgsAllPms.push_back(setHWMArgs);
                msgRecived++;
            }
        }
    }

    if (rc != 0)
        return rc;

    //set hwm
    std::vector<BRM::BulkSetHWMArg> allHwm;
    BulkSetHWMArgs::const_iterator itor;

    //cout << "total hwmArgsAllPms size " << hwmArgsAllPms.size() << endl;
    for (unsigned i = 0; i  < fWEClient->getPmCount(); i++)
    {
        itor = hwmArgsAllPms[i].begin();

        while (itor != hwmArgsAllPms[i].end())
        {
            allHwm.push_back(*itor);
            //cout << "received hwm info: " <<  itor->oid << ":" << itor->hwm << endl;
            itor++;
        }
    }

    //set CP data before hwm.

    //cout << "setting hwm allHwm size " << allHwm.size() << endl;
    vector<BRM::LBID_t> lbidList;

    if (idbdatafile::IDBPolicy::useHdfs())
    {
        BRM::LBID_t startLbid;

        for ( unsigned i = 0; i < allHwm.size(); i++)
        {
            rc = fDbrm->lookupLocalStartLbid(allHwm[i].oid, allHwm[i].partNum, allHwm[i].segNum, allHwm[i].hwm, startLbid);
            lbidList.push_back(startLbid);
        }
    }
    else
        fDbrm->getUncommittedExtentLBIDs(static_cast<BRM::VER_t>(txnID.id), lbidList);

    vector<BRM::LBID_t>::const_iterator iter = lbidList.begin();
    vector<BRM::LBID_t>::const_iterator end = lbidList.end();
    BRM::CPInfoList_t cpInfos;
    BRM::CPInfo aInfo;

    while (iter != end)
    {
        aInfo.firstLbid = *iter;
        aInfo.max = numeric_limits<int64_t>::min(); // Not used
        aInfo.min = numeric_limits<int64_t>::max(); // Not used
        aInfo.seqNum = -1;
        cpInfos.push_back(aInfo);
        ++iter;
    }

    std::vector<BRM::CPInfoMerge>  mergeCPDataArgs;
    rc = fDbrm->bulkSetHWMAndCP(allHwm, cpInfos, mergeCPDataArgs, txnID.id);
    fDbrm->takeSnapshot();
    //Set tablelock to rollforward remove meta files

    if (rc != 0)
        return rc;

    bool stateChanged = true;
    TablelockData* tablelockData = TablelockData::makeTablelockData(fSessionID);
    uint64_t tablelockId = tablelockData->getTablelockId(tableOid);

    try
    {
        stateChanged = fDbrm->changeState(tablelockId, BRM::CLEANUP);
    }
    catch (std::exception&)
    {
        errorMsg = IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE);
        stateChanged = false;
    }

    if (!stateChanged)
        return rc;

    bytestream.restart();
    //@Bug 4517 Remove meta data failure doesn't stop tablelock releasing.
    bytestream << (ByteStream::byte)WE_SVR_BATCH_AUTOON_REMOVE_META;
    bytestream << uniqueId;
    bytestream << tableOid;
    msgRecived = 0;
    fWEClient->write_to_all(bytestream);

    while (1)
    {
        if (msgRecived == fWEClient->getPmCount())
            break;

        fWEClient->read(uniqueId, bsIn);

        if ( bsIn->length() == 0 ) //read error
        {
            fWEClient->removeQueue(uniqueId);
            break;
        }
        else
        {
            *bsIn >> tmp8;
            msgRecived++;
        }
    }

    return rc;


}

int DMLPackageProcessor::rollBackBatchAutoOnTransaction(uint64_t uniqueId, BRM::TxnID txnID, uint32_t sessionID,
        const uint32_t tableOid, std::string& errorMsg)
{
    //Bulkrollback, rollback blocks, vbrollback, change state, remove meta file
    //cout << "In rollBackBatchAutoOnTransaction" << endl;
    std::vector<BRM::TableLockInfo> tableLocks;
    tableLocks = fDbrm->getAllTableLocks();
    //cout << " Got all tablelocks" << endl;
    unsigned idx = 0;
    string ownerName ("DMLProc batchinsert");
    uint64_t tableLockId = 0;
    int rc = 0;

    for (; idx < tableLocks.size(); idx++)
    {
        if ((tableLocks[idx].ownerName == ownerName) && (tableLocks[idx].tableOID == tableOid))
        {
            tableLockId = tableLocks[idx].id;
            break;
        }
    }

    if ((tableLockId == 0) || (tableOid == 0))
    {
        // table is not locked by DMLProc. Could happen if we failed to get lock
        // while inserting. Not an error during rollback, but we don't
        // want to do anything.
        return rc;
    }

    //cout << "sending to WES" << endl;
    ByteStream bytestream;
    fWEClient->addQueue(uniqueId);
    //cout << "adding queue id " << uniqueId << endl;
    bytestream << (ByteStream::byte) WE_SVR_ROLLBACK_BATCH_AUTO_ON;
    bytestream << uniqueId;
    bytestream << sessionID;
    bytestream << tableLockId;
    bytestream << tableOid;
    uint32_t msgRecived = 0;
    fWEClient->write_to_all(bytestream);
    boost::shared_ptr<messageqcpp::ByteStream> bsIn;
    bsIn.reset(new ByteStream());
    ByteStream::byte tmp8;

    //cout << "waiting for reply from WES" << endl;
    while (1)
    {
        if (msgRecived == fWEClient->getPmCount())
            break;

        fWEClient->read(uniqueId, bsIn);

        if ( bsIn->length() == 0 ) //read error
        {
            rc = NETWORK_ERROR;
            fWEClient->removeQueue(uniqueId);
            //cout << "erroring out remove queue id " << uniqueId << endl;
            break;
        }
        else
        {
            *bsIn >> tmp8;
            rc = tmp8;

            if (rc != 0)
            {
                *bsIn >> errorMsg;
                fWEClient->removeQueue(uniqueId);
                //cout << "erroring out remove queue id " << uniqueId << endl;
                break;
            }
            else
                msgRecived++;
        }
    }

    if (rc == 0) //change table lock state
    {
        bool stateChanged = true;

        //cout << "changing tablelock state" << endl;
        try
        {
            stateChanged = fDbrm->changeState(tableLockId, BRM::CLEANUP);
        }
        catch (std::exception&)
        {
            errorMsg = IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE);
            stateChanged = false;
        }

        if (!stateChanged)
        {
            rc = 1;
        }
    }

    if ( rc != 0 )
        return rc;

    bytestream.restart();
    bytestream << (ByteStream::byte)WE_SVR_BATCH_AUTOON_REMOVE_META;
    bytestream << uniqueId;
    bytestream << tableOid;
    msgRecived = 0;
    fWEClient->write_to_all(bytestream);

    while (1)
    {
        if (msgRecived == fWEClient->getPmCount())
            break;

        fWEClient->read(uniqueId, bsIn);

        if ( bsIn->length() == 0 ) //read error
        {
            fWEClient->removeQueue(uniqueId);
            //cout << "erroring out remove queue id " << uniqueId << endl;
            break;
        }
        else
        {
            *bsIn >> tmp8;
            msgRecived++;
        }
    }

    fWEClient->removeQueue(uniqueId);
    return rc;
}

int DMLPackageProcessor::commitBatchAutoOffTransaction(uint64_t uniqueId, BRM::TxnID txnID, const uint32_t tableOid, std::string& errorMsg)
{
    std::vector<BRM::TableLockInfo> tableLocks;
    tableLocks = fDbrm->getAllTableLocks();
    //cout << " Got all tablelocks" << endl;
    unsigned idx = 0;
    string ownerName ("DMLProc batchinsert");
    uint64_t tableLockId = 0;
    int rc = 0;
    boost::shared_ptr<messageqcpp::ByteStream> bsIn;
    bsIn.reset(new ByteStream());
    ByteStream::byte tmp8;

    for (; idx < tableLocks.size(); idx++)
    {
        if ((tableLocks[idx].ownerName == ownerName) && (tableLocks[idx].tableOID == tableOid))
        {
            tableLockId = tableLocks[idx].id;
            break;
        }
    }

    if ((tableLockId == 0) || (tableOid == 0))
    {
        // table is not locked by DMLProc. Could happen if we failed to get lock
        // while inserting. Not an error during rollback, but we don't
        // want to do anything.
        return rc;
    }

    bool stateChanged = true;

    //cout << "changing tablelock state" << endl;
    try
    {
        stateChanged = fDbrm->changeState(tableLockId, BRM::CLEANUP);
    }
    catch (std::exception&)
    {
        errorMsg = IDBErrorInfo::instance()->errorMsg(ERR_HARD_FAILURE);
        stateChanged = false;
    }

    if (!stateChanged)
    {
        rc = 1;
    }

    if ( rc != 0 )
        return rc;

    ByteStream bytestream;
    fWEClient->addQueue(uniqueId);
    bytestream << (ByteStream::byte)WE_SVR_BATCH_AUTOON_REMOVE_META;
    bytestream << uniqueId;
    bytestream << tableOid;
    uint32_t msgRecived = 0;
    fWEClient->write_to_all(bytestream);

    while (1)
    {
        if (msgRecived == fWEClient->getPmCount())
            break;

        fWEClient->read(uniqueId, bsIn);

        if ( bsIn->length() == 0 ) //read error
        {
            fWEClient->removeQueue(uniqueId);
            break;
        }
        else
        {
            *bsIn >> tmp8;
            msgRecived++;
        }
    }

    fWEClient->removeQueue(uniqueId);
    return rc;
}

int DMLPackageProcessor::rollBackBatchAutoOffTransaction(uint64_t uniqueId, BRM::TxnID txnID, uint32_t sessionID,
        const uint32_t tableOid, std::string& errorMsg)
{
    ByteStream bytestream;
    fWEClient->addQueue(uniqueId);
    bytestream << (ByteStream::byte) WE_SVR_ROLLBACK_BATCH_AUTO_OFF;
    bytestream << uniqueId;
    bytestream << sessionID;
    bytestream << (uint32_t)txnID.id;
    bytestream << tableOid;
    uint32_t msgRecived = 0;
    fWEClient->write_to_all(bytestream);
    boost::shared_ptr<messageqcpp::ByteStream> bsIn;
    bsIn.reset(new ByteStream());
    int rc = 0;
    ByteStream::byte tmp8;

    while (1)
    {
        if (msgRecived == fWEClient->getPmCount())
            break;

        fWEClient->read(uniqueId, bsIn);

        if ( bsIn->length() == 0 ) //read error
        {
            rc = NETWORK_ERROR;
            fWEClient->removeQueue(uniqueId);
            break;
        }
        else
        {
            *bsIn >> tmp8;
            rc = tmp8;

            if (rc != 0)
            {
                *bsIn >> errorMsg;
                fWEClient->removeQueue(uniqueId);
                break;
            }
            else
                msgRecived++;
        }
    }

    return rc;
}

int DMLPackageProcessor::flushDataFiles (int rcIn, std::map<FID, FID>& columnOids, uint64_t uniqueId, BRM::TxnID txnID, uint32_t tableOid)
{
//cout <<"in flushDataFiles" << endl;
    ByteStream bytestream;
    bytestream << (ByteStream::byte) WE_SVR_FLUSH_FILES;
    bytestream << uniqueId;
    bytestream << (uint32_t) rcIn;
    bytestream << (uint32_t)txnID.id;
    bytestream << tableOid;
    uint32_t msgRecived = 0;
    fWEClient->write_to_all(bytestream);
    boost::shared_ptr<messageqcpp::ByteStream> bsIn;
    bsIn.reset(new ByteStream());
    int rc = 0;
    ByteStream::byte tmp8;
    std::string errorMsg;

    try
    {
        while (1)
        {
            if (msgRecived == fWEClient->getPmCount())
                break;

            fWEClient->read(uniqueId, bsIn);

            if ( bsIn->length() == 0 ) //read error
            {
                rc = NETWORK_ERROR;
                break;
            }
            else
            {
                *bsIn >> tmp8;
                rc = tmp8;

                if (rc != 0)
                {
                    *bsIn >> errorMsg;
                    break;
                }
                else
                    msgRecived++;
            }
        }
    }
    catch (std::exception&)
    {
    }

    return rc;
}

int DMLPackageProcessor::endTransaction (uint64_t uniqueId, BRM::TxnID txnID, bool success)
{
//cout <<"in flushDataFiles" << endl;
    ByteStream bytestream;
    bytestream << (ByteStream::byte) WE_END_TRANSACTION;
    bytestream << uniqueId;
    bytestream << (uint32_t)txnID.id;
    bytestream << (ByteStream::byte)success;
    uint32_t msgRecived = 0;
    fWEClient->write_to_all(bytestream);
    boost::shared_ptr<messageqcpp::ByteStream> bsIn;
    bsIn.reset(new ByteStream());
    int rc = 0;
    ByteStream::byte tmp8;
    std::string errorMsg;

    try
    {
        while (1)
        {
            if (msgRecived == fWEClient->getPmCount())
                break;

            fWEClient->read(uniqueId, bsIn);

            if ( bsIn->length() == 0 ) //read error
            {
                rc = NETWORK_ERROR;
                break;
            }
            else
            {
                *bsIn >> tmp8;
                rc = tmp8;

                if (rc != 0)
                {
                    *bsIn >> errorMsg;
                    break;
                }
                else
                    msgRecived++;
            }
        }
    }
    catch (std::exception&)
    {
    }

    return rc;
}
}
// vim:ts=4 sw=4: