mariadb-columnstore-engine/utils/testbc/testbc3.cpp

/* Copyright (C) 2014 InfiniDB, Inc.

   This program is free software; you can redistribute it and/or
   modify it under the terms of the GNU General Public License
   as published by the Free Software Foundation; version 2 of
   the License.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with this program; if not, write to the Free Software
   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
   MA 02110-1301, USA. */

//
// C++ Implementation: bcTest
//
// Description:  A simple Test driver for the Disk Block Buffer Cache
//
// Author: Jason Rodriguez <jrodriguez@calpont.com>, (C) 2007
//
//

#include <vector>
#include <string>
#include <iomanip>
#include <iostream>
#include <sys/time.h>
#include <unistd.h>
#include <boost/thread/thread.hpp>

#include "blockrequestprocessor.h"
#include "blockcacheclient.h"
#include "stats.h"
#include "brm.h"
#include "logger.h"
#include "iomanager.h"

using namespace BRM;
using namespace dbbc;
using namespace std;
using namespace logging;
using namespace primitiveprocessor;

Stats* gPMStatsPtr = NULL;
bool gPMProfOn = false;
uint32_t gSession = 0;
uint32_t lastRangeListIdx = 0;
const uint32_t maxLoadBlocks(1024 * 1024);

void timespec_sub(const struct timespec& tv1,
                  const struct timespec& tv2,
                  double& tm)
{
    tm = (double)(tv2.tv_sec - tv1.tv_sec) + 1.e-9 * (tv2.tv_nsec - tv1.tv_nsec);
}

namespace primitiveprocessor
{
Logger ml;
}


class BCTest
{

public:

    struct OidRanges
    {
        OID_t oid;
        HWM_t hwm;
        LBIDRange_v ranges;
        OidRanges(const OID_t o, const HWM_t h, const LBIDRange_v r)
        {
            oid = o;
            hwm = h;
            ranges = r;
        }
    }; //struct OidRanges

    BCTest(const int cacheSz = 64 * 1024, int readThr = 2, int readAhead = 1024);

    typedef OidRanges OidRanges_t;
    typedef vector<OidRanges_t>OidRangesList_t;
    OidRangesList_t OidRangesList;

    DBRM dbrm;
    uint32_t extentSize;
    BRM::OID_t maxOid;

    int fCacheSz;
    int fReadThr;
    int fReadAhead;
    uint32_t maxBlocksAvailable;
    uint32_t fExtentSize;

    void setUp();
    int LoadOid(const OidRanges_t& o, uint32_t& loadCount);
    void LoadLbid(const BRM::LBID_t lbid, const BRM::VER_t ver);
    int ReadOidRanges(const OidRanges_t& v, uint32_t* hits, uint32_t* miss);
    void ReadOidLbids(const BRM::LBID_t lbid, const BRM::VER_t ver);

    BlockRequestProcessor BRP;

}; // class BCTest


BCTest::BCTest(int cacheSz, int readThr, int readAhead) :
    fCacheSz(cacheSz),
    fReadThr(readThr),
    fReadAhead(readAhead),
    BRP(fCacheSz, fReadThr, fReadAhead)
{
    setUp();
} // ctor

//
void BCTest::setUp()
{
    LBIDRange_v r;
    HWM_t hwm;
    OID_t oid = 1000;
    extentSize = dbrm.getExtentSize();
    maxBlocksAvailable = 0;
    int i = 0;
    fExtentSize = dbrm.getExtentSize();

    while ( oid < 5000 )
    {
        int ret = 0;
        ret = dbrm.lookup(oid, r);

        if (ret == 0 && r.size() > 0)
        {
            dbrm.getHWM(oid, hwm);
            maxBlocksAvailable += (r.size() * extentSize);
            OidRanges_t oid_range(oid, hwm, r);
            OidRangesList.push_back(oid_range);
            //cout << "Setup i: " << i++ << " o: " << oid
            //	<< " r: " << ret << " s: " << r.size()
            //	<< " m: " << maxBlocksAvailable
            //	<< endl;
            hwm = 0;
            r.clear();
        }

        oid++;
    }

    //cout << "\t" << OidRangesList.size() << " oid ranges loaded " << endl << endl;
    i = 0;
} // setUp()

int BCTest::LoadOid(const OidRanges_t& o, uint32_t& loadCount)
{
    blockCacheClient bc(BRP);
    uint32_t rCount = 0;

    for (uint32_t i = 0; i < o.ranges.size() ; i++)
    {
        const InlineLBIDRange r = {o.ranges[i].start, o.ranges[i].size};

        if (r.size > 0)
        {
            bc.check(r, 0, rCount);
            //cout <<  "i: " << i << " c: " <<  rCount << " " << o.ranges[i].size << endl;
            loadCount += rCount;
        }

        rCount = 0;
    } // for

    //cout << "hwm: " << o.hwm << " tot: " << loadCount <<  " " << o.ranges.size() << endl;

    return loadCount;

} // LoadOid

int BCTest::ReadOidRanges(const OidRanges_t& v, uint32_t* hits, uint32_t* miss)
{

    blockCacheClient bc(BRP);
    uint32_t readBlocks = 0;
    uint32_t missBlocks = 0;
    uint8_t inBuff[8192];
    int ret;

    for (uint32_t i = 0; i < v.ranges.size() && i < lastRangeListIdx; i++)
    {
        FileBuffer fb(-1, -1);
        const InlineLBIDRange r = {v.ranges[i].start, v.ranges[i].size};

        for (int j = r.start; j < r.start + r.size; j++)
        {
            ret = bc.read(j, 0, fb);
            FileBuffer* ptr = bc.getBlockPtr(j, 0);

            if (ptr)
            {
                readBlocks++;
                memcpy(inBuff, ptr->getData(), 8192);
            }
            else
                missBlocks++;
        }

        *hits += readBlocks;
        *miss += missBlocks;

        //cout << " -- Read range idx: " << i << " hits: " << readBlocks << " miss: " << missBlocks << " hwm: " << v.hwm << endl;
    }

    return readBlocks;

} // ReadRange

// add one block to block cache
//
void BCTest::LoadLbid(const BRM::LBID_t lbid, const BRM::VER_t ver)
{
    blockCacheClient bc(BRP);
    bool b;
    bc.check(lbid, ver, false, b);
} // LoadLbid

// get one block out of block cache
//
void BCTest::ReadOidLbids(const BRM::LBID_t lbid, const BRM::VER_t ver)
{
    uint8_t d[8192] = {'\0'};
    blockCacheClient bc(BRP);
    bc.read(lbid, ver, d);
} // ReadLbid


struct loadThr
{
    loadThr(BCTest& bc, int reps = 1) : fBC(bc), fReps(reps)
    {
    }

    void operator()()
    {
        uint32_t loadedBlocks = 0;
        uint32_t oidBlocks;
        uint32_t i = 0;
        uint32_t rc = 0;
        struct timespec tm1;
        struct timespec tm2;

        tm1.tv_sec = 0;
        tm1.tv_nsec = 0;
        tm2.tv_sec = 0;
        tm2.tv_nsec = 0;
        double tm3 = 0;
        clock_gettime(CLOCK_REALTIME, &tm1);

        for (i = 0; (loadedBlocks + (fBC.OidRangesList[i].ranges.size()*fBC.extentSize)) < fBC.fCacheSz && i < fBC.OidRangesList.size(); i++)
        {
            oidBlocks = 0;
            rc = fBC.LoadOid(fBC.OidRangesList[i], oidBlocks);
            //cout
            //	<< "-- Load " << i << " " << fBC.OidRangesList[i].oid
            //	<< " h: " << fBC.OidRangesList[i].hwm
            //	<< "/" << oidBlocks << " " << loadedBlocks
            //	<< endl;
            loadedBlocks += oidBlocks;
        } // for (i...

        lastRangeListIdx = i;

        clock_gettime(CLOCK_REALTIME, &tm2);
        timespec_sub(tm1, tm2, tm3);

        cout << "load ld: " << loadedBlocks
             << " sz: " << fBC.fCacheSz
             << " rng: " << lastRangeListIdx
             << " tm: " << right << setw(10) << fixed << tm3
             << endl;

    } // operator()

    BCTest& fBC;
    uint32_t fReps;

};

struct readThr
{

    readThr(BCTest& bc, int reps = 1) : fBC(bc), fReps(reps) {}

    void operator()()
    {
        uint32_t rc = 0;
        uint32_t readBlocks = 0;
        uint32_t hits = 0;
        uint32_t miss = 0;
        uint32_t hitsTot = 0;
        uint32_t missTot = 0;
        struct timespec tm1;
        struct timespec tm2;
        tm1.tv_sec = 0;
        tm1.tv_nsec = 0;
        tm2.tv_sec = 0;
        tm2.tv_nsec = 0;
        double tm3 = 0;
        clock_gettime(CLOCK_REALTIME, &tm1);

        for (uint32_t k = 0; k < fReps; k++)
        {
            for (uint32_t i = 0; i < lastRangeListIdx && i < fBC.OidRangesList.size() ; i++)
            {
                rc = fBC.ReadOidRanges(fBC.OidRangesList[i], &hits, &miss);
                //cout << "-- ReadTest " << fBC.OidRangesList[i].oid << " h: " << fBC.OidRangesList[i].hwm << "/" << rc << endl;
                readBlocks += rc;
                rc = 0;
            }

            hitsTot += hits;
            missTot += miss;
            hits = 0;
            miss = 0;
        }

        clock_gettime(CLOCK_REALTIME, &tm2);
        timespec_sub(tm1, tm2, tm3);

        cout << "readtest rd: " << hitsTot << "/" << missTot
             << " sz: " 		<< fBC.fCacheSz
             << " tm: " << right << setw(10) << fixed << tm3
             << endl;

    } // operator()


    BCTest& fBC;
    uint32_t fReps;

};

void usage()
{
    cout << "testbc <cacheSz/1024> <reader threads> <read ahead> <client threads> <reps>" << endl;
}

//
int main(int argc, char* argv[])
{

    int cacheSz = 128; // K number of blocks
    int thr = 1;
    int ra = 1024;
    int clients = 1;
    int reps = 1;

    if (argc > 1 && atoi(argv[1]) > 0)
        cacheSz = atoi(argv[1]) * 1024;

    if (argc > 2 && atoi(argv[2]) > 0)
        thr = atoi(argv[2]);

    if (argc > 3 && atoi(argv[3]) > 0)
        ra = atoi(argv[3]);

    if (argc > 4 && atoi(argv[4]) > 0)
        clients = atoi(argv[4]);

    if (argc > 5 && atoi(argv[5]) > 0)
        reps = atoi(argv[5]);

    BCTest bc(cacheSz, thr, ra);

    cout <<
         "Cache Size: " << cacheSz <<
         " read Threads: " << thr <<
         " read Ahead: " << ra <<
         " clients: " << clients <<
         " repetitions: " << reps <<
         " max Blocks: " << bc.maxBlocksAvailable <<
         endl;

    // loader cache
    struct loadThr loader1(bc, 1);
    struct readThr reader1(bc, reps);
    vector<boost::thread*> v;

    boost::thread* th1 = new boost::thread(loader1);
    th1->join();

    for (int i = 0; i < clients; i++)
    {
        boost::thread* rd1 = new boost::thread(reader1);
        v.push_back(rd1);
    }

    for (int i = 0; i < clients; i++)
    {
        boost::thread* rd1 = v[i];
        rd1->join();
        delete rd1;
    }

    v.clear();

    delete th1;

    return 0;

} // end main