mariadb-columnstore-engine/dbcon/joblist/tdriver-agg.cpp

/* Copyright (C) 2014 InfiniDB, Inc.
   Copyright (C) 2016 MariaDB Corporation

   This program is free software; you can redistribute it and/or
   modify it under the terms of the GNU General Public License
   as published by the Free Software Foundation; version 2 of
   the License.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with this program; if not, write to the Free Software
   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
   MA 02110-1301, USA. */

/***********************************************************************
 *   $Id: tdriver-agg.cpp 9210 2013-01-21 14:10:42Z rdempsey $
 *
 *
 ***********************************************************************/
#include <iostream>
#include <sstream>
#include <fstream>
#include <list>
#include <vector>

#include <pthread.h>
#include <time.h>
#include <sys/time.h>

#include <cppunit/extensions/HelperMacros.h>
#include <cppunit/extensions/TestFactoryRegistry.h>
#include <cppunit/ui/text/TestRunner.h>

#include "fifo.h"
#include "wsdl.h"
#include "constantdatalist.h"
#include "bucketdl.h"
#include "elementtype.h"
#include "zdl.h"
#include "stopwatch.cpp"
#include "jobstep.h"
//#include "aggregator.h"
#include "constantcolumn.h"
#include "simplefilter.h"
#include "aggregatecolumn.h"
#include "simplecolumn.h"
#include "dataconvert.h"
#include "largehashjoin.h"

using namespace dataconvert;

// #undef CPPUNIT_ASSERT
// #define CPPUNIT_ASSERT(x)

using namespace std;
using namespace joblist;
using namespace execplan;

Stopwatch timer1;
const uint32_t NUM_BUCKETS = 256;
const uint32_t MAX_SIZE = 0x100000;
const uint32_t MAX_ELEMENTS = 0x20000;
const uint32_t NUM_THREADS = 4;
const string datapath = "/home/zzhu/genii/tools/dbbuilder/lineitem.tbl";
// const string datapath="/usr/local/mariadb/columnstore/bin/lineitem.tbl";
int numConsumers = 1;
int numRuns = 1;
int printInterval = numRuns * 100000;
JSTimeStamp dlTimes;

void timespec_sub(const struct timespec& tv1, const struct timespec& tv2, struct timespec& diff)
{
  if (tv2.tv_nsec < tv1.tv_nsec)
  {
    diff.tv_sec = tv2.tv_sec - tv1.tv_sec - 1;
    diff.tv_nsec = tv1.tv_nsec - tv2.tv_nsec;
  }
  else
  {
    diff.tv_sec = tv2.tv_sec - tv1.tv_sec;
    diff.tv_nsec = tv2.tv_nsec - tv1.tv_nsec;
  }
}

struct ThreadParms
{
  BucketDL<ElementType>* bdl;
  TupleBucketDataList* tbdl;
  int count;
  int threadNumber;
};

void* bucketConsumer(void* arg)
{
  ThreadParms* parms = reinterpret_cast<ThreadParms*>(arg);
  // BucketDL<ElementType> *bdl = reinterpret_cast<BucketDL<ElementType> *>(arg);
  BucketDL<ElementType>* bdl = parms->bdl;
  int numBucketsToConsume = parms->count;
  int threadNumber = parms->threadNumber;
  int64_t i;
  ElementType val;
  uint32_t it;
  int bucketIndex;

  int cnt = 0;

  for (i = 0; i < numBucketsToConsume; i++)
  {
    bucketIndex = (threadNumber * numBucketsToConsume) + i;
    it = bdl->getIterator(bucketIndex);

    while (bdl->next(bucketIndex, it, &val))
      cnt++;
  }

  cout << "consumer " << threadNumber << "consumed " << cnt << endl;
  return NULL;
}

void* tupleConsumer(void* arg)
{
  ThreadParms* parms = reinterpret_cast<ThreadParms*>(arg);
  // BucketDL<ElementType> *bdl = reinterpret_cast<BucketDL<ElementType> *>(arg);
  TupleBucketDataList* tbdl = parms->tbdl;
  int numBucketsToConsume = parms->count;
  int threadNumber = parms->threadNumber;
  int i;
  // ElementType val;
  TupleType val;
  uint32_t it;
  int bucketIndex;

  int cnt = 0;

  for (i = 0; i < numBucketsToConsume; i++)
  {
    bucketIndex = (threadNumber * numBucketsToConsume) + i;
    it = tbdl->getIterator(bucketIndex);

    while (tbdl->next(bucketIndex, it, &val))
      cnt++;
  }

  cout << "consumer " << threadNumber << "consumed " << cnt << endl;
  return NULL;
}

/** @brief class TupleHasher
 *
 */
class TupleHasher1
{
 private:
  Hasher fHasher;
  uint32_t fHashLen;

 public:
  TupleHasher1(uint32_t len) : fHashLen(len)
  {
  }
  uint32_t operator()(const char* v) const
  {
    return fHasher(v, fHashLen);
  }
  /*
  uint32_t operator()(const uint64_t v) const
  {
      return v;
  }*/
};

/** @brief class TupleComparator
 *
 */
class TupleComparator1
{
 private:
  uint32_t fCmpLen;

 public:
  TupleComparator1(uint32_t len) : fCmpLen(len)
  {
  }
  bool operator()(const char* v1, const char* v2) const
  {
    return (memcmp(v1, v2, fCmpLen) == 0);
  }
  /*
  bool operator()(const uint64_t v1, const uint64_t v2) const
  {
      return (v1 == v2);
  }*/
};

void* aggregator(void* arg)
{
  uint32_t fHashLen = 4;
  TupleType tt;
  tt.second = new char[fHashLen];
  ThreadParms* parms = reinterpret_cast<ThreadParms*>(arg);
  int threadNumber = parms->threadNumber;
  struct timespec ts1, ts2, diff;
  uint32_t size = 585938;
#if 0
    //typedef std::vector<uint64_t> RIDVec;
    typedef std::pair<int64_t, Elem> Results;
    //typedef std::tr1::unordered_map<uint64_t, Results, TupleHasher, TupleComparator> shmp;
    //typedef std::tr1::unordered_map<uint64_t, Results, TupleHasher, TupleComparator>::iterator TupleHMIter;
    //typedef boost::shared_ptr<TupleHashMap> SHMP;
    typedef std::tr1::unordered_multimap<uint64_t, Results> TupleHashMap;
    typedef boost::shared_ptr<TupleHashMap> SHMP;
    typedef std::tr1::unordered_multimap<uint64_t, Results>::iterator TupleHMIter;
    typedef std::pair<TupleHMIter, TupleHMIter> HashItPair;
    HashItPair hashItPair;

    SHMP shmp(new TupleHashMap());
//        TupleHashMap *shmp = new TupleHashMap();
    TupleHMIter iter;
    uint32_t val = 0;
    uint64_t hv;
    Hasher hasher;
    bool flag = true;

    for (uint32_t k = 0; k < NUM_BUCKETS / 4; k++)
    {
        clock_gettime(CLOCK_REALTIME, &ts1);

        for (uint32_t j = 0; j < 4; j++)
        {
            for (uint32_t i = 0; i < size / 10; i++)
            {
                flag = true;
                memcpy(tt.second, &i, 4);
                hv = hasher(tt.second, fHashLen);
                iter = shmp->find(hv);

                if (iter != shmp->end())
                {
                    hashItPair = shmp->equal_range(hv);

                    for (iter = hashItPair.first; iter != hashItPair.second; iter++)
                    {
                        if (memcmp(iter->second.second.hashStr, tt.second, fHashLen) == 0)
                        {
                            //cout << "real hit" << endl;
                            //updateAggResult<result_t>(tt, hashIt->second.first);
                            iter->second.second.rids.push_back(tt.first);
                            flag = false;
                            break;
                        }
                    }
                }

                if (flag)
                {
                    Results rr;
                    rr.second.hashStr = new char[fHashLen];
                    //getAggResult<result_t>(tt, rr.first);
                    memcpy(rr.second.hashStr, tt.second, fHashLen);
                    rr.second.rids.push_back(tt.first);
                    shmp->insert(std::pair<uint64_t, Results> (hv, rr));
                }
            }

        }

        cout << "thread " << threadNumber << " vector " << k << " size=" << shmp->size() << endl;

        // clean up hashmap memory
        for (iter = shmp->begin(); iter != shmp->end(); iter++)
            delete [] iter->second.second.hashStr;

        shmp->clear();
//	shmp.reset(new TupleHashMap());
        clock_gettime(CLOCK_REALTIME, &ts2);
        timespec_sub(ts1, ts2, diff);
        cout << "thread " << threadNumber << "do aggregation took " << diff.tv_sec << "s " << diff.tv_nsec << " ns" << endl;
    }

#endif

#if 0
    typedef std::vector<uint64_t> RIDVec;
    typedef std::pair<uint64_t, RIDVec> Results;
    typedef std::tr1::unordered_map<char*, Results, TupleHasher1, TupleComparator1> TupleHashMap1;
    typedef std::tr1::unordered_map<char*, Results, TupleHasher1, TupleComparator1>::iterator TupleHMIter1;
    typedef boost::shared_ptr<TupleHashMap1> SHMP1;

    SHMP1 shmp;
    TupleHasher1 tupleHash(fHashLen);
    TupleComparator1 tupleComp(fHashLen);

    shmp.reset(new TupleHashMap1(1, tupleHash, tupleComp));
    TupleHMIter1 iter;
    Results rr;

    uint32_t val = 0;

    for (uint32_t k = 0; k < NUM_BUCKETS / 4; k++)
    {
        clock_gettime(CLOCK_REALTIME, &ts1);

        for (uint32_t j = 0; j < 4; j++)
        {
            for (uint32_t i = 0; i < size / 10; i++)
            {
                memcpy(tt.second, &i, 4);
                iter = shmp->find(tt.second);

                if (iter == shmp->end())
                {
                    rr.second.clear();
                    //getAggResult<result_t>(tt, rr.first);
                    rr.second.push_back(tt.first);
                    char*  hashStr = new char[fHashLen];
                    memcpy(hashStr, tt.second, fHashLen);
                    shmp->insert(std::pair<char*, Results> (hashStr, rr));
                }
                else
                {
                    //updateAggResult<result_t>(tt, hashIt->second.first);
                    iter->second.second.push_back(tt.first);
                }
            }
        }

        cout << "thread " << threadNumber << " hashmap " << k << " size=" << shmp->size() << endl;

        // clean up hashmap memory
        for (iter = shmp->begin(); iter != shmp->end(); iter++)
            delete [] iter->first;

        shmp->clear();
        clock_gettime(CLOCK_REALTIME, &ts2);
        timespec_sub(ts1, ts2, diff);
        cout << "thread " << threadNumber << "do aggregation took " << diff.tv_sec << "s " << diff.tv_nsec << " ns" << endl;
    }

#endif

#if 1
  // typedef std::vector<uint64_t> RIDVec;
  // typedef std::pair<uint64_t, RIDVec> Results;
  typedef std::tr1::unordered_map<char*, int64_t, TupleHasher1, TupleComparator1> TupleHashMap1;
  typedef std::tr1::unordered_map<char*, int64_t, TupleHasher1, TupleComparator1>::iterator TupleHMIter1;
  typedef boost::shared_ptr<TupleHashMap1> SHMP1;
  vector<TupleType> vt;
  vt.reserve(size / 10 * 4);

  SHMP1 shmp;
  TupleHasher1 tupleHash(fHashLen);
  TupleComparator1 tupleComp(fHashLen);

  shmp.reset(new TupleHashMap1(1, tupleHash, tupleComp));
  TupleHMIter1 iter;
  int64_t rr;

  uint32_t val = 0;

  for (uint32_t k = 0; k < NUM_BUCKETS / 4; k++)
  {
    clock_gettime(CLOCK_REALTIME, &ts1);

    for (uint32_t j = 0; j < 4; j++)
    {
      for (uint32_t i = 0; i < size / 10; i++)
      {
        memcpy(tt.second, &i, 4);
        vt.push_back(tt);

        iter = shmp->find(tt.second);

        if (iter == shmp->end())
        {
          // rr.second.clear();
          // getAggResult<result_t>(tt, rr.first);
          // rr.second.push_back(tt.first);
          char* hashStr = new char[fHashLen];
          memcpy(hashStr, tt.second, fHashLen);
          shmp->insert(std::pair<char*, int64_t>(hashStr, i));
        }
        else
        {
          // updateAggResult<result_t>(tt, hashIt->second.first);
          // iter->second.second.push_back(tt.first);
          iter->second += i;
        }
      }
    }

    cout << "thread " << threadNumber << " hashmap " << k << " size=" << shmp->size() << endl;

    // loop through vector to find match in map for rid
    vector<TupleType>::iterator it;
    int val = 0;

    for (it = vt.begin(); it != vt.end(); it++)
    {
      iter = shmp->find(it->second);

      if (iter != shmp->end())
        val++;
    }

    cout << "val = " << val << endl;

    // clean up hashmap memory
    for (iter = shmp->begin(); iter != shmp->end(); iter++)
      delete[] iter->first;

    shmp->clear();
    vt.clear();
    clock_gettime(CLOCK_REALTIME, &ts2);
    timespec_sub(ts1, ts2, diff);
    cout << "thread " << threadNumber << "do aggregation took " << diff.tv_sec << "s " << diff.tv_nsec
         << " ns" << endl;
  }

#endif
}

void* TBDL_producer(void* arg)
{
  ThreadParms* parms = reinterpret_cast<ThreadParms*>(arg);
  TupleBucketDataList* tbdl = parms->tbdl;
  TupleType t;
  vector<TupleType> vt;

  for (uint32_t i = 0; i < 150000000; i++)
  {
    t.first = i;
    t.second = new char[4];
    memcpy(t.second, &i, 4);
    vt.push_back(t);

    if (vt.size() == 8192)
    {
      tbdl->insert(vt);
      vt.clear();
    }
  }

  if (vt.size() > 0)
    tbdl->insert(vt);
}

void* BDL_producer(void* arg)
{
  ThreadParms* parms = reinterpret_cast<ThreadParms*>(arg);
  BucketDL<ElementType>* bdl = parms->bdl;
  TupleType t;
  ElementType e;
  vector<ElementType> vt;

  for (uint32_t i = 0; i < 150000000; i++)
  {
    e.first = i;
    e.second = i;
    // t.second = new char[4];
    // memcpy(t.second, &i, 4);
    vt.push_back(e);

    if (vt.size() == 8192)
    {
      bdl->insert(vt);
      vt.clear();
    }
  }

  if (vt.size() > 0)
    bdl->insert(vt);
}

class AggDriver : public CppUnit::TestFixture
{
  CPPUNIT_TEST_SUITE(AggDriver);
  // CPPUNIT_TEST(aggFilter_group2);
  // CPPUNIT_TEST(tuplewsdl);
  // CPPUNIT_TEST(bucketdl);
  // CPPUNIT_TEST(aggFilter_group1);
  // CPPUNIT_TEST(hashjoin);
  // CPPUNIT_TEST(hashmap_tuple);
  // CPPUNIT_TEST(hashmap_tuple_multi);
  // CPPUNIT_TEST(hashmap_ET);
  // CPPUNIT_TEST(tuplewsdl_multi);
  CPPUNIT_TEST(bdl_multi);
  CPPUNIT_TEST_SUITE_END();
  typedef boost::shared_ptr<SimpleFilter> SSFP;

 private:
  ResourceManager fRm;

 public:
  void setUp1(JobStepAssociation& in, JobStepAssociation& out)
  {
    // sleep(20);
    // input - TupleBucket
    AnyDataListSPtr adl1(new AnyDataList());
    TupleBucketDataList* tbdl = new TupleBucketDataList(NUM_BUCKETS, numConsumers, MAX_SIZE, fRm);
    tbdl->setMultipleProducers(0);
    // tbdl->setElementMode(1);

    adl1->tupleBucketDL(tbdl);
    in.outAdd(adl1);

    // output - ElementType Bucket
    AnyDataListSPtr adl2(new AnyDataList());
    BucketDataList* bdl = new BucketDataList(NUM_BUCKETS, numConsumers, MAX_ELEMENTS, fRm);
    bdl->setElementMode(1);
    adl2->bucketDL(bdl);
    out.outAdd(adl2);

    // prepare the input datalist
    // ifstream ifs(datapath.c_str(), ios::in);
    uint64_t l_orderkey;
    uint64_t l_quantity;
    char data[1000];
    // char row[50];
    char* tok;
    uint64_t count = 0;
    uint32_t rid = 0;
    TupleType* tt = new TupleType();
    tbdl->hashLen(sizeof(l_orderkey));
    uint32_t size = sizeof(l_orderkey) + sizeof(l_quantity);
    // tbdl->elementLen(size + sizeof(rid));
    tbdl->elementLen(sizeof(rid), size);
    // char* second = new char[size];
    vector<TupleType> v;

    timer1.start("input");
    ifstream ifs(datapath.c_str(), ios::in);

    while (!ifs.eof())
    {
      ifs.getline(data, 1000);
      tok = strtok(data, "|");
      count = 0;

      while (tok != NULL)
      {
        if (count == 0)
          l_orderkey = atol(tok);

        if (count == 4)
          l_quantity = atof(tok) * 100;

        count++;
        tok = strtok(NULL, "|");
      }

      for (int i = 0; i < numRuns; i++)
      {
        l_orderkey += 6000000 * i;
        tt->first = rid;
        tt->second = new char[size];
        memcpy(tt->second, &l_orderkey, sizeof(l_orderkey));
        memcpy(tt->second + sizeof(l_orderkey), &l_quantity, sizeof(l_quantity));
        v.push_back(*tt);

        if (v.size() == 2048)
        {
          tbdl->insert(v);
          v.clear();
        }

        rid++;
      }

      if (rid % printInterval == 0)
        cout << rid << " " << l_orderkey << " " << l_quantity << endl;
    }

    ifs.close();

    if (v.size() > 0)
    {
      tbdl->insert(v);
      // for (uint32_t i = 0; i < v.size(); i++)
      //    delete [] (v[i].second);
      v.clear();
    }

    tbdl->endOfInput();
    timer1.stop("input");
    cout << "input size=" << tbdl->totalSize() << endl;
    // ifs.close();
    delete tt;
  }

  int64_t convertValueNum(const string& str, const CalpontSystemCatalog::ColType& ct, bool isNull)
  {
    // if (str.size() == 0 || isNull ) return valueNullNum(ct);

    int64_t v = 0;

    boost::any anyVal = DataConvert::convertColumnData(ct, str, false);

    switch (ct.colDataType)
    {
      case CalpontSystemCatalog::BIT: v = boost::any_cast<bool>(anyVal); break;

      case CalpontSystemCatalog::TINYINT: v = boost::any_cast<char>(anyVal); break;

      case CalpontSystemCatalog::SMALLINT: v = boost::any_cast<int16_t>(anyVal); break;

      case CalpontSystemCatalog::MEDINT:
      case CalpontSystemCatalog::INT: v = boost::any_cast<int32_t>(anyVal); break;

      case CalpontSystemCatalog::BIGINT: v = boost::any_cast<long long>(anyVal); break;

      case CalpontSystemCatalog::FLOAT:
      {
        float i = boost::any_cast<float>(anyVal);
        v = (Int64)i;
      }
      break;

      case CalpontSystemCatalog::DOUBLE:
      {
        double i = boost::any_cast<double>(anyVal);
        v = (Int64)i;
      }
      break;

      case CalpontSystemCatalog::CHAR:
      case CalpontSystemCatalog::VARCHAR:
      case CalpontSystemCatalog::BLOB:
      case CalpontSystemCatalog::CLOB:
      {
        // v = boost::any_cast<string>(anyVal);
        string i = boost::any_cast<string>(anyVal);
        v = *((Int64*)i.c_str());
      }

      break;

      case CalpontSystemCatalog::DATE: v = boost::any_cast<uint32_t>(anyVal); break;

      case CalpontSystemCatalog::DATETIME: v = boost::any_cast<uint64_t>(anyVal); break;

      case CalpontSystemCatalog::DECIMAL: v = boost::any_cast<long long>(anyVal); break;

      default: break;
    }

    return v;
  }

  void tuplewsdl_multi()
  {
    TupleBucketDataList* tbdl = new TupleBucketDataList(NUM_BUCKETS, 1, MAX_SIZE, fRm);
    tbdl->setMultipleProducers(true);
    tbdl->setElementMode(1);  // RID_VALUE
    tbdl->hashLen(4);
    tbdl->elementLen(4, 12);
    pthread_t producer[4];
    ThreadParms producerThreadParms[4];

    timer1.start("insert-tbdl");

    for (uint32_t i = 0; i < 4; i++)
    {
      producerThreadParms[i].tbdl = tbdl;
      producerThreadParms[i].threadNumber = i;
      // producerThreadParms[i].count        = ::count;
      pthread_create(&producer[i], NULL, TBDL_producer, &producerThreadParms[i]);
    }

    for (uint32_t i = 0; i < 4; i++)
      pthread_join(producer[i], NULL);

    tbdl->endOfInput();
    timer1.stop("insert-tbdl");
    cout << "tbdl finish insert " << tbdl->totalSize() << endl;
    // timer1.finish();
  }

  void bdl_multi()
  {
    BucketDL<ElementType>* bdl = new BucketDL<ElementType>(128, 1, MAX_ELEMENTS, fRm);
    bdl->setMultipleProducers(true);
    bdl->setElementMode(1);  // RID_VALUE
    bdl->setDiskElemSize(4, 4);
    pthread_t producer[4];
    ThreadParms producerThreadParms[4];

    timer1.start("insert-bdl");

    for (uint32_t i = 0; i < 4; i++)
    {
      producerThreadParms[i].bdl = bdl;
      producerThreadParms[i].threadNumber = i;
      // producerThreadParms[i].count        = ::count;
      pthread_create(&producer[i], NULL, BDL_producer, &producerThreadParms[i]);
    }

    for (uint32_t i = 0; i < 4; i++)
      pthread_join(producer[i], NULL);

    bdl->endOfInput();
    timer1.stop("insert-bdl");
    cout << "bdl finish insert " << bdl->totalSize() << endl;
    timer1.finish();
  }

  void tuplewsdl()
  {
    TupleBucketDataList* tbdl = new TupleBucketDataList(NUM_BUCKETS, numConsumers, MAX_SIZE, fRm);
    // prepare the input datalist
    // ifstream ifs(datapath.c_str(), ios::in);
    uint64_t l_orderkey, l_quantity;
    char data[1000];
    // char row[50];
    char* tok;
    uint64_t count = 0, rid = 0;
    TupleType* tt = new TupleType();
    tbdl->hashLen(sizeof(l_orderkey));
    uint32_t size = sizeof(l_orderkey) + sizeof(l_quantity);
    // tbdl->elementLen(size + sizeof(rid));
    tbdl->elementLen(sizeof(rid), size);
    // char* second = new char[size];
    vector<TupleType> v;

    timer1.start("tuple-input");
    int i;

    for (i = 0; i < numRuns; i++)
    {
      ifstream ifs(datapath.c_str(), ios::in);

      while (!ifs.eof())
      {
        ifs.getline(data, 1000);
        tok = strtok(data, "|");
        count = 0;

        while (tok != NULL)
        {
          if (count == 0)
            l_orderkey = 6000000 * i + atol(tok);

          if (count == 4)
            l_quantity = atof(tok) * 100;

          count++;
          tok = strtok(NULL, "|");
        }

        tt->first = rid;
        tt->second = new char[size];
        memcpy(tt->second, &l_orderkey, sizeof(l_orderkey));
        memcpy(tt->second + sizeof(l_orderkey), &l_quantity, sizeof(l_quantity));
        v.push_back(*tt);

        if (v.size() == 2048)
        {
          tbdl->insert(v);

          for (uint32_t i = 0; i < v.size(); i++)
            v[i].deleter();

          v.clear();
        }

        rid++;

        if (rid % printInterval == 0)
          cout << rid << " " << l_orderkey << " " << l_quantity << endl;
      }

      ifs.close();
    }

    if (v.size() > 0)
    {
      tbdl->insert(v);

      for (uint32_t i = 0; i < v.size(); i++)
        delete[](v[i].second);

      v.clear();
    }

    tbdl->endOfInput();
    timer1.stop("tuple-input");
    cout << "input size=" << tbdl->totalSize() << endl;
    // ifs.close();
    delete tt;

    ThreadParms thparms;
    thparms.count = NUM_BUCKETS / NUM_THREADS;
    thparms.tbdl = tbdl;
    pthread_t consumer[NUM_THREADS];

    ThreadParms thParms[NUM_THREADS];
    timer1.start("tuple-consumer");

    for (i = 0; i < NUM_THREADS; i++)
    {
      thparms.threadNumber = i;
      thParms[i] = thparms;
      pthread_create(&consumer[i], NULL, tupleConsumer, &thParms[i]);
    }

    for (i = 0; i < NUM_THREADS; i++)
      pthread_join(consumer[i], NULL);

    timer1.stop("tuple-consumer");

    timer1.finish();
    delete tbdl;
  }

  void aggFilter_group1()
  {
    /*
                int tm;
              for (int m = 0; m < 1024; m++)
              {
                char *a = new char[5*1024*1024];
                memset(a,0,5*1024*1024);
                memcpy(&tm, a, 4);
              }
    */
    // for (int i = 0; i < 10; i++) {
    JobStepAssociation in, out;
    setUp1(in, out);
    CalpontSystemCatalog::TableName tn("test", "lineitem");
    string s_l_orderkey = "test.lineitem.l_orderkey";
    string s_l_quantity = "test.lineitem.l_quantity";
    uint32_t sessionid = 1;
    uint32_t txnId = 1;
    uint32_t verId = 1;
    uint16_t stepID = 0;
    uint32_t statementID = 1;
    boost::shared_ptr<CalpontSystemCatalog> csc = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionid);

    SRCP srcp;
    SimpleColumn* l_orderkey = new SimpleColumn(s_l_orderkey, sessionid);
    SimpleColumn* l_quantity = new SimpleColumn(s_l_quantity, sessionid);
    CalpontSystemCatalog::ROPair ropair = csc->tableRID(tn);
    CalpontSystemCatalog::OID tableoid = ropair.objnum;

    // sum(l_quantity) group by l_ordereky
    srcp.reset(l_quantity);
    AggregateColumn* ac = new AggregateColumn("sum", srcp.get()->clone(), sessionid);

    ac->addProjectCol(srcp);
    srcp.reset(l_orderkey);
    ac->addGroupByCol(srcp);

    // constant column 318
    ConstantColumn* cc = new ConstantColumn(318, ConstantColumn::NUM);
    // >
    Operator* op = new Operator(">");
    SOP sop(op);
    // sum(l_quantity) > 318
    SSFP ssfp;
    SimpleFilter* sf = new SimpleFilter(sop, ac, cc);
    ssfp.reset(sf);
    SJSTEP step;

    AggregateFilterStep* afs = new AggregateFilterStep(in, out, ac->functionName(), ac->groupByColList(),
                                                       ac->projectColList(), ac->functionParms(), tableoid,
                                                       sessionid, txnId, verId, stepID, statementID, fRm);

    // output column l_orderkey;
    afs->outputCol(dynamic_cast<SimpleColumn*>(srcp.get())->oid());
    step.reset(afs);
    // one column case, nomalize filter value
    CalpontSystemCatalog::ColType ct;
    int64_t intVal;
    string strVal;

    // int8_t cop = op2num(sop);
    if (typeid((*ac->functionParms().get())) == typeid(SimpleColumn))
    {
      SimpleColumn* sc = reinterpret_cast<SimpleColumn*>(ac->functionParms().get());
      ct = csc->colType(sc->oid());
      intVal = convertValueNum(cc->constval(), ct, false);
      afs->addFilter(COMPARE_GT, intVal);
    }
    else
    {
      if (cc->type() == ConstantColumn::NUM)
      {
        intVal = atol(cc->constval().c_str());
        afs->addFilter(COMPARE_GT, intVal, false);
      }
      else if (cc->type() == ConstantColumn::LITERAL)
        afs->addFilter(COMPARE_GT, cc->constval(), false);
    }

    timer1.start("agg");
    afs->run();
    afs->join();
    timer1.stop("agg");
    timer1.finish();
    //}
  }

  void bucketdl()
  {
    BucketDL<ElementType>* bdl = new BucketDL<ElementType>(NUM_BUCKETS, 1, MAX_ELEMENTS, fRm);
    bdl->setElementMode(1);
    ElementType* et = new ElementType();

    uint64_t l_orderkey, l_quantity;
    char data[1000];
    char* tok;
    uint64_t count = 0, rid = 0;
    TupleType* tt = new TupleType();

    vector<ElementType> v;
    timer1.start("bucket-input");
    int i;

    for (i = 0; i < numRuns; i++)
    {
      ifstream ifs(datapath.c_str(), ios::in);

      while (!ifs.eof())
      {
        ifs.getline(data, 1000);
        tok = strtok(data, "|");
        count = 0;

        while (tok != NULL)
        {
          if (count == 0)
            l_orderkey = 6000000 * i + atol(tok);

          if (count == 4)
            l_quantity = atof(tok) * 100;

          count++;
          tok = strtok(NULL, "|");
        }

        et->first = rid;
        et->second = rid;
        // tt->second = new char[16];
        // memcpy(tt->second, &l_orderkey, sizeof(l_orderkey));
        // memcpy(tt->second+sizeof(l_orderkey), &l_quantity, sizeof(l_quantity));
        v.push_back(*et);

        if (v.size() == 2048)
        {
          bdl->insert(v);
          v.clear();
        }

        rid++;

        // delete [] tt->second;
        if (rid % printInterval == 0)
          cout << rid << " " << l_orderkey << " " << l_quantity << endl;
      }

      ifs.close();
    }

    if (v.size() > 0)
    {
      bdl->insert(v);
      v.clear();
    }

    bdl->endOfInput();
    timer1.stop("bucket-input");
    cout << "input size=" << bdl->totalSize() << endl;
    delete tt;
    delete et;

    ThreadParms thparms;
    thparms.count = NUM_BUCKETS / NUM_THREADS;
    thparms.bdl = bdl;
    pthread_t consumer[NUM_THREADS];

    ThreadParms thParms[NUM_THREADS];
    timer1.start("bucket-consumer");

    for (i = 0; i < NUM_THREADS; i++)
    {
      thparms.threadNumber = i;
      thParms[i] = thparms;
      pthread_create(&consumer[i], NULL, bucketConsumer, &thParms[i]);
    }

    for (i = 0; i < NUM_THREADS; i++)
      pthread_join(consumer[i], NULL);

    timer1.stop("bucket-consumer");

    timer1.finish();
    delete bdl;
  }

  void aggFilter_group2()
  {
    CalpontSystemCatalog::TableName tn("test", "lineitem");
    string s_l_orderkey = "test.lineitem.l_orderkey";
    string s_l_quantity = "test.lineitem.l_quantity";
    string s_l_partkey = "test.lineitem.l_partkey";
    uint32_t sessionid = 1;
    uint32_t txnId = 1;
    uint32_t verId = 1;
    uint16_t stepID = 0;
    uint32_t statementID = 1;
    boost::shared_ptr<CalpontSystemCatalog> csc = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionid);

    SRCP srcp;
    SimpleColumn* l_orderkey = new SimpleColumn(s_l_orderkey, sessionid);
    SimpleColumn* l_quantity = new SimpleColumn(s_l_quantity, sessionid);
    SimpleColumn* l_partkey = new SimpleColumn(s_l_partkey, sessionid);
    CalpontSystemCatalog::ROPair ropair = csc->tableRID(tn);
    CalpontSystemCatalog::OID tableoid = ropair.objnum;

    // sum(l_quantity) group by l_ordereky
    AggregateColumn* ac = new AggregateColumn("sum", l_quantity, sessionid);
    srcp.reset(l_quantity);
    ac->addProjectCol(srcp);
    srcp.reset(l_orderkey);
    ac->addGroupByCol(srcp);
    srcp.reset(l_partkey);
    ac->addGroupByCol(srcp);

    // constant column 318
    ConstantColumn* cc = new ConstantColumn(318, ConstantColumn::NUM);
    // >
    Operator* op = new Operator(">");
    SOP sop(op);
    // sum(l_quantity) > 318
    SimpleFilter* sf = new SimpleFilter(sop, ac, cc);

    // aggregate filter step
    /** @brief Constructor */
    JobStepAssociation in;
    JobStepAssociation out;
    AggregateFilterStep* afs = new AggregateFilterStep(in, out, ac->functionName(), ac->groupByColList(),
                                                       ac->projectColList(), ac->functionParms(), tableoid,
                                                       sessionid, txnId, verId, stepID, statementID, fRm);
    afs->run();
    afs->join();
  }

  void hashjoin()
  {
    BucketDL<ElementType> A(NUM_BUCKETS, 1, MAX_ELEMENTS, fRm);
    BucketDL<ElementType> B(NUM_BUCKETS, 1, MAX_ELEMENTS, fRm);
    BucketDL<ElementType> C(NUM_BUCKETS, 1, MAX_ELEMENTS, fRm);
    BucketDL<ElementType> D(NUM_BUCKETS, 1, MAX_ELEMENTS, fRm);
    A.setDiskElemSize(4, 4);
    B.setDiskElemSize(4, 4);
    A.setElementMode(1);
    A.setHashMode(1);
    B.setElementMode(1);
    B.setHashMode(1);
    int i;
    timer1.start("insert");

    for (i = 0; i < 6000000 * numRuns; i++)
      A.insert(ElementType(i, i));

    A.endOfInput();

    for (i = 0; i < 1500000 * numRuns; i++)
      B.insert(ElementType(i, i + 1000000));

    B.endOfInput();
    timer1.stop("insert");
    C.setElementMode(1);
    D.setElementMode(1);
    BDLWrapper<ElementType> setA(&A);
    BDLWrapper<ElementType> setB(&B);
    DataList<ElementType>* resultA(&C);
    DataList<ElementType>* resultB(&D);
    timer1.start("hashjoin");
    HashJoin<ElementType>* hj = new HashJoin<ElementType>(setA, setB, resultA, resultB, INNER, &dlTimes);
    hj->performJoin();
    timer1.stop("hashjoin");
    timer1.finish();
    cout << "hash join output size: " << resultA->totalSize() << "/" << resultB->totalSize() << endl;
  }

  typedef struct Elem
  {
    char* hashStr;
    // int64_t result;
    vector<uint64_t> rids;
  } Elem;

  void hashmap_tuple()
  {
    int tm;

    for (int m = 0; m < 1024; m++)
    {
      char* a = new char[5 * 1024 * 1024];
      memset(a, 0, 5 * 1024 * 1024);
      memcpy(&tm, a, 4);
    }

    uint32_t fHashLen = 4;
    TupleType tt;
    tt.second = new char[fHashLen];
    uint64_t size = 585937;
    uint64_t i = 0, j = 0;
#if 1
    pthread_t agg[NUM_THREADS];
    ThreadParms thParms[NUM_THREADS];
    ThreadParms thParm;
    struct timespec ts1, ts2, diff;

    clock_gettime(CLOCK_REALTIME, &ts1);

    for (i = 0; i < NUM_THREADS; i++)
    {
      thParm.threadNumber = i;
      thParms[i] = thParm;
      pthread_create(&agg[i], NULL, aggregator, &thParms[i]);
    }

    for (i = 0; i < NUM_THREADS; i++)
      pthread_join(agg[i], NULL);

    clock_gettime(CLOCK_REALTIME, &ts2);
    timespec_sub(ts1, ts2, diff);
    cout << "aggregation took " << diff.tv_sec << "s " << diff.tv_nsec << "ns" << endl;
#endif
#if 0
        //typedef std::vector<uint64_t> RIDVec;
        typedef std::pair<int64_t, Elem> Results;
        //typedef std::tr1::unordered_map<uint64_t, Results, TupleHasher, TupleComparator> shmp;
        //typedef std::tr1::unordered_map<uint64_t, Results, TupleHasher, TupleComparator>::iterator TupleHMIter;
        //typedef boost::shared_ptr<TupleHashMap> SHMP;
        typedef std::tr1::unordered_multimap<uint64_t, Results> TupleHashMap;
        typedef boost::shared_ptr<TupleHashMap> SHMP;
        typedef std::tr1::unordered_multimap<uint64_t, Results>::iterator TupleHMIter;
        typedef std::pair<TupleHMIter, TupleHMIter> HashItPair;
        HashItPair hashItPair;

        SHMP shmp(new TupleHashMap());
        //TupleHashMap *shmp = new TupleHashMap();
        TupleHMIter iter;
        uint32_t val = 0;
        uint64_t hv;
        Elem elem;
        Results rr;
        Hasher hasher;
        bool flag = true;
        timer1.start("vectormap");

        for (int k = 0; k < 1; k++)
        {
            for (j = 0; j < 4; j++)
            {
                for (i = 0; i < size / 10; i++)
                {
                    flag = true;
                    memcpy(tt.second, &i, 4);
                    hv = hasher(tt.second, fHashLen);
                    iter = shmp->find(hv);

                    if (iter != shmp->end())
                    {
                        hashItPair = shmp->equal_range(hv);

                        for (iter = hashItPair.first; iter != hashItPair.second; iter++)
                        {
                            if (memcmp(iter->second.second.hashStr, tt.second, fHashLen) == 0)
                            {
                                //cout << "real hit" << endl;
                                //updateAggResult<result_t>(tt, hashIt->second.first);
                                iter->second.second.rids.push_back(tt.first);
                                flag = false;
                                break;
                            }
                        }
                    }

                    if (flag)
                    {
                        rr.second.hashStr = new char[fHashLen];
                        memcpy(rr.second.hashStr, tt.second, fHashLen);
                        rr.second.rids.clear();
                        rr.second.rids.push_back(tt.first);
                        shmp->insert(std::pair<uint64_t, Results> (hv, rr));
                    }
                }
            }

            cout << "vector size=" << shmp->size() << endl;

            // clean up hashmap memory
            for (iter = shmp->begin(); iter != shmp->end(); iter++)
                delete [] iter->second.second.hashStr;

            shmp->clear();
        }

        timer1.stop("vectormap");
        timer1.finish();

#endif
#if 0
        typedef std::vector<uint64_t> RIDVec;
        typedef std::pair<uint64_t, RIDVec> Results;
        typedef std::tr1::unordered_map<char*, Results, TupleHasher, TupleComparator> TupleHashMap1;
        typedef std::tr1::unordered_map<char*, Results, TupleHasher, TupleComparator>::iterator TupleHMIter1;
        typedef boost::shared_ptr<TupleHashMap1> SHMP1;

        SHMP1 shmp;
        TupleHasher tupleHash(fHashLen);
        TupleComparator tupleComp(fHashLen);

        shmp.reset(new TupleHashMap1(1, tupleHash, tupleComp));
        TupleHMIter1 iter;
        Results rr;

        uint32_t val = 0;
        timer1.start("uniquemap");

        for (j = 0; j < 4; j++)
            for (i = 0; i < size / 10; i++)
            {
                memcpy(tt.second, &i, 4);
                iter = shmp->find(tt.second);

                if (iter == shmp->end())
                {
                    rr.second.clear();
                    //getAggResult<result_t>(tt, rr.first);
                    rr.second.push_back(tt.first);
                    char*  hashStr = new char[fHashLen];
                    memcpy(hashStr, tt.second, fHashLen);
                    shmp->insert(std::pair<char*, Results> (hashStr, rr));
                }
                else
                {
                    //updateAggResult<result_t>(tt, hashIt->second.first);
                    iter->second.second.push_back(tt.first);
                }
            }

        timer1.stop("uniquemap");
        timer1.finish();
        cout << "hashmap size=" << shmp->size() << endl;

        // clean up hashmap memory
        for (iter = shmp->begin(); iter != shmp->end(); iter++)
            delete [] iter->first;

        shmp->clear();
#endif
  }

#if 0
    void hashmap_tuple_multi()
    {
        uint32_t fHashLen = 4;
        TupleHasher tupleHash(fHashLen);
        TupleComparator tupleComp(fHashLen);
        TupleType tt;

        //typedef std::vector<uint64_t> RIDVec;
        typedef std::pair<uint64_t, uint64_t> Results;
        typedef std::tr1::unordered_multimap<char*, Results, TupleHasher, TupleComparator> TupleHashMap;
        typedef std::tr1::unordered_multimap<char*, Results, TupleHasher, TupleComparator>::iterator TupleHMIter;
        typedef boost::shared_ptr<TupleHashMap> SHMP;
        std::pair<TupleHMIter, TupleHMIter> hashItPair;

        SHMP shmp;
        shmp.reset(new TupleHashMap(1, tupleHash, tupleComp));
        TupleHMIter iter;
        Results rr;
        Hasher fHasher;

        timer1.start("multi_insert");
        uint32_t i = 0, j = 0;
        uint32_t val = 0;

        for ( j = 0; j < 4; j++)
            for ( i = 0; i < 1171875; i++)
            {
                tt.second = new char[fHashLen];
                memcpy(tt.second, &i, fHashLen);
                rr.first = j;
                rr.second = i;
                shmp->insert(std::pair<char*, Results> (tt.second, rr));
            }

        timer1.stop("multi_insert");
        cout << "hashmap size=" << shmp->size() << endl;

        tt.second = new char[fHashLen];
        timer1.start("multi_result");

        iter = shmp->begin();
        char* key;

        if (iter != shmp->end())
            key = iter->first;

        uint32_t ct = 0;

        for (iter = ++iter; iter != shmp->end(); iter++)
        {
            if (memcmp(iter->first, key, 4) == 0)
                val += iter->second.first;
            else
            {
                key = iter->first;
                val = iter->second.first;
                ct++;
            }
        }

        timer1.stop("multi_result");

        timer1.start("base");

        for ( j = 0; j < 4; j++)
            for ( i = 0; i < 1171875; i++)
            {
                tt.second = new char[fHashLen];
                memcpy(tt.second, &i, fHashLen);
            }

        timer1.stop("base");
        timer1.finish();
        cout << "group val=" << val << endl;

        // clean up hashmap memory
        for (iter = shmp->begin(); iter != shmp->end(); iter++)
            delete [] iter->first;
    }
#endif
  void hashmap_ET()
  {
    typedef std::list<ElementType> hashList_t;
    typedef std::list<ElementType>::iterator hashListIter_t;

    // typedef std::tr1::unordered_multimap<ElementType::second_type, ElementType::first_type> hash_t;
    // typedef std::tr1::unordered_multimap<ElementType::second_type, ElementType::first_type>::iterator
    // hashIter_t; typedef std::tr1::unordered_multimap<ElementType::second_type,
    // ElementType::first_type>::value_type hashPair_t;

    typedef std::tr1::unordered_map<ElementType::second_type, hashList_t> hash_t;
    typedef std::tr1::unordered_map<ElementType::second_type, hashList_t>::iterator hashIter_t;
    typedef std::tr1::unordered_map<ElementType::second_type, hashList_t>::value_type hashPair_t;

    hash_t hashmap;
    hashIter_t it;
    hashList_t list;

    timer1.start("et_insert");
    ElementType et;

    // while (true)
    for (int i = 0; i < 1200000; i++)
    {
      list.push_back(et);
      et.second = i;
      hashmap.insert(hashPair_t(et.second, list));
      list.clear();
    }

    timer1.stop("et_insert");
    cout << "hashmap size=" << hashmap.size() << endl;

    int val = 0, i, j;
    timer1.start("et_find");

    for (j = 0; j < 4; j++)
      for (i = 0; i < 1200000; i++)
      {
        et.second = i;
        it = hashmap.find(i);

        if (it == hashmap.end())
        {
          list.clear();
          // getAggResult<result_t>(tt, rr.first);
          list.push_back(et);
          hashmap.insert(hashPair_t(et.second, list));
        }
        else
        {
          it->second.push_back(et);
        }
      }

    timer1.stop("et_find");
    timer1.finish();
    cout << "hashmap size=" << hashmap.size() << endl;
  }
};

CPPUNIT_TEST_SUITE_REGISTRATION(AggDriver);

int main(int argc, char** argv)
{
  CppUnit::TextUi::TestRunner runner;
  CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry();
  runner.addTest(registry.makeTest());
  bool wasSuccessful = runner.run("", false);
  return (wasSuccessful ? 0 : 1);
}