mariadb-columnstore-engine/writeengine/bulk/parsetest.cpp

/* Copyright (C) 2014 InfiniDB, Inc.

   This program is free software; you can redistribute it and/or
   modify it under the terms of the GNU General Public License
   as published by the Free Software Foundation; version 2 of
   the License.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with this program; if not, write to the Free Software
   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
   MA 02110-1301, USA. */

/******************************************************************************************
 * $Id: cpimport.cpp 33 2006-08-24 14:36:17Z wzhou $
 *
 ******************************************************************************************/
#include <iostream>
#include <we_bulkload.h>

#define ENV_BULK_DIR "CP_BULK_DIR"
#include <string>
#include <boost/progress.hpp>

using namespace std;
using namespace WriteEngine;

#define MAXSTRINGS 100000
string Lines[MAXSTRINGS];
typedef std::vector<std::string> LineFldList;

const int parseStr(const string& instr, LineFldList fields)
{
  typedef boost::tokenizer<boost::char_separator<char> > tokenizer;
  boost::char_separator<char> sep("|");
  tokenizer tokens(instr, sep);

  for (tokenizer::iterator tok_iter = tokens.begin(); tok_iter != tokens.end(); ++tok_iter)
  {
    //        std::cout << "<" << *tok_iter << "> ";
    fields.push_back(*tok_iter);
  }

  // std::cout << "\n";
  return EXIT_SUCCESS;
}
int strtok_test(const string& instr)
{
  char* fragment;
  char* search = (char*)malloc(instr.length());

  memcpy(search, instr.c_str(), instr.length());

  fragment = strtok(search, "|");

  do
  {
    // printf("Token: %s\n", fragment);
    fragment = strtok(NULL, "|");
  } while (fragment);

  free(search);
  return EXIT_SUCCESS;
}

int handrolled_test(const string& instr)
{
  char* search = (char*)malloc(instr.length());
  char* pos;
  int count = 0;
  int span;
  string temp;
  string results[10];

  memcpy(search, instr.data(), instr.length());

  if (search[0] == '|')
  {
    pos = search + 1;
  }
  else
  {
    pos = search;
  }

  while (pos)
  {
    span = strcspn(pos, "|");

    if (span)
    {
      temp.assign(pos, span);
      results[count++].assign(temp);
    }

    pos = index(pos + 1, '|');

    if (pos)
    {
      pos++;
    }
  };

  free(search);

  // printf("\n%i dips", count);
  return EXIT_SUCCESS;
}

int handrolled_test2(string& instr, string Fields[])
{
  char* search = (char*)malloc(instr.length() + 1);
  char* pos;      // pos is used to step inside the search string
  int count = 0;  // keeps track of fields found
  int charspan;
  int num_bars;

  strcpy(search, instr.c_str());
  pos = search;

  if (search[0] == '|')
  {
    pos = search + 1;
    Fields[count++].assign("");  // a leading bar indicates an opening blank
  }
  else
  {
    pos = search;
  }

  while (pos < search + instr.length() - 1)
  {
    charspan = strcspn(pos, "|");

    if (charspan)
    {
      Fields[count++].assign(pos, charspan);
      pos += charspan + 1;
    }
    else
    {
      Fields[count++].assign("");
      pos++;
    }

    num_bars = strspn(pos, "|");
    pos += num_bars;

    for (; num_bars > 0; num_bars--)
    {
      Fields[count++].assign("");
    }
  };

  free(search);

  return count;
}

int parseToken()
{
  return 1;
}

int build_data()
{
  int idx;

  for (idx = 0; idx < MAXSTRINGS; idx++)
  {
    // tpch data files are of the form
    // item|item|item and the line may end with |
    // even though this may wrongly suggest a blank value at the end
    Lines[idx] = "12345|abcdef|banana|banana|";  // 'item item item item'
  }

  // std::cout  << Lines[idx-1] << endl;
  return 0;
}

int main(int argc, char** argv)
{
  string sJobIdStr, sBulkDir = "", sDbDir = "", sFileName, sTmp;
  int fcount;
  string Fields[1000];
  string search;
  string searches[] = {"",
                       "|",
                       "|||||||||||||||",
                       "12345|abcdef|banana|",
                       "123456789012345678901234567890",
                       "|12345678901234567890|12345678901234567890|12345678901234567890|12345678901234567890|"
                       "12345678901234567890|12345678901234567890|12345678901234567890|12345678901234567890|"
                       "12345678901234567890|12345678901234567890|12345678901234567890",
                       "|12345|abcdef|banana|bank123",
                       "|123456789012345678901234567890",
                       "12345|abcdef|banana|bank123",
                       "12345||abcdef||banana|bank",
                       "|12345||abcdef|banana|bank",
                       "|12345|abcdef|banana|bank|",
                       "|12345|abcdef|banana||",
                       "|12345|abcdef|banana|||"};
  // 14 elements
  printf("\nAccuracy:");

  for (int test = 0; test < 14; test++)
  {
    printf("\n\nSearch string %i: %s", test, searches[test].c_str());
    fcount = handrolled_test2(searches[test], Fields);

    for (int idx = 0; idx < fcount; idx++)
    {
      printf("\nString %i: %s$", idx, Fields[idx].c_str());
    }
  }

  printf("\n\nSpeed:\n");

  build_data();
  boost::timer t;

  LineFldList parseFields;

  for (int idx = 0; idx < MAXSTRINGS; idx++)
  {
    parseStr(Lines[idx], parseFields);
  }

  printf("Boost Parse Timer: %lf\n", t.elapsed());
  t.restart();

  for (int idx = 0; idx < MAXSTRINGS; idx++)
  {
    strtok_test(Lines[idx]);
  }

  printf("Strtok Timer: %lf\n", t.elapsed());

  t.restart();

  for (int idx = 0; idx < MAXSTRINGS; idx++)
  {
    handrolled_test(Lines[idx]);
  }

  printf("Handrolled Timer: %lf\n", t.elapsed());

  t.restart();

  for (int idx = 0; idx < MAXSTRINGS; idx++)
  {
    fcount = handrolled_test2(Lines[idx], Fields);
  }

  printf("Handrolled2 Timer: %lf\n", t.elapsed());

  printf("\n");
  return 0;
}