mariadb-columnstore-engine/writeengine/bulk/parsetest.cpp

/* Copyright (C) 2014 InfiniDB, Inc.

   This program is free software; you can redistribute it and/or
   modify it under the terms of the GNU General Public License
   as published by the Free Software Foundation; version 2 of
   the License.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with this program; if not, write to the Free Software
   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
   MA 02110-1301, USA. */

/******************************************************************************************
* $Id: cpimport.cpp 33 2006-08-24 14:36:17Z wzhou $
*
******************************************************************************************/
#include <iostream>
#include <we_bulkload.h>

#define     ENV_BULK_DIR               "CP_BULK_DIR"
#include <string>
#include <boost/progress.hpp>

using namespace std;
using namespace WriteEngine;

#define MAXSTRINGS 100000
string Lines[MAXSTRINGS];
typedef std::vector<std::string>          LineFldList;


const int parseStr( const string& instr,  LineFldList fields)
{
    typedef boost::tokenizer<boost::char_separator<char> >
    tokenizer;
    boost::char_separator<char> sep("|");
    tokenizer tokens(instr, sep);

    for (tokenizer::iterator tok_iter = tokens.begin(); tok_iter != tokens.end(); ++tok_iter)
    {
//        std::cout << "<" << *tok_iter << "> ";
        fields.push_back( *tok_iter );
    }

    //std::cout << "\n";
    return EXIT_SUCCESS;

}
int strtok_test( const string& instr )
{
    char* fragment;
    char* search = (char*)malloc( instr.length());

    memcpy( search, instr.c_str(), instr.length());

    fragment = strtok(search, "|");

    do
    {
        // printf("Token: %s\n", fragment);
        fragment = strtok(NULL, "|");
    }
    while (fragment);

    free( search );
    return EXIT_SUCCESS;
}

int handrolled_test( const string& instr )
{
    char* search = (char*)malloc( instr.length());
    char* pos;
    int count = 0;
    int span;
    string temp;
    string results[10];

    memcpy( search, instr.data(), instr.length());

    if (search[0] == '|')
    {
        pos = search + 1;
    }
    else
    {
        pos = search;
    }

    while (pos)
    {
        span = strcspn(pos, "|");

        if (span)
        {
            temp.assign(pos, span);
            results[count++].assign(temp);
        }

        pos = index(pos + 1, '|');

        if (pos)
        {
            pos++ ;
        }
    };

    free( search );

    //printf("\n%i dips", count);
    return EXIT_SUCCESS;
}

int handrolled_test2( string& instr, string Fields[] )
{

    char* search = (char*)malloc( instr.length() + 1 );
    char* pos; // pos is used to step inside the search string
    int count = 0; // keeps track of fields found
    int charspan;
    int num_bars;

    strcpy( search, instr.c_str() );
    pos = search;


    if (search[0] == '|')
    {
        pos = search + 1;
        Fields[count++].assign(""); // a leading bar indicates an opening blank
    }
    else
    {
        pos = search;
    }

    while (pos < search + instr.length() - 1 )
    {

        charspan = strcspn(pos, "|");

        if (charspan)
        {
            Fields[count++].assign(pos, charspan);
            pos += charspan + 1;
        }
        else
        {
            Fields[count++].assign("");
            pos++;
        }

        num_bars = strspn(pos, "|");
        pos += num_bars;

        for ( ; num_bars > 0; num_bars--)
        {
            Fields[count++].assign("");
        }
    };

    free( search );

    return count;
}

int parseToken()
{
    return 1;
}


int build_data()
{
    int idx;

    for (idx = 0; idx < MAXSTRINGS; idx++)
    {
        //tpch data files are of the form
        // item|item|item and the line may end with |
        // even though this may wrongly suggest a blank value at the end
        Lines[idx] = "12345|abcdef|banana|banana|";  // 'item item item item'
    }

    //std::cout  << Lines[idx-1] << endl;
    return 0;
}

int main(int argc, char** argv)
{

    string   sJobIdStr, sBulkDir = "", sDbDir = "", sFileName, sTmp;
    int fcount;
    string Fields[1000] ;
    string search;
    string searches[] =
    {
        "", "|", "|||||||||||||||", "12345|abcdef|banana|", "123456789012345678901234567890",
        "|12345678901234567890|12345678901234567890|12345678901234567890|12345678901234567890|12345678901234567890|12345678901234567890|12345678901234567890|12345678901234567890|12345678901234567890|12345678901234567890|12345678901234567890",
        "|12345|abcdef|banana|bank123", "|123456789012345678901234567890", "12345|abcdef|banana|bank123",
        "12345||abcdef||banana|bank", "|12345||abcdef|banana|bank", "|12345|abcdef|banana|bank|",
        "|12345|abcdef|banana||", "|12345|abcdef|banana|||"
    };
    // 14 elements
    printf("\nAccuracy:");


    for (int test = 0; test < 14; test++)
    {
        printf("\n\nSearch string %i: %s", test, searches[test].c_str());
        fcount = handrolled_test2(searches[test], Fields);

        for (int idx = 0; idx < fcount; idx++)
        {
            printf("\nString %i: %s$", idx, Fields[idx].c_str());
        }
    }

    printf("\n\nSpeed:\n");

    build_data();
    boost::timer t;

    LineFldList parseFields;

    for (int idx = 0; idx < MAXSTRINGS; idx++)
    {
        parseStr(Lines[idx], parseFields);
    }

    printf("Boost Parse Timer: %lf\n", t.elapsed());
    t.restart();

    for (int idx = 0; idx < MAXSTRINGS; idx++)
    {
        strtok_test(Lines[idx]);
    }

    printf("Strtok Timer: %lf\n", t.elapsed());

    t.restart();

    for (int idx = 0; idx < MAXSTRINGS; idx++)
    {
        handrolled_test(Lines[idx]);
    }

    printf("Handrolled Timer: %lf\n", t.elapsed());

    t.restart();

    for (int idx = 0; idx < MAXSTRINGS; idx++)
    {
        fcount = handrolled_test2(Lines[idx], Fields);
    }

    printf("Handrolled2 Timer: %lf\n", t.elapsed());

    printf("\n");
    return 0;
}