Merge branch 'develop-1.1' into 1.1-merge-up-20180509-a2

dbcon/ddlpackage/ddl-gram-win.cpp

@@ -857,7 +857,7 @@ static const yytype_int16 yydefgoto[] =
         254,   255,   167,   168,   219,   220,   169,   256,   257,   152,
         130,   131,   132,   133,   178,   172,   134,   190,   304,   135,
         185,   237,    51,   145,    52,    71
-};
+    };
 
 /* YYPACT[STATE-NUM] -- Index in YYTABLE of the portion describing
    STATE-NUM.  */
@@ -917,7 +917,7 @@ static const yytype_int16 yypgoto[] =
         -179,  -223,  -157,  -270,  -270,  -270,   216,    90,    -5,  -202,
         -270,  -270,  -270,  -270,   161,   116,  -270,   166,  -270,  -270,
         -270,  -270,  -270,   119,  -270,  -270
-};
+    };
 
 /* YYTABLE[YYPACT[STATE-NUM]].  What to do in state STATE-NUM.  If
    positive, shift that token.  If negative, reduce the rule which

dbcon/ddlpackage/ddl.l

@@ -18,6 +18,7 @@
 
 /* $Id: ddl.l 9341 2013-03-27 14:10:35Z chao $ */
 %{
+#include <string.h>
 #include <iostream>
 #include <vector>
 #include <stdio.h>
@@ -31,7 +32,7 @@
 #endif
 
 using namespace ddlpackage;
-typedef enum { NOOP, STRIP_QUOTES } copy_action_t;
+typedef enum { NOOP, STRIP_QUOTES, STRIP_QUOTES_FQ } copy_action_t;
 int lineno = 1;
 void ddlerror(struct pass_to_bison* x, char const *s);
 
@@ -71,8 +72,10 @@ identifier {ident_start}{ident_cont}*
 extended_identifier {ident_start}{extended_ident_cont}*
 /* fully qualified names regexes */
 fq_identifier {identifier}\.{identifier}
-identifer_quoted {grave_accent}{extended_identifier}{grave_accent}
+identifier_quoted {grave_accent}{extended_identifier}{grave_accent}
-identifer_double_quoted {double_quote}{extended_identifier}{double_quote}
+identifier_double_quoted {double_quote}{extended_identifier}{double_quote}
+fq_quoted ({identifier_quoted}|{extended_identifier})\.({identifier_quoted}|{identifier})
+fq_double_quoted ({identifier_double_quoted}|{extended_identifier})\.({identifier_double_quoted}|{identifier})
 
 integer			[-+]?{digit}+
 decimal			([-+]?({digit}*\.{digit}+)|({digit}+\.{digit}*))
@@ -84,9 +87,11 @@ realfail2		({integer}|{decimal})[Ee][-+]
 %%
 
 
-{identifer_quoted} { ddlget_lval(yyscanner)->str = scanner_copy( ddlget_text(yyscanner), yyscanner, STRIP_QUOTES ); return IDENT; }
+{identifier_quoted} { ddlget_lval(yyscanner)->str = scanner_copy( ddlget_text(yyscanner), yyscanner, STRIP_QUOTES ); return IDENT; }
-{identifer_double_quoted} { ddlget_lval(yyscanner)->str = scanner_copy( ddlget_text(yyscanner), yyscanner, STRIP_QUOTES ); return IDENT; }
+{identifier_double_quoted} { ddlget_lval(yyscanner)->str = scanner_copy( ddlget_text(yyscanner), yyscanner, STRIP_QUOTES ); return IDENT; }
 {fq_identifier} { ddlget_lval(yyscanner)->str = scanner_copy(ddlget_text(yyscanner), yyscanner); return FQ_IDENT; }
+{fq_quoted} { ddlget_lval(yyscanner)->str = scanner_copy(ddlget_text(yyscanner), yyscanner, STRIP_QUOTES_FQ); return FQ_IDENT; }
+{fq_double_quoted} { ddlget_lval(yyscanner)->str = scanner_copy(ddlget_text(yyscanner), yyscanner, STRIP_QUOTES_FQ); return FQ_IDENT; }
 
 ACTION {return ACTION;}
 ADD {return ADD;}
@@ -281,6 +286,57 @@ char* scanner_copy (char *str, yyscan_t yyscanner, copy_action_t action)
         nv[strlen(str) - 1] = '\0';
         result = nv + 1;
     }
+    else if (action == STRIP_QUOTES_FQ)
+    {
+        bool move_left = false;
+        bool move_right = false;
+        char* left = nv;
+        char* tmp_first = nv;
+        // MCOL-1384 Loop through all comas in this quoted fq id
+        // looking for $quote_sign.$quote_sign sequence.
+        char* fq_delimiter;
+        int tmp_pos = 0;
+        while((fq_delimiter = strchr(tmp_first, '.')) != NULL)
+        {
+            if( (*(fq_delimiter -1) == '`' && *(fq_delimiter + 1) == '`') ||
+              (*(fq_delimiter -1) == '"' && *(fq_delimiter + 1) == '"') )
+            {
+                tmp_pos += fq_delimiter - tmp_first;
+                break;
+            }
+            tmp_first = fq_delimiter;
+        }
+
+        char* fq_delimiter_orig = str + tmp_pos;
+        char* right = fq_delimiter + 1;
+        char* right_orig = fq_delimiter_orig + 1;
+        // MCOL-1384 Strip quotes from the left part.
+        if(*left == '"' || *left == '`')
+        {
+            result = left + 1;
+            *(fq_delimiter - 1) = '.';
+            move_left = true;
+        }
+        else
+        {
+            fq_delimiter += 1;
+        }
+
+        int right_length = strlen(right);
+        // MCOL-1384 Strip quotes from the right part.
+        if(*right == '`' || *right == '"')
+        {
+            right += 1; right_orig += 1;
+            right_length -= 2;
+            move_right = true;
+            *(fq_delimiter + right_length) = '\0';
+        }
+
+        if(move_left || move_right)
+        {
+            strncpy(fq_delimiter, right_orig, right_length);
+        }
+    }
 
     return result;
 }

dbcon/dmlpackage/dml-gram-win.cpp

@@ -875,7 +875,7 @@ static const yytype_int16 yydefgoto[] =
         149,   278,   150,   243,   151,   192,   152,    63,    64,    65,
         66,    67,   216,    44,    68,   335,   156,    33,    69,   259,
         340,    80
-};
+    };
 
 /* YYPACT[STATE-NUM] -- Index in YYTABLE of the portion describing
    STATE-NUM.  */
@@ -939,7 +939,7 @@ static const yytype_int16 yypgoto[] =
         -270,   244,  -270,  -270,  -270,     8,   -24,  -270,  -189,    61,
         -270,   140,   -11,   484,   -97,  -270,   -72,     4,    47,  -270,
         449,   443
-};
+    };
 
 /* YYTABLE[YYPACT[STATE-NUM]].  What to do in state STATE-NUM.  If
    positive, shift that token.  If negative, reduce the rule which

dbcon/joblist/jlf_tuplejoblist.cpp

@@ -493,10 +493,11 @@ void adjustLastStep(JobStepVector& querySteps, DeliveredTableMap& deliverySteps,
     TupleAnnexStep* tas = dynamic_cast<TupleAnnexStep*>(jobInfo.annexStep.get());
     tas->setLimit(jobInfo.limitStart, jobInfo.limitCount);
 
- //       if (jobInfo.limitCount != (uint64_t) - 1)
+//       if (jobInfo.limitCount != (uint64_t) - 1)
 //        {
     if (jobInfo.orderByColVec.size() > 0)
         tas->addOrderBy(new LimitedOrderBy());
+
 //        }
 
     if (jobInfo.constantCol == CONST_COL_EXIST)
@@ -504,6 +505,7 @@ void adjustLastStep(JobStepVector& querySteps, DeliveredTableMap& deliverySteps,
 
     if (jobInfo.hasDistinct)
         tas->setDistinct();
+
 //    }
 
     if (jobInfo.annexStep)

dbcon/joblist/limitedorderby.cpp

@@ -76,12 +76,14 @@ void LimitedOrderBy::initialize(const RowGroup& rg, const JobInfo& jobInfo)
     {
         map<uint32_t, uint32_t>::iterator j = keyToIndexMap.find(i->first);
         idbassert(j != keyToIndexMap.end());
+
         // MCOL-1052 Ordering direction in CSEP differs from
         // internal direction representation.
         if (i->second)
             fOrderByCond.push_back(IdbSortSpec(j->second, false));
         else
             fOrderByCond.push_back(IdbSortSpec(j->second, true));
+
         //fOrderByCond.push_back(IdbSortSpec(j->second, i->second));
     }
 

dbcon/mysql/ha_calpont.cpp

@@ -123,8 +123,8 @@ static int calpont_rollback(handlerton* hton, THD* thd, bool all);
 static int calpont_close_connection ( handlerton* hton, THD* thd );
 handlerton* calpont_hton;
 
-static group_by_handler *
+static group_by_handler*
-create_calpont_group_by_handler(THD *thd, Query *query);
+create_calpont_group_by_handler(THD* thd, Query* query);
 
 /* Variables for example share methods */
 
@@ -1151,10 +1151,10 @@ static MYSQL_SYSVAR_ULONG(
  *    group_by_handler if success
  *    NULL in other case
  ***********************************************************/
-static group_by_handler *
+static group_by_handler*
-create_calpont_group_by_handler(THD *thd, Query *query)
+create_calpont_group_by_handler(THD* thd, Query* query)
 {
-    ha_calpont_group_by_handler *handler = NULL;
+    ha_calpont_group_by_handler* handler = NULL;
 
     if ( thd->infinidb_vtable.vtable_state == THD::INFINIDB_DISABLE_VTABLE )
     {

dbcon/mysql/ha_calpont.h

@@ -274,8 +274,8 @@ public:
  ***********************************************************/
 class ha_calpont_group_by_handler: public group_by_handler
 {
-    public:
+public:
-        ha_calpont_group_by_handler(THD *thd_arg, Query *query)
+    ha_calpont_group_by_handler(THD* thd_arg, Query* query)
         : group_by_handler(thd_arg, calpont_hton),
           select(query->select),
           table_list(query->from),
@@ -290,13 +290,13 @@ class ha_calpont_group_by_handler: public group_by_handler
     int next_row();
     int end_scan();
 
-        List<Item> *select;
+    List<Item>* select;
-        TABLE_LIST *table_list;
+    TABLE_LIST* table_list;
     bool        distinct;
-        Item       *where;
+    Item*       where;
-        ORDER      *group_by;
+    ORDER*      group_by;
-        ORDER      *order_by;
+    ORDER*      order_by;
-        Item       *having;
+    Item*       having;
 };
 #endif //HA_CALPONT_H__
 

dbcon/mysql/ha_calpont_ddl.cpp

@@ -174,10 +174,6 @@ uint32_t convertDataType(int dataType)
             calpontDataType = CalpontSystemCatalog::DATETIME;
             break;
 
-        case ddlpackage::DDL_TIME:
-            calpontDataType = CalpontSystemCatalog::TIME;
-            break;
-
         case ddlpackage::DDL_CLOB:
             calpontDataType = CalpontSystemCatalog::CLOB;
             break;
@@ -1872,21 +1868,17 @@ int ProcessDDLStatement(string& ddlStatement, string& schema, const string& tabl
         if (ddlStatement.find("AUTO_INCREMENT") != string::npos)
         {
             thd->raise_error_printf(ER_CHECK_NOT_IMPLEMENTED, "Use of the MySQL auto_increment syntax is not supported in Columnstore. If you wish to create an auto increment column in Columnstore, please consult the Columnstore SQL Syntax Guide for the correct usage.");
-        }
+            ci->alterTableState = cal_connection_info::NOT_ALTER;
-        // MCOL-867. MariaDB RENAME TABLE statement supports WAIT|NOWAIT options since 10.3.0 but Columnstore isn't yet.
+            ci->isAlter = false;
-        else if (ddlStatement.find("WAIT") != string::npos || ddlStatement.find("NOWAIT") != string::npos)
-        {
-            thd->raise_error_printf(ER_CHECK_NOT_IMPLEMENTED, "WAIT and NOWAIT options are not supported in Columnstore. Please consult the Columnstore SQL Syntax Guide for the correct usage.");
         }
         else
         {
             //@Bug 1888,1885. update error message
             thd->raise_error_printf(ER_CHECK_NOT_IMPLEMENTED, "The syntax or the data type(s) is not supported by Columnstore. Please check the Columnstore syntax guide for supported syntax or data types.");
-        }
-
             ci->alterTableState = cal_connection_info::NOT_ALTER;
             ci->isAlter = false;
         }
+    }
 
     return rc;
 }
@@ -2187,7 +2179,6 @@ int ha_calpont_impl_rename_table_(const char* from, const char* to, cal_connecti
     THD* thd = current_thd;
     string emsg;
 
-    ostringstream stmt1;
     pair<string, string> fromPair;
     pair<string, string> toPair;
     string stmt;
@@ -2215,20 +2206,21 @@ int ha_calpont_impl_rename_table_(const char* from, const char* to, cal_connecti
         return -1;
     }
 
-    stmt1 << "alter table " << fromPair.second << " rename to " << toPair.second << ";";
+    // This explicitely shields both db objects with quotes that the lexer strips down later.
-
+    stmt = "alter table `" + fromPair.second + "` rename to `" + toPair.second + "`;";
-    stmt = stmt1.str();
     string db;
 
-    if ( fromPair.first.length() != 0 )
+    if ( thd->db )
-        db = fromPair.first;
-    else if ( thd->db )
         db = thd->db;
+    else if ( fromPair.first.length() != 0 )
+        db = fromPair.first;
+    else
+        db = toPair.first;
 
     int rc = ProcessDDLStatement(stmt, db, "", tid2sid(thd->thread_id), emsg);
 
     if (rc != 0)
-        push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, emsg.c_str());
+        push_warning(thd, Sql_condition::WARN_LEVEL_ERROR, 9999, emsg.c_str());
 
     return rc;
 }

dbcon/mysql/ha_calpont_impl.cpp

@@ -427,7 +427,7 @@ int vbin2hex(const uint8_t* p, const unsigned l, char* o)
     return 0;
 }
 
-int fetchNextRow(uchar* buf, cal_table_info& ti, cal_connection_info* ci, bool handler_flag=false)
+int fetchNextRow(uchar* buf, cal_table_info& ti, cal_connection_info* ci, bool handler_flag = false)
 {
     int rc = HA_ERR_END_OF_FILE;
     int num_attr = ti.msTablePtr->s->fields;
@@ -469,6 +469,7 @@ int fetchNextRow(uchar* buf, cal_table_info& ti, cal_connection_info* ci, bool h
     {
         Field** f;
         f = ti.msTablePtr->field;
+
         //set all fields to null in null col bitmap
         if (!handler_flag)
             memset(buf, -1, ti.msTablePtr->s->null_bytes);
@@ -476,6 +477,7 @@ int fetchNextRow(uchar* buf, cal_table_info& ti, cal_connection_info* ci, bool h
         {
             memset(ti.msTablePtr->null_flags, -1, ti.msTablePtr->s->null_bytes);
         }
+
         std::vector<CalpontSystemCatalog::ColType>& colTypes = ti.tpl_scan_ctx->ctp;
         int64_t intColVal = 0;
         uint64_t uintColVal = 0;
@@ -5274,15 +5276,17 @@ int ha_calpont_impl_group_by_init(ha_calpont_group_by_handler* group_hand, TABLE
             execplan::CalpontSelectExecutionPlan::ColumnMap::iterator condColMapIter;
             execplan::ParseTree* ptIt;
             execplan::ReturnedColumn* rcIt;
-            for(TABLE_LIST* tl = gi.groupByTables; tl; tl=tl->next_local)
+
+            for (TABLE_LIST* tl = gi.groupByTables; tl; tl = tl->next_local)
             {
                 mapiter = ci->tableMap.find(tl->table);
-                if(mapiter != ci->tableMap.end() && mapiter->second.condInfo != NULL 
+
+                if (mapiter != ci->tableMap.end() && mapiter->second.condInfo != NULL
                         && mapiter->second.condInfo->condPush)
                 {
-                    while(!mapiter->second.condInfo->ptWorkStack.empty())
+                    while (!mapiter->second.condInfo->ptWorkStack.empty())
                     {
-                        ptIt=mapiter->second.condInfo->ptWorkStack.top();
+                        ptIt = mapiter->second.condInfo->ptWorkStack.top();
                         mapiter->second.condInfo->ptWorkStack.pop();
                         gi.pushedPts.push_back(ptIt);
                     }
@@ -5316,14 +5320,14 @@ int ha_calpont_impl_group_by_init(ha_calpont_group_by_handler* group_hand, TABLE
             push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, msg.c_str());
         }
 
-    #ifdef PLAN_HEX_FILE
+#ifdef PLAN_HEX_FILE
         // plan serialization
         ifstream ifs("/tmp/li1-plan.hex");
         ByteStream bs1;
         ifs >> bs1;
         ifs.close();
         csep->unserialize(bs1);
-    #endif
+#endif
 
         if (ci->traceFlags & 1)
         {

dbcon/mysql/ha_window_function.cpp

@@ -518,6 +518,7 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n
                 if (!srcp)
                 {
                     orderItem = orderCol->item_ptr;
+
                     if (orderItem)
                     {
                         gwi.fatalParseError = false;

dbcon/mysql/is_columnstore_columns.cpp

@@ -27,6 +27,8 @@
 #include <boost/shared_ptr.hpp>
 #include "calpontsystemcatalog.h"
 #include "dataconvert.h"
+#include "exceptclasses.h"
+using namespace logging;
 
 
 // Required declaration as it isn't in a MairaDB include
@@ -70,7 +72,25 @@ static int is_columnstore_columns_fill(THD* thd, TABLE_LIST* tables, COND* cond)
     for (std::vector<std::pair<execplan::CalpontSystemCatalog::OID, execplan::CalpontSystemCatalog::TableName> >::const_iterator it = catalog_tables.begin();
             it != catalog_tables.end(); ++it)
     {
-        execplan::CalpontSystemCatalog::RIDList column_rid_list = systemCatalogPtr->columnRIDs((*it).second, true);
+        execplan::CalpontSystemCatalog::RIDList column_rid_list;
+
+        // Note a table may get dropped as you iterate over the list of tables.
+        // So simply ignore the dropped table.
+        try
+        {
+            column_rid_list = systemCatalogPtr->columnRIDs((*it).second, true);
+        }
+        catch (IDBExcept& ex)
+        {
+            if (ex.errorCode() == ERR_TABLE_NOT_IN_CATALOG)
+            {
+                continue;
+            }
+            else
+            {
+                return 1;
+            }
+        }
 
         for (size_t col_num = 0; col_num < column_rid_list.size(); col_num++)
         {

oam/install_scripts/syslogSetup.sh

@@ -13,6 +13,7 @@ syslog_conf=nofile
 rsyslog7=0
 
 user=`whoami 2>/dev/null`
+group=user
 
 SUDO=" "
 if [ "$user" != "root" ]; then
@@ -167,9 +168,24 @@ if [ ! -z "$syslog_conf" ] ; then
 		# remove older version incase it was installed by previous build
 		$SUDO rm -rf /etc/rsyslog.d/columnstore.conf
 
+		// determine username/groupname
+		
+		if [ -f /var/log/messages ]; then
+		      user=`stat -c "%U %G" /var/log/messages | awk '{print $1}'`
+		      group=`stat -c "%U %G" /var/log/messages | awk '{print $2}'`
+		fi
+		
+		if [ -f /var/log/syslog ]; then
+		      user=`stat -c "%U %G" /var/log/syslog | awk '{print $1}'`
+		      group=`stat -c "%U %G" /var/log/syslog | awk '{print $2}'`
+		fi
+		
+		//set permissions
+		$SUDO chown $user:$group -R /var/log/mariadb > /dev/null 2>&1
+		
 		if [ $rsyslog7 == 1 ]; then
-			sed -i -e s/groupname/adm/g ${columnstoreSyslogFile7}
+			sed -i -e s/groupname/$group/g ${columnstoreSyslogFile7}
-			sed -i -e s/username/syslog/g ${columnstoreSyslogFile7}
+			sed -i -e s/username/$user/g ${columnstoreSyslogFile7}
 			
 			$SUDO rm -f /etc/rsyslog.d/49-columnstore.conf
 			$SUDO cp  ${columnstoreSyslogFile7} ${syslog_conf}

tools/qfe/bison-win.cpp

@@ -556,7 +556,7 @@ static const yytype_int8 yydefgoto[] =
     -1,     2,     6,     7,     8,     9,    18,    19,    23,    24,
         25,    33,    36,    10,    11,    52,    39,    45,    46,    48,
         61,    56
-};
+    };
 
 /* YYPACT[STATE-NUM] -- Index in YYTABLE of the portion describing
    STATE-NUM.  */
@@ -578,7 +578,7 @@ static const yytype_int8 yypgoto[] =
     -20,   -20,   -20,    37,   -20,    32,    36,   -20,   -20,   -20,
         -20,   -20,   -20,   -15,   -19,   -10,   -20,     6,    -1,   -20,
         -20,   -20
-};
+    };
 
 /* YYTABLE[YYPACT[STATE-NUM]].  What to do in state STATE-NUM.  If
    positive, shift that token.  If negative, reduce the rule which

utils/rowgroup/rowgroup.cpp

@@ -82,10 +82,10 @@ StringStore::~StringStore()
 #endif
 }
 
-uint32_t StringStore::storeString(const uint8_t* data, uint32_t len)
+uint64_t StringStore::storeString(const uint8_t* data, uint32_t len)
 {
     MemChunk* lastMC = NULL;
-    uint32_t ret = 0;
+    uint64_t ret = 0;
 
     empty = false;   // At least a NULL is being stored.
 
@@ -95,7 +95,7 @@ uint32_t StringStore::storeString(const uint8_t* data, uint32_t len)
 
     if ((len == 8 || len == 9) &&
             *((uint64_t*) data) == *((uint64_t*) joblist::CPNULLSTRMARK.c_str()))
-        return numeric_limits<uint32_t>::max();
+        return numeric_limits<uint64_t>::max();
 
     //@bug6065, make StringStore::storeString() thread safe
     boost::mutex::scoped_lock lk(fMutex, defer_lock);
@@ -106,20 +106,21 @@ uint32_t StringStore::storeString(const uint8_t* data, uint32_t len)
     if (mem.size() > 0)
         lastMC = (MemChunk*) mem.back().get();
 
-    if (len >= CHUNK_SIZE)
+    if ((len + 4) >= CHUNK_SIZE)
     {
-        shared_array<uint8_t> newOne(new uint8_t[len + sizeof(MemChunk)]);
+        shared_array<uint8_t> newOne(new uint8_t[len + sizeof(MemChunk) + 4]);
         longStrings.push_back(newOne);
         lastMC = (MemChunk*) longStrings.back().get();
-        lastMC->capacity = lastMC->currentSize = len;
+        lastMC->capacity = lastMC->currentSize = len + 4;
-        memcpy(lastMC->data, data, len);
+        memcpy(lastMC->data, &len, 4);
+        memcpy(lastMC->data + 4, data, len);
         // High bit to mark a long string
-        ret = 0x80000000;
+        ret = 0x8000000000000000;
         ret += longStrings.size() - 1;
     }
     else
     {
-        if ((lastMC == NULL) || (lastMC->capacity - lastMC->currentSize < len))
+        if ((lastMC == NULL) || (lastMC->capacity - lastMC->currentSize < (len + 4)))
         {
             // mem usage debugging
             //if (lastMC)
@@ -134,7 +135,13 @@ uint32_t StringStore::storeString(const uint8_t* data, uint32_t len)
 
 
         ret = ((mem.size() - 1) * CHUNK_SIZE) + lastMC->currentSize;
-        memcpy(&(lastMC->data[lastMC->currentSize]), data, len);
+
+        // If this ever happens then we have big problems
+        if (ret & 0x8000000000000000)
+            throw logic_error("StringStore memory exceeded.");
+
+        memcpy(&(lastMC->data[lastMC->currentSize]), &len, 4);
+        memcpy(&(lastMC->data[lastMC->currentSize]) + 4, data, len);
         /*
         cout << "stored: '" << hex;
         for (uint32_t i = 0; i < len ; i++) {
@@ -142,7 +149,7 @@ uint32_t StringStore::storeString(const uint8_t* data, uint32_t len)
         }
         cout << "' at position " << lastMC->currentSize << " len " << len << dec << endl;
         */
-        lastMC->currentSize += len;
+        lastMC->currentSize += len + 4;
     }
 
     return ret;
@@ -150,35 +157,35 @@ uint32_t StringStore::storeString(const uint8_t* data, uint32_t len)
 
 void StringStore::serialize(ByteStream& bs) const
 {
-    uint32_t i;
+    uint64_t i;
     MemChunk* mc;
 
-    bs << (uint32_t) mem.size();
+    bs << (uint64_t) mem.size();
     bs << (uint8_t) empty;
 
     for (i = 0; i < mem.size(); i++)
     {
         mc = (MemChunk*) mem[i].get();
-        bs << (uint32_t) mc->currentSize;
+        bs << (uint64_t) mc->currentSize;
         //cout << "serialized " << mc->currentSize << " bytes\n";
         bs.append(mc->data, mc->currentSize);
     }
 
-    bs << (uint32_t) longStrings.size();
+    bs << (uint64_t) longStrings.size();
 
     for (i = 0; i < longStrings.size(); i++)
     {
         mc = (MemChunk*) longStrings[i].get();
-        bs << (uint32_t) mc->currentSize;
+        bs << (uint64_t) mc->currentSize;
         bs.append(mc->data, mc->currentSize);
     }
 }
 
 void StringStore::deserialize(ByteStream& bs)
 {
-    uint32_t i;
+    uint64_t i;
-    uint32_t count;
+    uint64_t count;
-    uint32_t size;
+    uint64_t size;
     uint8_t* buf;
     MemChunk* mc;
     uint8_t tmp8;
@@ -856,10 +863,9 @@ bool Row::isNullValue(uint32_t colIndex) const
 
             if (inStringTable(colIndex))
             {
-                uint32_t offset, length;
+                uint64_t offset;
-                offset = *((uint32_t*) &data[offsets[colIndex]]);
+                offset = *((uint64_t*) &data[offsets[colIndex]]);
-                length = *((uint32_t*) &data[offsets[colIndex] + 4]);
+                return strings->isNullValue(offset);
-                return strings->isNullValue(offset, length);
             }
 
             if (data[offsets[colIndex]] == 0)   // empty string
@@ -922,10 +928,9 @@ bool Row::isNullValue(uint32_t colIndex) const
 
             if (inStringTable(colIndex))
             {
-                uint32_t offset, length;
+                uint64_t offset;
-                offset = *((uint32_t*) &data[pos]);
+                offset = *((uint64_t*) &data[pos]);
-                length = *((uint32_t*) &data[pos + 4]);
+                return strings->isNullValue(offset);
-                return strings->isNullValue(offset, length);
             }
 
             if (*((uint16_t*) &data[pos]) == 0)
@@ -1720,8 +1725,8 @@ RGData RowGroup::duplicate()
 
 void Row::setStringField(const std::string& val, uint32_t colIndex)
 {
-    uint32_t length;
+    uint64_t offset;
-    uint32_t offset;
+    uint64_t length;
 
     //length = strlen(val.c_str()) + 1;
     length = val.length();
@@ -1732,8 +1737,7 @@ void Row::setStringField(const std::string& val, uint32_t colIndex)
     if (inStringTable(colIndex))
     {
         offset = strings->storeString((const uint8_t*) val.data(), length);
-        *((uint32_t*) &data[offsets[colIndex]]) = offset;
+        *((uint64_t*) &data[offsets[colIndex]]) = offset;
-        *((uint32_t*) &data[offsets[colIndex] + 4]) = length;
 //		cout << " -- stored offset " << *((uint32_t *) &data[offsets[colIndex]])
 //				<< " length " << *((uint32_t *) &data[offsets[colIndex] + 4])
 //				<< endl;

utils/rowgroup/rowgroup.h

@@ -92,13 +92,14 @@ public:
     StringStore();
     virtual ~StringStore();
 
-    inline std::string getString(uint32_t offset, uint32_t length) const;
+    inline std::string getString(uint64_t offset) const;
-    uint32_t storeString(const uint8_t* data, uint32_t length);  //returns the offset
+    uint64_t storeString(const uint8_t* data, uint32_t length);  //returns the offset
-    inline const uint8_t* getPointer(uint32_t offset) const;
+    inline const uint8_t* getPointer(uint64_t offset) const;
+    inline uint32_t getStringLength(uint64_t offset);
     inline bool isEmpty() const;
     inline uint64_t getSize() const;
-    inline bool isNullValue(uint32_t offset, uint32_t length) const;
+    inline bool isNullValue(uint64_t offset) const;
-    inline bool equals(const std::string& str, uint32_t offset, uint32_t length) const;
+    inline bool equals(const std::string& str, uint64_t offset) const;
 
     void clear();
 
@@ -615,9 +616,8 @@ inline bool Row::equals(const std::string& val, uint32_t colIndex) const
 {
     if (inStringTable(colIndex))
     {
-        uint32_t offset = *((uint32_t*) &data[offsets[colIndex]]);
+        uint64_t offset = *((uint64_t*) &data[offsets[colIndex]]);
-        uint32_t length = *((uint32_t*) &data[offsets[colIndex] + 4]);
+        return strings->equals(val, offset);
-        return strings->equals(val, offset, length);
     }
     else
         return (strncmp(val.c_str(), (char*) &data[offsets[colIndex]], getColumnWidth(colIndex)) == 0);
@@ -719,7 +719,7 @@ inline int64_t Row::getIntField(uint32_t colIndex) const
 inline const uint8_t* Row::getStringPointer(uint32_t colIndex) const
 {
     if (inStringTable(colIndex))
-        return strings->getPointer(*((uint32_t*) &data[offsets[colIndex]]));
+        return strings->getPointer(*((uint64_t*) &data[offsets[colIndex]]));
 
     return &data[offsets[colIndex]];
 }
@@ -727,14 +727,14 @@ inline const uint8_t* Row::getStringPointer(uint32_t colIndex) const
 inline uint32_t Row::getStringLength(uint32_t colIndex) const
 {
     if (inStringTable(colIndex))
-        return *((uint32_t*) &data[offsets[colIndex] + 4]);
+        return strings->getStringLength(*((uint64_t*) &data[offsets[colIndex]]));
 
     return strnlen((char*) &data[offsets[colIndex]], getColumnWidth(colIndex));
 }
 
 inline void Row::setStringField(const uint8_t* strdata, uint32_t length, uint32_t colIndex)
 {
-    uint32_t offset;
+    uint64_t offset;
 
     if (length > getColumnWidth(colIndex))
         length = getColumnWidth(colIndex);
@@ -742,8 +742,7 @@ inline void Row::setStringField(const uint8_t* strdata, uint32_t length, uint32_
     if (inStringTable(colIndex))
     {
         offset = strings->storeString(strdata, length);
-        *((uint32_t*) &data[offsets[colIndex]]) = offset;
+        *((uint64_t*) &data[offsets[colIndex]]) = offset;
-        *((uint32_t*) &data[offsets[colIndex] + 4]) = length;
 //		cout << " -- stored offset " << *((uint32_t *) &data[offsets[colIndex]])
 //				<< " length " << *((uint32_t *) &data[offsets[colIndex] + 4])
 //				<< endl;
@@ -759,8 +758,7 @@ inline void Row::setStringField(const uint8_t* strdata, uint32_t length, uint32_
 inline std::string Row::getStringField(uint32_t colIndex) const
 {
     if (inStringTable(colIndex))
-        return strings->getString(*((uint32_t*) &data[offsets[colIndex]]),
+        return strings->getString(*((uint64_t*) &data[offsets[colIndex]]));
-                                  *((uint32_t*) &data[offsets[colIndex] + 4]));
 
     // Not all CHAR/VARCHAR are NUL terminated so use length
     return std::string((char*) &data[offsets[colIndex]],
@@ -778,7 +776,7 @@ inline std::string Row::getVarBinaryStringField(uint32_t colIndex) const
 inline uint32_t Row::getVarBinaryLength(uint32_t colIndex) const
 {
     if (inStringTable(colIndex))
-        return *((uint32_t*) &data[offsets[colIndex] + 4]);
+        return strings->getStringLength(*((uint64_t*) &data[offsets[colIndex]]));;
 
     return *((uint16_t*) &data[offsets[colIndex]]);
 }
@@ -786,7 +784,7 @@ inline uint32_t Row::getVarBinaryLength(uint32_t colIndex) const
 inline const uint8_t* Row::getVarBinaryField(uint32_t colIndex) const
 {
     if (inStringTable(colIndex))
-        return strings->getPointer(*((uint32_t*) &data[offsets[colIndex]]));
+        return strings->getPointer(*((uint64_t*) &data[offsets[colIndex]]));
 
     return &data[offsets[colIndex] + 2];
 }
@@ -795,7 +793,7 @@ inline const uint8_t* Row::getVarBinaryField(uint32_t& len, uint32_t colIndex) c
 {
     if (inStringTable(colIndex))
     {
-        len = *((uint32_t*) &data[offsets[colIndex] + 4]);
+        len = strings->getStringLength(*((uint64_t*) &data[offsets[colIndex]]));
         return getVarBinaryField(colIndex);
     }
     else
@@ -1043,9 +1041,8 @@ inline void Row::setVarBinaryField(const uint8_t* val, uint32_t len, uint32_t co
 
     if (inStringTable(colIndex))
     {
-        uint32_t offset = strings->storeString(val, len);
+        uint64_t offset = strings->storeString(val, len);
-        *((uint32_t*) &data[offsets[colIndex]]) = offset;
+        *((uint64_t*) &data[offsets[colIndex]]) = offset;
-        *((uint32_t*) &data[offsets[colIndex] + 4]) = len;
     }
     else
     {
@@ -1762,26 +1759,29 @@ inline void copyRow(const Row& in, Row* out)
     copyRow(in, out, std::min(in.getColumnCount(), out->getColumnCount()));
 }
 
-inline std::string StringStore::getString(uint32_t off, uint32_t len) const
+inline std::string StringStore::getString(uint64_t off) const
 {
-    if (off == std::numeric_limits<uint32_t>::max())
+    uint32_t length;
+
+    if (off == std::numeric_limits<uint64_t>::max())
         return joblist::CPNULLSTRMARK;
 
     MemChunk* mc;
 
-    if (off & 0x80000000)
+    if (off & 0x8000000000000000)
     {
-        off = off - 0x80000000;
+        off = off - 0x8000000000000000;
 
         if (longStrings.size() <= off)
             return joblist::CPNULLSTRMARK;
 
         mc = (MemChunk*) longStrings[off].get();
-        return std::string((char*) mc->data, len);
+        memcpy(&length, mc->data, 4);
+        return std::string((char*) mc->data + 4, length);
     }
 
-    uint32_t chunk = off / CHUNK_SIZE;
+    uint64_t chunk = off / CHUNK_SIZE;
-    uint32_t offset = off % CHUNK_SIZE;
+    uint64_t offset = off % CHUNK_SIZE;
 
     // this has to handle uninitialized data as well.  If it's uninitialized it doesn't matter
     // what gets returned, it just can't go out of bounds.
@@ -1790,30 +1790,32 @@ inline std::string StringStore::getString(uint32_t off, uint32_t len) const
 
     mc = (MemChunk*) mem[chunk].get();
 
-    if ((offset + len) > mc->currentSize)
+    memcpy(&length, &mc->data[offset], 4);
+
+    if ((offset + length) > mc->currentSize)
         return joblist::CPNULLSTRMARK;
 
-    return std::string((char*) & (mc->data[offset]), len);
+    return std::string((char*) & (mc->data[offset]) + 4, length);
 }
 
-inline const uint8_t* StringStore::getPointer(uint32_t off) const
+inline const uint8_t* StringStore::getPointer(uint64_t off) const
 {
-    if (off == std::numeric_limits<uint32_t>::max())
+    if (off == std::numeric_limits<uint64_t>::max())
         return (const uint8_t*) joblist::CPNULLSTRMARK.c_str();
 
-    uint32_t chunk = off / CHUNK_SIZE;
+    uint64_t chunk = off / CHUNK_SIZE;
-    uint32_t offset = off % CHUNK_SIZE;
+    uint64_t offset = off % CHUNK_SIZE;
     MemChunk* mc;
 
-    if (off & 0x80000000)
+    if (off & 0x8000000000000000)
     {
-        off = off - 0x80000000;
+        off = off - 0x8000000000000000;
 
         if (longStrings.size() <= off)
             return (const uint8_t*) joblist::CPNULLSTRMARK.c_str();
 
         mc = (MemChunk*) longStrings[off].get();
-        return mc->data;
+        return mc->data + 4;
     }
 
     // this has to handle uninitialized data as well.  If it's uninitialized it doesn't matter
@@ -1826,19 +1828,18 @@ inline const uint8_t* StringStore::getPointer(uint32_t off) const
     if (offset > mc->currentSize)
         return (const uint8_t*) joblist::CPNULLSTRMARK.c_str();
 
-    return &(mc->data[offset]);
+    return &(mc->data[offset]) + 4;
 }
 
-inline bool StringStore::isNullValue(uint32_t off, uint32_t len) const
+inline bool StringStore::isNullValue(uint64_t off) const
 {
-    if (off == std::numeric_limits<uint32_t>::max() || len == 0)
+    uint32_t length;
+
+    if (off == std::numeric_limits<uint64_t>::max())
         return true;
 
-    if (len < 8)
-        return false;
-
     // Long strings won't be NULL
-    if (off & 0x80000000)
+    if (off & 0x8000000000000000)
         return false;
 
     uint32_t chunk = off / CHUNK_SIZE;
@@ -1849,35 +1850,46 @@ inline bool StringStore::isNullValue(uint32_t off, uint32_t len) const
         return true;
 
     mc = (MemChunk*) mem[chunk].get();
+    memcpy(&length, &mc->data[offset], 4);
 
-    if ((offset + len) > mc->currentSize)
+    if (length == 0)
         return true;
 
-    if (mc->data[offset] == 0)    // "" = NULL string for some reason...
+    if (length < 8)
+        return false;
+
+    if ((offset + length) > mc->currentSize)
         return true;
 
-    return (*((uint64_t*) &mc->data[offset]) == *((uint64_t*) joblist::CPNULLSTRMARK.c_str()));
+    if (mc->data[offset + 4] == 0)  // "" = NULL string for some reason...
+        return true;
+
+    return (*((uint64_t*) &mc->data[offset] + 4) == *((uint64_t*) joblist::CPNULLSTRMARK.c_str()));
 }
 
-inline bool StringStore::equals(const std::string& str, uint32_t off, uint32_t len) const
+inline bool StringStore::equals(const std::string& str, uint64_t off) const
 {
-    if (off == std::numeric_limits<uint32_t>::max() || len == 0)
+    uint32_t length;
+
+    if (off == std::numeric_limits<uint64_t>::max())
         return str == joblist::CPNULLSTRMARK;
 
     MemChunk* mc;
 
-    if (off & 0x80000000)
+    if (off & 0x8000000000000000)
     {
-        if (longStrings.size() <= (off - 0x80000000))
+        if (longStrings.size() <= (off - 0x8000000000000000))
             return false;
 
-        mc = (MemChunk*) longStrings[off - 0x80000000].get();
+        mc = (MemChunk*) longStrings[off - 0x8000000000000000].get();
+
+        memcpy(&length, mc->data, 4);
 
         // Not sure if this check it needed, but adds safety
-        if (len > mc->currentSize)
+        if (length > mc->currentSize)
             return false;
 
-        return (strncmp(str.c_str(), (const char*) mc->data, len) == 0);
+        return (strncmp(str.c_str(), (const char*) mc->data + 4, length) == 0);
     }
 
     uint32_t chunk = off / CHUNK_SIZE;
@@ -1887,11 +1899,44 @@ inline bool StringStore::equals(const std::string& str, uint32_t off, uint32_t l
         return false;
 
     mc = (MemChunk*) mem[chunk].get();
+    memcpy(&length, &mc->data[offset], 4);
 
-    if ((offset + len) > mc->currentSize)
+    if ((offset + length) > mc->currentSize)
         return false;
 
-    return (strncmp(str.c_str(), (const char*) &mc->data[offset], len) == 0);
+    return (strncmp(str.c_str(), (const char*) &mc->data[offset] + 4, length) == 0);
+}
+inline uint32_t StringStore::getStringLength(uint64_t off)
+{
+    uint32_t length;
+    MemChunk* mc;
+
+    if (off == std::numeric_limits<uint64_t>::max())
+        return 0;
+
+    if (off & 0x8000000000000000)
+    {
+        off = off - 0x8000000000000000;
+
+        if (longStrings.size() <= off)
+            return 0;
+
+        mc = (MemChunk*) longStrings[off].get();
+        memcpy(&length, mc->data, 4);
+    }
+    else
+    {
+        uint64_t chunk = off / CHUNK_SIZE;
+        uint64_t offset = off % CHUNK_SIZE;
+
+        if (mem.size() <= chunk)
+            return 0;
+
+        mc = (MemChunk*) mem[chunk].get();
+        memcpy(&length, &mc->data[offset], 4);
+    }
+
+    return length;
 }
 
 inline bool StringStore::isEmpty() const

writeengine/bulk/we_bulkloadbuffer.cpp

@@ -1490,7 +1490,7 @@ int  BulkLoadBuffer::parseCol(ColumnInfo& columnInfo)
         }
 
         // create a buffer for the size of the rows being written.
-        unsigned char* buf = new unsigned char[fTotalReadRowsParser *
+        unsigned char* buf = new unsigned char[fTotalReadRowsParser*
                                                columnInfo.column.width];
         char*  field = new char[MAX_FIELD_SIZE + 1];