database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-11-24 08:41:09 +03:00
Code Activity

Merge branch 'develop-1.1' into 1.1-merge-up-20180509-a2

Browse Source
This commit is contained in:
Andrew Hutchings
2018-05-09 09:20:55 +01:00
parent 24b8c790d5 3227421dea
commit 1a582eed4a
20 changed files with 3430 additions and 3306 deletions
Download Patch File Download Diff File Expand all files Collapse all files

66
dbcon/ddlpackage/ddl.l
View File

@@ -18,6 +18,7 @@
/* $Id: ddl.l 9341 2013-03-27 14:10:35Z chao $ */
%{
#include <string.h>
#include <iostream>
#include <vector>
#include <stdio.h>
@@ -31,7 +32,7 @@
#endif
using namespace ddlpackage;
typedef enum { NOOP, STRIP_QUOTES } copy_action_t;
typedef enum { NOOP, STRIP_QUOTES, STRIP_QUOTES_FQ } copy_action_t;
int lineno = 1;
void ddlerror(struct pass_to_bison* x, char const *s);
@@ -71,8 +72,10 @@ identifier {ident_start}{ident_cont}*
extended_identifier {ident_start}{extended_ident_cont}*
/* fully qualified names regexes */
fq_identifier {identifier}\.{identifier}
identifer_quoted {grave_accent}{extended_identifier}{grave_accent}
identifer_double_quoted {double_quote}{extended_identifier}{double_quote}
identifier_quoted {grave_accent}{extended_identifier}{grave_accent}
identifier_double_quoted {double_quote}{extended_identifier}{double_quote}
fq_quoted ({identifier_quoted}|{extended_identifier})\.({identifier_quoted}|{identifier})
fq_double_quoted ({identifier_double_quoted}|{extended_identifier})\.({identifier_double_quoted}|{identifier})
integer [-+]?{digit}+
decimal ([-+]?({digit}*\.{digit}+)|({digit}+\.{digit}*))
@@ -84,9 +87,11 @@ realfail2 ({integer}|{decimal})[Ee][-+]
%%
{identifer_quoted} { ddlget_lval(yyscanner)->str = scanner_copy( ddlget_text(yyscanner), yyscanner, STRIP_QUOTES ); return IDENT; }
{identifer_double_quoted} { ddlget_lval(yyscanner)->str = scanner_copy( ddlget_text(yyscanner), yyscanner, STRIP_QUOTES ); return IDENT; }
{identifier_quoted} { ddlget_lval(yyscanner)->str = scanner_copy( ddlget_text(yyscanner), yyscanner, STRIP_QUOTES ); return IDENT; }
{identifier_double_quoted} { ddlget_lval(yyscanner)->str = scanner_copy( ddlget_text(yyscanner), yyscanner, STRIP_QUOTES ); return IDENT; }
{fq_identifier} { ddlget_lval(yyscanner)->str = scanner_copy(ddlget_text(yyscanner), yyscanner); return FQ_IDENT; }
{fq_quoted} { ddlget_lval(yyscanner)->str = scanner_copy(ddlget_text(yyscanner), yyscanner, STRIP_QUOTES_FQ); return FQ_IDENT; }
{fq_double_quoted} { ddlget_lval(yyscanner)->str = scanner_copy(ddlget_text(yyscanner), yyscanner, STRIP_QUOTES_FQ); return FQ_IDENT; }
ACTION {return ACTION;}
ADD {return ADD;}
@@ -281,6 +286,57 @@ char* scanner_copy (char *str, yyscan_t yyscanner, copy_action_t action)
nv[strlen(str) - 1] = '\0';
result = nv + 1;
}
else if (action == STRIP_QUOTES_FQ)
{
bool move_left = false;
bool move_right = false;
char* left = nv;
char* tmp_first = nv;
// MCOL-1384 Loop through all comas in this quoted fq id
// looking for $quote_sign.$quote_sign sequence.
char* fq_delimiter;
int tmp_pos = 0;
while((fq_delimiter = strchr(tmp_first, '.')) != NULL)
{
if( (*(fq_delimiter -1) == '`' && *(fq_delimiter + 1) == '`') ||
(*(fq_delimiter -1) == '"' && *(fq_delimiter + 1) == '"') )
{
tmp_pos += fq_delimiter - tmp_first;
break;
}
tmp_first = fq_delimiter;
}
char* fq_delimiter_orig = str + tmp_pos;
char* right = fq_delimiter + 1;
char* right_orig = fq_delimiter_orig + 1;
// MCOL-1384 Strip quotes from the left part.
if(*left == '"' || *left == '`')
{
result = left + 1;
*(fq_delimiter - 1) = '.';
move_left = true;
}
else
{
fq_delimiter += 1;
}
int right_length = strlen(right);
// MCOL-1384 Strip quotes from the right part.
if(*right == '`' || *right == '"')
{
right += 1; right_orig += 1;
right_length -= 2;
move_right = true;
*(fq_delimiter + right_length) = '\0';
}
if(move_left || move_right)
{
strncpy(fq_delimiter, right_orig, right_length);
}
}
return result;
}

2
dbcon/joblist/jlf_tuplejoblist.cpp
View File

@@ -497,6 +497,7 @@ void adjustLastStep(JobStepVector& querySteps, DeliveredTableMap& deliverySteps,
// {
if (jobInfo.orderByColVec.size() > 0)
tas->addOrderBy(new LimitedOrderBy());
// }
if (jobInfo.constantCol == CONST_COL_EXIST)
@@ -504,6 +505,7 @@ void adjustLastStep(JobStepVector& querySteps, DeliveredTableMap& deliverySteps,
if (jobInfo.hasDistinct)
tas->setDistinct();
// }
if (jobInfo.annexStep)

2
dbcon/joblist/limitedorderby.cpp
View File

@@ -76,12 +76,14 @@ void LimitedOrderBy::initialize(const RowGroup& rg, const JobInfo& jobInfo)
{
map<uint32_t, uint32_t>::iterator j = keyToIndexMap.find(i->first);
idbassert(j != keyToIndexMap.end());
// MCOL-1052 Ordering direction in CSEP differs from
// internal direction representation.
if (i->second)
fOrderByCond.push_back(IdbSortSpec(j->second, false));
else
fOrderByCond.push_back(IdbSortSpec(j->second, true));
//fOrderByCond.push_back(IdbSortSpec(j->second, i->second));
}

30
dbcon/mysql/ha_calpont_ddl.cpp
View File

@@ -174,10 +174,6 @@ uint32_t convertDataType(int dataType)
calpontDataType = CalpontSystemCatalog::DATETIME;
break;
case ddlpackage::DDL_TIME:
calpontDataType = CalpontSystemCatalog::TIME;
break;
case ddlpackage::DDL_CLOB:
calpontDataType = CalpontSystemCatalog::CLOB;
break;
@@ -1872,21 +1868,17 @@ int ProcessDDLStatement(string& ddlStatement, string& schema, const string& tabl
if (ddlStatement.find("AUTO_INCREMENT") != string::npos)
{
thd->raise_error_printf(ER_CHECK_NOT_IMPLEMENTED, "Use of the MySQL auto_increment syntax is not supported in Columnstore. If you wish to create an auto increment column in Columnstore, please consult the Columnstore SQL Syntax Guide for the correct usage.");
}
// MCOL-867. MariaDB RENAME TABLE statement supports WAIT|NOWAIT options since 10.3.0 but Columnstore isn't yet.
else if (ddlStatement.find("WAIT") != string::npos || ddlStatement.find("NOWAIT") != string::npos)
{
thd->raise_error_printf(ER_CHECK_NOT_IMPLEMENTED, "WAIT and NOWAIT options are not supported in Columnstore. Please consult the Columnstore SQL Syntax Guide for the correct usage.");
ci->alterTableState = cal_connection_info::NOT_ALTER;
ci->isAlter = false;
}
else
{
//@Bug 1888,1885. update error message
thd->raise_error_printf(ER_CHECK_NOT_IMPLEMENTED, "The syntax or the data type(s) is not supported by Columnstore. Please check the Columnstore syntax guide for supported syntax or data types.");
}
ci->alterTableState = cal_connection_info::NOT_ALTER;
ci->isAlter = false;
}
}
return rc;
}
@@ -2187,7 +2179,6 @@ int ha_calpont_impl_rename_table_(const char* from, const char* to, cal_connecti
THD* thd = current_thd;
string emsg;
ostringstream stmt1;
pair<string, string> fromPair;
pair<string, string> toPair;
string stmt;
@@ -2215,20 +2206,21 @@ int ha_calpont_impl_rename_table_(const char* from, const char* to, cal_connecti
return -1;
}
stmt1 << "alter table " << fromPair.second << " rename to " << toPair.second << ";";
stmt = stmt1.str();
// This explicitely shields both db objects with quotes that the lexer strips down later.
stmt = "alter table `" + fromPair.second + "` rename to `" + toPair.second + "`;";
string db;
if ( fromPair.first.length() != 0 )
db = fromPair.first;
else if ( thd->db )
if ( thd->db )
db = thd->db;
else if ( fromPair.first.length() != 0 )
db = fromPair.first;
else
db = toPair.first;
int rc = ProcessDDLStatement(stmt, db, "", tid2sid(thd->thread_id), emsg);
if (rc != 0)
push_warning(thd, Sql_condition::WARN_LEVEL_WARN, 9999, emsg.c_str());
push_warning(thd, Sql_condition::WARN_LEVEL_ERROR, 9999, emsg.c_str());
return rc;
}

368
dbcon/mysql/ha_calpont_execplan.cpp
View File

File diff suppressed because it is too large Load Diff

4
dbcon/mysql/ha_calpont_impl.cpp
View File

@@ -469,6 +469,7 @@ int fetchNextRow(uchar* buf, cal_table_info& ti, cal_connection_info* ci, bool h
{
Field** f;
f = ti.msTablePtr->field;
//set all fields to null in null col bitmap
if (!handler_flag)
memset(buf, -1, ti.msTablePtr->s->null_bytes);
@@ -476,6 +477,7 @@ int fetchNextRow(uchar* buf, cal_table_info& ti, cal_connection_info* ci, bool h
{
memset(ti.msTablePtr->null_flags, -1, ti.msTablePtr->s->null_bytes);
}
std::vector<CalpontSystemCatalog::ColType>& colTypes = ti.tpl_scan_ctx->ctp;
int64_t intColVal = 0;
uint64_t uintColVal = 0;
@@ -5274,9 +5276,11 @@ int ha_calpont_impl_group_by_init(ha_calpont_group_by_handler* group_hand, TABLE
execplan::CalpontSelectExecutionPlan::ColumnMap::iterator condColMapIter;
execplan::ParseTree* ptIt;
execplan::ReturnedColumn* rcIt;
for (TABLE_LIST* tl = gi.groupByTables; tl; tl = tl->next_local)
{
mapiter = ci->tableMap.find(tl->table);
if (mapiter != ci->tableMap.end() && mapiter->second.condInfo != NULL
&& mapiter->second.condInfo->condPush)
{

1
dbcon/mysql/ha_window_function.cpp
View File

@@ -518,6 +518,7 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n
if (!srcp)
{
orderItem = orderCol->item_ptr;
if (orderItem)
{
gwi.fatalParseError = false;

22
dbcon/mysql/is_columnstore_columns.cpp
View File

@@ -27,6 +27,8 @@
#include <boost/shared_ptr.hpp>
#include "calpontsystemcatalog.h"
#include "dataconvert.h"
#include "exceptclasses.h"
using namespace logging;
// Required declaration as it isn't in a MairaDB include
@@ -70,7 +72,25 @@ static int is_columnstore_columns_fill(THD* thd, TABLE_LIST* tables, COND* cond)
for (std::vector<std::pair<execplan::CalpontSystemCatalog::OID, execplan::CalpontSystemCatalog::TableName> >::const_iterator it = catalog_tables.begin();
it != catalog_tables.end(); ++it)
{
execplan::CalpontSystemCatalog::RIDList column_rid_list = systemCatalogPtr->columnRIDs((*it).second, true);
execplan::CalpontSystemCatalog::RIDList column_rid_list;
// Note a table may get dropped as you iterate over the list of tables.
// So simply ignore the dropped table.
try
{
column_rid_list = systemCatalogPtr->columnRIDs((*it).second, true);
}
catch (IDBExcept& ex)
{
if (ex.errorCode() == ERR_TABLE_NOT_IN_CATALOG)
{
continue;
}
else
{
return 1;
}
}
for (size_t col_num = 0; col_num < column_rid_list.size(); col_num++)
{

20
oam/install_scripts/syslogSetup.sh
View File

@@ -13,6 +13,7 @@ syslog_conf=nofile
rsyslog7=0
user=`whoami 2>/dev/null`
group=user
SUDO=" "
if [ "$user" != "root" ]; then
@@ -167,9 +168,24 @@ if [ ! -z "$syslog_conf" ] ; then
# remove older version incase it was installed by previous build
$SUDO rm -rf /etc/rsyslog.d/columnstore.conf
// determine username/groupname
if [ -f /var/log/messages ]; then
user=`stat -c "%U %G" /var/log/messages | awk '{print $1}'`
group=`stat -c "%U %G" /var/log/messages | awk '{print $2}'`
fi
if [ -f /var/log/syslog ]; then
user=`stat -c "%U %G" /var/log/syslog | awk '{print $1}'`
group=`stat -c "%U %G" /var/log/syslog | awk '{print $2}'`
fi
//set permissions
$SUDO chown $user:$group -R /var/log/mariadb > /dev/null 2>&1
if [ $rsyslog7 == 1 ]; then
sed -i -e s/groupname/adm/g ${columnstoreSyslogFile7}
sed -i -e s/username/syslog/g ${columnstoreSyslogFile7}
sed -i -e s/groupname/$group/g ${columnstoreSyslogFile7}
sed -i -e s/username/$user/g ${columnstoreSyslogFile7}
$SUDO rm -f /etc/rsyslog.d/49-columnstore.conf
$SUDO cp ${columnstoreSyslogFile7} ${syslog_conf}

66
utils/rowgroup/rowgroup.cpp
View File

@@ -82,10 +82,10 @@ StringStore::~StringStore()
#endif
}
uint32_t StringStore::storeString(const uint8_t* data, uint32_t len)
uint64_t StringStore::storeString(const uint8_t* data, uint32_t len)
{
MemChunk* lastMC = NULL;
uint32_t ret = 0;
uint64_t ret = 0;
empty = false; // At least a NULL is being stored.
@@ -95,7 +95,7 @@ uint32_t StringStore::storeString(const uint8_t* data, uint32_t len)
if ((len == 8 || len == 9) &&
*((uint64_t*) data) == *((uint64_t*) joblist::CPNULLSTRMARK.c_str()))
return numeric_limits<uint32_t>::max();
return numeric_limits<uint64_t>::max();
//@bug6065, make StringStore::storeString() thread safe
boost::mutex::scoped_lock lk(fMutex, defer_lock);
@@ -106,20 +106,21 @@ uint32_t StringStore::storeString(const uint8_t* data, uint32_t len)
if (mem.size() > 0)
lastMC = (MemChunk*) mem.back().get();
if (len >= CHUNK_SIZE)
if ((len + 4) >= CHUNK_SIZE)
{
shared_array<uint8_t> newOne(new uint8_t[len + sizeof(MemChunk)]);
shared_array<uint8_t> newOne(new uint8_t[len + sizeof(MemChunk) + 4]);
longStrings.push_back(newOne);
lastMC = (MemChunk*) longStrings.back().get();
lastMC->capacity = lastMC->currentSize = len;
memcpy(lastMC->data, data, len);
lastMC->capacity = lastMC->currentSize = len + 4;
memcpy(lastMC->data, &len, 4);
memcpy(lastMC->data + 4, data, len);
// High bit to mark a long string
ret = 0x80000000;
ret = 0x8000000000000000;
ret += longStrings.size() - 1;
}
else
{
if ((lastMC == NULL) || (lastMC->capacity - lastMC->currentSize < len))
if ((lastMC == NULL) || (lastMC->capacity - lastMC->currentSize < (len + 4)))
{
// mem usage debugging
//if (lastMC)
@@ -134,7 +135,13 @@ uint32_t StringStore::storeString(const uint8_t* data, uint32_t len)
ret = ((mem.size() - 1) * CHUNK_SIZE) + lastMC->currentSize;
memcpy(&(lastMC->data[lastMC->currentSize]), data, len);
// If this ever happens then we have big problems
if (ret & 0x8000000000000000)
throw logic_error("StringStore memory exceeded.");
memcpy(&(lastMC->data[lastMC->currentSize]), &len, 4);
memcpy(&(lastMC->data[lastMC->currentSize]) + 4, data, len);
/*
cout << "stored: '" << hex;
for (uint32_t i = 0; i < len ; i++) {
@@ -142,7 +149,7 @@ uint32_t StringStore::storeString(const uint8_t* data, uint32_t len)
}
cout << "' at position " << lastMC->currentSize << " len " << len << dec << endl;
*/
lastMC->currentSize += len;
lastMC->currentSize += len + 4;
}
return ret;
@@ -150,35 +157,35 @@ uint32_t StringStore::storeString(const uint8_t* data, uint32_t len)
void StringStore::serialize(ByteStream& bs) const
{
uint32_t i;
uint64_t i;
MemChunk* mc;
bs << (uint32_t) mem.size();
bs << (uint64_t) mem.size();
bs << (uint8_t) empty;
for (i = 0; i < mem.size(); i++)
{
mc = (MemChunk*) mem[i].get();
bs << (uint32_t) mc->currentSize;
bs << (uint64_t) mc->currentSize;
//cout << "serialized " << mc->currentSize << " bytes\n";
bs.append(mc->data, mc->currentSize);
}
bs << (uint32_t) longStrings.size();
bs << (uint64_t) longStrings.size();
for (i = 0; i < longStrings.size(); i++)
{
mc = (MemChunk*) longStrings[i].get();
bs << (uint32_t) mc->currentSize;
bs << (uint64_t) mc->currentSize;
bs.append(mc->data, mc->currentSize);
}
}
void StringStore::deserialize(ByteStream& bs)
{
uint32_t i;
uint32_t count;
uint32_t size;
uint64_t i;
uint64_t count;
uint64_t size;
uint8_t* buf;
MemChunk* mc;
uint8_t tmp8;
@@ -856,10 +863,9 @@ bool Row::isNullValue(uint32_t colIndex) const
if (inStringTable(colIndex))
{
uint32_t offset, length;
offset = *((uint32_t*) &data[offsets[colIndex]]);
length = *((uint32_t*) &data[offsets[colIndex] + 4]);
return strings->isNullValue(offset, length);
uint64_t offset;
offset = *((uint64_t*) &data[offsets[colIndex]]);
return strings->isNullValue(offset);
}
if (data[offsets[colIndex]] == 0) // empty string
@@ -922,10 +928,9 @@ bool Row::isNullValue(uint32_t colIndex) const
if (inStringTable(colIndex))
{
uint32_t offset, length;
offset = *((uint32_t*) &data[pos]);
length = *((uint32_t*) &data[pos + 4]);
return strings->isNullValue(offset, length);
uint64_t offset;
offset = *((uint64_t*) &data[pos]);
return strings->isNullValue(offset);
}
if (*((uint16_t*) &data[pos]) == 0)
@@ -1720,8 +1725,8 @@ RGData RowGroup::duplicate()
void Row::setStringField(const std::string& val, uint32_t colIndex)
{
uint32_t length;
uint32_t offset;
uint64_t offset;
uint64_t length;
//length = strlen(val.c_str()) + 1;
length = val.length();
@@ -1732,8 +1737,7 @@ void Row::setStringField(const std::string& val, uint32_t colIndex)
if (inStringTable(colIndex))
{
offset = strings->storeString((const uint8_t*) val.data(), length);
*((uint32_t*) &data[offsets[colIndex]]) = offset;
*((uint32_t*) &data[offsets[colIndex] + 4]) = length;
*((uint64_t*) &data[offsets[colIndex]]) = offset;
// cout << " -- stored offset " << *((uint32_t *) &data[offsets[colIndex]])
// << " length " << *((uint32_t *) &data[offsets[colIndex] + 4])
// << endl;

157
utils/rowgroup/rowgroup.h
View File

@@ -92,13 +92,14 @@ public:
StringStore();
virtual ~StringStore();
inline std::string getString(uint32_t offset, uint32_t length) const;
uint32_t storeString(const uint8_t* data, uint32_t length); //returns the offset
inline const uint8_t* getPointer(uint32_t offset) const;
inline std::string getString(uint64_t offset) const;
uint64_t storeString(const uint8_t* data, uint32_t length); //returns the offset
inline const uint8_t* getPointer(uint64_t offset) const;
inline uint32_t getStringLength(uint64_t offset);
inline bool isEmpty() const;
inline uint64_t getSize() const;
inline bool isNullValue(uint32_t offset, uint32_t length) const;
inline bool equals(const std::string& str, uint32_t offset, uint32_t length) const;
inline bool isNullValue(uint64_t offset) const;
inline bool equals(const std::string& str, uint64_t offset) const;
void clear();
@@ -615,9 +616,8 @@ inline bool Row::equals(const std::string& val, uint32_t colIndex) const
{
if (inStringTable(colIndex))
{
uint32_t offset = *((uint32_t*) &data[offsets[colIndex]]);
uint32_t length = *((uint32_t*) &data[offsets[colIndex] + 4]);
return strings->equals(val, offset, length);
uint64_t offset = *((uint64_t*) &data[offsets[colIndex]]);
return strings->equals(val, offset);
}
else
return (strncmp(val.c_str(), (char*) &data[offsets[colIndex]], getColumnWidth(colIndex)) == 0);
@@ -719,7 +719,7 @@ inline int64_t Row::getIntField(uint32_t colIndex) const
inline const uint8_t* Row::getStringPointer(uint32_t colIndex) const
{
if (inStringTable(colIndex))
return strings->getPointer(*((uint32_t*) &data[offsets[colIndex]]));
return strings->getPointer(*((uint64_t*) &data[offsets[colIndex]]));
return &data[offsets[colIndex]];
}
@@ -727,14 +727,14 @@ inline const uint8_t* Row::getStringPointer(uint32_t colIndex) const
inline uint32_t Row::getStringLength(uint32_t colIndex) const
{
if (inStringTable(colIndex))
return *((uint32_t*) &data[offsets[colIndex] + 4]);
return strings->getStringLength(*((uint64_t*) &data[offsets[colIndex]]));
return strnlen((char*) &data[offsets[colIndex]], getColumnWidth(colIndex));
}
inline void Row::setStringField(const uint8_t* strdata, uint32_t length, uint32_t colIndex)
{
uint32_t offset;
uint64_t offset;
if (length > getColumnWidth(colIndex))
length = getColumnWidth(colIndex);
@@ -742,8 +742,7 @@ inline void Row::setStringField(const uint8_t* strdata, uint32_t length, uint32_
if (inStringTable(colIndex))
{
offset = strings->storeString(strdata, length);
*((uint32_t*) &data[offsets[colIndex]]) = offset;
*((uint32_t*) &data[offsets[colIndex] + 4]) = length;
*((uint64_t*) &data[offsets[colIndex]]) = offset;
// cout << " -- stored offset " << *((uint32_t *) &data[offsets[colIndex]])
// << " length " << *((uint32_t *) &data[offsets[colIndex] + 4])
// << endl;
@@ -759,8 +758,7 @@ inline void Row::setStringField(const uint8_t* strdata, uint32_t length, uint32_
inline std::string Row::getStringField(uint32_t colIndex) const
{
if (inStringTable(colIndex))
return strings->getString(*((uint32_t*) &data[offsets[colIndex]]),
*((uint32_t*) &data[offsets[colIndex] + 4]));
return strings->getString(*((uint64_t*) &data[offsets[colIndex]]));
// Not all CHAR/VARCHAR are NUL terminated so use length
return std::string((char*) &data[offsets[colIndex]],
@@ -778,7 +776,7 @@ inline std::string Row::getVarBinaryStringField(uint32_t colIndex) const
inline uint32_t Row::getVarBinaryLength(uint32_t colIndex) const
{
if (inStringTable(colIndex))
return *((uint32_t*) &data[offsets[colIndex] + 4]);
return strings->getStringLength(*((uint64_t*) &data[offsets[colIndex]]));;
return *((uint16_t*) &data[offsets[colIndex]]);
}
@@ -786,7 +784,7 @@ inline uint32_t Row::getVarBinaryLength(uint32_t colIndex) const
inline const uint8_t* Row::getVarBinaryField(uint32_t colIndex) const
{
if (inStringTable(colIndex))
return strings->getPointer(*((uint32_t*) &data[offsets[colIndex]]));
return strings->getPointer(*((uint64_t*) &data[offsets[colIndex]]));
return &data[offsets[colIndex] + 2];
}
@@ -795,7 +793,7 @@ inline const uint8_t* Row::getVarBinaryField(uint32_t& len, uint32_t colIndex) c
{
if (inStringTable(colIndex))
{
len = *((uint32_t*) &data[offsets[colIndex] + 4]);
len = strings->getStringLength(*((uint64_t*) &data[offsets[colIndex]]));
return getVarBinaryField(colIndex);
}
else
@@ -1043,9 +1041,8 @@ inline void Row::setVarBinaryField(const uint8_t* val, uint32_t len, uint32_t co
if (inStringTable(colIndex))
{
uint32_t offset = strings->storeString(val, len);
*((uint32_t*) &data[offsets[colIndex]]) = offset;
*((uint32_t*) &data[offsets[colIndex] + 4]) = len;
uint64_t offset = strings->storeString(val, len);
*((uint64_t*) &data[offsets[colIndex]]) = offset;
}
else
{
@@ -1762,26 +1759,29 @@ inline void copyRow(const Row& in, Row* out)
copyRow(in, out, std::min(in.getColumnCount(), out->getColumnCount()));
}
inline std::string StringStore::getString(uint32_t off, uint32_t len) const
inline std::string StringStore::getString(uint64_t off) const
{
if (off == std::numeric_limits<uint32_t>::max())
uint32_t length;
if (off == std::numeric_limits<uint64_t>::max())
return joblist::CPNULLSTRMARK;
MemChunk* mc;
if (off & 0x80000000)
if (off & 0x8000000000000000)
{
off = off - 0x80000000;
off = off - 0x8000000000000000;
if (longStrings.size() <= off)
return joblist::CPNULLSTRMARK;
mc = (MemChunk*) longStrings[off].get();
return std::string((char*) mc->data, len);
memcpy(&length, mc->data, 4);
return std::string((char*) mc->data + 4, length);
}
uint32_t chunk = off / CHUNK_SIZE;
uint32_t offset = off % CHUNK_SIZE;
uint64_t chunk = off / CHUNK_SIZE;
uint64_t offset = off % CHUNK_SIZE;
// this has to handle uninitialized data as well. If it's uninitialized it doesn't matter
// what gets returned, it just can't go out of bounds.
@@ -1790,30 +1790,32 @@ inline std::string StringStore::getString(uint32_t off, uint32_t len) const
mc = (MemChunk*) mem[chunk].get();
if ((offset + len) > mc->currentSize)
memcpy(&length, &mc->data[offset], 4);
if ((offset + length) > mc->currentSize)
return joblist::CPNULLSTRMARK;
return std::string((char*) & (mc->data[offset]), len);
return std::string((char*) & (mc->data[offset]) + 4, length);
}
inline const uint8_t* StringStore::getPointer(uint32_t off) const
inline const uint8_t* StringStore::getPointer(uint64_t off) const
{
if (off == std::numeric_limits<uint32_t>::max())
if (off == std::numeric_limits<uint64_t>::max())
return (const uint8_t*) joblist::CPNULLSTRMARK.c_str();
uint32_t chunk = off / CHUNK_SIZE;
uint32_t offset = off % CHUNK_SIZE;
uint64_t chunk = off / CHUNK_SIZE;
uint64_t offset = off % CHUNK_SIZE;
MemChunk* mc;
if (off & 0x80000000)
if (off & 0x8000000000000000)
{
off = off - 0x80000000;
off = off - 0x8000000000000000;
if (longStrings.size() <= off)
return (const uint8_t*) joblist::CPNULLSTRMARK.c_str();
mc = (MemChunk*) longStrings[off].get();
return mc->data;
return mc->data + 4;
}
// this has to handle uninitialized data as well. If it's uninitialized it doesn't matter
@@ -1826,19 +1828,18 @@ inline const uint8_t* StringStore::getPointer(uint32_t off) const
if (offset > mc->currentSize)
return (const uint8_t*) joblist::CPNULLSTRMARK.c_str();
return &(mc->data[offset]);
return &(mc->data[offset]) + 4;
}
inline bool StringStore::isNullValue(uint32_t off, uint32_t len) const
inline bool StringStore::isNullValue(uint64_t off) const
{
if (off == std::numeric_limits<uint32_t>::max() || len == 0)
uint32_t length;
if (off == std::numeric_limits<uint64_t>::max())
return true;
if (len < 8)
return false;
// Long strings won't be NULL
if (off & 0x80000000)
if (off & 0x8000000000000000)
return false;
uint32_t chunk = off / CHUNK_SIZE;
@@ -1849,35 +1850,46 @@ inline bool StringStore::isNullValue(uint32_t off, uint32_t len) const
return true;
mc = (MemChunk*) mem[chunk].get();
memcpy(&length, &mc->data[offset], 4);
if ((offset + len) > mc->currentSize)
if (length == 0)
return true;
if (mc->data[offset] == 0) // "" = NULL string for some reason...
if (length < 8)
return false;
if ((offset + length) > mc->currentSize)
return true;
return (*((uint64_t*) &mc->data[offset]) == *((uint64_t*) joblist::CPNULLSTRMARK.c_str()));
if (mc->data[offset + 4] == 0) // "" = NULL string for some reason...
return true;
return (*((uint64_t*) &mc->data[offset] + 4) == *((uint64_t*) joblist::CPNULLSTRMARK.c_str()));
}
inline bool StringStore::equals(const std::string& str, uint32_t off, uint32_t len) const
inline bool StringStore::equals(const std::string& str, uint64_t off) const
{
if (off == std::numeric_limits<uint32_t>::max() || len == 0)
uint32_t length;
if (off == std::numeric_limits<uint64_t>::max())
return str == joblist::CPNULLSTRMARK;
MemChunk* mc;
if (off & 0x80000000)
if (off & 0x8000000000000000)
{
if (longStrings.size() <= (off - 0x80000000))
if (longStrings.size() <= (off - 0x8000000000000000))
return false;
mc = (MemChunk*) longStrings[off - 0x80000000].get();
mc = (MemChunk*) longStrings[off - 0x8000000000000000].get();
memcpy(&length, mc->data, 4);
// Not sure if this check it needed, but adds safety
if (len > mc->currentSize)
if (length > mc->currentSize)
return false;
return (strncmp(str.c_str(), (const char*) mc->data, len) == 0);
return (strncmp(str.c_str(), (const char*) mc->data + 4, length) == 0);
}
uint32_t chunk = off / CHUNK_SIZE;
@@ -1887,11 +1899,44 @@ inline bool StringStore::equals(const std::string& str, uint32_t off, uint32_t l
return false;
mc = (MemChunk*) mem[chunk].get();
memcpy(&length, &mc->data[offset], 4);
if ((offset + len) > mc->currentSize)
if ((offset + length) > mc->currentSize)
return false;
return (strncmp(str.c_str(), (const char*) &mc->data[offset], len) == 0);
return (strncmp(str.c_str(), (const char*) &mc->data[offset] + 4, length) == 0);
}
inline uint32_t StringStore::getStringLength(uint64_t off)
{
uint32_t length;
MemChunk* mc;
if (off == std::numeric_limits<uint64_t>::max())
return 0;
if (off & 0x8000000000000000)
{
off = off - 0x8000000000000000;
if (longStrings.size() <= off)
return 0;
mc = (MemChunk*) longStrings[off].get();
memcpy(&length, mc->data, 4);
}
else
{
uint64_t chunk = off / CHUNK_SIZE;
uint64_t offset = off % CHUNK_SIZE;
if (mem.size() <= chunk)
return 0;
mc = (MemChunk*) mem[chunk].get();
memcpy(&length, &mc->data[offset], 4);
}
return length;
}
inline bool StringStore::isEmpty() const