database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-08-01 06:46:55 +03:00
Code Activity

MCOL-392 Add initial TIME datatype support

Browse Source
This commit is contained in:
Andrew Hutchings
2018-04-23 19:20:31 +01:00
parent b584a7f555
commit 3c1ebd8b94
109 changed files with 2241 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

378
utils/dataconvert/dataconvert.cpp
View File

@ -859,6 +859,153 @@ bool mysql_str_to_datetime( const string& input, DateTime& output, bool& isDate
return true;
}
bool mysql_str_to_time( const string& input, Time& output )
{
int32_t datesepct = 0;
uint32_t dtend = 0;
/**
* We need to deal with the time portion.
* The rules are:
* - Time portion may be empty
* - Time portion may start with 'T'
* - Time portion always ends with '\0'
* - Time portion always starts with hour
* - Without time separators (':'):
* HHMMSS
* - All Times can end with option .[microseconds]
* - With time separators there are no specific field length
* requirements
*/
while ( input[dtend] == ' ' && dtend < input.length() )
{
++dtend;
}
if ( dtend == input.length() )
{
return false;
}
uint32_t timesep_ct = 0;
bool has_usec = false;
uint32_t len_before_msec = 0;
uint32_t tmstart = ( input[dtend] == ' ' || input[dtend] == 'T' ) ? dtend + 1 : dtend;
uint32_t tmend = tmstart;
for ( ; tmend < input.length(); ++tmend )
{
char c = input[tmend];
if ( isdigit( c ) )
{
// digits always ok
continue;
}
// else if( c == ':' )
// {
// timesep_ct++;
// }
// else if( c == '.' )
// {
// len_before_msec = ( tmend - tmstart );
// has_usec = true;
// }
else if ( ispunct(c) )
{
if ( c == '.' && timesep_ct == 2 )
{
len_before_msec = ( tmend - tmstart );
has_usec = true;
}
else
{
timesep_ct++;
}
}
else
{
// some other character showed up
output.reset();
return false;
}
}
if ( !len_before_msec )
len_before_msec = ( tmend - tmstart );
int32_t hour = -1;
int32_t min = 0;
int32_t sec = 0;
int32_t usec = 0;
const char* tstart = input.c_str() + tmstart;
if ( timesep_ct == 2 )
{
readDecimal(tstart, hour);
++tstart; // skip one separator
readDecimal(tstart, min);
++tstart; // skip one separator
readDecimal(tstart, sec);
}
else if ( timesep_ct == 1 )
{
readDecimal(tstart, hour);
++tstart; // skip one separator
readDecimal(tstart, min);
}
else if ( timesep_ct == 0 && len_before_msec == 6 )
{
readDecimal(tstart, hour, 2);
readDecimal(tstart, min, 2);
readDecimal(tstart, sec, 2);
}
else if ( timesep_ct == 0 && len_before_msec == 4 )
{
readDecimal(tstart, hour, 2);
readDecimal(tstart, min, 2);
}
else if ( timesep_ct == 0 && len_before_msec == 2 )
{
readDecimal(tstart, hour, 2);
}
else
{
output.reset();
return false;
}
if ( has_usec )
{
++tstart; // skip '.' character. We could error check if we wanted to
uint32_t numread = readDecimal(tstart, usec);
if ( numread > 6 || numread < 1 )
{
// don't allow more than 6 digits when specifying microseconds
output.reset();
return false;
}
// usec have to be scaled up so that it always represents microseconds
for ( int i = numread; i < 6; i++ )
usec *= 10;
}
if ( !isTimeValid( hour, min, sec, usec ) )
{
output.reset();
return false;
}
output.hour = hour;
output.minute = min;
output.second = sec;
output.msecond = usec;
return true;
}
bool stringToDateStruct( const string& data, Date& date )
{
bool isDate;
@ -894,6 +1041,14 @@ bool stringToDatetimeStruct(const string& data, DateTime& dtime, bool* date)
return true;
}
bool stringToTimeStruct(const string& data, Time& dtime)
{
if ( !mysql_str_to_time( data, dtime ) )
return false;
return true;
}
boost::any
DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType,
const std::string& dataOrig, bool& pushWarning, bool nulFlag, bool noRoundup, bool isUpdate )
@ -1229,6 +1384,23 @@ DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType,
}
break;
case CalpontSystemCatalog::TIME:
{
Time aTime;
if (stringToTimeStruct(data, aTime))
{
value = (int64_t) *(reinterpret_cast<int64_t*>(&aTime));
}
else
{
value = (int64_t) 0;
pushWarning = true;
}
}
break;
case CalpontSystemCatalog::BLOB:
case CalpontSystemCatalog::CLOB:
value = data;
@ -1345,6 +1517,13 @@ DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType,
}
break;
case CalpontSystemCatalog::TIME:
{
uint64_t d = joblist::TIMENULL;
value = d;
}
break;
case CalpontSystemCatalog::CHAR:
{
std::string charnull;
@ -1692,6 +1871,91 @@ int64_t DataConvert::convertColumnDatetime(
return value;
}
//------------------------------------------------------------------------------
// Convert time string to binary time. Used by BulkLoad.
// Most of this is taken from str_to_time in sql-common/my_time.c
//------------------------------------------------------------------------------
int64_t DataConvert::convertColumnTime(
const char* dataOrg,
CalpontDateTimeFormat datetimeFormat,
int& status,
unsigned int dataOrgLen )
{
status = 0;
const char* p;
p = dataOrg;
char fld[10];
int16_t value = 0;
int inYear, inMonth, inDay, inHour, inMinute, inSecond, inMicrosecond;
inHour = 0;
inMinute = 0;
inSecond = 0;
inMicrosecond = 0;
if ( datetimeFormat != CALPONTTIME_ENUM )
{
status = -1;
return value;
}
memcpy( fld, p, 2);
fld[2] = '\0';
inHour = strtol(fld, 0, 10);
if (!isdigit(p[2]) || !isdigit(p[3]))
{
status = -1;
return value;
}
memcpy( fld, p + 2, 2);
fld[2] = '\0';
inMinute = strtol(fld, 0, 10);
if (!isdigit(p[4]) || !isdigit(p[5]))
{
status = -1;
return value;
}
memcpy( fld, p + 4, 2);
fld[2] = '\0';
inSecond = strtol(fld, 0, 10);
if (dataOrgLen > 9)
{
unsigned int microFldLen = dataOrgLen - 9;
if (microFldLen > (sizeof(fld) - 1))
microFldLen = sizeof(fld) - 1;
memcpy( fld, p + 9, microFldLen);
fld[microFldLen] = '\0';
inMicrosecond = strtol(fld, 0, 10);
}
if ( isTimeValid (inHour, inMinute, inSecond, inMicrosecond) )
{
Time atime;
atime.hour = inHour;
atime.minute = inMinute;
atime.second = inSecond;
atime.msecond = inMicrosecond;
memcpy( &value, &atime, 8);
}
else
{
status = -1;
}
return value;
}
//------------------------------------------------------------------------------
// Verify that specified datetime is valid
//------------------------------------------------------------------------------
@ -1706,6 +1970,14 @@ bool DataConvert::isColumnDateTimeValid( int64_t dateTime )
return false;
}
bool DataConvert::isColumnTimeValid( int64_t time )
{
Time dt;
memcpy(&dt, &time, sizeof(uint64_t));
return isTimeValid(dt.hour, dt.minute, dt.second, dt.msecond);
}
std::string DataConvert::dateToString( int datevalue )
{
// @bug 4703 abandon multiple ostringstream's for conversion
@ -1742,6 +2014,32 @@ std::string DataConvert::datetimeToString( long long datetimevalue, long decima
return buf;
}
std::string DataConvert::timeToString( long long timevalue, long decimals )
{
// 10 is default which means we don't need microseconds
if (decimals > 6 || decimals < 0)
{
decimals = 0;
}
// @bug 4703 abandon multiple ostringstream's for conversion
Time dt(timevalue);
const int TIMETOSTRING_LEN = 19; // (-H)HH:MM:SS.mmmmmm\0
char buf[TIMETOSTRING_LEN];
sprintf(buf, "%02d:%02d:%02d", dt.hour, dt.minute, dt.second);
if (dt.msecond && decimals)
{
size_t start = strlen(buf);
snprintf(buf + strlen(buf), 12 + decimals, ".%d", dt.msecond);
// Pad end with zeros
if (strlen(buf) - start < decimals)
{
sprintf(buf + strlen(buf), "%0*d", decimals - (strlen(buf) - start), 0);
}
}
return buf;
}
std::string DataConvert::dateToString1( int datevalue )
{
// @bug 4703 abandon multiple ostringstream's for conversion
@ -1763,6 +2061,18 @@ std::string DataConvert::datetimeToString1( long long datetimevalue )
sprintf(buf, "%04d%02d%02d%02d%02d%02d%06d", dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second, dt.msecond);
return buf;
}
std::string DataConvert::timeToString1( long long datetimevalue )
{
// @bug 4703 abandon multiple ostringstream's for conversion
DateTime dt(datetimevalue);
const int TIMETOSTRING1_LEN = 14; // HHMMSSmmmmmm\0
char buf[TIMETOSTRING1_LEN];
sprintf(buf, "%02d%02d%02d%06d", dt.hour, dt.minute, dt.second, dt.msecond);
return buf;
}
#if 0
bool DataConvert::isNullData(ColumnResult* cr, int rownum, CalpontSystemCatalog::ColType colType)
{
@ -1983,6 +2293,11 @@ int64_t DataConvert::datetimeToInt(const string& datetime)
return stringToDatetime(datetime);
}
int64_t DataConvert::timeToInt(const string& time)
{
return stringToTime(time);
}
int64_t DataConvert::stringToDate(const string& data)
{
Date aDay;
@ -2256,6 +2571,66 @@ int64_t DataConvert::intToDatetime(int64_t data, bool* date)
return *(reinterpret_cast<uint64_t*>(&adaytime));
}
int64_t DataConvert::intToTime(int64_t data)
{
char buf[21] = {0};
Time atime;
if (data == 0)
{
atime.hour = 0;
atime.minute = 0;
atime.second = 0;
atime.msecond = 0;
return *(reinterpret_cast<uint64_t*>(&atime));
}
snprintf( buf, 15, "%llu", (long long unsigned int)data);
//string date = buf;
string hour, min, sec, msec;
int64_t h = 0, minute = 0, s = 0, ms = 0;
switch (strlen(buf))
{
case 6:
hour = string(buf, 2);
min = string(buf + 2, 2);
sec = string(buf + 4, 2);
msec = string(buf + 6, 6);
break;
case 4:
min = string(buf, 2);
sec = string(buf + 2, 2);
msec = string(buf + 4, 6);
break;
case 2:
sec = string(buf, 2);
msec = string(buf + 2, 6);
break;
default:
return -1;
}
h = atoi(hour.c_str());
minute = atoi(min.c_str());
s = atoi(sec.c_str());
ms = atoi(msec.c_str());
if (!isTimeValid(h, minute, s, ms))
return -1;
atime.hour = h;
atime.minute = minute;
atime.second = s;
atime.msecond = ms;
return *(reinterpret_cast<uint64_t*>(&atime));
}
int64_t DataConvert::stringToTime(const string& data)
{
// MySQL supported time value format 'D HHH:MM:SS.fraction'
@ -2402,6 +2777,7 @@ CalpontSystemCatalog::ColType DataConvert::convertUnionColType(vector<CalpontSys
unionedType.colWidth = 20;
break;
case CalpontSystemCatalog::TIME:
case CalpontSystemCatalog::DATETIME:
unionedType.colDataType = CalpontSystemCatalog::CHAR;
unionedType.colWidth = 26;
@ -2469,6 +2845,7 @@ CalpontSystemCatalog::ColType DataConvert::convertUnionColType(vector<CalpontSys
case CalpontSystemCatalog::DATE:
case CalpontSystemCatalog::DATETIME:
case CalpontSystemCatalog::TIME:
default:
break;
}
@ -2496,6 +2873,7 @@ CalpontSystemCatalog::ColType DataConvert::convertUnionColType(vector<CalpontSys
case CalpontSystemCatalog::UDECIMAL:
case CalpontSystemCatalog::UFLOAT:
case CalpontSystemCatalog::UDOUBLE:
case CalpontSystemCatalog::TIME:
unionedType.colDataType = CalpontSystemCatalog::CHAR;
unionedType.scale = 0;
unionedType.colWidth = 26;

116
utils/dataconvert/dataconvert.h
View File

@ -112,7 +112,8 @@ namespace dataconvert
enum CalpontDateTimeFormat
{
CALPONTDATE_ENUM = 1, // date format is: "YYYY-MM-DD"
CALPONTDATETIME_ENUM = 2 // date format is: "YYYY-MM-DD HH:MI:SS"
CALPONTDATETIME_ENUM = 2, // date format is: "YYYY-MM-DD HH:MI:SS"
CALPONTTIME_ENUM = 3
};
@ -172,7 +173,7 @@ struct DateTime
inline
int64_t DateTime::convertToMySQLint() const
{
return (int64_t) (year * 10000000000LL) + (month * 100000000) + (day * 1000000) + (hour * 10000) + (minute * 100) + second;
return (int64_t) (year * 10000000000000000LL) + (month * 100000000000000) + (day * 1000000000000) + (hour * 10000000000) + (minute * 100000000) + (second * 1000000) + msecond;
}
inline
@ -213,8 +214,30 @@ struct Time
hour((val >> 40) & 0xfff),
day((val >> 52) & 0xfff)
{}
Time(signed d, signed h, signed min, signed sec, signed msec) :
msecond(sec), second(sec), minute(min), hour(h), day(d) {}
int64_t convertToMySQLint() const;
void reset();
};
inline
void Time::reset()
{
msecond = 0xFFFFFE;
second = 0xFF;
minute = 0xFF;
hour = 0xFFF;
day = 0xFFF;
}
inline
int64_t Time::convertToMySQLint() const
{
return (int64_t) (hour * 10000000000) + (minute * 100000000) + (second * 1000000) + msecond;
}
static uint32_t daysInMonth[13] = {31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31, 0};
inline uint32_t getDaysInMonth(uint32_t month)
@ -281,6 +304,28 @@ bool isDateTimeValid ( int hour, int minute, int second, int microSecond)
return valid;
}
inline
bool isTimeValid ( int hour, int minute, int second, int microSecond)
{
bool valid = false;
if ( hour >= -838 && hour <= 838 )
{
if ( minute >= 0 && minute < 60 )
{
if ( second >= 0 && second < 60 )
{
if ( microSecond >= 0 && microSecond <= 999999 )
{
valid = true;
}
}
}
}
return valid;
}
inline
int64_t string_to_ll( const std::string& data, bool& bSaturate )
{
@ -364,6 +409,15 @@ public:
EXPORT static std::string datetimeToString( long long datetimevalue, long decimals = 0 );
static inline void datetimeToString( long long datetimevalue, char* buf, unsigned int buflen );
/**
* @brief convert a columns data from native format to a string
*
* @param type the columns database type
* @param data the columns string representation of it's data
*/
EXPORT static std::string timeToString( long long timevalue, long decimals = 0 );
static inline void timeToString( long long timevalue, char* buf, unsigned int buflen );
/**
* @brief convert a columns data from native format to a string
*
@ -382,6 +436,15 @@ public:
EXPORT static std::string datetimeToString1( long long datetimevalue );
static inline void datetimeToString1( long long datetimevalue, char* buf, unsigned int buflen );
/**
* @brief convert a columns data from native format to a string
*
* @param type the columns database type
* @param data the columns string representation of it's data
*/
EXPORT static std::string timeToString1( long long timevalue );
static inline void timeToString1( long long timevalue, char* buf, unsigned int buflen );
/**
* @brief convert a date column data, represnted as a string, to it's native
* format. This function is for bulkload to use.
@ -415,10 +478,25 @@ public:
CalpontDateTimeFormat datetimeFormat,
int& status, unsigned int dataOrgLen );
/**
* @brief convert a time column data, represented as a string,
* to it's native format. This function is for bulkload to use.
*
* @param type the columns data type
* @param dataOrig the columns string representation of it's data
* @param timeFormat the format the time value in
* @param status 0 - success, -1 - fail
* @param dataOrgLen length specification of dataOrg
*/
EXPORT static int64_t convertColumnTime( const char* dataOrg,
CalpontDateTimeFormat datetimeFormat,
int& status, unsigned int dataOrgLen );
/**
* @brief Is specified datetime valid; used by binary bulk load
*/
EXPORT static bool isColumnDateTimeValid( int64_t dateTime );
EXPORT static bool isColumnTimeValid( int64_t time );
EXPORT static bool isNullData(execplan::ColumnResult* cr, int rownum, execplan::CalpontSystemCatalog::ColType colType);
static inline std::string decimalToString(int64_t value, uint8_t scale, execplan::CalpontSystemCatalog::ColDataType colDataType);
@ -438,11 +516,13 @@ public:
EXPORT static int64_t intToDate(int64_t data);
// convert integer to datetime
EXPORT static int64_t intToDatetime(int64_t data, bool* isDate = NULL);
// convert integer to date
EXPORT static int64_t intToTime(int64_t data);
// convert string to date. alias to stringToDate
EXPORT static int64_t dateToInt(const std::string& date);
// convert string to datetime. alias to datetimeToInt
EXPORT static int64_t datetimeToInt(const std::string& datetime);
EXPORT static int64_t timeToInt(const std::string& time);
EXPORT static int64_t stringToTime (const std::string& data);
// bug4388, union type conversion
EXPORT static execplan::CalpontSystemCatalog::ColType convertUnionColType(std::vector<execplan::CalpontSystemCatalog::ColType>&);
@ -484,6 +564,27 @@ inline void DataConvert::datetimeToString( long long datetimevalue, char* buf, u
}
}
inline void DataConvert::timeToString( long long timevalue, char* buf, unsigned int buflen )
{
if ((timevalue & 0xffffff) > 0)
{
snprintf( buf, buflen, "%02d:%02d:%02d.%d",
(unsigned)((timevalue >> 40) & 0xfff),
(unsigned)((timevalue >> 32) & 0xff),
(unsigned)((timevalue >> 24) & 0xff),
(unsigned)((timevalue) & 0xffffff)
);
}
else
{
snprintf( buf, buflen, "%02d:%02d:%02d",
(unsigned)((timevalue >> 40) & 0xfff),
(unsigned)((timevalue >> 32) & 0xff),
(unsigned)((timevalue >> 24) & 0xff)
);
}
}
inline void DataConvert::dateToString1( int datevalue, char* buf, unsigned int buflen)
{
snprintf( buf, buflen, "%04d%02d%02d",
@ -505,6 +606,15 @@ inline void DataConvert::datetimeToString1( long long datetimevalue, char* buf,
);
}
inline void DataConvert::timeToString1( long long timevalue, char* buf, unsigned int buflen )
{
snprintf( buf, buflen, "%02d%02d%02d",
(unsigned)((timevalue >> 40) & 0xfff),
(unsigned)((timevalue >> 32) & 0xff),
(unsigned)((timevalue >> 14) & 0xff)
);
}
inline std::string DataConvert::decimalToString(int64_t value, uint8_t scale, execplan::CalpontSystemCatalog::ColDataType colDataType)
{
char buf[80];