1
0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-08-01 06:46:55 +03:00

MCOL-265 Add support for TIMESTAMP data type

This commit is contained in:
Gagan Goel
2019-03-17 14:14:03 -04:00
parent 8a7ccd7d93
commit e89d1ac3cf
167 changed files with 4346 additions and 250 deletions

View File

@ -106,6 +106,22 @@ const int64_t IDB_pow[19] =
};
const int32_t SECS_PER_MIN = 60;
const int32_t MINS_PER_HOUR = 60;
const int32_t HOURS_PER_DAY = 24;
const int32_t DAYS_PER_WEEK = 7;
const int32_t DAYS_PER_NYEAR = 365;
const int32_t DAYS_PER_LYEAR = 366;
const int32_t SECS_PER_HOUR = SECS_PER_MIN * MINS_PER_HOUR;
const int32_t SECS_PER_DAY = SECS_PER_HOUR * HOURS_PER_DAY;
const int32_t EPOCH_YEAR = 1970;
const int32_t MONS_PER_YEAR = 12;
const int32_t MAX_TIMESTAMP_YEAR = 2038;
const int32_t MIN_TIMESTAMP_YEAR = 1969;
const int32_t MAX_TIMESTAMP_VALUE = (1ULL << 31) - 1;
const int32_t MIN_TIMESTAMP_VALUE = 0;
namespace dataconvert
{
@ -116,6 +132,485 @@ enum CalpontDateTimeFormat
CALPONTTIME_ENUM = 3
};
/** @brief a structure that represents a timestamp in broken down
* representation
*/
struct MySQLTime
{
unsigned int year, month, day, hour, minute, second;
unsigned long second_part;
CalpontDateTimeFormat time_type;
void reset()
{
year = month = day = 0;
hour = minute = second = second_part = 0;
time_type = CALPONTDATETIME_ENUM;
}
};
/**
* This function converts the timezone represented as a string
* in the format "+HH:MM" or "-HH:MM" to a signed offset in seconds
* Most of this code is taken from tztime.cc:str_to_offset
*/
inline
bool timeZoneToOffset(const char *str, std::string::size_type length, long *offset)
{
const char *end = str + length;
bool negative;
unsigned long number_tmp;
long offset_tmp;
if (length < 4)
return 1;
if (*str == '+')
negative = 0;
else if (*str == '-')
negative = 1;
else
return 1;
str++;
number_tmp = 0;
while (str < end && isdigit(*str))
{
number_tmp = number_tmp * 10 + *str - '0';
str++;
}
if (str + 1 >= end || *str != ':')
return 1;
str++;
offset_tmp = number_tmp * 60L;
number_tmp = 0;
while (str < end && isdigit(*str))
{
number_tmp = number_tmp * 10 + *str - '0';
str++;
}
if (str != end)
return 1;
offset_tmp = (offset_tmp + number_tmp) * 60L;
if (negative)
offset_tmp = -offset_tmp;
/*
Check if offset is in range prescribed by standard
(from -12:59 to 13:00).
*/
if (number_tmp > 59 || offset_tmp < -13 * 3600L + 1 ||
offset_tmp > 13 * 3600L)
return 1;
*offset = offset_tmp;
return 0;
}
const int32_t year_lengths[2] =
{
DAYS_PER_NYEAR, DAYS_PER_LYEAR
};
const unsigned int mon_lengths[2][MONS_PER_YEAR]=
{
{ 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 },
{ 31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 }
};
const unsigned int mon_starts[2][MONS_PER_YEAR]=
{
{ 0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334 },
{ 0, 31, 60, 91, 121, 152, 182, 213, 244, 274, 305, 335 }
};
inline int32_t leapsThruEndOf(int32_t year)
{
return (year / 4 - year / 100 + year / 400);
}
inline bool isLeapYear ( int year)
{
if ( year % 400 == 0 )
return true;
if ( ( year % 4 == 0 ) && ( year % 100 != 0 ) )
return true;
return false;
}
static uint32_t daysInMonth[13] = {31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31, 0};
inline uint32_t getDaysInMonth(uint32_t month, int year)
{
if (month < 1 || month > 12)
return 0;
uint32_t days = daysInMonth[month - 1];
if ((month == 2) && isLeapYear(year))
days++;
return days;
}
inline
bool isDateValid ( int day, int month, int year)
{
bool valid = true;
if ( year == 0 && month == 0 && year == 0 )
{
return true;
}
int daycheck = getDaysInMonth( month, year );
if ( ( year < 1000 ) || ( year > 9999 ) )
valid = false;
else if ( month < 1 || month > 12 )
valid = false;
else if ( day < 1 || day > daycheck )
valid = false;
return ( valid );
}
inline
bool isDateTimeValid ( int hour, int minute, int second, int microSecond)
{
bool valid = false;
if ( hour >= 0 && hour <= 24 )
{
if ( minute >= 0 && minute < 60 )
{
if ( second >= 0 && second < 60 )
{
if ( microSecond >= 0 && microSecond <= 999999 )
{
valid = true;
}
}
}
}
return valid;
}
inline
bool isTimeValid ( int hour, int minute, int second, int microSecond)
{
bool valid = false;
if ( hour >= -838 && hour <= 838 )
{
if ( minute >= 0 && minute < 60 )
{
if ( second >= 0 && second < 60 )
{
if ( microSecond >= 0 && microSecond <= 999999 )
{
valid = true;
}
}
}
}
return valid;
}
inline
bool isTimestampValid ( uint64_t second, uint64_t microsecond )
{
bool valid = false;
// MariaDB server currently sets the upper limit on timestamp to
// 0x7FFFFFFF. So enforce the same restriction here.
// TODO: We however store the seconds portion of the timestamp in
// 44 bits, so change this limit when the server supports higher values.
if ( second >= MIN_TIMESTAMP_VALUE && second <= MAX_TIMESTAMP_VALUE )
{
if ( microsecond >= 0 && microsecond <= 999999 )
{
valid = true;
}
}
return valid;
}
/**
* @brief converts a timestamp (seconds in UTC since Epoch)
* to broken-down representation. Most of this code is taken
* from sec_to_TIME and Time_zone_system::gmt_sec_to_TIME
* functions in tztime.cc in the server
*
* @param seconds the value to be converted
* @param time the broken-down representation of the timestamp
* @param timeZone a string with the server timezone of the machine
* which initiated the query
*/
inline void gmtSecToMySQLTime(int64_t seconds, MySQLTime& time,
const std::string& timeZone)
{
if (seconds == 0)
{
time.reset();
return;
}
if (timeZone == "SYSTEM")
{
struct tm tmp_tm;
time_t tmp_t = (time_t)seconds;
localtime_r(&tmp_t, &tmp_tm);
time.second_part = 0;
time.year = (int) ((tmp_tm.tm_year + 1900) % 10000);
time.month = (int) tmp_tm.tm_mon + 1;
time.day = (int) tmp_tm.tm_mday;
time.hour = (int) tmp_tm.tm_hour;
time.minute = (int) tmp_tm.tm_min;
time.second = (int) tmp_tm.tm_sec;
time.time_type = CALPONTDATETIME_ENUM;
if (time.second == 60 || time.second == 61)
time.second = 59;
}
else
{
long offset;
if (timeZoneToOffset(timeZone.c_str(), timeZone.size(), &offset))
{
time.reset();
return;
}
int64_t days;
int32_t rem;
int32_t y;
int32_t yleap;
const unsigned int *ip;
days = (int64_t) (seconds / SECS_PER_DAY);
rem = (int32_t) (seconds % SECS_PER_DAY);
rem += offset;
while (rem < 0)
{
rem += SECS_PER_DAY;
days--;
}
while (rem >= SECS_PER_DAY)
{
rem -= SECS_PER_DAY;
days++;
}
time.hour = (unsigned int) (rem / SECS_PER_HOUR);
rem = rem % SECS_PER_HOUR;
time.minute = (unsigned int) (rem / SECS_PER_MIN);
time.second = (unsigned int) (rem % SECS_PER_MIN);
y = EPOCH_YEAR;
while (days < 0 || days >= (int64_t) (year_lengths[yleap = isLeapYear(y)]))
{
int32_t newy;
newy = y + days / DAYS_PER_NYEAR;
if (days < 0)
newy--;
days -= (newy - y) * DAYS_PER_NYEAR +
leapsThruEndOf(newy - 1) -
leapsThruEndOf(y - 1);
y = newy;
}
time.year = y;
ip = mon_lengths[yleap];
for (time.month = 0; days >= (int64_t) ip[time.month]; time.month++)
days -= (int64_t) ip[time.month];
time.month++;
time.day = (unsigned int) (days + 1);
time.second_part = 0;
time.time_type = CALPONTDATETIME_ENUM;
}
}
/**
* @brief function that provides a rough estimate if a broken-down
* representation of timestamp is in range
*
* @param t the broken-down representation of timestamp
*/
inline bool validateTimestampRange(const MySQLTime& t)
{
if ((t.year > MAX_TIMESTAMP_YEAR || t.year < MIN_TIMESTAMP_YEAR) ||
(t.year == MAX_TIMESTAMP_YEAR && (t.month > 1 || t.day > 19)))
return false;
return true;
}
inline
int64_t secSinceEpoch(int year, int month, int day, int hour, int min, int sec)
{
int64_t days = (year - EPOCH_YEAR) * DAYS_PER_NYEAR +
leapsThruEndOf(year - 1) -
leapsThruEndOf(EPOCH_YEAR - 1);
days += mon_starts[isLeapYear(year)][month - 1];
days += day - 1;
return ((days * HOURS_PER_DAY + hour) * MINS_PER_HOUR + min) *
SECS_PER_MIN + sec;
}
// This is duplicate of funchelpers.h:calc_mysql_daynr,
// with one additional function parameter
inline uint32_t calc_mysql_daynr( uint32_t year, uint32_t month, uint32_t day, bool& isValid )
{
int temp;
int y = year;
long delsum;
if ( !isDateValid( day, month, year ) )
{
isValid = false;
return 0;
}
delsum = (long) (365 * y + 31 * ((int) month - 1) + (int) day);
if (month <= 2)
y--;
else
delsum -= (long) ((int) month * 4 + 23) / 10;
temp = (int) ((y / 100 + 1) * 3) / 4;
return delsum + (int) y / 4 - temp;
}
/**
* @brief converts a timestamp from broken-down representation
* to seconds since UTC epoch
*
* @param time the broken-down representation of the timestamp
@param timeZone a string with the server timezone of the machine
which initiated the query
*/
inline int64_t mySQLTimeToGmtSec(const MySQLTime& time,
const std::string& timeZone, bool& isValid)
{
int64_t seconds;
if (!validateTimestampRange(time))
{
isValid = false;
return 0;
}
if (timeZone == "SYSTEM")
{
// This is mirror of code in func_unix_timestamp.cpp
uint32_t loop;
time_t tmp_t = 0;
int shift = 0;
struct tm* l_time, tm_tmp;
int64_t diff;
localtime_r(&tmp_t, &tm_tmp);
// Get the system timezone offset at 0 seconds since epoch
int64_t my_time_zone = tm_tmp.tm_gmtoff;
int day = time.day;
if ((time.year == MAX_TIMESTAMP_YEAR) && (time.month == 1) && (day > 4))
{
day -= 2;
shift = 2;
}
tmp_t = (time_t)(((calc_mysql_daynr(time.year, time.month, day, isValid) -
719528) * 86400L + (int64_t)time.hour * 3600L +
(int64_t)(time.minute * 60 + time.second)) - (time_t)my_time_zone);
if (!isValid)
return 0;
localtime_r(&tmp_t, &tm_tmp);
l_time = &tm_tmp;
for (loop = 0; loop < 2 && (time.hour != (uint32_t) l_time->tm_hour ||
time.minute != (uint32_t) l_time->tm_min ||
time.second != (uint32_t)l_time->tm_sec); loop++)
{
int days = day - l_time->tm_mday;
if (days < -1)
days = 1; /* Month has wrapped */
else if (days > 1)
days = -1;
diff = (3600L * (int64_t) (days * 24 + ((int) time.hour - (int) l_time->tm_hour)) +
(int64_t) (60 * ((int) time.minute - (int) l_time->tm_min)) +
(int64_t) ((int) time.second - (int) l_time->tm_sec));
tmp_t += (time_t) diff;
localtime_r(&tmp_t, &tm_tmp);
l_time = &tm_tmp;
}
if (loop == 2 && time.hour != (uint32_t)l_time->tm_hour)
{
int days = day - l_time->tm_mday;
if (days < -1)
days = 1; /* Month has wrapped */
else if (days > 1)
days = -1;
diff = (3600L * (int64_t) (days * 24 + ((int) time.hour - (int) l_time->tm_hour)) +
(int64_t) (60 * ((int) time.minute - (int) l_time->tm_min)) +
(int64_t) ((int) time.second - (int) l_time->tm_sec));
if (diff == 3600)
tmp_t += 3600 - time.minute * 60 - time.second; /* Move to next hour */
else if (diff == -3600)
tmp_t -= time.minute * 60 + time.second; /* Move to previous hour */
}
/* shift back, if we were dealing with boundary dates */
tmp_t += shift * 86400L;
seconds = (int64_t)tmp_t;
}
else
{
long offset;
if (timeZoneToOffset(timeZone.c_str(), timeZone.size(), &offset))
{
isValid = false;
return 0;
}
seconds = secSinceEpoch(time.year, time.month, time.day,
time.hour, time.minute, time.second) - offset;
}
/* make sure we have legit timestamps (i.e. we didn't over/underflow anywhere above) */
if (seconds >= MIN_TIMESTAMP_VALUE && seconds <= MAX_TIMESTAMP_VALUE)
return seconds;
isValid = false;
return 0;
}
/** @brief a structure to hold a date
*/
@ -258,96 +753,44 @@ int64_t Time::convertToMySQLint() const
}
}
static uint32_t daysInMonth[13] = {31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31, 0};
inline bool isLeapYear ( int year)
/** @brief a structure to hold a timestamp
*/
struct TimeStamp
{
if ( year % 400 == 0 )
return true;
unsigned msecond : 20;
unsigned long long second : 44;
// NULL column value = 0xFFFFFFFFFFFFFFFE
TimeStamp( ) :
msecond(0xFFFFE), second(0xFFFFFFFFFFF) {}
// Construct a TimeStamp from a 64 bit integer Calpont timestamp.
TimeStamp(uint64_t val) :
msecond(val & 0xFFFFF), second(val >> 20) {}
TimeStamp(unsigned msec, unsigned long long sec) :
msecond(msec), second(sec) {}
if ( ( year % 4 == 0 ) && ( year % 100 != 0 ) )
return true;
int64_t convertToMySQLint(const std::string& timeZone) const;
void reset();
};
return false;
}
inline uint32_t getDaysInMonth(uint32_t month, int year)
inline
int64_t TimeStamp::convertToMySQLint(const std::string& timeZone) const
{
if (month < 1 || month > 12)
return 0;
const int TIMESTAMPTOSTRING1_LEN = 22; // YYYYMMDDHHMMSSmmmmmm\0
char buf[TIMESTAMPTOSTRING1_LEN];
uint32_t days = daysInMonth[month - 1];
MySQLTime time;
gmtSecToMySQLTime(second, time, timeZone);
if ((month == 2) && isLeapYear(year))
days++;
sprintf(buf, "%04d%02d%02d%02d%02d%02d", time.year, time.month, time.day, time.hour, time.minute, time.second);
return days;
return (int64_t) atoll(buf);
}
inline
bool isDateValid ( int day, int month, int year)
void TimeStamp::reset()
{
bool valid = true;
if ( year == 0 && month == 0 && year == 0 )
{
return true;
}
int daycheck = getDaysInMonth( month, year );
if ( ( year < 1000 ) || ( year > 9999 ) )
valid = false;
else if ( month < 1 || month > 12 )
valid = false;
else if ( day < 1 || day > daycheck )
valid = false;
return ( valid );
}
inline
bool isDateTimeValid ( int hour, int minute, int second, int microSecond)
{
bool valid = false;
if ( hour >= 0 && hour <= 24 )
{
if ( minute >= 0 && minute < 60 )
{
if ( second >= 0 && second < 60 )
{
if ( microSecond >= 0 && microSecond <= 999999 )
{
valid = true;
}
}
}
}
return valid;
}
inline
bool isTimeValid ( int hour, int minute, int second, int microSecond)
{
bool valid = false;
if ( hour >= -838 && hour <= 838 )
{
if ( minute >= 0 && minute < 60 )
{
if ( second >= 0 && second < 60 )
{
if ( microSecond >= 0 && microSecond <= 999999 )
{
valid = true;
}
}
}
}
return valid;
msecond = 0xFFFFE;
second = 0xFFFFFFFFFFF;
}
inline
@ -412,7 +855,7 @@ public:
* @param data the columns string representation of it's data
*/
EXPORT static boost::any convertColumnData( const execplan::CalpontSystemCatalog::ColType& colType,
const std::string& dataOrig, bool& bSaturate,
const std::string& dataOrig, bool& bSaturate, const std::string& timeZone,
bool nulFlag = false, bool noRoundup = false, bool isUpdate = false);
/**
@ -433,6 +876,15 @@ public:
EXPORT static std::string datetimeToString( long long datetimevalue, long decimals = 0 );
static inline void datetimeToString( long long datetimevalue, char* buf, unsigned int buflen, long decimals = 0 );
/**
* @brief convert a columns data from native format to a string
*
* @param type the columns database type
* @param data the columns string representation of it's data
*/
EXPORT static std::string timestampToString( long long timestampvalue, const std::string& timezone, long decimals = 0 );
static inline void timestampToString( long long timestampvalue, char* buf, unsigned int buflen, const std::string& timezone, long decimals = 0 );
/**
* @brief convert a columns data from native format to a string
*
@ -460,6 +912,15 @@ public:
EXPORT static std::string datetimeToString1( long long datetimevalue );
static inline void datetimeToString1( long long datetimevalue, char* buf, unsigned int buflen );
/**
* @brief convert a columns data from native format to a string
*
* @param type the columns database type
* @param data the columns string representation of it's data
*/
EXPORT static std::string timestampToString1( long long timestampvalue, const std::string& timezone );
static inline void timestampToString1( long long timestampvalue, char* buf, unsigned int buflen, const std::string& timezone );
/**
* @brief convert a columns data from native format to a string
*
@ -502,6 +963,21 @@ public:
CalpontDateTimeFormat datetimeFormat,
int& status, unsigned int dataOrgLen );
/**
* @brief convert a timestamp column data, represented as a string,
* to it's native format. This function is for bulkload to use.
*
* @param dataOrg the columns string representation of it's data
* @param datetimeFormat the format the date value in
* @param status 0 - success, -1 - fail
* @param dataOrgLen length specification of dataOrg
* @param timeZone the timezone used for conversion to native format
*/
EXPORT static int64_t convertColumnTimestamp( const char* dataOrg,
CalpontDateTimeFormat datetimeFormat,
int& status, unsigned int dataOrgLen,
const std::string& timeZone );
/**
* @brief convert a time column data, represented as a string,
* to it's native format. This function is for bulkload to use.
@ -521,6 +997,7 @@ public:
*/
EXPORT static bool isColumnDateTimeValid( int64_t dateTime );
EXPORT static bool isColumnTimeValid( int64_t time );
EXPORT static bool isColumnTimeStampValid( int64_t timeStamp );
EXPORT static bool isNullData(execplan::ColumnResult* cr, int rownum, execplan::CalpontSystemCatalog::ColType colType);
static inline std::string decimalToString(int64_t value, uint8_t scale, execplan::CalpontSystemCatalog::ColDataType colDataType);
@ -536,6 +1013,8 @@ public:
EXPORT static int64_t stringToDate(const std::string& data);
// convert string to datetime
EXPORT static int64_t stringToDatetime(const std::string& data, bool* isDate = NULL);
// convert string to timestamp
EXPORT static int64_t stringToTimestamp(const std::string& data, const std::string& timeZone);
// convert integer to date
EXPORT static int64_t intToDate(int64_t data);
// convert integer to datetime
@ -546,6 +1025,7 @@ public:
EXPORT static int64_t dateToInt(const std::string& date);
// convert string to datetime. alias to datetimeToInt
EXPORT static int64_t datetimeToInt(const std::string& datetime);
EXPORT static int64_t timestampToInt(const std::string& timestamp, const std::string& timeZone);
EXPORT static int64_t timeToInt(const std::string& time);
EXPORT static int64_t stringToTime (const std::string& data);
// bug4388, union type conversion
@ -591,6 +1071,31 @@ inline void DataConvert::datetimeToString( long long datetimevalue, char* buf, u
}
}
inline void DataConvert::timestampToString( long long timestampvalue, char* buf, unsigned int buflen, const std::string& timezone, long decimals )
{
// 10 is default which means we don't need microseconds
if (decimals > 6 || decimals < 0)
{
decimals = 0;
}
TimeStamp timestamp(timestampvalue);
int64_t seconds = timestamp.second;
MySQLTime time;
gmtSecToMySQLTime(seconds, time, timezone);
snprintf( buf, buflen, "%04d-%02d-%02d %02d:%02d:%02d",
time.year, time.month, time.day,
time.hour, time.minute, time.second
);
if (timestamp.msecond || decimals)
{
snprintf(buf + strlen(buf), buflen - strlen(buf), ".%0*d", (int)decimals, timestamp.msecond);
}
}
inline void DataConvert::timeToString( long long timevalue, char* buf, unsigned int buflen, long decimals )
{
// 10 is default which means we don't need microseconds
@ -655,6 +1160,20 @@ inline void DataConvert::datetimeToString1( long long datetimevalue, char* buf,
);
}
inline void DataConvert::timestampToString1( long long timestampvalue, char* buf, unsigned int buflen, const std::string& timezone )
{
TimeStamp timestamp(timestampvalue);
int64_t seconds = timestamp.second;
MySQLTime time;
gmtSecToMySQLTime(seconds, time, timezone);
snprintf( buf, buflen, "%04d%02d%02d%02d%02d%02d",
time.year, time.month, time.day,
time.hour, time.minute, time.second
);
}
inline void DataConvert::timeToString1( long long timevalue, char* buf, unsigned int buflen )
{
// Handle negative correctly