database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-29 08:21:15 +03:00
Code Activity

MCOL-5573 Fix cpimport truncation of TEXT columns.

Browse Source 
1. Restore the utf8_truncate_point() function in utils/common/utils_utf8.h
that I removed as part of the patch for MCOL-4931.

2. As per the definition of TEXT columns, the default column width represents
the maximum number of bytes that can be stored in the TEXT column. So the
effective maximum length is less if the value contains multi-byte characters.
However, if the user explicitly specifies the length of the TEXT column in a
table DDL, such as TEXT(65535), then the DDL logic ensures that enough number
of bytes are allocated (upto a system maximum) to allow upto that many number
of characters (multi-byte characters if the charset for the column is multi-byte,
such as utf8mb3).
This commit is contained in:
Gagan Goel
2023-09-18 13:55:24 -04:00
parent 39a31fe064
commit 7f9c624626
4 changed files with 58 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
writeengine/bulk/we_columninfo.cpp
View File

@ -1697,7 +1697,7 @@ int ColumnInfo::updateDctnryStore(char* buf, ColPosPair** pos, const int totalRo
Stats::stopParseEvent(WE_STATS_WAIT_TO_PARSE_DCT);
#endif
int rc = fStore->insertDctnry(buf, pos, totalRow, id, tokenBuf, truncCount, column.cs);
int rc = fStore->insertDctnry(buf, pos, totalRow, id, tokenBuf, truncCount, column.cs, column.weType);
if (rc != NO_ERROR)
{

39
writeengine/dictionary/we_dctnry.cpp
View File

@ -49,6 +49,7 @@ using namespace BRM;
#include "cacheutils.h"
using namespace idbdatafile;
#include "checks.h"
#include "utils_utf8.h" // for utf8_truncate_point()
namespace
{
@ -763,7 +764,8 @@ int Dctnry::insertDctnry2(Signature& sig)
* failure - it did not write the header to block
******************************************************************************/
int Dctnry::insertDctnry(const char* buf, ColPosPair** pos, const int totalRow, const int col, char* tokenBuf,
long long& truncCount, const CHARSET_INFO* cs)
long long& truncCount, const CHARSET_INFO* cs,
const WriteEngine::ColType& weType)
{
#ifdef PROFILE
Stats::startParseEvent(WE_STATS_PARSE_DCT);
@ -838,17 +840,32 @@ int Dctnry::insertDctnry(const char* buf, ColPosPair** pos, const int totalRow,
if (cs->mbmaxlen > 1)
{
const char* start = (const char*) curSig.signature;
const char* end = (const char*)(curSig.signature + curSig.size);
size_t numChars = cs->numchars(start, end);
size_t maxCharLength = m_colWidth / cs->mbmaxlen;
if (numChars > maxCharLength)
// For TEXT columns, we truncate based on the number of bytes,
// and not based on the number of characters, as for CHAR/VARCHAR
// columns in the else block.
if (weType == WriteEngine::WR_TEXT)
{
MY_STRCOPY_STATUS status;
cs->well_formed_char_length(start, end, maxCharLength, &status);
curSig.size = status.m_source_end_pos - start;
truncCount++;
if (curSig.size > m_colWidth)
{
uint8_t truncate_point = utf8::utf8_truncate_point((const char*)curSig.signature, m_colWidth);
curSig.size = m_colWidth - truncate_point;
truncCount++;
}
}
else
{
const char* start = (const char*) curSig.signature;
const char* end = (const char*)(curSig.signature + curSig.size);
size_t numChars = cs->numchars(start, end);
size_t maxCharLength = m_colWidth / cs->mbmaxlen;
if (numChars > maxCharLength)
{
MY_STRCOPY_STATUS status;
cs->well_formed_char_length(start, end, maxCharLength, &status);
curSig.size = status.m_source_end_pos - start;
truncCount++;
}
}
}
else // cs->mbmaxlen == 1

3
writeengine/dictionary/we_dctnry.h
View File

@ -168,7 +168,8 @@ class Dctnry : public DbFileOp
* @param tokenBuf - (output) list of tokens for the parsed strings
*/
EXPORT int insertDctnry(const char* buf, ColPosPair** pos, const int totalRow, const int col,
char* tokenBuf, long long& truncCount, const CHARSET_INFO* cs);
char* tokenBuf, long long& truncCount, const CHARSET_INFO* cs,
const WriteEngine::ColType& weType);
/**
* @brief Update dictionary store with tokenized strings (for DDL/DML use)