diff --git a/src/backend/tsearch/to_tsany.c b/src/backend/tsearch/to_tsany.c index d98def5b63f..46a92a8105a 100644 --- a/src/backend/tsearch/to_tsany.c +++ b/src/backend/tsearch/to_tsany.c @@ -165,8 +165,8 @@ TSVector make_tsvector(ParsedText *prs) { int i, - j, - lenstr = 0, + j; + size_t lenstr = 0, totallen; TSVector in; WordEntry *ptr; @@ -177,10 +177,22 @@ make_tsvector(ParsedText *prs) if (prs->curwords > 0) prs->curwords = uniqueWORD(prs->words, prs->curwords); - /* Determine space needed */ + /* + * Determine space needed. Since what we are calculating is equivalent to + * the size of a portion of the input data structure, lenstr surely can't + * overflow size_t. + */ for (i = 0; i < prs->curwords; i++) { - lenstr += prs->words[i].len; + int toklen = prs->words[i].len; + + /* Double-check that caller passed only lexemes of valid lengths */ + if (toklen <= 0 || toklen > MAXSTRLEN) + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("lexeme is too long for tsvector (%zu bytes, max %zu bytes)", + (size_t) toklen, (size_t) MAXSTRLEN))); + lenstr += toklen; if (prs->words[i].alen) { lenstr = SHORTALIGN(lenstr); @@ -191,7 +203,8 @@ make_tsvector(ParsedText *prs) if (lenstr > MAXSTRPOS) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), - errmsg("string is too long for tsvector (%d bytes, max %d bytes)", lenstr, MAXSTRPOS))); + errmsg("string is too long for tsvector (%zu bytes, max %zu bytes)", + lenstr, (size_t) MAXSTRPOS))); totallen = CALCDATASIZE(prs->curwords, lenstr); in = (TSVector) palloc0(totallen); diff --git a/src/backend/tsearch/ts_parse.c b/src/backend/tsearch/ts_parse.c index 64b62dcd3c0..cb69e9899c5 100644 --- a/src/backend/tsearch/ts_parse.c +++ b/src/backend/tsearch/ts_parse.c @@ -401,12 +401,30 @@ parsetext(Oid cfgId, ParsedText *prs, char *buf, int buflen) while ((norms = LexizeExec(&ldata, NULL)) != NULL) { - TSLexeme *ptr = norms; - prs->pos++; /* set pos */ - while (ptr->lexeme) + for (TSLexeme *ptr = norms; ptr->lexeme; ptr++) { + size_t lexeme_len = strlen(ptr->lexeme); + + if (lexeme_len > MAXSTRLEN) + { +#ifdef IGNORE_LONGLEXEME + ereport(NOTICE, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("word is too long to be indexed"), + errdetail("Words longer than %d characters are ignored.", + MAXSTRLEN))); + continue; +#else + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("word is too long to be indexed"), + errdetail("Words longer than %d characters are ignored.", + MAXSTRLEN))); +#endif + } + if (prs->curwords == prs->lenwords) { prs->lenwords *= 2; @@ -415,13 +433,12 @@ parsetext(Oid cfgId, ParsedText *prs, char *buf, int buflen) if (ptr->flags & TSL_ADDPOS) prs->pos++; - prs->words[prs->curwords].len = strlen(ptr->lexeme); + prs->words[prs->curwords].len = lexeme_len; prs->words[prs->curwords].word = ptr->lexeme; prs->words[prs->curwords].nvariant = ptr->nvariant; prs->words[prs->curwords].flags = ptr->flags & TSL_PREFIX; prs->words[prs->curwords].alen = 0; prs->words[prs->curwords].pos.pos = LIMITPOS(prs->pos); - ptr++; prs->curwords++; } pfree(norms); diff --git a/src/backend/utils/adt/tsvector.c b/src/backend/utils/adt/tsvector.c index 024f5160cd4..40be82b890a 100644 --- a/src/backend/utils/adt/tsvector.c +++ b/src/backend/utils/adt/tsvector.c @@ -217,8 +217,8 @@ tsvectorin(PG_FUNCTION_ARGS) if (cur - tmpbuf > MAXSTRPOS) ereturn(escontext, (Datum) 0, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), - errmsg("string is too long for tsvector (%ld bytes, max %ld bytes)", - (long) (cur - tmpbuf), (long) MAXSTRPOS))); + errmsg("string is too long for tsvector (%zu bytes, max %zu bytes)", + (size_t) (cur - tmpbuf), (size_t) MAXSTRPOS))); /* * Enlarge buffers if needed @@ -271,7 +271,8 @@ tsvectorin(PG_FUNCTION_ARGS) if (buflen > MAXSTRPOS) ereturn(escontext, (Datum) 0, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), - errmsg("string is too long for tsvector (%d bytes, max %d bytes)", buflen, MAXSTRPOS))); + errmsg("string is too long for tsvector (%zu bytes, max %zu bytes)", + (size_t) buflen, (size_t) MAXSTRPOS))); totallen = CALCDATASIZE(len, buflen); in = (TSVector) palloc0(totallen); @@ -316,8 +317,8 @@ tsvectorout(PG_FUNCTION_ARGS) TSVector out = PG_GETARG_TSVECTOR(0); char *outbuf; int32 i, - lenbuf = 0, pp; + size_t lenbuf; WordEntry *ptr = ARRPTR(out); char *curin, *curout; @@ -326,7 +327,7 @@ tsvectorout(PG_FUNCTION_ARGS) lenbuf = out->size * 2 /* '' */ + out->size - 1 /* space */ + 2 /* \0 */ ; for (i = 0; i < out->size; i++) { - lenbuf += ptr[i].len * 2 * pg_database_encoding_max_length() /* for escape */ ; + lenbuf += ptr[i].len * 2 /* allow for escapes */ ; if (ptr[i].haspos) lenbuf += 1 /* : */ + 7 /* int2 + , + weight */ * POSDATALEN(out, &(ptr[i])); } @@ -458,12 +459,14 @@ tsvectorrecv(PG_FUNCTION_ARGS) bool needSort = false; nentries = pq_getmsgint(buf, sizeof(int32)); - if (nentries < 0 || nentries > (MaxAllocSize / sizeof(WordEntry))) + + /* We disallow empty lexemes, so more than MAXSTRPOS of them can't fit */ + if (nentries < 0 || nentries > MAXSTRPOS) elog(ERROR, "invalid size of tsvector"); hdrlen = DATAHDRSIZE + sizeof(WordEntry) * nentries; - len = hdrlen * 2; /* times two to make room for lexemes */ + len = hdrlen * 2; /* times two to make some room for lexemes */ vec = (TSVector) palloc0(len); vec->size = nentries; @@ -480,6 +483,8 @@ tsvectorrecv(PG_FUNCTION_ARGS) /* sanity checks */ lex_len = strlen(lexeme); + if (lex_len == 0) + elog(ERROR, "invalid tsvector: empty lexeme"); if (lex_len > MAXSTRLEN) elog(ERROR, "invalid tsvector: lexeme too long"); @@ -545,6 +550,15 @@ tsvectorrecv(PG_FUNCTION_ARGS) } } + /* + * Enforce that datalen is still within MAXSTRPOS, ie the last lexeme + * didn't go past that. We could allow that, since no "pos" field + * overflowed, but tsvectorrecv shouldn't accept values that other + * tsvector-constructing routines wouldn't. + */ + if (datalen > MAXSTRPOS) + elog(ERROR, "invalid tsvector: maximum total lexeme length exceeded"); + SET_VARSIZE(vec, hdrlen + datalen); if (needSort) diff --git a/src/backend/utils/adt/tsvector_op.c b/src/backend/utils/adt/tsvector_op.c index 53a9541e89f..935a7fae538 100644 --- a/src/backend/utils/adt/tsvector_op.c +++ b/src/backend/utils/adt/tsvector_op.c @@ -175,6 +175,7 @@ tsvector_strip(PG_FUNCTION_ARGS) *arrout; char *cur; + /* Output can't be bigger than input, so no need for overflow checks */ for (i = 0; i < in->size; i++) len += arrin[i].len; @@ -492,6 +493,8 @@ tsvector_delete_by_indices(TSVector tsv, int *indices_to_delete, /* * Copy tsv to tsout, skipping lexemes listed in indices_to_delete. + * + * Output can't be bigger than input, so no need for overflow checks. */ arrout = ARRPTR(tsout); dataout = STRPTR(tsout); @@ -721,7 +724,7 @@ tsvector_to_array(PG_FUNCTION_ARGS) int i; ArrayType *array; - elements = palloc(tsin->size * sizeof(Datum)); + elements = palloc_array(Datum, tsin->size); for (i = 0; i < tsin->size; i++) { @@ -756,20 +759,29 @@ array_to_tsvector(PG_FUNCTION_ARGS) deconstruct_array_builtin(v, TEXTOID, &dlexemes, &nulls, &nitems); /* - * Reject nulls and zero length strings (maybe we should just ignore them, - * instead?) + * Reject nulls and zero-length or over-length strings (maybe we should + * just ignore them, instead?) */ for (i = 0; i < nitems; i++) { + int toklen; + if (nulls[i]) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("lexeme array may not contain nulls"))); - if (VARSIZE(DatumGetPointer(dlexemes[i])) - VARHDRSZ == 0) + toklen = VARSIZE(DatumGetPointer(dlexemes[i])) - VARHDRSZ; + if (toklen == 0) ereport(ERROR, (errcode(ERRCODE_ZERO_LENGTH_CHARACTER_STRING), errmsg("lexeme array may not contain empty strings"))); + if (toklen >= MAXSTRLEN) + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("word is too long (%d bytes, max %d bytes)", + toklen, + MAXSTRLEN - 1))); } /* Sort and de-dup, because this is required for a valid tsvector. */ @@ -783,6 +795,11 @@ array_to_tsvector(PG_FUNCTION_ARGS) /* Calculate space needed for surviving lexemes. */ for (i = 0; i < nitems; i++) datalen += VARSIZE(DatumGetPointer(dlexemes[i])) - VARHDRSZ; + if (datalen > MAXSTRPOS) + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("string is too long for tsvector (%zu bytes, max %zu bytes)", + (size_t) datalen, (size_t) MAXSTRPOS))); tslen = CALCDATASIZE(nitems, datalen); /* Allocate and fill tsvector. */ @@ -844,9 +861,15 @@ tsvector_filter(PG_FUNCTION_ARGS) mask |= 1 << parse_weight(char_weight); } + /* + * The output tsvector might be smaller than the input, but it can't be + * bigger, so VARSIZE(tsin) is surely enough space. Also, we don't need + * to worry about overflows below. + */ tsout = (TSVector) palloc0(VARSIZE(tsin)); tsout->size = tsin->size; arrout = ARRPTR(tsout); + /* worst-case location of output's lexemes; we may need to adjust below */ dataout = STRPTR(tsout); for (i = j = 0; i < tsin->size; i++) @@ -946,6 +969,12 @@ tsvector_concat(PG_FUNCTION_ARGS) * Conservative estimate of space needed. We might need all the data in * both inputs, and conceivably add a pad byte before position data for * each item where there was none before. + * + * Note: since the MAXSTRPOS limit constrains each input tsvector to be + * considerably less than MaxAllocSize, we don't need to worry about + * integer overflow here, nor in the data-copying steps below. We do need + * to enforce that the result meets the MAXSTRPOS limit, but we check that + * once at the end. */ output_bytes = VARSIZE(in1) + VARSIZE(in2) + i1 + i2; @@ -1097,7 +1126,8 @@ tsvector_concat(PG_FUNCTION_ARGS) if (dataoff > MAXSTRPOS) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), - errmsg("string is too long for tsvector (%d bytes, max %d bytes)", dataoff, MAXSTRPOS))); + errmsg("string is too long for tsvector (%zu bytes, max %zu bytes)", + (size_t) dataoff, (size_t) MAXSTRPOS))); /* * Adjust sizes (asserting that we didn't overrun the original estimates) diff --git a/src/include/tsearch/ts_type.h b/src/include/tsearch/ts_type.h index 0226220e421..ee1e061c506 100644 --- a/src/include/tsearch/ts_type.h +++ b/src/include/tsearch/ts_type.h @@ -35,7 +35,9 @@ * * The positions for each lexeme must be sorted. * - * Note, tsvectorsend/recv believe that sizeof(WordEntry) == 4 + * Note that while the WordEntry items must be sorted per tsCompareString(), + * the per-lexeme data storage could be in some other order, ie the series + * of WordEntry->pos values need not be strictly ascending. */ typedef struct @@ -46,13 +48,15 @@ typedef struct pos:20; /* MAX 1Mb */ } WordEntry; -#define MAXSTRLEN ( (1<<11) - 1) -#define MAXSTRPOS ( (1<<20) - 1) +#define MAXSTRLEN ( (1<<11) - 1) /* maximum value of WordEntry.len */ +#define MAXSTRPOS ( (1<<20) - 1) /* maximum value of WordEntry.pos */ extern int compareWordEntryPos(const void *a, const void *b); /* - * Equivalent to + * Representation of positions (and weights) associated with a lexeme. + * + * WordEntryPos is equivalent to * typedef struct { * uint16 * weight:2, @@ -75,40 +79,53 @@ typedef struct WordEntryPos pos[1]; } WordEntryPosVector1; +#define MAXNUMPOS (256) /* semi-arbitrary limit on npos */ +/* Macros for getting/setting the fields of a WordEntryPos */ #define WEP_GETWEIGHT(x) ( (x) >> 14 ) #define WEP_GETPOS(x) ( (x) & 0x3fff ) #define WEP_SETWEIGHT(x,v) ( (x) = ( (v) << 14 ) | ( (x) & 0x3fff ) ) #define WEP_SETPOS(x,v) ( (x) = ( (x) & 0xc000 ) | ( (v) & 0x3fff ) ) -#define MAXENTRYPOS (1<<14) -#define MAXNUMPOS (256) +#define MAXENTRYPOS (1<<14) /* max value of WordEntryPos pos field, +1 */ +/* Macro for clamping a position to what will fit in WordEntryPos pos field */ #define LIMITPOS(x) ( ( (x) >= MAXENTRYPOS ) ? (MAXENTRYPOS-1) : (x) ) /* This struct represents a complete tsvector datum */ typedef struct { int32 vl_len_; /* varlena header (do not touch directly!) */ - int32 size; + int32 size; /* number of entries[] items */ WordEntry entries[FLEXIBLE_ARRAY_MEMBER]; /* lexemes follow the entries[] array */ } TSVectorData; typedef TSVectorData *TSVector; +/* + * Calculate the size of a TSVector given the number of WordEntries and + * the total space needed for lexeme text and positions. NOTE: callers + * must enforce lenstr <= MAXSTRPOS, which ensures that WordEntry.pos + * fields will not overflow, and also protects against integer overflow here. + * (Since we prohibit empty lexemes, nentries can't exceed lenstr.) + */ #define DATAHDRSIZE (offsetof(TSVectorData, entries)) #define CALCDATASIZE(nentries, lenstr) (DATAHDRSIZE + (nentries) * sizeof(WordEntry) + (lenstr) ) /* pointer to start of a tsvector's WordEntry array */ -#define ARRPTR(x) ( (x)->entries ) +#define ARRPTR(tsv) ( (tsv)->entries ) /* pointer to start of a tsvector's lexeme storage */ -#define STRPTR(x) ( (char *) &(x)->entries[(x)->size] ) +#define STRPTR(tsv) ( (char *) &(tsv)->entries[(tsv)->size] ) -#define _POSVECPTR(x, e) ((WordEntryPosVector *)(STRPTR(x) + SHORTALIGN((e)->pos + (e)->len))) -#define POSDATALEN(x,e) ( ( (e)->haspos ) ? (_POSVECPTR(x,e)->npos) : 0 ) -#define POSDATAPTR(x,e) (_POSVECPTR(x,e)->pos) +/* pointer to WordEntryPosVector for a WordEntry */ +#define _POSVECPTR(tsv,we) ((WordEntryPosVector *) \ + (STRPTR(tsv) + SHORTALIGN((we)->pos + (we)->len))) +/* number of positions stored for a WordEntry */ +#define POSDATALEN(tsv,we) ( (we)->haspos ? _POSVECPTR(tsv,we)->npos : 0 ) +/* pointer to start of positions stored for a WordEntry */ +#define POSDATAPTR(tsv,we) (_POSVECPTR(tsv,we)->pos) /* * fmgr interface functions