database/postgres
1
0
Fork 0
mirror of https://github.com/postgres/postgres.git synced 2025-11-10 17:42:29 +03:00
Code Activity
Files
14ff44f80c09718d43d853363941457f5468cc03
postgres/src/backend/snowball/dict_snowball.c
Tom Lane 1058555a5e In the Snowball dictionary, don't try to stem excessively-long words. 
If the input word exceeds 1000 bytes, don't pass it to the stemmer;
just return it as-is after case folding.  Such an input is surely
not a word in any human language, so whatever the stemmer might
do to it would be pretty dubious in the first place.  Adding this
restriction protects us against a known recursion-to-stack-overflow
problem in the Turkish stemmer, and it seems like good insurance
against any other safety or performance issues that may exist in
the Snowball stemmers.  (I note, for example, that they contain no
CHECK_FOR_INTERRUPTS calls, so we really don't want them running
for a long time.)  The threshold of 1000 bytes is arbitrary.

An alternative definition could have been to treat such words as
stopwords, but that seems like a bigger break from the old behavior.

Per report from Egor Chindyaskin and Alexander Lakhin.
Thanks to Olly Betts for the recommendation to fix it this way.

Discussion: https://postgr.es/m/1661334672.728714027@f473.i.mail.ru
2022-08-31 10:42:05 -04:00

348 lines
10 KiB
C
Raw Blame History

/*-------------------------------------------------------------------------
*
* dict_snowball.c
* Snowball dictionary
*
* Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group
*
* IDENTIFICATION
* src/backend/snowball/dict_snowball.c
*
*-------------------------------------------------------------------------
*/
#include "postgres.h"
#include "commands/defrem.h"
#include "tsearch/ts_locale.h"
#include "tsearch/ts_utils.h"
/* Some platforms define MAXINT and/or MININT, causing conflicts */
#ifdef MAXINT
#undef MAXINT
#endif
#ifdef MININT
#undef MININT
#endif
/* Now we can include the original Snowball header.h */
#include "snowball/libstemmer/header.h"
#include "snowball/libstemmer/stem_ISO_8859_1_basque.h"
#include "snowball/libstemmer/stem_ISO_8859_1_catalan.h"
#include "snowball/libstemmer/stem_ISO_8859_1_danish.h"
#include "snowball/libstemmer/stem_ISO_8859_1_dutch.h"
#include "snowball/libstemmer/stem_ISO_8859_1_english.h"
#include "snowball/libstemmer/stem_ISO_8859_1_finnish.h"
#include "snowball/libstemmer/stem_ISO_8859_1_french.h"
#include "snowball/libstemmer/stem_ISO_8859_1_german.h"
#include "snowball/libstemmer/stem_ISO_8859_1_indonesian.h"
#include "snowball/libstemmer/stem_ISO_8859_1_irish.h"
#include "snowball/libstemmer/stem_ISO_8859_1_italian.h"
#include "snowball/libstemmer/stem_ISO_8859_1_norwegian.h"
#include "snowball/libstemmer/stem_ISO_8859_1_porter.h"
#include "snowball/libstemmer/stem_ISO_8859_1_portuguese.h"
#include "snowball/libstemmer/stem_ISO_8859_1_spanish.h"
#include "snowball/libstemmer/stem_ISO_8859_1_swedish.h"
#include "snowball/libstemmer/stem_ISO_8859_2_hungarian.h"
#include "snowball/libstemmer/stem_ISO_8859_2_romanian.h"
#include "snowball/libstemmer/stem_KOI8_R_russian.h"
#include "snowball/libstemmer/stem_UTF_8_arabic.h"
#include "snowball/libstemmer/stem_UTF_8_armenian.h"
#include "snowball/libstemmer/stem_UTF_8_basque.h"
#include "snowball/libstemmer/stem_UTF_8_catalan.h"
#include "snowball/libstemmer/stem_UTF_8_danish.h"
#include "snowball/libstemmer/stem_UTF_8_dutch.h"
#include "snowball/libstemmer/stem_UTF_8_english.h"
#include "snowball/libstemmer/stem_UTF_8_finnish.h"
#include "snowball/libstemmer/stem_UTF_8_french.h"
#include "snowball/libstemmer/stem_UTF_8_german.h"
#include "snowball/libstemmer/stem_UTF_8_greek.h"
#include "snowball/libstemmer/stem_UTF_8_hindi.h"
#include "snowball/libstemmer/stem_UTF_8_hungarian.h"
#include "snowball/libstemmer/stem_UTF_8_indonesian.h"
#include "snowball/libstemmer/stem_UTF_8_irish.h"
#include "snowball/libstemmer/stem_UTF_8_italian.h"
#include "snowball/libstemmer/stem_UTF_8_lithuanian.h"
#include "snowball/libstemmer/stem_UTF_8_nepali.h"
#include "snowball/libstemmer/stem_UTF_8_norwegian.h"
#include "snowball/libstemmer/stem_UTF_8_porter.h"
#include "snowball/libstemmer/stem_UTF_8_portuguese.h"
#include "snowball/libstemmer/stem_UTF_8_romanian.h"
#include "snowball/libstemmer/stem_UTF_8_russian.h"
#include "snowball/libstemmer/stem_UTF_8_serbian.h"
#include "snowball/libstemmer/stem_UTF_8_spanish.h"
#include "snowball/libstemmer/stem_UTF_8_swedish.h"
#include "snowball/libstemmer/stem_UTF_8_tamil.h"
#include "snowball/libstemmer/stem_UTF_8_turkish.h"
#include "snowball/libstemmer/stem_UTF_8_yiddish.h"
PG_MODULE_MAGIC;
PG_FUNCTION_INFO_V1(dsnowball_init);
PG_FUNCTION_INFO_V1(dsnowball_lexize);
/* List of supported modules */
typedef struct stemmer_module
{
const char *name;
pg_enc enc;
struct SN_env *(*create) (void);
void (*close) (struct SN_env *);
int (*stem) (struct SN_env *);
} stemmer_module;
/* Args: stemmer name, PG code for encoding, Snowball's name for encoding */
#define STEMMER_MODULE(name,enc,senc) \
{#name, enc, name##_##senc##_create_env, name##_##senc##_close_env, name##_##senc##_stem}
static const stemmer_module stemmer_modules[] =
{
/*
* Stemmers list from Snowball distribution
*/
STEMMER_MODULE(basque, PG_LATIN1, ISO_8859_1),
STEMMER_MODULE(catalan, PG_LATIN1, ISO_8859_1),
STEMMER_MODULE(danish, PG_LATIN1, ISO_8859_1),
STEMMER_MODULE(dutch, PG_LATIN1, ISO_8859_1),
STEMMER_MODULE(english, PG_LATIN1, ISO_8859_1),
STEMMER_MODULE(finnish, PG_LATIN1, ISO_8859_1),
STEMMER_MODULE(french, PG_LATIN1, ISO_8859_1),
STEMMER_MODULE(german, PG_LATIN1, ISO_8859_1),
STEMMER_MODULE(indonesian, PG_LATIN1, ISO_8859_1),
STEMMER_MODULE(irish, PG_LATIN1, ISO_8859_1),
STEMMER_MODULE(italian, PG_LATIN1, ISO_8859_1),
STEMMER_MODULE(norwegian, PG_LATIN1, ISO_8859_1),
STEMMER_MODULE(porter, PG_LATIN1, ISO_8859_1),
STEMMER_MODULE(portuguese, PG_LATIN1, ISO_8859_1),
STEMMER_MODULE(spanish, PG_LATIN1, ISO_8859_1),
STEMMER_MODULE(swedish, PG_LATIN1, ISO_8859_1),
STEMMER_MODULE(hungarian, PG_LATIN2, ISO_8859_2),
STEMMER_MODULE(romanian, PG_LATIN2, ISO_8859_2),
STEMMER_MODULE(russian, PG_KOI8R, KOI8_R),
STEMMER_MODULE(arabic, PG_UTF8, UTF_8),
STEMMER_MODULE(armenian, PG_UTF8, UTF_8),
STEMMER_MODULE(basque, PG_UTF8, UTF_8),
STEMMER_MODULE(catalan, PG_UTF8, UTF_8),
STEMMER_MODULE(danish, PG_UTF8, UTF_8),
STEMMER_MODULE(dutch, PG_UTF8, UTF_8),
STEMMER_MODULE(english, PG_UTF8, UTF_8),
STEMMER_MODULE(finnish, PG_UTF8, UTF_8),
STEMMER_MODULE(french, PG_UTF8, UTF_8),
STEMMER_MODULE(german, PG_UTF8, UTF_8),
STEMMER_MODULE(greek, PG_UTF8, UTF_8),
STEMMER_MODULE(hindi, PG_UTF8, UTF_8),
STEMMER_MODULE(hungarian, PG_UTF8, UTF_8),
STEMMER_MODULE(indonesian, PG_UTF8, UTF_8),
STEMMER_MODULE(irish, PG_UTF8, UTF_8),
STEMMER_MODULE(italian, PG_UTF8, UTF_8),
STEMMER_MODULE(lithuanian, PG_UTF8, UTF_8),
STEMMER_MODULE(nepali, PG_UTF8, UTF_8),
STEMMER_MODULE(norwegian, PG_UTF8, UTF_8),
STEMMER_MODULE(porter, PG_UTF8, UTF_8),
STEMMER_MODULE(portuguese, PG_UTF8, UTF_8),
STEMMER_MODULE(romanian, PG_UTF8, UTF_8),
STEMMER_MODULE(russian, PG_UTF8, UTF_8),
STEMMER_MODULE(serbian, PG_UTF8, UTF_8),
STEMMER_MODULE(spanish, PG_UTF8, UTF_8),
STEMMER_MODULE(swedish, PG_UTF8, UTF_8),
STEMMER_MODULE(tamil, PG_UTF8, UTF_8),
STEMMER_MODULE(turkish, PG_UTF8, UTF_8),
STEMMER_MODULE(yiddish, PG_UTF8, UTF_8),
/*
* Stemmer with PG_SQL_ASCII encoding should be valid for any server
* encoding
*/
STEMMER_MODULE(english, PG_SQL_ASCII, ISO_8859_1),
{NULL, 0, NULL, NULL, NULL} /* list end marker */
};
typedef struct DictSnowball
{
struct SN_env *z;
StopList stoplist;
bool needrecode; /* needs recoding before/after call stem */
int (*stem) (struct SN_env *z);
/*
* snowball saves alloced memory between calls, so we should run it in our
* private memory context. Note, init function is executed in long lived
* context, so we just remember CurrentMemoryContext
*/
MemoryContext dictCtx;
} DictSnowball;
static void
locate_stem_module(DictSnowball *d, const char *lang)
{
const stemmer_module *m;
/*
* First, try to find exact match of stemmer module. Stemmer with
* PG_SQL_ASCII encoding is treated as working with any server encoding
*/
for (m = stemmer_modules; m->name; m++)
{
if ((m->enc == PG_SQL_ASCII || m->enc == GetDatabaseEncoding()) &&
pg_strcasecmp(m->name, lang) == 0)
{
d->stem = m->stem;
d->z = m->create();
d->needrecode = false;
return;
}
}
/*
* Second, try to find stemmer for needed language for UTF8 encoding.
*/
for (m = stemmer_modules; m->name; m++)
{
if (m->enc == PG_UTF8 && pg_strcasecmp(m->name, lang) == 0)
{
d->stem = m->stem;
d->z = m->create();
d->needrecode = true;
return;
}
}
ereport(ERROR,
(errcode(ERRCODE_UNDEFINED_OBJECT),
errmsg("no Snowball stemmer available for language \"%s\" and encoding \"%s\"",
lang, GetDatabaseEncodingName())));
}
Datum
dsnowball_init(PG_FUNCTION_ARGS)
{
List *dictoptions = (List *) PG_GETARG_POINTER(0);
DictSnowball *d;
bool stoploaded = false;
ListCell *l;
d = (DictSnowball *) palloc0(sizeof(DictSnowball));
foreach(l, dictoptions)
{
DefElem *defel = (DefElem *) lfirst(l);
if (strcmp(defel->defname, "stopwords") == 0)
{
if (stoploaded)
ereport(ERROR,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
errmsg("multiple StopWords parameters")));
readstoplist(defGetString(defel), &d->stoplist, lowerstr);
stoploaded = true;
}
else if (strcmp(defel->defname, "language") == 0)
{
if (d->stem)
ereport(ERROR,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
errmsg("multiple Language parameters")));
locate_stem_module(d, defGetString(defel));
}
else
{
ereport(ERROR,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
errmsg("unrecognized Snowball parameter: \"%s\"",
defel->defname)));
}
}
if (!d->stem)
ereport(ERROR,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
errmsg("missing Language parameter")));
d->dictCtx = CurrentMemoryContext;
PG_RETURN_POINTER(d);
}
Datum
dsnowball_lexize(PG_FUNCTION_ARGS)
{
DictSnowball *d = (DictSnowball *) PG_GETARG_POINTER(0);
char *in = (char *) PG_GETARG_POINTER(1);
int32 len = PG_GETARG_INT32(2);
char *txt = lowerstr_with_len(in, len);
TSLexeme *res = palloc0(sizeof(TSLexeme) * 2);
/*
* Do not pass strings exceeding 1000 bytes to the stemmer, as they're
* surely not words in any human language. This restriction avoids
* wasting cycles on stuff like base64-encoded data, and it protects us
* against possible inefficiency or misbehavior in the stemmer. (For
* example, the Turkish stemmer has an indefinite recursion, so it can
* crash on long-enough strings.) However, Snowball dictionaries are
* defined to recognize all strings, so we can't reject the string as an
* unknown word.
*/
if (len > 1000)
{
/* return the lexeme lowercased, but otherwise unmodified */
res->lexeme = txt;
}
else if (*txt == '\0' || searchstoplist(&(d->stoplist), txt))
{
/* empty or stopword, so report as stopword */
pfree(txt);
}
else
{
MemoryContext saveCtx;
/*
* recode to utf8 if stemmer is utf8 and doesn't match server encoding
*/
if (d->needrecode)
{
char *recoded;
recoded = pg_server_to_any(txt, strlen(txt), PG_UTF8);
if (recoded != txt)
{
pfree(txt);
txt = recoded;
}
}
/* see comment about d->dictCtx */
saveCtx = MemoryContextSwitchTo(d->dictCtx);
SN_set_current(d->z, strlen(txt), (symbol *) txt);
d->stem(d->z);
MemoryContextSwitchTo(saveCtx);
if (d->z->p && d->z->l)
{
txt = repalloc(txt, d->z->l + 1);
memcpy(txt, d->z->p, d->z->l);
txt[d->z->l] = '\0';
}
/* back recode if needed */
if (d->needrecode)
{
char *recoded;
recoded = pg_any_to_server(txt, strlen(txt), PG_UTF8);
if (recoded != txt)
{
pfree(txt);
txt = recoded;
}
}
res->lexeme = txt;
}
PG_RETURN_POINTER(res);
}
View Git Blame Copy Permalink