database/postgres
1
0
Fork 0
mirror of https://github.com/postgres/postgres.git synced 2025-09-02 04:21:28 +03:00
Code Activity

Create a type-specific typanalyze routine for tsvector, which collects stats

Browse Source 
on the most common individual lexemes in place of the mostly-useless default
behavior of counting duplicate tsvectors.  Future work: create selectivity
estimation functions that actually do something with these stats.

(Some other things we ought to look at doing: using the Lossy Counting
algorithm in compute_minimal_stats, and using the element-counting idea for
stats on regular arrays.)

Jan Urbanski
This commit is contained in:
Tom Lane
2008-07-14 00:51:46 +00:00
parent 6816577a78
commit 6f6d863258
11 changed files with 467 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/backend/tsearch/Makefile
View File

@@ -4,7 +4,7 @@
#
# Copyright (c) 2006-2008, PostgreSQL Global Development Group
#
# $PostgreSQL: pgsql/src/backend/tsearch/Makefile,v 1.6 2008/02/19 10:30:08 petere Exp $
# $PostgreSQL: pgsql/src/backend/tsearch/Makefile,v 1.7 2008/07/14 00:51:45 tgl Exp $
#
#-------------------------------------------------------------------------
subdir = src/backend/tsearch
@@ -19,7 +19,7 @@ DICTFILES=synonym_sample.syn thesaurus_sample.ths hunspell_sample.affix \
OBJS = ts_locale.o ts_parse.o wparser.o wparser_def.o dict.o \
dict_simple.o dict_synonym.o dict_thesaurus.o \
dict_ispell.o regis.o spell.o \
to_tsany.o ts_utils.o
to_tsany.o ts_typanalyze.o ts_utils.o
include $(top_srcdir)/src/backend/common.mk

403
src/backend/tsearch/ts_typanalyze.c Normal file
View File

@@ -0,0 +1,403 @@
/*-------------------------------------------------------------------------
*
* ts_typanalyze.c
* functions for gathering statistics from tsvector columns
*
* Portions Copyright (c) 1996-2008, PostgreSQL Global Development Group
*
*
* IDENTIFICATION
* $PostgreSQL: pgsql/src/backend/tsearch/ts_typanalyze.c,v 1.1 2008/07/14 00:51:45 tgl Exp $
*
*-------------------------------------------------------------------------
*/
#include "postgres.h"
#include "access/hash.h"
#include "catalog/pg_operator.h"
#include "commands/vacuum.h"
#include "tsearch/ts_type.h"
#include "utils/builtins.h"
#include "utils/hsearch.h"
/* A hash key for lexemes */
typedef struct
{
char *lexeme; /* lexeme (not NULL terminated!) */
int length; /* its length in bytes */
} LexemeHashKey;
/* A hash table entry for the Lossy Counting algorithm */
typedef struct
{
LexemeHashKey key; /* This is 'e' from the LC algorithm. */
int frequency; /* This is 'f'. */
int delta; /* And this is 'delta'. */
} TrackItem;
static void compute_tsvector_stats(VacAttrStats *stats,
AnalyzeAttrFetchFunc fetchfunc,
int samplerows,
double totalrows);
static void prune_lexemes_hashtable(HTAB *lexemes_tab, int b_current);
static uint32 lexeme_hash(const void *key, Size keysize);
static int lexeme_match(const void *key1, const void *key2, Size keysize);
static int trackitem_compare_desc(const void *e1, const void *e2);
/*
* ts_typanalyze -- a custom typanalyze function for tsvector columns
*/
Datum
ts_typanalyze(PG_FUNCTION_ARGS)
{
VacAttrStats *stats = (VacAttrStats *) PG_GETARG_POINTER(0);
Form_pg_attribute attr = stats->attr;
/* If the attstattarget column is negative, use the default value */
/* NB: it is okay to scribble on stats->attr since it's a copy */
if (attr->attstattarget < 0)
attr->attstattarget = default_statistics_target;
stats->compute_stats = compute_tsvector_stats;
/* see comment about the choice of minrows from analyze.c */
stats->minrows = 300 * attr->attstattarget;
PG_RETURN_BOOL(true);
}
/*
* compute_tsvector_stats() -- compute statistics for a tsvector column
*
* This functions computes statistics that are useful for determining @@
* operations' selectivity, along with the fraction of non-null rows and
* average width.
*
* Instead of finding the most common values, as we do for most datatypes,
* we're looking for the most common lexemes. This is more useful, because
* there most probably won't be any two rows with the same tsvector and thus
* the notion of a MCV is a bit bogus with this datatype. With a list of the
* most common lexemes we can do a better job at figuring out @@ selectivity.
*
* For the same reasons we assume that tsvector columns are unique when
* determining the number of distinct values.
*
* The algorithm used is Lossy Counting, as proposed in the paper "Approximate
* frequency counts over data streams" by G. S. Manku and R. Motwani, in
* Proceedings of the 28th International Conference on Very Large Data Bases,
* Hong Kong, China, August 2002, section 4.2. The paper is available at
* http://www.vldb.org/conf/2002/S10P03.pdf
*
* The Lossy Counting (aka LC) algorithm goes like this:
* Let D be a set of triples (e, f, d), where e is an element value, f is
* that element's frequency (occurrence count) and d is the maximum error in
* f. We start with D empty and process the elements in batches of size
* w. (The batch size is also known as "bucket size".) Let the current batch
* number be b_current, starting with 1. For each element e we either
* increment its f count, if it's already in D, or insert a new triple into D
* with values (e, 1, b_current - 1). After processing each batch we prune D,
* by removing from it all elements with f + d <= b_current. Finally, we
* gather elements with largest f. The LC paper proves error bounds on f
* dependent on the batch size w, and shows that the required table size
* is no more than a few times w.
*
* We use a hashtable for the D structure and a bucket width of
* statistic_target * 100, where 100 is an arbitrarily chosen constant, meant
* to approximate the number of lexemes in a single tsvector.
*/
static void
compute_tsvector_stats(VacAttrStats *stats,
AnalyzeAttrFetchFunc fetchfunc,
int samplerows,
double totalrows)
{
int num_mcelem;
int null_cnt = 0;
double total_width = 0;
/* This is D from the LC algorithm. */
HTAB *lexemes_tab;
HASHCTL hash_ctl;
HASH_SEQ_STATUS scan_status;
/* This is the current bucket number from the LC algorithm */
int b_current;
/* This is 'w' from the LC algorithm */
int bucket_width;
int vector_no,
lexeme_no;
LexemeHashKey hash_key;
TrackItem *item;
/* We want statistic_target * 100 lexemes in the MCELEM array */
num_mcelem = stats->attr->attstattarget * 100;
/*
* We set bucket width equal to the target number of result lexemes.
* This is probably about right but perhaps might need to be scaled
* up or down a bit?
*/
bucket_width = num_mcelem;
/*
* Create the hashtable. It will be in local memory, so we don't need to
* worry about initial size too much. Also we don't need to pay any
* attention to locking and memory management.
*/
MemSet(&hash_ctl, 0, sizeof(hash_ctl));
hash_ctl.keysize = sizeof(LexemeHashKey);
hash_ctl.entrysize = sizeof(TrackItem);
hash_ctl.hash = lexeme_hash;
hash_ctl.match = lexeme_match;
hash_ctl.hcxt = CurrentMemoryContext;
lexemes_tab = hash_create("Analyzed lexemes table",
bucket_width * 4,
&hash_ctl,
HASH_ELEM | HASH_FUNCTION | HASH_COMPARE | HASH_CONTEXT);
/* Initialize counters. */
b_current = 1;
lexeme_no = 1;
/* Loop over the tsvectors. */
for (vector_no = 0; vector_no < samplerows; vector_no++)
{
Datum value;
bool isnull;
TSVector vector;
WordEntry *curentryptr;
char *lexemesptr;
int j;
vacuum_delay_point();
value = fetchfunc(stats, vector_no, &isnull);
/*
* Check for null/nonnull.
*/
if (isnull)
{
null_cnt++;
continue;
}
/*
* Add up widths for average-width calculation. Since it's a
* tsvector, we know it's varlena. As in the regular
* compute_minimal_stats function, we use the toasted width for this
* calculation.
*/
total_width += VARSIZE_ANY(DatumGetPointer(value));
/*
* Now detoast the tsvector if needed.
*/
vector = DatumGetTSVector(value);
/*
* We loop through the lexemes in the tsvector and add them to our
* tracking hashtable. Note: the hashtable entries will point into
* the (detoasted) tsvector value, therefore we cannot free that
* storage until we're done.
*/
lexemesptr = STRPTR(vector);
curentryptr = ARRPTR(vector);
for (j = 0; j < vector->size; j++)
{
bool found;
/* Construct a hash key */
hash_key.lexeme = lexemesptr + curentryptr->pos;
hash_key.length = curentryptr->len;
/* Lookup current lexeme in hashtable, adding it if new */
item = (TrackItem *) hash_search(lexemes_tab,
(const void *) &hash_key,
HASH_ENTER, &found);
if (found)
{
/* The lexeme is already on the tracking list */
item->frequency++;
}
else
{
/* Initialize new tracking list element */
item->frequency = 1;
item->delta = b_current - 1;
}
/* We prune the D structure after processing each bucket */
if (lexeme_no % bucket_width == 0)
{
prune_lexemes_hashtable(lexemes_tab, b_current);
b_current++;
}
/* Advance to the next WordEntry in the tsvector */
lexeme_no++;
curentryptr++;
}
}
/* We can only compute real stats if we found some non-null values. */
if (null_cnt < samplerows)
{
int nonnull_cnt = samplerows - null_cnt;
int i;
TrackItem **sort_table;
int track_len;
stats->stats_valid = true;
/* Do the simple null-frac and average width stats */
stats->stanullfrac = (double) null_cnt / (double) samplerows;
stats->stawidth = total_width / (double) nonnull_cnt;
/* Assume it's a unique column (see notes above) */
stats->stadistinct = -1.0;
/*
* Determine the top-N lexemes by simply copying pointers from the
* hashtable into an array and applying qsort()
*/
track_len = hash_get_num_entries(lexemes_tab);
sort_table = (TrackItem **) palloc(sizeof(TrackItem *) * track_len);
hash_seq_init(&scan_status, lexemes_tab);
i = 0;
while ((item = (TrackItem *) hash_seq_search(&scan_status)) != NULL)
{
sort_table[i++] = item;
}
Assert(i == track_len);
qsort(sort_table, track_len, sizeof(TrackItem *),
trackitem_compare_desc);
/* Suppress any single-occurrence items */
while (track_len > 0)
{
if (sort_table[track_len-1]->frequency > 1)
break;
track_len--;
}
/* Determine the number of most common lexemes to be stored */
if (num_mcelem > track_len)
num_mcelem = track_len;
/* Generate MCELEM slot entry */
if (num_mcelem > 0)
{
MemoryContext old_context;
Datum *mcelem_values;
float4 *mcelem_freqs;
/* Must copy the target values into anl_context */
old_context = MemoryContextSwitchTo(stats->anl_context);
mcelem_values = (Datum *) palloc(num_mcelem * sizeof(Datum));
mcelem_freqs = (float4 *) palloc(num_mcelem * sizeof(float4));
for (i = 0; i < num_mcelem; i++)
{
TrackItem *item = sort_table[i];
mcelem_values[i] =
PointerGetDatum(cstring_to_text_with_len(item->key.lexeme,
item->key.length));
mcelem_freqs[i] = (double) item->frequency / (double) nonnull_cnt;
}
MemoryContextSwitchTo(old_context);
stats->stakind[0] = STATISTIC_KIND_MCELEM;
stats->staop[0] = TextEqualOperator;
stats->stanumbers[0] = mcelem_freqs;
stats->numnumbers[0] = num_mcelem;
stats->stavalues[0] = mcelem_values;
stats->numvalues[0] = num_mcelem;
/* We are storing text values */
stats->statypid[0] = TEXTOID;
stats->statyplen[0] = -1; /* typlen, -1 for varlena */
stats->statypbyval[0] = false;
stats->statypalign[0] = 'i';
}
}
else
{
/* We found only nulls; assume the column is entirely null */
stats->stats_valid = true;
stats->stanullfrac = 1.0;
stats->stawidth = 0; /* "unknown" */
stats->stadistinct = 0.0; /* "unknown" */
}
/*
* We don't need to bother cleaning up any of our temporary palloc's.
* The hashtable should also go away, as it used a child memory context.
*/
}
/*
* A function to prune the D structure from the Lossy Counting algorithm.
* Consult compute_tsvector_stats() for wider explanation.
*/
static void
prune_lexemes_hashtable(HTAB *lexemes_tab, int b_current)
{
HASH_SEQ_STATUS scan_status;
TrackItem *item;
hash_seq_init(&scan_status, lexemes_tab);
while ((item = (TrackItem *) hash_seq_search(&scan_status)) != NULL)
{
if (item->frequency + item->delta <= b_current)
{
if (hash_search(lexemes_tab, (const void *) &item->key,
HASH_REMOVE, NULL) == NULL)
elog(ERROR, "hash table corrupted");
}
}
}
/*
* Hash functions for lexemes. They are strings, but not NULL terminated,
* so we need a special hash function.
*/
static uint32
lexeme_hash(const void *key, Size keysize)
{
const LexemeHashKey *l = (const LexemeHashKey *) key;
return DatumGetUInt32(hash_any((const unsigned char *) l->lexeme,
l->length));
}
/*
* Matching function for lexemes, to be used in hashtable lookups.
*/
static int
lexeme_match(const void *key1, const void *key2, Size keysize)
{
const LexemeHashKey *d1 = (const LexemeHashKey *) key1;
const LexemeHashKey *d2 = (const LexemeHashKey *) key2;
/* The lexemes need to have the same length, and be memcmp-equal */
if (d1->length == d2->length &&
memcmp(d1->lexeme, d2->lexeme, d1->length) == 0)
return 0;
else
return 1;
}
/*
* qsort() comparator for TrackItems - LC style (descending sort)
*/
static int
trackitem_compare_desc(const void *e1, const void *e2)
{
const TrackItem * const *t1 = (const TrackItem * const *) e1;
const TrackItem * const *t2 = (const TrackItem * const *) e2;
return (*t2)->frequency - (*t1)->frequency;
}