1
0
mirror of https://github.com/postgres/postgres.git synced 2025-11-15 03:41:20 +03:00

Rewrite of planner statistics-gathering code. ANALYZE is now available as

a separate statement (though it can still be invoked as part of VACUUM, too).
pg_statistic redesigned to be more flexible about what statistics are
stored.  ANALYZE now collects a list of several of the most common values,
not just one, plus a histogram (not just the min and max values).  Random
sampling is used to make the process reasonably fast even on very large
tables.  The number of values and histogram bins collected is now
user-settable via an ALTER TABLE command.

There is more still to do; the new stats are not being used everywhere
they could be in the planner.  But the remaining changes for this project
should be localized, and the behavior is already better than before.

A not-very-related change is that sorting now makes use of btree comparison
routines if it can find one, rather than invoking '<' twice.
This commit is contained in:
Tom Lane
2001-05-07 00:43:27 +00:00
parent 9583aea9d0
commit f905d65ee3
66 changed files with 4131 additions and 2063 deletions

File diff suppressed because it is too large Load Diff

View File

@@ -7,7 +7,7 @@
* Portions Copyright (c) 1994, Regents of the University of California
*
* IDENTIFICATION
* $Header: /cvsroot/pgsql/src/backend/utils/cache/lsyscache.c,v 1.52 2001/03/23 04:49:55 momjian Exp $
* $Header: /cvsroot/pgsql/src/backend/utils/cache/lsyscache.c,v 1.53 2001/05/07 00:43:24 tgl Exp $
*
* NOTES
* Eventually, the index information should go through here, too.
@@ -18,7 +18,10 @@
#include "access/tupmacs.h"
#include "catalog/pg_operator.h"
#include "catalog/pg_proc.h"
#include "catalog/pg_statistic.h"
#include "catalog/pg_type.h"
#include "utils/array.h"
#include "utils/builtins.h"
#include "utils/lsyscache.h"
#include "utils/syscache.h"
@@ -182,106 +185,6 @@ get_atttypmod(Oid relid, AttrNumber attnum)
return -1;
}
/*
* get_attdispersion
*
* Retrieve the dispersion statistic for an attribute,
* or produce an estimate if no info is available.
*
* min_estimate is the minimum estimate to return if insufficient data
* is available to produce a reliable value. This value may vary
* depending on context. (For example, when deciding whether it is
* safe to use a hashjoin, we want to be more conservative than when
* estimating the number of tuples produced by an equijoin.)
*/
double
get_attdispersion(Oid relid, AttrNumber attnum, double min_estimate)
{
HeapTuple atp;
Form_pg_attribute att_tup;
double dispersion;
Oid atttypid;
int32 ntuples;
atp = SearchSysCache(ATTNUM,
ObjectIdGetDatum(relid),
Int16GetDatum(attnum),
0, 0);
if (!HeapTupleIsValid(atp))
{
/* this should not happen */
elog(ERROR, "get_attdispersion: no attribute tuple %u %d",
relid, attnum);
return min_estimate;
}
att_tup = (Form_pg_attribute) GETSTRUCT(atp);
dispersion = att_tup->attdispersion;
atttypid = att_tup->atttypid;
ReleaseSysCache(atp);
if (dispersion > 0.0)
return dispersion; /* we have a specific estimate from VACUUM */
/*
* Special-case boolean columns: the dispersion of a boolean is highly
* unlikely to be anywhere near 1/numtuples, instead it's probably
* more like 0.5.
*
* Are there any other cases we should wire in special estimates for?
*/
if (atttypid == BOOLOID)
return 0.5;
/*
* Dispersion is either 0 (no data available) or -1 (dispersion is
* 1/numtuples). Either way, we need the relation size.
*/
atp = SearchSysCache(RELOID,
ObjectIdGetDatum(relid),
0, 0, 0);
if (!HeapTupleIsValid(atp))
{
/* this should not happen */
elog(ERROR, "get_attdispersion: no relation tuple %u", relid);
return min_estimate;
}
ntuples = ((Form_pg_class) GETSTRUCT(atp))->reltuples;
ReleaseSysCache(atp);
if (ntuples == 0)
return min_estimate; /* no data available */
if (dispersion < 0.0) /* VACUUM thinks there are no duplicates */
return 1.0 / (double) ntuples;
/*
* VACUUM ANALYZE does not compute dispersion for system attributes,
* but some of them can reasonably be assumed unique anyway.
*/
if (attnum == ObjectIdAttributeNumber ||
attnum == SelfItemPointerAttributeNumber)
return 1.0 / (double) ntuples;
if (attnum == TableOidAttributeNumber)
return 1.0;
/*
* VACUUM ANALYZE has not been run for this table. Produce an estimate
* of 1/numtuples. This may produce unreasonably small estimates for
* large tables, so limit the estimate to no less than min_estimate.
*/
dispersion = 1.0 / (double) ntuples;
if (dispersion < min_estimate)
dispersion = min_estimate;
return dispersion;
}
/* ---------- INDEX CACHE ---------- */
/* watch this space...
@@ -876,3 +779,157 @@ get_typtype(Oid typid)
}
#endif
/* ---------- STATISTICS CACHE ---------- */
/*
* get_attstatsslot
*
* Extract the contents of a "slot" of a pg_statistic tuple.
* Returns TRUE if requested slot type was found, else FALSE.
*
* Unlike other routines in this file, this takes a pointer to an
* already-looked-up tuple in the pg_statistic cache. We do this since
* most callers will want to extract more than one value from the cache
* entry, and we don't want to repeat the cache lookup unnecessarily.
*
* statstuple: pg_statistics tuple to be examined.
* atttype: type OID of attribute.
* atttypmod: typmod of attribute.
* reqkind: STAKIND code for desired statistics slot kind.
* reqop: STAOP value wanted, or InvalidOid if don't care.
* values, nvalues: if not NULL, the slot's stavalues are extracted.
* numbers, nnumbers: if not NULL, the slot's stanumbers are extracted.
*
* If assigned, values and numbers are set to point to palloc'd arrays.
* If the attribute type is pass-by-reference, the values referenced by
* the values array are themselves palloc'd. The palloc'd stuff can be
* freed by calling free_attstatsslot.
*/
bool
get_attstatsslot(HeapTuple statstuple,
Oid atttype, int32 atttypmod,
int reqkind, Oid reqop,
Datum **values, int *nvalues,
float4 **numbers, int *nnumbers)
{
Form_pg_statistic stats = (Form_pg_statistic) GETSTRUCT(statstuple);
int i,
j;
Datum val;
bool isnull;
ArrayType *statarray;
int narrayelem;
HeapTuple typeTuple;
FmgrInfo inputproc;
Oid typelem;
for (i = 0; i < STATISTIC_NUM_SLOTS; i++)
{
if ((&stats->stakind1)[i] == reqkind &&
(reqop == InvalidOid || (&stats->staop1)[i] == reqop))
break;
}
if (i >= STATISTIC_NUM_SLOTS)
return false; /* not there */
if (values)
{
val = SysCacheGetAttr(STATRELATT, statstuple,
Anum_pg_statistic_stavalues1 + i,
&isnull);
if (isnull)
elog(ERROR, "get_attstatsslot: stavalues is null");
statarray = DatumGetArrayTypeP(val);
/*
* Do initial examination of the array. This produces a list
* of text Datums --- ie, pointers into the text array value.
*/
deconstruct_array(statarray, false, -1, 'i', values, nvalues);
narrayelem = *nvalues;
/*
* We now need to replace each text Datum by its internal equivalent.
*
* Get the type input proc and typelem for the column datatype.
*/
typeTuple = SearchSysCache(TYPEOID,
ObjectIdGetDatum(atttype),
0, 0, 0);
if (!HeapTupleIsValid(typeTuple))
elog(ERROR, "get_attstatsslot: Cache lookup failed for type %u",
atttype);
fmgr_info(((Form_pg_type) GETSTRUCT(typeTuple))->typinput, &inputproc);
typelem = ((Form_pg_type) GETSTRUCT(typeTuple))->typelem;
ReleaseSysCache(typeTuple);
/*
* Do the conversions. The palloc'd array of Datums is reused
* in place.
*/
for (j = 0; j < narrayelem; j++)
{
char *strval;
strval = DatumGetCString(DirectFunctionCall1(textout,
(*values)[j]));
(*values)[j] = FunctionCall3(&inputproc,
CStringGetDatum(strval),
ObjectIdGetDatum(typelem),
Int32GetDatum(atttypmod));
pfree(strval);
}
/*
* Free statarray if it's a detoasted copy.
*/
if ((Pointer) statarray != DatumGetPointer(val))
pfree(statarray);
}
if (numbers)
{
val = SysCacheGetAttr(STATRELATT, statstuple,
Anum_pg_statistic_stanumbers1 + i,
&isnull);
if (isnull)
elog(ERROR, "get_attstatsslot: stanumbers is null");
statarray = DatumGetArrayTypeP(val);
/*
* We expect the array to be a 1-D float4 array; verify that.
* We don't need to use deconstruct_array() since the array
* data is just going to look like a C array of float4 values.
*/
narrayelem = ARR_DIMS(statarray)[0];
if (ARR_NDIM(statarray) != 1 || narrayelem <= 0 ||
ARR_SIZE(statarray) != (ARR_OVERHEAD(1) + narrayelem * sizeof(float4)))
elog(ERROR, "get_attstatsslot: stanumbers is bogus");
*numbers = (float4 *) palloc(narrayelem * sizeof(float4));
memcpy(*numbers, ARR_DATA_PTR(statarray), narrayelem * sizeof(float4));
*nnumbers = narrayelem;
/*
* Free statarray if it's a detoasted copy.
*/
if ((Pointer) statarray != DatumGetPointer(val))
pfree(statarray);
}
return true;
}
void
free_attstatsslot(Oid atttype,
Datum *values, int nvalues,
float4 *numbers, int nnumbers)
{
if (values)
{
if (! get_typbyval(atttype))
{
int i;
for (i = 0; i < nvalues; i++)
pfree(DatumGetPointer(values[i]));
}
pfree(values);
}
if (numbers)
pfree(numbers);
}

View File

@@ -8,7 +8,7 @@
*
*
* IDENTIFICATION
* $Header: /cvsroot/pgsql/src/backend/utils/cache/syscache.c,v 1.60 2001/03/22 03:59:57 momjian Exp $
* $Header: /cvsroot/pgsql/src/backend/utils/cache/syscache.c,v 1.61 2001/05/07 00:43:24 tgl Exp $
*
* NOTES
* These routines allow the parser/planner/executor to perform
@@ -313,7 +313,7 @@ static struct cachedesc cacheinfo[] = {
0,
0
}},
{StatisticRelationName, /* STATRELID */
{StatisticRelationName, /* STATRELATT */
StatisticRelidAttnumIndex,
2,
{

View File

@@ -78,7 +78,7 @@
* Portions Copyright (c) 1994, Regents of the University of California
*
* IDENTIFICATION
* $Header: /cvsroot/pgsql/src/backend/utils/sort/tuplesort.c,v 1.15 2001/03/23 04:49:55 momjian Exp $
* $Header: /cvsroot/pgsql/src/backend/utils/sort/tuplesort.c,v 1.16 2001/05/07 00:43:24 tgl Exp $
*
*-------------------------------------------------------------------------
*/
@@ -87,7 +87,11 @@
#include "access/heapam.h"
#include "access/nbtree.h"
#include "catalog/catname.h"
#include "catalog/pg_amop.h"
#include "catalog/pg_amproc.h"
#include "miscadmin.h"
#include "utils/fmgroids.h"
#include "utils/logtape.h"
#include "utils/lsyscache.h"
#include "utils/tuplesort.h"
@@ -263,6 +267,7 @@ struct Tuplesortstate
TupleDesc tupDesc;
int nKeys;
ScanKey scanKeys;
SortFunctionKind *sortFnKinds;
/*
* These variables are specific to the IndexTuple case; they are set
@@ -279,6 +284,7 @@ struct Tuplesortstate
Oid datumType;
Oid sortOperator;
FmgrInfo sortOpFn; /* cached lookup data for sortOperator */
SortFunctionKind sortFnKind;
/* we need typelen and byval in order to know how to copy the Datums. */
int datumTypeLen;
bool datumTypeByVal;
@@ -458,14 +464,14 @@ tuplesort_begin_common(bool randomAccess)
Tuplesortstate *
tuplesort_begin_heap(TupleDesc tupDesc,
int nkeys, ScanKey keys,
int nkeys,
Oid *sortOperators, AttrNumber *attNums,
bool randomAccess)
{
Tuplesortstate *state = tuplesort_begin_common(randomAccess);
int i;
AssertArg(nkeys >= 1);
AssertArg(keys[0].sk_attno != 0);
AssertArg(keys[0].sk_procedure != 0);
AssertArg(nkeys > 0);
state->comparetup = comparetup_heap;
state->copytup = copytup_heap;
@@ -475,7 +481,29 @@ tuplesort_begin_heap(TupleDesc tupDesc,
state->tupDesc = tupDesc;
state->nKeys = nkeys;
state->scanKeys = keys;
state->scanKeys = (ScanKey) palloc(nkeys * sizeof(ScanKeyData));
MemSet(state->scanKeys, 0, nkeys * sizeof(ScanKeyData));
state->sortFnKinds = (SortFunctionKind *)
palloc(nkeys * sizeof(SortFunctionKind));
MemSet(state->sortFnKinds, 0, nkeys * sizeof(SortFunctionKind));
for (i = 0; i < nkeys; i++)
{
RegProcedure sortFunction;
AssertArg(sortOperators[i] != 0);
AssertArg(attNums[i] != 0);
/* select a function that implements the sort operator */
SelectSortFunction(sortOperators[i], &sortFunction,
&state->sortFnKinds[i]);
ScanKeyEntryInitialize(&state->scanKeys[i],
0x0,
attNums[i],
sortFunction,
(Datum) 0);
}
return state;
}
@@ -507,6 +535,7 @@ tuplesort_begin_datum(Oid datumType,
bool randomAccess)
{
Tuplesortstate *state = tuplesort_begin_common(randomAccess);
RegProcedure sortFunction;
int16 typlen;
bool typbyval;
@@ -518,8 +547,12 @@ tuplesort_begin_datum(Oid datumType,
state->datumType = datumType;
state->sortOperator = sortOperator;
/* lookup the function that implements the sort operator */
fmgr_info(get_opcode(sortOperator), &state->sortOpFn);
/* select a function that implements the sort operator */
SelectSortFunction(sortOperator, &sortFunction, &state->sortFnKind);
/* and look up the function */
fmgr_info(sortFunction, &state->sortOpFn);
/* lookup necessary attributes of the datum type */
get_typlenbyval(datumType, &typlen, &typbyval);
state->datumTypeLen = typlen;
@@ -548,6 +581,13 @@ tuplesort_end(Tuplesortstate *state)
}
if (state->memtupindex)
pfree(state->memtupindex);
/* this stuff might better belong in a variant-specific shutdown routine */
if (state->scanKeys)
pfree(state->scanKeys);
if (state->sortFnKinds)
pfree(state->sortFnKinds);
pfree(state);
}
@@ -1692,6 +1732,7 @@ comparetup_heap(Tuplesortstate *state, const void *a, const void *b)
for (nkey = 0; nkey < state->nKeys; nkey++)
{
ScanKey scanKey = state->scanKeys + nkey;
SortFunctionKind fnKind = state->sortFnKinds[nkey];
AttrNumber attno = scanKey->sk_attno;
Datum lattr,
rattr;
@@ -1708,23 +1749,36 @@ comparetup_heap(Tuplesortstate *state, const void *a, const void *b)
}
else if (isnull2)
return -1;
else if (scanKey->sk_flags & SK_COMMUTE)
{
if (DatumGetBool(FunctionCall2(&scanKey->sk_func,
rattr, lattr)))
return -1; /* a < b after commute */
if (DatumGetBool(FunctionCall2(&scanKey->sk_func,
lattr, rattr)))
return 1; /* a > b after commute */
}
else
{
if (DatumGetBool(FunctionCall2(&scanKey->sk_func,
lattr, rattr)))
return -1; /* a < b */
if (DatumGetBool(FunctionCall2(&scanKey->sk_func,
rattr, lattr)))
return 1; /* a > b */
int32 compare;
if (fnKind == SORTFUNC_LT)
{
if (DatumGetBool(FunctionCall2(&scanKey->sk_func,
lattr, rattr)))
compare = -1; /* a < b */
else if (DatumGetBool(FunctionCall2(&scanKey->sk_func,
rattr, lattr)))
compare = 1; /* a > b */
else
compare = 0;
}
else
{
/* sort function is CMP or REVCMP */
compare = DatumGetInt32(FunctionCall2(&scanKey->sk_func,
lattr, rattr));
if (fnKind == SORTFUNC_REVCMP)
compare = -compare;
}
if (compare != 0)
{
if (scanKey->sk_flags & SK_COMMUTE)
compare = -compare;
return compare;
}
}
}
@@ -1852,8 +1906,10 @@ comparetup_index(Tuplesortstate *state, const void *a, const void *b)
}
else
{
/* the comparison function is always of CMP type */
compare = DatumGetInt32(FunctionCall2(&entry->sk_func,
attrDatum1, attrDatum2));
attrDatum1,
attrDatum2));
}
if (compare != 0)
@@ -1954,7 +2010,7 @@ comparetup_datum(Tuplesortstate *state, const void *a, const void *b)
}
else if (rtup->isNull)
return -1;
else
else if (state->sortFnKind == SORTFUNC_LT)
{
if (DatumGetBool(FunctionCall2(&state->sortOpFn,
ltup->val, rtup->val)))
@@ -1964,6 +2020,17 @@ comparetup_datum(Tuplesortstate *state, const void *a, const void *b)
return 1; /* a > b */
return 0;
}
else
{
/* sort function is CMP or REVCMP */
int32 compare;
compare = DatumGetInt32(FunctionCall2(&state->sortOpFn,
ltup->val, rtup->val));
if (state->sortFnKind == SORTFUNC_REVCMP)
compare = -compare;
return compare;
}
}
static void *
@@ -2032,3 +2099,119 @@ tuplesize_datum(Tuplesortstate *state, void *tup)
return (unsigned int) tuplelen;
}
}
/*
* This routine selects an appropriate sorting function to implement
* a sort operator as efficiently as possible. The straightforward
* method is to use the operator's implementation proc --- ie, "<"
* comparison. However, that way often requires two calls of the function
* per comparison. If we can find a btree three-way comparator function
* associated with the operator, we can use it to do the comparisons
* more efficiently. We also support the possibility that the operator
* is ">" (descending sort), in which case we have to reverse the output
* of the btree comparator.
*
* Possibly this should live somewhere else (backend/catalog/, maybe?).
*/
void
SelectSortFunction(Oid sortOperator,
RegProcedure *sortFunction,
SortFunctionKind *kind)
{
Relation relation;
HeapScanDesc scan;
ScanKeyData skey[3];
HeapTuple tuple;
Oid opclass = InvalidOid;
/*
* Scan pg_amop to see if the target operator is registered as the
* "<" or ">" operator of any btree opclass. It's possible that it
* might be registered both ways (eg, if someone were to build a
* "reverse sort" opclass for some reason); prefer the "<" case if so.
* If the operator is registered the same way in multiple opclasses,
* assume we can use the associated comparator function from any one.
*/
relation = heap_openr(AccessMethodOperatorRelationName,
AccessShareLock);
ScanKeyEntryInitialize(&skey[0], 0,
Anum_pg_amop_amopid,
F_OIDEQ,
ObjectIdGetDatum(BTREE_AM_OID));
ScanKeyEntryInitialize(&skey[1], 0,
Anum_pg_amop_amopopr,
F_OIDEQ,
ObjectIdGetDatum(sortOperator));
scan = heap_beginscan(relation, false, SnapshotNow, 2, skey);
while (HeapTupleIsValid(tuple = heap_getnext(scan, 0)))
{
Form_pg_amop aform = (Form_pg_amop) GETSTRUCT(tuple);
if (aform->amopstrategy == BTLessStrategyNumber)
{
opclass = aform->amopclaid;
*kind = SORTFUNC_CMP;
break; /* done looking */
}
else if (aform->amopstrategy == BTGreaterStrategyNumber)
{
opclass = aform->amopclaid;
*kind = SORTFUNC_REVCMP;
/* keep scanning in hopes of finding a BTLess entry */
}
}
heap_endscan(scan);
heap_close(relation, AccessShareLock);
if (OidIsValid(opclass))
{
/* Found a suitable opclass, get its comparator support function */
relation = heap_openr(AccessMethodProcedureRelationName,
AccessShareLock);
ScanKeyEntryInitialize(&skey[0], 0,
Anum_pg_amproc_amid,
F_OIDEQ,
ObjectIdGetDatum(BTREE_AM_OID));
ScanKeyEntryInitialize(&skey[1], 0,
Anum_pg_amproc_amopclaid,
F_OIDEQ,
ObjectIdGetDatum(opclass));
ScanKeyEntryInitialize(&skey[2], 0,
Anum_pg_amproc_amprocnum,
F_INT2EQ,
Int16GetDatum(BTORDER_PROC));
scan = heap_beginscan(relation, false, SnapshotNow, 3, skey);
*sortFunction = InvalidOid;
if (HeapTupleIsValid(tuple = heap_getnext(scan, 0)))
{
Form_pg_amproc aform = (Form_pg_amproc) GETSTRUCT(tuple);
*sortFunction = aform->amproc;
}
heap_endscan(scan);
heap_close(relation, AccessShareLock);
if (RegProcedureIsValid(*sortFunction))
return;
}
/* Can't find a comparator, so use the operator as-is */
*kind = SORTFUNC_LT;
*sortFunction = get_opcode(sortOperator);
if (!RegProcedureIsValid(*sortFunction))
elog(ERROR, "SelectSortFunction: operator %u has no implementation",
sortOperator);
}