1
0
mirror of https://github.com/postgres/postgres.git synced 2025-11-19 13:42:17 +03:00

Widen query numbers-of-tuples-processed counters to uint64.

This patch widens SPI_processed, EState's es_processed field, PortalData's
portalPos field, FuncCallContext's call_cntr and max_calls fields,
ExecutorRun's count argument, PortalRunFetch's result, and the max number
of rows in a SPITupleTable to uint64, and deals with (I hope) all the
ensuing fallout.  Some of these values were declared uint32 before, and
others "long".

I also removed PortalData's posOverflow field, since that logic seems
pretty useless given that portalPos is now always 64 bits.

The user-visible results are that command tags for SELECT etc will
correctly report tuple counts larger than 4G, as will plpgsql's GET
GET DIAGNOSTICS ... ROW_COUNT command.  Queries processing more tuples
than that are still not exactly the norm, but they're becoming more
common.

Most values associated with FETCH/MOVE distances, such as PortalRun's count
argument and the count argument of most SPI functions that have one, remain
declared as "long".  It's not clear whether it would be worth promoting
those to int64; but it would definitely be a large dollop of additional
API churn on top of this, and it would only help 32-bit platforms which
seem relatively less likely to see any benefit.

Andreas Scherbaum, reviewed by Christian Ullrich, additional hacking by me
This commit is contained in:
Tom Lane
2016-03-12 16:05:10 -05:00
parent e01157500f
commit 23a27b039d
32 changed files with 273 additions and 181 deletions

View File

@@ -197,7 +197,7 @@ ExecCreateTableAs(CreateTableAsStmt *stmt, const char *queryString,
/* save the rowcount if we're given a completionTag to fill */
if (completionTag)
snprintf(completionTag, COMPLETION_TAG_BUFSIZE,
"SELECT %u", queryDesc->estate->es_processed);
"SELECT " UINT64_FORMAT, queryDesc->estate->es_processed);
/* and clean up */
ExecutorFinish(queryDesc);

View File

@@ -148,7 +148,7 @@ PerformPortalFetch(FetchStmt *stmt,
char *completionTag)
{
Portal portal;
long nprocessed;
uint64 nprocessed;
/*
* Disallow empty-string cursor name (conflicts with protocol-level
@@ -181,7 +181,7 @@ PerformPortalFetch(FetchStmt *stmt,
/* Return command status if wanted */
if (completionTag)
snprintf(completionTag, COMPLETION_TAG_BUFSIZE, "%s %ld",
snprintf(completionTag, COMPLETION_TAG_BUFSIZE, "%s " UINT64_FORMAT,
stmt->ismove ? "MOVE" : "FETCH",
nprocessed);
}
@@ -392,20 +392,14 @@ PersistHoldablePortal(Portal portal)
if (portal->atEnd)
{
/*
* We can handle this case even if posOverflow: just force the
* tuplestore forward to its end. The size of the skip request
* here is arbitrary.
* Just force the tuplestore forward to its end. The size of the
* skip request here is arbitrary.
*/
while (tuplestore_skiptuples(portal->holdStore, 1000000, true))
/* continue */ ;
}
else
{
if (portal->posOverflow) /* oops, cannot trust portalPos */
ereport(ERROR,
(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
errmsg("could not reposition held cursor")));
tuplestore_rescan(portal->holdStore);
if (!tuplestore_skiptuples(portal->holdStore,

View File

@@ -79,7 +79,7 @@ static void ExecutePlan(EState *estate, PlanState *planstate,
bool use_parallel_mode,
CmdType operation,
bool sendTuples,
long numberTuples,
uint64 numberTuples,
ScanDirection direction,
DestReceiver *dest);
static bool ExecCheckRTEPerms(RangeTblEntry *rte);
@@ -278,7 +278,7 @@ standard_ExecutorStart(QueryDesc *queryDesc, int eflags)
*/
void
ExecutorRun(QueryDesc *queryDesc,
ScanDirection direction, long count)
ScanDirection direction, uint64 count)
{
if (ExecutorRun_hook)
(*ExecutorRun_hook) (queryDesc, direction, count);
@@ -288,7 +288,7 @@ ExecutorRun(QueryDesc *queryDesc,
void
standard_ExecutorRun(QueryDesc *queryDesc,
ScanDirection direction, long count)
ScanDirection direction, uint64 count)
{
EState *estate;
CmdType operation;
@@ -1521,12 +1521,12 @@ ExecutePlan(EState *estate,
bool use_parallel_mode,
CmdType operation,
bool sendTuples,
long numberTuples,
uint64 numberTuples,
ScanDirection direction,
DestReceiver *dest)
{
TupleTableSlot *slot;
long current_tuple_count;
uint64 current_tuple_count;
/*
* initialize local variables
@@ -1542,7 +1542,7 @@ ExecutePlan(EState *estate,
* If a tuple count was supplied, we must force the plan to run without
* parallelism, because we might exit early.
*/
if (numberTuples != 0)
if (numberTuples)
use_parallel_mode = false;
/*

View File

@@ -853,7 +853,7 @@ postquel_getnext(execution_state *es, SQLFunctionCachePtr fcache)
else
{
/* Run regular commands to completion unless lazyEval */
long count = (es->lazyEval) ? 1L : 0L;
uint64 count = (es->lazyEval) ? 1 : 0;
ExecutorRun(es->qd, ForwardScanDirection, count);
@@ -861,7 +861,7 @@ postquel_getnext(execution_state *es, SQLFunctionCachePtr fcache)
* If we requested run to completion OR there was no tuple returned,
* command must be complete.
*/
result = (count == 0L || es->qd->estate->es_processed == 0);
result = (count == 0 || es->qd->estate->es_processed == 0);
}
return result;

View File

@@ -36,7 +36,7 @@
#include "utils/typcache.h"
uint32 SPI_processed = 0;
uint64 SPI_processed = 0;
Oid SPI_lastoid = InvalidOid;
SPITupleTable *SPI_tuptable = NULL;
int SPI_result;
@@ -56,12 +56,12 @@ static void _SPI_prepare_oneshot_plan(const char *src, SPIPlanPtr plan);
static int _SPI_execute_plan(SPIPlanPtr plan, ParamListInfo paramLI,
Snapshot snapshot, Snapshot crosscheck_snapshot,
bool read_only, bool fire_triggers, long tcount);
bool read_only, bool fire_triggers, uint64 tcount);
static ParamListInfo _SPI_convert_params(int nargs, Oid *argtypes,
Datum *Values, const char *Nulls);
static int _SPI_pquery(QueryDesc *queryDesc, bool fire_triggers, long tcount);
static int _SPI_pquery(QueryDesc *queryDesc, bool fire_triggers, uint64 tcount);
static void _SPI_error_callback(void *arg);
@@ -1991,10 +1991,10 @@ _SPI_prepare_oneshot_plan(const char *src, SPIPlanPtr plan)
static int
_SPI_execute_plan(SPIPlanPtr plan, ParamListInfo paramLI,
Snapshot snapshot, Snapshot crosscheck_snapshot,
bool read_only, bool fire_triggers, long tcount)
bool read_only, bool fire_triggers, uint64 tcount)
{
int my_res = 0;
uint32 my_processed = 0;
uint64 my_processed = 0;
Oid my_lastoid = InvalidOid;
SPITupleTable *my_tuptable = NULL;
int res = 0;
@@ -2218,8 +2218,8 @@ _SPI_execute_plan(SPIPlanPtr plan, ParamListInfo paramLI,
if (IsA(stmt, CreateTableAsStmt))
{
Assert(strncmp(completionTag, "SELECT ", 7) == 0);
_SPI_current->processed = strtoul(completionTag + 7,
NULL, 10);
_SPI_current->processed = pg_strtouint64(completionTag + 7,
NULL, 10);
/*
* For historical reasons, if CREATE TABLE AS was spelled
@@ -2231,8 +2231,8 @@ _SPI_execute_plan(SPIPlanPtr plan, ParamListInfo paramLI,
else if (IsA(stmt, CopyStmt))
{
Assert(strncmp(completionTag, "COPY ", 5) == 0);
_SPI_current->processed = strtoul(completionTag + 5,
NULL, 10);
_SPI_current->processed = pg_strtouint64(completionTag + 5,
NULL, 10);
}
}
@@ -2348,7 +2348,7 @@ _SPI_convert_params(int nargs, Oid *argtypes,
}
static int
_SPI_pquery(QueryDesc *queryDesc, bool fire_triggers, long tcount)
_SPI_pquery(QueryDesc *queryDesc, bool fire_triggers, uint64 tcount)
{
int operation = queryDesc->operation;
int eflags;
@@ -2460,7 +2460,7 @@ static void
_SPI_cursor_operation(Portal portal, FetchDirection direction, long count,
DestReceiver *dest)
{
long nfetched;
uint64 nfetched;
/* Check that the portal is valid */
if (!PortalIsValid(portal))
@@ -2563,7 +2563,7 @@ _SPI_end_call(bool procmem)
static bool
_SPI_checktuples(void)
{
uint32 processed = _SPI_current->processed;
uint64 processed = _SPI_current->processed;
SPITupleTable *tuptable = _SPI_current->tuptable;
bool failed = false;

View File

@@ -39,16 +39,16 @@ static void ProcessQuery(PlannedStmt *plan,
DestReceiver *dest,
char *completionTag);
static void FillPortalStore(Portal portal, bool isTopLevel);
static uint32 RunFromStore(Portal portal, ScanDirection direction, long count,
static uint64 RunFromStore(Portal portal, ScanDirection direction, uint64 count,
DestReceiver *dest);
static long PortalRunSelect(Portal portal, bool forward, long count,
static uint64 PortalRunSelect(Portal portal, bool forward, long count,
DestReceiver *dest);
static void PortalRunUtility(Portal portal, Node *utilityStmt, bool isTopLevel,
DestReceiver *dest, char *completionTag);
static void PortalRunMulti(Portal portal, bool isTopLevel,
DestReceiver *dest, DestReceiver *altdest,
char *completionTag);
static long DoPortalRunFetch(Portal portal,
static uint64 DoPortalRunFetch(Portal portal,
FetchDirection fdirection,
long count,
DestReceiver *dest);
@@ -195,7 +195,8 @@ ProcessQuery(PlannedStmt *plan,
{
case CMD_SELECT:
snprintf(completionTag, COMPLETION_TAG_BUFSIZE,
"SELECT %u", queryDesc->estate->es_processed);
"SELECT " UINT64_FORMAT,
queryDesc->estate->es_processed);
break;
case CMD_INSERT:
if (queryDesc->estate->es_processed == 1)
@@ -203,15 +204,18 @@ ProcessQuery(PlannedStmt *plan,
else
lastOid = InvalidOid;
snprintf(completionTag, COMPLETION_TAG_BUFSIZE,
"INSERT %u %u", lastOid, queryDesc->estate->es_processed);
"INSERT %u " UINT64_FORMAT,
lastOid, queryDesc->estate->es_processed);
break;
case CMD_UPDATE:
snprintf(completionTag, COMPLETION_TAG_BUFSIZE,
"UPDATE %u", queryDesc->estate->es_processed);
"UPDATE " UINT64_FORMAT,
queryDesc->estate->es_processed);
break;
case CMD_DELETE:
snprintf(completionTag, COMPLETION_TAG_BUFSIZE,
"DELETE %u", queryDesc->estate->es_processed);
"DELETE " UINT64_FORMAT,
queryDesc->estate->es_processed);
break;
default:
strcpy(completionTag, "???");
@@ -548,7 +552,6 @@ PortalStart(Portal portal, ParamListInfo params,
portal->atStart = true;
portal->atEnd = false; /* allow fetches */
portal->portalPos = 0;
portal->posOverflow = false;
PopActiveSnapshot();
break;
@@ -576,7 +579,6 @@ PortalStart(Portal portal, ParamListInfo params,
portal->atStart = true;
portal->atEnd = false; /* allow fetches */
portal->portalPos = 0;
portal->posOverflow = false;
break;
case PORTAL_UTIL_SELECT:
@@ -598,7 +600,6 @@ PortalStart(Portal portal, ParamListInfo params,
portal->atStart = true;
portal->atEnd = false; /* allow fetches */
portal->portalPos = 0;
portal->posOverflow = false;
break;
case PORTAL_MULTI_QUERY:
@@ -708,7 +709,7 @@ PortalRun(Portal portal, long count, bool isTopLevel,
char *completionTag)
{
bool result;
uint32 nprocessed;
uint64 nprocessed;
ResourceOwner saveTopTransactionResourceOwner;
MemoryContext saveTopTransactionContext;
Portal saveActivePortal;
@@ -794,7 +795,7 @@ PortalRun(Portal portal, long count, bool isTopLevel,
{
if (strcmp(portal->commandTag, "SELECT") == 0)
snprintf(completionTag, COMPLETION_TAG_BUFSIZE,
"SELECT %u", nprocessed);
"SELECT " UINT64_FORMAT, nprocessed);
else
strcpy(completionTag, portal->commandTag);
}
@@ -877,14 +878,14 @@ PortalRun(Portal portal, long count, bool isTopLevel,
*
* count <= 0 is interpreted as a no-op: the destination gets started up
* and shut down, but nothing else happens. Also, count == FETCH_ALL is
* interpreted as "all rows".
* interpreted as "all rows". (cf FetchStmt.howMany)
*
* Caller must already have validated the Portal and done appropriate
* setup (cf. PortalRun).
*
* Returns number of rows processed (suitable for use in result tag)
*/
static long
static uint64
PortalRunSelect(Portal portal,
bool forward,
long count,
@@ -892,7 +893,7 @@ PortalRunSelect(Portal portal,
{
QueryDesc *queryDesc;
ScanDirection direction;
uint32 nprocessed;
uint64 nprocessed;
/*
* NB: queryDesc will be NULL if we are fetching from a held cursor or a
@@ -926,7 +927,10 @@ PortalRunSelect(Portal portal,
if (forward)
{
if (portal->atEnd || count <= 0)
{
direction = NoMovementScanDirection;
count = 0; /* don't pass negative count to executor */
}
else
direction = ForwardScanDirection;
@@ -935,29 +939,22 @@ PortalRunSelect(Portal portal,
count = 0;
if (portal->holdStore)
nprocessed = RunFromStore(portal, direction, count, dest);
nprocessed = RunFromStore(portal, direction, (uint64) count, dest);
else
{
PushActiveSnapshot(queryDesc->snapshot);
ExecutorRun(queryDesc, direction, count);
ExecutorRun(queryDesc, direction, (uint64) count);
nprocessed = queryDesc->estate->es_processed;
PopActiveSnapshot();
}
if (!ScanDirectionIsNoMovement(direction))
{
long oldPos;
if (nprocessed > 0)
portal->atStart = false; /* OK to go backward now */
if (count == 0 ||
(unsigned long) nprocessed < (unsigned long) count)
if (count == 0 || nprocessed < (uint64) count)
portal->atEnd = true; /* we retrieved 'em all */
oldPos = portal->portalPos;
portal->portalPos += nprocessed;
/* portalPos doesn't advance when we fall off the end */
if (portal->portalPos < oldPos)
portal->posOverflow = true;
}
}
else
@@ -969,7 +966,10 @@ PortalRunSelect(Portal portal,
errhint("Declare it with SCROLL option to enable backward scan.")));
if (portal->atStart || count <= 0)
{
direction = NoMovementScanDirection;
count = 0; /* don't pass negative count to executor */
}
else
direction = BackwardScanDirection;
@@ -978,11 +978,11 @@ PortalRunSelect(Portal portal,
count = 0;
if (portal->holdStore)
nprocessed = RunFromStore(portal, direction, count, dest);
nprocessed = RunFromStore(portal, direction, (uint64) count, dest);
else
{
PushActiveSnapshot(queryDesc->snapshot);
ExecutorRun(queryDesc, direction, count);
ExecutorRun(queryDesc, direction, (uint64) count);
nprocessed = queryDesc->estate->es_processed;
PopActiveSnapshot();
}
@@ -994,22 +994,14 @@ PortalRunSelect(Portal portal,
portal->atEnd = false; /* OK to go forward now */
portal->portalPos++; /* adjust for endpoint case */
}
if (count == 0 ||
(unsigned long) nprocessed < (unsigned long) count)
if (count == 0 || nprocessed < (uint64) count)
{
portal->atStart = true; /* we retrieved 'em all */
portal->portalPos = 0;
portal->posOverflow = false;
}
else
{
long oldPos;
oldPos = portal->portalPos;
portal->portalPos -= nprocessed;
if (portal->portalPos > oldPos ||
portal->portalPos <= 0)
portal->posOverflow = true;
}
}
}
@@ -1083,11 +1075,11 @@ FillPortalStore(Portal portal, bool isTopLevel)
* are run in the caller's memory context (since we have no estate). Watch
* out for memory leaks.
*/
static uint32
RunFromStore(Portal portal, ScanDirection direction, long count,
static uint64
RunFromStore(Portal portal, ScanDirection direction, uint64 count,
DestReceiver *dest)
{
long current_tuple_count = 0;
uint64 current_tuple_count = 0;
TupleTableSlot *slot;
slot = MakeSingleTupleTableSlot(portal->tupDesc);
@@ -1136,7 +1128,7 @@ RunFromStore(Portal portal, ScanDirection direction, long count,
ExecDropSingleTupleTableSlot(slot);
return (uint32) current_tuple_count;
return current_tuple_count;
}
/*
@@ -1375,15 +1367,19 @@ PortalRunMulti(Portal portal, bool isTopLevel,
*
* Note: we presently assume that no callers of this want isTopLevel = true.
*
* count <= 0 is interpreted as a no-op: the destination gets started up
* and shut down, but nothing else happens. Also, count == FETCH_ALL is
* interpreted as "all rows". (cf FetchStmt.howMany)
*
* Returns number of rows processed (suitable for use in result tag)
*/
long
uint64
PortalRunFetch(Portal portal,
FetchDirection fdirection,
long count,
DestReceiver *dest)
{
long result;
uint64 result;
Portal saveActivePortal;
ResourceOwner saveResourceOwner;
MemoryContext savePortalContext;
@@ -1470,9 +1466,13 @@ PortalRunFetch(Portal portal,
* DoPortalRunFetch
* Guts of PortalRunFetch --- the portal context is already set up
*
* count <= 0 is interpreted as a no-op: the destination gets started up
* and shut down, but nothing else happens. Also, count == FETCH_ALL is
* interpreted as "all rows". (cf FetchStmt.howMany)
*
* Returns number of rows processed (suitable for use in result tag)
*/
static long
static uint64
DoPortalRunFetch(Portal portal,
FetchDirection fdirection,
long count,
@@ -1508,13 +1508,21 @@ DoPortalRunFetch(Portal portal,
{
/*
* Definition: Rewind to start, advance count-1 rows, return
* next row (if any). In practice, if the goal is less than
* halfway back to the start, it's better to scan from where
* we are. In any case, we arrange to fetch the target row
* going forwards.
* next row (if any).
*
* In practice, if the goal is less than halfway back to the
* start, it's better to scan from where we are.
*
* Also, if current portalPos is outside the range of "long",
* do it the hard way to avoid possible overflow of the count
* argument to PortalRunSelect. We must exclude exactly
* LONG_MAX, as well, lest the count look like FETCH_ALL.
*
* In any case, we arrange to fetch the target row going
* forwards.
*/
if (portal->posOverflow || portal->portalPos == LONG_MAX ||
count - 1 <= portal->portalPos / 2)
if ((uint64) (count - 1) <= portal->portalPos / 2 ||
portal->portalPos >= (uint64) LONG_MAX)
{
DoPortalRewind(portal);
if (count > 1)
@@ -1523,7 +1531,7 @@ DoPortalRunFetch(Portal portal,
}
else
{
long pos = portal->portalPos;
long pos = (long) portal->portalPos;
if (portal->atEnd)
pos++; /* need one extra fetch if off end */
@@ -1609,7 +1617,7 @@ DoPortalRunFetch(Portal portal,
if (dest->mydest == DestNone)
{
/* MOVE 0 returns 0/1 based on if FETCH 0 would return a row */
return on_row ? 1L : 0L;
return on_row ? 1 : 0;
}
else
{
@@ -1635,12 +1643,11 @@ DoPortalRunFetch(Portal portal,
*/
if (!forward && count == FETCH_ALL && dest->mydest == DestNone)
{
long result = portal->portalPos;
uint64 result = portal->portalPos;
if (result > 0 && !portal->atEnd)
result--;
DoPortalRewind(portal);
/* result is bogus if pos had overflowed, but it's best we can do */
return result;
}
@@ -1677,5 +1684,4 @@ DoPortalRewind(Portal portal)
portal->atStart = true;
portal->atEnd = false;
portal->portalPos = 0;
portal->posOverflow = false;
}

View File

@@ -388,3 +388,25 @@ pg_ltostr(char *str, int32 value)
return end;
}
/*
* pg_strtouint64
* Converts 'str' into an unsigned 64-bit integer.
*
* This has the identical API to strtoul(3), except that it will handle
* 64-bit ints even where "long" is narrower than that.
*
* For the moment it seems sufficient to assume that the platform has
* such a function somewhere; let's not roll our own.
*/
uint64
pg_strtouint64(const char *str, char **endptr, int base)
{
#ifdef WIN32
return _strtoui64(str, endptr, base);
#elif defined(HAVE_STRTOULL) && SIZEOF_LONG < 8
return strtoull(str, endptr, base);
#else
return strtoul(str, endptr, base);
#endif
}

View File

@@ -260,7 +260,6 @@ tsquery_rewrite_query(PG_FUNCTION_ARGS)
SPIPlanPtr plan;
Portal portal;
bool isnull;
int i;
if (query->size == 0)
{
@@ -294,6 +293,8 @@ tsquery_rewrite_query(PG_FUNCTION_ARGS)
while (SPI_processed > 0 && tree)
{
uint64 i;
for (i = 0; i < SPI_processed && tree; i++)
{
Datum qdata = SPI_getbinval(SPI_tuptable->vals[i], SPI_tuptable->tupdesc, 1, &isnull);

View File

@@ -1682,7 +1682,6 @@ static TSVectorStat *
ts_stat_sql(MemoryContext persistentContext, text *txt, text *ws)
{
char *query = text_to_cstring(txt);
int i;
TSVectorStat *stat;
bool isnull;
Portal portal;
@@ -1746,6 +1745,8 @@ ts_stat_sql(MemoryContext persistentContext, text *txt, text *ws)
while (SPI_processed > 0)
{
uint64 i;
for (i = 0; i < SPI_processed; i++)
{
Datum data = SPI_getbinval(SPI_tuptable->vals[i], SPI_tuptable->tupdesc, 1, &isnull);

View File

@@ -161,7 +161,7 @@ static const char *map_sql_catalog_to_xmlschema_types(List *nspid_list,
static const char *map_sql_type_to_xml_name(Oid typeoid, int typmod);
static const char *map_sql_typecoll_to_xmlschema_types(List *tupdesc_list);
static const char *map_sql_type_to_xmlschema_type(Oid typeoid, int typmod);
static void SPI_sql_row_to_xmlelement(int rownum, StringInfo result,
static void SPI_sql_row_to_xmlelement(uint64 rownum, StringInfo result,
char *tablename, bool nulls, bool tableforest,
const char *targetns, bool top_level);
@@ -2260,7 +2260,7 @@ _SPI_strdup(const char *s)
static List *
query_to_oid_list(const char *query)
{
int i;
uint64 i;
List *list = NIL;
SPI_execute(query, true, 0);
@@ -2379,7 +2379,7 @@ cursor_to_xml(PG_FUNCTION_ARGS)
StringInfoData result;
Portal portal;
int i;
uint64 i;
initStringInfo(&result);
@@ -2454,7 +2454,7 @@ query_to_xml_internal(const char *query, char *tablename,
{
StringInfo result;
char *xmltn;
int i;
uint64 i;
if (tablename)
xmltn = map_sql_identifier_to_xml_name(tablename, true, false);
@@ -3532,7 +3532,7 @@ map_sql_type_to_xmlschema_type(Oid typeoid, int typmod)
* SPI cursor. See also SQL/XML:2008 section 9.10.
*/
static void
SPI_sql_row_to_xmlelement(int rownum, StringInfo result, char *tablename,
SPI_sql_row_to_xmlelement(uint64 rownum, StringInfo result, char *tablename,
bool nulls, bool tableforest,
const char *targetns, bool top_level)
{