1
0
mirror of https://github.com/postgres/postgres.git synced 2025-11-10 17:42:29 +03:00

Convert the arithmetic for shared memory size calculation from 'int'

to 'Size' (that is, size_t), and install overflow detection checks in it.
This allows us to remove the former arbitrary restrictions on NBuffers
etc.  It won't make any difference in a 32-bit machine, but in a 64-bit
machine you could theoretically have terabytes of shared buffers.
(How efficiently we could manage 'em remains to be seen.)  Similarly,
num_temp_buffers, work_mem, and maintenance_work_mem can be set above
2Gb on a 64-bit machine.  Original patch from Koichi Suzuki, additional
work by moi.
This commit is contained in:
Tom Lane
2005-08-20 23:26:37 +00:00
parent 2299ceab1c
commit 0007490e09
50 changed files with 774 additions and 275 deletions

View File

@@ -8,7 +8,7 @@
*
*
* IDENTIFICATION
* $PostgreSQL: pgsql/src/backend/storage/buffer/buf_init.c,v 1.75 2005/08/12 05:05:50 tgl Exp $
* $PostgreSQL: pgsql/src/backend/storage/buffer/buf_init.c,v 1.76 2005/08/20 23:26:17 tgl Exp $
*
*-------------------------------------------------------------------------
*/
@@ -89,7 +89,7 @@ InitBufferPool(void)
BufferBlocks = (char *)
ShmemInitStruct("Buffer Blocks",
NBuffers * BLCKSZ, &foundBufs);
NBuffers * (Size) BLCKSZ, &foundBufs);
if (foundDescs || foundBufs)
{
@@ -155,8 +155,11 @@ InitBufferPoolAccess(void)
/*
* Allocate and zero local arrays of per-buffer info.
*/
PrivateRefCount = (int32 *) calloc(NBuffers,
sizeof(*PrivateRefCount));
PrivateRefCount = (int32 *) calloc(NBuffers, sizeof(int32));
if (!PrivateRefCount)
ereport(FATAL,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
}
/*
@@ -165,19 +168,19 @@ InitBufferPoolAccess(void)
* compute the size of shared memory for the buffer pool including
* data pages, buffer descriptors, hash tables, etc.
*/
int
Size
BufferShmemSize(void)
{
int size = 0;
Size size = 0;
/* size of buffer descriptors */
size += MAXALIGN(NBuffers * sizeof(BufferDesc));
size = add_size(size, mul_size(NBuffers, sizeof(BufferDesc)));
/* size of data pages */
size += NBuffers * MAXALIGN(BLCKSZ);
size = add_size(size, mul_size(NBuffers, BLCKSZ));
/* size of stuff controlled by freelist.c */
size += StrategyShmemSize();
size = add_size(size, StrategyShmemSize());
return size;
}

View File

@@ -13,7 +13,7 @@
*
*
* IDENTIFICATION
* $PostgreSQL: pgsql/src/backend/storage/buffer/buf_table.c,v 1.41 2005/05/29 04:23:04 tgl Exp $
* $PostgreSQL: pgsql/src/backend/storage/buffer/buf_table.c,v 1.42 2005/08/20 23:26:17 tgl Exp $
*
*-------------------------------------------------------------------------
*/
@@ -37,7 +37,7 @@ static HTAB *SharedBufHash;
* Estimate space needed for mapping hashtable
* size is the desired hash table size (possibly more than NBuffers)
*/
int
Size
BufTableShmemSize(int size)
{
return hash_estimate_size(size, sizeof(BufferLookupEnt));

View File

@@ -9,7 +9,7 @@
*
*
* IDENTIFICATION
* $PostgreSQL: pgsql/src/backend/storage/buffer/freelist.c,v 1.51 2005/03/04 20:21:06 tgl Exp $
* $PostgreSQL: pgsql/src/backend/storage/buffer/freelist.c,v 1.52 2005/08/20 23:26:17 tgl Exp $
*
*-------------------------------------------------------------------------
*/
@@ -208,16 +208,16 @@ StrategyHintVacuum(bool vacuum_active)
* Note: for somewhat historical reasons, the buffer lookup hashtable size
* is also determined here.
*/
int
Size
StrategyShmemSize(void)
{
int size = 0;
Size size = 0;
/* size of lookup hash table */
size += BufTableShmemSize(NBuffers);
size = add_size(size, BufTableShmemSize(NBuffers));
/* size of the shared replacement strategy control block */
size += MAXALIGN(sizeof(BufferStrategyControl));
size = add_size(size, MAXALIGN(sizeof(BufferStrategyControl)));
return size;
}

View File

@@ -9,7 +9,7 @@
*
*
* IDENTIFICATION
* $PostgreSQL: pgsql/src/backend/storage/buffer/localbuf.c,v 1.68 2005/08/08 19:44:22 tgl Exp $
* $PostgreSQL: pgsql/src/backend/storage/buffer/localbuf.c,v 1.69 2005/08/20 23:26:17 tgl Exp $
*
*-------------------------------------------------------------------------
*/
@@ -255,15 +255,13 @@ InitLocalBuffers(void)
int i;
/* Allocate and zero buffer headers and auxiliary arrays */
LocalBufferDescriptors = (BufferDesc *)
MemoryContextAllocZero(TopMemoryContext,
nbufs * sizeof(BufferDesc));
LocalBufferBlockPointers = (Block *)
MemoryContextAllocZero(TopMemoryContext,
nbufs * sizeof(Block));
LocalRefCount = (int32 *)
MemoryContextAllocZero(TopMemoryContext,
nbufs * sizeof(int32));
LocalBufferDescriptors = (BufferDesc *) calloc(nbufs, sizeof(BufferDesc));
LocalBufferBlockPointers = (Block *) calloc(nbufs, sizeof(Block));
LocalRefCount = (int32 *) calloc(nbufs, sizeof(int32));
if (!LocalBufferDescriptors || !LocalBufferBlockPointers || !LocalRefCount)
ereport(FATAL,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("out of memory")));
nextFreeLocalBuf = 0;

View File

@@ -8,7 +8,7 @@
* Portions Copyright (c) 1994, Regents of the University of California
*
* IDENTIFICATION
* $PostgreSQL: pgsql/src/backend/storage/freespace/freespace.c,v 1.47 2005/08/17 03:50:59 momjian Exp $
* $PostgreSQL: pgsql/src/backend/storage/freespace/freespace.c,v 1.48 2005/08/20 23:26:20 tgl Exp $
*
*
* NOTES:
@@ -271,11 +271,13 @@ InitFreeSpaceMap(void)
bool found;
/* Create table header */
FreeSpaceMap = (FSMHeader *) ShmemInitStruct("Free Space Map Header", sizeof(FSMHeader), &found);
FreeSpaceMap = (FSMHeader *) ShmemInitStruct("Free Space Map Header",
sizeof(FSMHeader),
&found);
if (FreeSpaceMap == NULL)
ereport(FATAL,
(errcode(ERRCODE_OUT_OF_MEMORY),
errmsg("insufficient shared memory for free space map")));
errmsg("insufficient shared memory for free space map")));
if (!found)
MemSet(FreeSpaceMap, 0, sizeof(FSMHeader));
@@ -308,7 +310,7 @@ InitFreeSpaceMap(void)
errmsg("max_fsm_pages must exceed max_fsm_relations * %d",
CHUNKPAGES)));
FreeSpaceMap->arena = (char *) ShmemAlloc(nchunks * CHUNKBYTES);
FreeSpaceMap->arena = (char *) ShmemAlloc((Size) nchunks * CHUNKBYTES);
if (FreeSpaceMap->arena == NULL)
ereport(FATAL,
(errcode(ERRCODE_OUT_OF_MEMORY),
@@ -322,27 +324,22 @@ InitFreeSpaceMap(void)
/*
* Estimate amount of shmem space needed for FSM.
*/
int
Size
FreeSpaceShmemSize(void)
{
int size;
Size size;
int nchunks;
/* table header */
size = MAXALIGN(sizeof(FSMHeader));
/* hash table, including the FSMRelation objects */
size += hash_estimate_size(MaxFSMRelations + 1, sizeof(FSMRelation));
size = add_size(size, hash_estimate_size(MaxFSMRelations + 1,
sizeof(FSMRelation)));
/* page-storage arena */
nchunks = (MaxFSMPages - 1) / CHUNKPAGES + 1;
if (nchunks >= (INT_MAX / CHUNKBYTES))
ereport(FATAL,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
errmsg("max_fsm_pages is too large")));
size += MAXALIGN(nchunks * CHUNKBYTES);
size = add_size(size, mul_size(nchunks, CHUNKBYTES));
return size;
}

View File

@@ -8,7 +8,7 @@
*
*
* IDENTIFICATION
* $PostgreSQL: pgsql/src/backend/storage/ipc/ipci.c,v 1.77 2005/06/17 22:32:45 tgl Exp $
* $PostgreSQL: pgsql/src/backend/storage/ipc/ipci.c,v 1.78 2005/08/20 23:26:20 tgl Exp $
*
*-------------------------------------------------------------------------
*/
@@ -61,36 +61,44 @@ CreateSharedMemoryAndSemaphores(bool makePrivate, int port)
if (!IsUnderPostmaster)
{
int size;
Size size;
int numSemas;
/*
* Size of the Postgres shared-memory block is estimated via
* moderately-accurate estimates for the big hogs, plus 100K for
* the stuff that's too small to bother with estimating.
*
* We take some care during this phase to ensure that the total
* size request doesn't overflow size_t. If this gets through,
* we don't need to be so careful during the actual allocation
* phase.
*/
size = hash_estimate_size(SHMEM_INDEX_SIZE, sizeof(ShmemIndexEnt));
size += BufferShmemSize();
size += LockShmemSize();
size += ProcGlobalShmemSize();
size += XLOGShmemSize();
size += CLOGShmemSize();
size += SUBTRANSShmemSize();
size += TwoPhaseShmemSize();
size += MultiXactShmemSize();
size += LWLockShmemSize();
size += ProcArrayShmemSize();
size += SInvalShmemSize(MaxBackends);
size += FreeSpaceShmemSize();
size += BgWriterShmemSize();
size = 100000;
size = add_size(size, hash_estimate_size(SHMEM_INDEX_SIZE,
sizeof(ShmemIndexEnt)));
size = add_size(size, BufferShmemSize());
size = add_size(size, LockShmemSize());
size = add_size(size, ProcGlobalShmemSize());
size = add_size(size, XLOGShmemSize());
size = add_size(size, CLOGShmemSize());
size = add_size(size, SUBTRANSShmemSize());
size = add_size(size, TwoPhaseShmemSize());
size = add_size(size, MultiXactShmemSize());
size = add_size(size, LWLockShmemSize());
size = add_size(size, ProcArrayShmemSize());
size = add_size(size, SInvalShmemSize());
size = add_size(size, FreeSpaceShmemSize());
size = add_size(size, BgWriterShmemSize());
#ifdef EXEC_BACKEND
size += ShmemBackendArraySize();
size = add_size(size, ShmemBackendArraySize());
#endif
size += 100000;
/* might as well round it off to a multiple of a typical page size */
size += 8192 - (size % 8192);
elog(DEBUG3, "invoking IpcMemoryCreate(size=%d)", size);
/* might as well round it off to a multiple of a typical page size */
size = add_size(size, 8192 - (size % 8192));
elog(DEBUG3, "invoking IpcMemoryCreate(size=%lu)",
(unsigned long) size);
/*
* Create the shmem segment
@@ -163,7 +171,7 @@ CreateSharedMemoryAndSemaphores(bool makePrivate, int port)
/*
* Set up shared-inval messaging
*/
CreateSharedInvalidationState(MaxBackends);
CreateSharedInvalidationState();
/*
* Set up free-space map

View File

@@ -8,7 +8,7 @@
* Portions Copyright (c) 1994, Regents of the University of California
*
* IDENTIFICATION
* $PostgreSQL: pgsql/src/backend/storage/ipc/pmsignal.c,v 1.18 2004/12/31 22:00:56 pgsql Exp $
* $PostgreSQL: pgsql/src/backend/storage/ipc/pmsignal.c,v 1.19 2005/08/20 23:26:20 tgl Exp $
*
*-------------------------------------------------------------------------
*/
@@ -48,7 +48,9 @@ PMSignalInit(void)
bool found;
PMSignalFlags = (sig_atomic_t *)
ShmemInitStruct("PMSignalFlags", NUM_PMSIGNALS * sizeof(sig_atomic_t), &found);
ShmemInitStruct("PMSignalFlags",
NUM_PMSIGNALS * sizeof(sig_atomic_t),
&found);
if (!found)
MemSet(PMSignalFlags, 0, NUM_PMSIGNALS * sizeof(sig_atomic_t));

View File

@@ -23,7 +23,7 @@
*
*
* IDENTIFICATION
* $PostgreSQL: pgsql/src/backend/storage/ipc/procarray.c,v 1.5 2005/08/20 01:26:36 ishii Exp $
* $PostgreSQL: pgsql/src/backend/storage/ipc/procarray.c,v 1.6 2005/08/20 23:26:20 tgl Exp $
*
*-------------------------------------------------------------------------
*/
@@ -81,11 +81,16 @@ static void DisplayXidCache(void);
/*
* Report shared-memory space needed by CreateSharedProcArray.
*/
int
Size
ProcArrayShmemSize(void)
{
return MAXALIGN(offsetof(ProcArrayStruct, procs) +
(MaxBackends + max_prepared_xacts) * sizeof(PGPROC *));
Size size;
size = offsetof(ProcArrayStruct, procs);
size = add_size(size, mul_size(sizeof(PGPROC *),
add_size(MaxBackends, max_prepared_xacts)));
return size;
}
/*

View File

@@ -8,7 +8,7 @@
*
*
* IDENTIFICATION
* $PostgreSQL: pgsql/src/backend/storage/ipc/shmem.c,v 1.84 2005/05/29 04:23:04 tgl Exp $
* $PostgreSQL: pgsql/src/backend/storage/ipc/shmem.c,v 1.85 2005/08/20 23:26:20 tgl Exp $
*
*-------------------------------------------------------------------------
*/
@@ -141,8 +141,8 @@ InitShmemAllocation(void *seghdr, bool init)
void *
ShmemAlloc(Size size)
{
uint32 newStart;
uint32 newFree;
Size newStart;
Size newFree;
void *newSpace;
/* use volatile pointer to prevent code rearrangement */
@@ -415,3 +415,40 @@ ShmemInitStruct(const char *name, Size size, bool *foundPtr)
SpinLockRelease(ShmemIndexLock);
return structPtr;
}
/*
* Add two Size values, checking for overflow
*/
Size
add_size(Size s1, Size s2)
{
Size result;
result = s1 + s2;
/* We are assuming Size is an unsigned type here... */
if (result < s1 || result < s2)
ereport(ERROR,
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
errmsg("requested shared memory size overflows size_t")));
return result;
}
/*
* Multiply two Size values, checking for overflow
*/
Size
mul_size(Size s1, Size s2)
{
Size result;
if (s1 == 0 || s2 == 0)
return 0;
result = s1 * s2;
/* We are assuming Size is an unsigned type here... */
if (result / s2 != s1)
ereport(ERROR,
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
errmsg("requested shared memory size overflows size_t")));
return result;
}

View File

@@ -8,7 +8,7 @@
*
*
* IDENTIFICATION
* $PostgreSQL: pgsql/src/backend/storage/ipc/sinval.c,v 1.76 2005/05/19 21:35:46 tgl Exp $
* $PostgreSQL: pgsql/src/backend/storage/ipc/sinval.c,v 1.77 2005/08/20 23:26:21 tgl Exp $
*
*-------------------------------------------------------------------------
*/
@@ -55,10 +55,10 @@ static void ProcessCatchupEvent(void);
/* should be called only by the POSTMASTER */
/****************************************************************************/
void
CreateSharedInvalidationState(int maxBackends)
CreateSharedInvalidationState(void)
{
/* SInvalLock must be initialized already, during LWLock init */
SIBufferInit(maxBackends);
SIBufferInit();
}
/*

View File

@@ -8,7 +8,7 @@
*
*
* IDENTIFICATION
* $PostgreSQL: pgsql/src/backend/storage/ipc/sinvaladt.c,v 1.59 2005/05/19 21:35:46 tgl Exp $
* $PostgreSQL: pgsql/src/backend/storage/ipc/sinvaladt.c,v 1.60 2005/08/20 23:26:21 tgl Exp $
*
*-------------------------------------------------------------------------
*/
@@ -32,14 +32,15 @@ static void SISetProcStateInvalid(SISeg *segP);
/*
* SInvalShmemSize --- return shared-memory space needed
*/
int
SInvalShmemSize(int maxBackends)
Size
SInvalShmemSize(void)
{
/*
* Figure space needed. Note sizeof(SISeg) includes the first
* ProcState entry.
*/
return sizeof(SISeg) + sizeof(ProcState) * (maxBackends - 1);
Size size;
size = offsetof(SISeg, procState);
size = add_size(size, mul_size(sizeof(ProcState), MaxBackends));
return size;
}
/*
@@ -47,16 +48,15 @@ SInvalShmemSize(int maxBackends)
* Create and initialize a new SI message buffer
*/
void
SIBufferInit(int maxBackends)
SIBufferInit(void)
{
int segSize;
SISeg *segP;
int i;
bool found;
/* Allocate space in shared memory */
segSize = SInvalShmemSize(maxBackends);
shmInvalBuffer = segP = (SISeg *) ShmemInitStruct("shmInvalBuffer", segSize, &found);
shmInvalBuffer = segP = (SISeg *)
ShmemInitStruct("shmInvalBuffer", SInvalShmemSize(), &found);
if (found)
return;
@@ -64,13 +64,13 @@ SIBufferInit(int maxBackends)
segP->minMsgNum = 0;
segP->maxMsgNum = 0;
segP->lastBackend = 0;
segP->maxBackends = maxBackends;
segP->freeBackends = maxBackends;
segP->maxBackends = MaxBackends;
segP->freeBackends = MaxBackends;
/* The buffer[] array is initially all unused, so we need not fill it */
/* Mark all backends inactive */
for (i = 0; i < maxBackends; i++)
for (i = 0; i < segP->maxBackends; i++)
{
segP->procState[i].nextMsgNum = -1; /* inactive */
segP->procState[i].resetState = false;

View File

@@ -8,7 +8,7 @@
*
*
* IDENTIFICATION
* $PostgreSQL: pgsql/src/backend/storage/lmgr/lock.c,v 1.156 2005/06/17 22:32:45 tgl Exp $
* $PostgreSQL: pgsql/src/backend/storage/lmgr/lock.c,v 1.157 2005/08/20 23:26:23 tgl Exp $
*
* NOTES
* Outside modules can create a lock table and acquire/release
@@ -46,7 +46,8 @@
/* This configuration variable is used to set the lock table size */
int max_locks_per_xact; /* set by guc.c */
#define NLOCKENTS() (max_locks_per_xact * (MaxBackends + max_prepared_xacts))
#define NLOCKENTS() \
mul_size(max_locks_per_xact, add_size(MaxBackends, max_prepared_xacts))
/* Record that's written to 2PC state file when a lock is persisted */
@@ -1864,20 +1865,20 @@ next_item:
/*
* Estimate shared-memory space used for lock tables
*/
int
Size
LockShmemSize(void)
{
int size = 0;
Size size;
long max_table_size = NLOCKENTS();
/* lock method headers */
size += MAX_LOCK_METHODS * MAXALIGN(sizeof(LockMethodData));
size = MAX_LOCK_METHODS * MAXALIGN(sizeof(LockMethodData));
/* lockHash table */
size += hash_estimate_size(max_table_size, sizeof(LOCK));
size = add_size(size, hash_estimate_size(max_table_size, sizeof(LOCK)));
/* proclockHash table */
size += hash_estimate_size(max_table_size, sizeof(PROCLOCK));
size = add_size(size, hash_estimate_size(max_table_size, sizeof(PROCLOCK)));
/*
* Note we count only one pair of hash tables, since the userlocks
@@ -1886,7 +1887,7 @@ LockShmemSize(void)
* Since the lockHash entry count above is only an estimate, add 10%
* safety margin.
*/
size += size / 10;
size = add_size(size, size / 10);
return size;
}

View File

@@ -15,7 +15,7 @@
* Portions Copyright (c) 1994, Regents of the University of California
*
* IDENTIFICATION
* $PostgreSQL: pgsql/src/backend/storage/lmgr/lwlock.c,v 1.28 2005/04/28 21:47:15 tgl Exp $
* $PostgreSQL: pgsql/src/backend/storage/lmgr/lwlock.c,v 1.29 2005/08/20 23:26:24 tgl Exp $
*
*-------------------------------------------------------------------------
*/
@@ -129,17 +129,18 @@ NumLWLocks(void)
/*
* Compute shmem space needed for LWLocks.
*/
int
Size
LWLockShmemSize(void)
{
Size size;
int numLocks = NumLWLocks();
uint32 spaceLocks;
/* Allocate the LWLocks plus space for shared allocation counter. */
spaceLocks = numLocks * sizeof(LWLock) + 2 * sizeof(int);
spaceLocks = MAXALIGN(spaceLocks);
size = mul_size(numLocks, sizeof(LWLock));
return (int) spaceLocks;
size = add_size(size, 2 * sizeof(int));
return size;
}
@@ -150,7 +151,7 @@ void
CreateLWLocks(void)
{
int numLocks = NumLWLocks();
uint32 spaceLocks = LWLockShmemSize();
Size spaceLocks = LWLockShmemSize();
LWLock *lock;
int id;

View File

@@ -8,7 +8,7 @@
*
*
* IDENTIFICATION
* $PostgreSQL: pgsql/src/backend/storage/lmgr/proc.c,v 1.162 2005/08/08 03:11:55 tgl Exp $
* $PostgreSQL: pgsql/src/backend/storage/lmgr/proc.c,v 1.163 2005/08/20 23:26:24 tgl Exp $
*
*-------------------------------------------------------------------------
*/
@@ -91,15 +91,19 @@ static bool CheckStatementTimeout(void);
/*
* Report shared-memory space needed by InitProcGlobal.
*/
int
Size
ProcGlobalShmemSize(void)
{
int size = 0;
Size size = 0;
size += MAXALIGN(sizeof(PROC_HDR)); /* ProcGlobal */
size += MAXALIGN(NUM_DUMMY_PROCS * sizeof(PGPROC)); /* DummyProcs */
size += MAXALIGN(MaxBackends * sizeof(PGPROC)); /* MyProcs */
size += MAXALIGN(sizeof(slock_t)); /* ProcStructLock */
/* ProcGlobal */
size = add_size(size, sizeof(PROC_HDR));
/* DummyProcs */
size = add_size(size, mul_size(NUM_DUMMY_PROCS, sizeof(PGPROC)));
/* MyProcs */
size = add_size(size, mul_size(MaxBackends, sizeof(PGPROC)));
/* ProcStructLock */
size = add_size(size, sizeof(slock_t));
return size;
}