diff --git a/src/backend/utils/mmgr/README b/src/backend/utils/mmgr/README index 777481c83d3..b20b9d48526 100644 --- a/src/backend/utils/mmgr/README +++ b/src/backend/utils/mmgr/README @@ -384,35 +384,55 @@ MemoryContext like the parent and child contexts, and the name of the context. This is essentially an abstract superclass, and the behavior is -determined by the "methods" pointer is its virtual function table -(struct MemoryContextMethods). Specific memory context types will use -derived structs having these fields as their first fields. All the +determined by the "methods" pointer which references which set of +MemoryContextMethods are to be used. Specific memory context types will +use derived structs having these fields as their first fields. All the contexts of a specific type will have methods pointers that point to -the same static table of function pointers. +the corresponding element in the mcxt_methods[] array as defined in mcxt.c. While operations like allocating from and resetting a context take the relevant MemoryContext as a parameter, operations like free and realloc are trickier. To make those work, we require all memory context types to produce allocated chunks that are immediately, -without any padding, preceded by a pointer to the corresponding -MemoryContext. +without any padding, preceded by a uint64 value of which the least +significant 3 bits are set to the owning context's MemoryContextMethodID. +This allows the code to determine the correct MemoryContextMethods to +use by looking up the mcxt_methods[] array using the 3 bits as an index +into that array. If a type of allocator needs additional information about its chunks, like e.g. the size of the allocation, that information can in turn -precede the MemoryContext. This means the only overhead implied by -the memory context mechanism is a pointer to its context, so we're not -constraining context-type designers very much. +either be encoded into the remaining 61 bits of the preceding uint64 value +or if more space is required, additional values may be stored directly prior +to the uint64 value. It is up to the context implementation to manage this. -Given this, routines like pfree determine their corresponding context -with an operation like (although that is usually encapsulated in -GetMemoryChunkContext()) +Given this, routines like pfree can determine which set of +MemoryContextMethods to call the free_p function for by calling +GetMemoryChunkMethodID() and finding the corresponding MemoryContextMethods +in the mcxt_methods[] array. For convenience, the MCXT_METHOD() macro is +provided, making the code as simple as: - MemoryContext context = *(MemoryContext*) (((char *) pointer) - sizeof(void *)); - -and then invoke the corresponding method for the context - - context->methods->free_p(pointer); +void +pfree(void *pointer) +{ + MCXT_METHOD(pointer, free_p)(pointer); +} +All of the current memory contexts make use of the MemoryChunk header type +which is defined in memutils_memorychunk.h. This suits all of the existing +context types well as it makes use of the remaining 61-bits of the uint64 +header to efficiently encode the size of the chunk of memory (or freelist +index, in the case of aset.c) and the number of bytes which must be subtracted +from the chunk in order to obtain a reference to the block that the chunk +belongs to. 30 bits are used for each of these. If more than 30 bits are +required then the memory context must manage that itself. This can be done by +calling the MemoryChunkSetHdrMaskExternal() function on the given chunk. +Currently, each memory context type stores large allocations on dedicated +blocks (which always contain only a single chunk). For these, finding the +block is simple as we know that the chunk must be the first on the given +block, so the block is always at a fixed offset to the chunk. For these, +finding the size of the chunk is also simple as the block always stores an +endptr which we can use to calculate the size of the chunk. More Control Over aset.c Behavior --------------------------------- diff --git a/src/backend/utils/mmgr/aset.c b/src/backend/utils/mmgr/aset.c index ec3e264a733..b6eeb8abab1 100644 --- a/src/backend/utils/mmgr/aset.c +++ b/src/backend/utils/mmgr/aset.c @@ -49,6 +49,8 @@ #include "port/pg_bitutils.h" #include "utils/memdebug.h" #include "utils/memutils.h" +#include "utils/memutils_memorychunk.h" +#include "utils/memutils_internal.h" /*-------------------- * Chunk freelist k holds chunks of size 1 << (k + ALLOC_MINBITS), @@ -66,7 +68,9 @@ * CAUTION: ALLOC_MINBITS must be large enough so that * 1<= 1024); @@ -409,6 +373,7 @@ AllocSetContextCreateInternal(MemoryContext parent, (minContextSize == MAXALIGN(minContextSize) && minContextSize >= 1024 && minContextSize <= maxBlockSize)); + Assert(maxBlockSize <= MEMORYCHUNK_MAX_BLOCKOFFSET); /* * Check whether the parameters match either available freelist. We do @@ -443,7 +408,7 @@ AllocSetContextCreateInternal(MemoryContext parent, /* Reinitialize its header, installing correct name and parent */ MemoryContextCreate((MemoryContext) set, T_AllocSetContext, - &AllocSetMethods, + MCTX_ASET_ID, parent, name); @@ -517,15 +482,20 @@ AllocSetContextCreateInternal(MemoryContext parent, * requests that are all the maximum chunk size we will waste at most * 1/8th of the allocated space. * - * We have to have allocChunkLimit a power of two, because the requested - * and actually-allocated sizes of any chunk must be on the same side of - * the limit, else we get confused about whether the chunk is "big". - * * Also, allocChunkLimit must not exceed ALLOCSET_SEPARATE_THRESHOLD. */ StaticAssertStmt(ALLOC_CHUNK_LIMIT == ALLOCSET_SEPARATE_THRESHOLD, "ALLOC_CHUNK_LIMIT != ALLOCSET_SEPARATE_THRESHOLD"); + /* + * Determine the maximum size that a chunk can be before we allocate an + * entire AllocBlock dedicated for that chunk. We set the absolute limit + * of that size as ALLOC_CHUNK_LIMIT but we reduce it further so that we + * can fit about ALLOC_CHUNK_FRACTION chunks this size on a maximally + * sized block. (We opt to keep allocChunkLimit a power-of-2 value + * primarily for legacy reasons rather than calculating it so that exactly + * ALLOC_CHUNK_FRACTION chunks fit on a maximally sized block.) + */ set->allocChunkLimit = ALLOC_CHUNK_LIMIT; while ((Size) (set->allocChunkLimit + ALLOC_CHUNKHDRSZ) > (Size) ((maxBlockSize - ALLOC_BLOCKHDRSZ) / ALLOC_CHUNK_FRACTION)) @@ -534,7 +504,7 @@ AllocSetContextCreateInternal(MemoryContext parent, /* Finally, do the type-independent part of context creation */ MemoryContextCreate((MemoryContext) set, T_AllocSetContext, - &AllocSetMethods, + MCTX_ASET_ID, parent, name); @@ -555,7 +525,7 @@ AllocSetContextCreateInternal(MemoryContext parent, * thrash malloc() when a context is repeatedly reset after small allocations, * which is typical behavior for per-tuple contexts. */ -static void +void AllocSetReset(MemoryContext context) { AllocSet set = (AllocSet) context; @@ -623,7 +593,7 @@ AllocSetReset(MemoryContext context) * * Unlike AllocSetReset, this *must* free all resources of the set. */ -static void +void AllocSetDelete(MemoryContext context) { AllocSet set = (AllocSet) context; @@ -717,12 +687,12 @@ AllocSetDelete(MemoryContext context) * is marked, as mcxt.c will set it to UNDEFINED. In some paths we will * return space that is marked NOACCESS - AllocSetRealloc has to beware! */ -static void * +void * AllocSetAlloc(MemoryContext context, Size size) { AllocSet set = (AllocSet) context; AllocBlock block; - AllocChunk chunk; + MemoryChunk *chunk; int fidx; Size chunk_size; Size blksize; @@ -746,18 +716,20 @@ AllocSetAlloc(MemoryContext context, Size size) block->aset = set; block->freeptr = block->endptr = ((char *) block) + blksize; - chunk = (AllocChunk) (((char *) block) + ALLOC_BLOCKHDRSZ); - chunk->aset = set; - chunk->size = chunk_size; + chunk = (MemoryChunk *) (((char *) block) + ALLOC_BLOCKHDRSZ); + + /* mark the MemoryChunk as externally managed */ + MemoryChunkSetHdrMaskExternal(chunk, MCTX_ASET_ID); + #ifdef MEMORY_CONTEXT_CHECKING chunk->requested_size = size; /* set mark to catch clobber of "unused" space */ if (size < chunk_size) - set_sentinel(AllocChunkGetPointer(chunk), size); + set_sentinel(MemoryChunkGetPointer(chunk), size); #endif #ifdef RANDOMIZE_ALLOCATED_MEMORY /* fill the allocated space with junk */ - randomize_mem((char *) AllocChunkGetPointer(chunk), size); + randomize_mem((char *) MemoryChunkGetPointer(chunk), size); #endif /* @@ -780,13 +752,13 @@ AllocSetAlloc(MemoryContext context, Size size) } /* Ensure any padding bytes are marked NOACCESS. */ - VALGRIND_MAKE_MEM_NOACCESS((char *) AllocChunkGetPointer(chunk) + size, + VALGRIND_MAKE_MEM_NOACCESS((char *) MemoryChunkGetPointer(chunk) + size, chunk_size - size); /* Disallow external access to private part of chunk header. */ VALGRIND_MAKE_MEM_NOACCESS(chunk, ALLOCCHUNK_PRIVATE_LEN); - return AllocChunkGetPointer(chunk); + return MemoryChunkGetPointer(chunk); } /* @@ -799,37 +771,40 @@ AllocSetAlloc(MemoryContext context, Size size) chunk = set->freelist[fidx]; if (chunk != NULL) { - Assert(chunk->size >= size); + AllocFreeListLink *link = GetFreeListLink(chunk); - set->freelist[fidx] = (AllocChunk) chunk->aset; + Assert(fidx == MemoryChunkGetValue(chunk)); - chunk->aset = (void *) set; + /* pop this chunk off the freelist */ + VALGRIND_MAKE_MEM_DEFINED(link, sizeof(AllocFreeListLink)); + set->freelist[fidx] = link->next; + VALGRIND_MAKE_MEM_NOACCESS(link, sizeof(AllocFreeListLink)); #ifdef MEMORY_CONTEXT_CHECKING chunk->requested_size = size; /* set mark to catch clobber of "unused" space */ - if (size < chunk->size) - set_sentinel(AllocChunkGetPointer(chunk), size); + if (size < GetChunkSizeFromFreeListIdx(fidx)) + set_sentinel(MemoryChunkGetPointer(chunk), size); #endif #ifdef RANDOMIZE_ALLOCATED_MEMORY /* fill the allocated space with junk */ - randomize_mem((char *) AllocChunkGetPointer(chunk), size); + randomize_mem((char *) MemoryChunkGetPointer(chunk), size); #endif /* Ensure any padding bytes are marked NOACCESS. */ - VALGRIND_MAKE_MEM_NOACCESS((char *) AllocChunkGetPointer(chunk) + size, - chunk->size - size); + VALGRIND_MAKE_MEM_NOACCESS((char *) MemoryChunkGetPointer(chunk) + size, + GetChunkSizeFromFreeListIdx(fidx) - size); /* Disallow external access to private part of chunk header. */ VALGRIND_MAKE_MEM_NOACCESS(chunk, ALLOCCHUNK_PRIVATE_LEN); - return AllocChunkGetPointer(chunk); + return MemoryChunkGetPointer(chunk); } /* * Choose the actual chunk size to allocate. */ - chunk_size = (1 << ALLOC_MINBITS) << fidx; + chunk_size = GetChunkSizeFromFreeListIdx(fidx); Assert(chunk_size >= size); /* @@ -856,6 +831,7 @@ AllocSetAlloc(MemoryContext context, Size size) */ while (availspace >= ((1 << ALLOC_MINBITS) + ALLOC_CHUNKHDRSZ)) { + AllocFreeListLink *link; Size availchunk = availspace - ALLOC_CHUNKHDRSZ; int a_fidx = AllocSetFreeIndex(availchunk); @@ -864,29 +840,34 @@ AllocSetAlloc(MemoryContext context, Size size) * freelist than the one we need to put this chunk on. The * exception is when availchunk is exactly a power of 2. */ - if (availchunk != ((Size) 1 << (a_fidx + ALLOC_MINBITS))) + if (availchunk != GetChunkSizeFromFreeListIdx(a_fidx)) { a_fidx--; Assert(a_fidx >= 0); - availchunk = ((Size) 1 << (a_fidx + ALLOC_MINBITS)); + availchunk = GetChunkSizeFromFreeListIdx(a_fidx); } - chunk = (AllocChunk) (block->freeptr); + chunk = (MemoryChunk *) (block->freeptr); /* Prepare to initialize the chunk header. */ VALGRIND_MAKE_MEM_UNDEFINED(chunk, ALLOC_CHUNKHDRSZ); - block->freeptr += (availchunk + ALLOC_CHUNKHDRSZ); availspace -= (availchunk + ALLOC_CHUNKHDRSZ); - chunk->size = availchunk; + /* store the freelist index in the value field */ + MemoryChunkSetHdrMask(chunk, block, a_fidx, MCTX_ASET_ID); #ifdef MEMORY_CONTEXT_CHECKING - chunk->requested_size = 0; /* mark it free */ + chunk->requested_size = InvalidAllocSize; /* mark it free */ #endif - chunk->aset = (void *) set->freelist[a_fidx]; + /* push this chunk onto the free list */ + link = GetFreeListLink(chunk); + + VALGRIND_MAKE_MEM_DEFINED(link, sizeof(AllocFreeListLink)); + link->next = set->freelist[a_fidx]; + VALGRIND_MAKE_MEM_NOACCESS(link, sizeof(AllocFreeListLink)); + set->freelist[a_fidx] = chunk; } - /* Mark that we need to create a new block */ block = NULL; } @@ -954,7 +935,7 @@ AllocSetAlloc(MemoryContext context, Size size) /* * OK, do the allocation */ - chunk = (AllocChunk) (block->freeptr); + chunk = (MemoryChunk *) (block->freeptr); /* Prepare to initialize the chunk header. */ VALGRIND_MAKE_MEM_UNDEFINED(chunk, ALLOC_CHUNKHDRSZ); @@ -962,67 +943,68 @@ AllocSetAlloc(MemoryContext context, Size size) block->freeptr += (chunk_size + ALLOC_CHUNKHDRSZ); Assert(block->freeptr <= block->endptr); - chunk->aset = (void *) set; - chunk->size = chunk_size; + /* store the free list index in the value field */ + MemoryChunkSetHdrMask(chunk, block, fidx, MCTX_ASET_ID); + #ifdef MEMORY_CONTEXT_CHECKING chunk->requested_size = size; /* set mark to catch clobber of "unused" space */ - if (size < chunk->size) - set_sentinel(AllocChunkGetPointer(chunk), size); + if (size < chunk_size) + set_sentinel(MemoryChunkGetPointer(chunk), size); #endif #ifdef RANDOMIZE_ALLOCATED_MEMORY /* fill the allocated space with junk */ - randomize_mem((char *) AllocChunkGetPointer(chunk), size); + randomize_mem((char *) MemoryChunkGetPointer(chunk), size); #endif /* Ensure any padding bytes are marked NOACCESS. */ - VALGRIND_MAKE_MEM_NOACCESS((char *) AllocChunkGetPointer(chunk) + size, + VALGRIND_MAKE_MEM_NOACCESS((char *) MemoryChunkGetPointer(chunk) + size, chunk_size - size); /* Disallow external access to private part of chunk header. */ VALGRIND_MAKE_MEM_NOACCESS(chunk, ALLOCCHUNK_PRIVATE_LEN); - return AllocChunkGetPointer(chunk); + return MemoryChunkGetPointer(chunk); } /* * AllocSetFree * Frees allocated memory; memory is removed from the set. */ -static void -AllocSetFree(MemoryContext context, void *pointer) +void +AllocSetFree(void *pointer) { - AllocSet set = (AllocSet) context; - AllocChunk chunk = AllocPointerGetChunk(pointer); + AllocSet set; + MemoryChunk *chunk = PointerGetMemoryChunk(pointer); /* Allow access to private part of chunk header. */ VALGRIND_MAKE_MEM_DEFINED(chunk, ALLOCCHUNK_PRIVATE_LEN); + if (MemoryChunkIsExternal(chunk)) + { + + AllocBlock block = ExternalChunkGetBlock(chunk); + + set = block->aset; + #ifdef MEMORY_CONTEXT_CHECKING - /* Test for someone scribbling on unused space in chunk */ - if (chunk->requested_size < chunk->size) - if (!sentinel_ok(pointer, chunk->requested_size)) - elog(WARNING, "detected write past chunk end in %s %p", - set->header.name, chunk); + { + Size chunk_size = block->endptr - (char *) pointer; + + /* Test for someone scribbling on unused space in chunk */ + if (chunk->requested_size < chunk_size) + if (!sentinel_ok(pointer, chunk->requested_size)) + elog(WARNING, "detected write past chunk end in %s %p", + set->header.name, chunk); + } #endif - if (chunk->size > set->allocChunkLimit) - { - /* - * Big chunks are certain to have been allocated as single-chunk - * blocks. Just unlink that block and return it to malloc(). - */ - AllocBlock block = (AllocBlock) (((char *) chunk) - ALLOC_BLOCKHDRSZ); /* - * Try to verify that we have a sane block pointer: it should - * reference the correct aset, and freeptr and endptr should point - * just past the chunk. + * Try to verify that we have a sane block pointer, the freeptr should + * match the endptr. */ - if (block->aset != set || - block->freeptr != block->endptr || - block->freeptr != ((char *) block) + - (chunk->size + ALLOC_BLOCKHDRSZ + ALLOC_CHUNKHDRSZ)) + if (block->freeptr != block->endptr) elog(ERROR, "could not find block containing chunk %p", chunk); /* OK, remove block from aset's list and free it */ @@ -1033,7 +1015,7 @@ AllocSetFree(MemoryContext context, void *pointer) if (block->next) block->next->prev = block->prev; - context->mem_allocated -= block->endptr - ((char *) block); + set->header.mem_allocated -= block->endptr - ((char *) block); #ifdef CLOBBER_FREED_MEMORY wipe_mem(block, block->freeptr - ((char *) block)); @@ -1042,20 +1024,37 @@ AllocSetFree(MemoryContext context, void *pointer) } else { - /* Normal case, put the chunk into appropriate freelist */ - int fidx = AllocSetFreeIndex(chunk->size); + int fidx = MemoryChunkGetValue(chunk); + AllocBlock block = MemoryChunkGetBlock(chunk); + AllocFreeListLink *link = GetFreeListLink(chunk); - chunk->aset = (void *) set->freelist[fidx]; - -#ifdef CLOBBER_FREED_MEMORY - wipe_mem(pointer, chunk->size); -#endif + set = block->aset; #ifdef MEMORY_CONTEXT_CHECKING - /* Reset requested_size to 0 in chunks that are on freelist */ - chunk->requested_size = 0; + /* Test for someone scribbling on unused space in chunk */ + if (chunk->requested_size < GetChunkSizeFromFreeListIdx(fidx)) + if (!sentinel_ok(pointer, chunk->requested_size)) + elog(WARNING, "detected write past chunk end in %s %p", + set->header.name, chunk); #endif + +#ifdef CLOBBER_FREED_MEMORY + wipe_mem(pointer, GetChunkSizeFromFreeListIdx(fidx)); +#endif + /* push this chunk onto the top of the free list */ + VALGRIND_MAKE_MEM_DEFINED(link, sizeof(AllocFreeListLink)); + link->next = set->freelist[fidx]; + VALGRIND_MAKE_MEM_NOACCESS(link, sizeof(AllocFreeListLink)); set->freelist[fidx] = chunk; + +#ifdef MEMORY_CONTEXT_CHECKING + + /* + * Reset requested_size to InvalidAllocSize in chunks that are on free + * list. + */ + chunk->requested_size = InvalidAllocSize; +#endif } } @@ -1071,57 +1070,48 @@ AllocSetFree(MemoryContext context, void *pointer) * (In principle, we could use VALGRIND_GET_VBITS() to rediscover the old * request size.) */ -static void * -AllocSetRealloc(MemoryContext context, void *pointer, Size size) +void * +AllocSetRealloc(void *pointer, Size size) { - AllocSet set = (AllocSet) context; - AllocChunk chunk = AllocPointerGetChunk(pointer); + AllocBlock block; + AllocSet set; + MemoryChunk *chunk = PointerGetMemoryChunk(pointer); Size oldsize; /* Allow access to private part of chunk header. */ VALGRIND_MAKE_MEM_DEFINED(chunk, ALLOCCHUNK_PRIVATE_LEN); - oldsize = chunk->size; - -#ifdef MEMORY_CONTEXT_CHECKING - /* Test for someone scribbling on unused space in chunk */ - if (chunk->requested_size < oldsize) - if (!sentinel_ok(pointer, chunk->requested_size)) - elog(WARNING, "detected write past chunk end in %s %p", - set->header.name, chunk); -#endif - - if (oldsize > set->allocChunkLimit) + if (MemoryChunkIsExternal(chunk)) { /* * The chunk must have been allocated as a single-chunk block. Use * realloc() to make the containing block bigger, or smaller, with * minimum space wastage. */ - AllocBlock block = (AllocBlock) (((char *) chunk) - ALLOC_BLOCKHDRSZ); Size chksize; Size blksize; Size oldblksize; - /* - * Try to verify that we have a sane block pointer: it should - * reference the correct aset, and freeptr and endptr should point - * just past the chunk. - */ - if (block->aset != set || - block->freeptr != block->endptr || - block->freeptr != ((char *) block) + - (oldsize + ALLOC_BLOCKHDRSZ + ALLOC_CHUNKHDRSZ)) - elog(ERROR, "could not find block containing chunk %p", chunk); + block = ExternalChunkGetBlock(chunk); + oldsize = block->endptr - (char *) pointer; + set = block->aset; + +#ifdef MEMORY_CONTEXT_CHECKING + /* Test for someone scribbling on unused space in chunk */ + if (chunk->requested_size < oldsize) + if (!sentinel_ok(pointer, chunk->requested_size)) + elog(WARNING, "detected write past chunk end in %s %p", + set->header.name, chunk); +#endif /* - * Even if the new request is less than set->allocChunkLimit, we stick - * with the single-chunk block approach. Therefore we need - * chunk->size to be bigger than set->allocChunkLimit, so we don't get - * confused about the chunk's status in future calls. + * Try to verify that we have a sane block pointer, the freeptr should + * match the endptr. */ - chksize = Max(size, set->allocChunkLimit + 1); - chksize = MAXALIGN(chksize); + if (block->freeptr != block->endptr) + elog(ERROR, "could not find block containing chunk %p", chunk); + + chksize = MAXALIGN(size); /* Do the realloc */ blksize = chksize + ALLOC_BLOCKHDRSZ + ALLOC_CHUNKHDRSZ; @@ -1136,21 +1126,20 @@ AllocSetRealloc(MemoryContext context, void *pointer, Size size) } /* updated separately, not to underflow when (oldblksize > blksize) */ - context->mem_allocated -= oldblksize; - context->mem_allocated += blksize; + set->header.mem_allocated -= oldblksize; + set->header.mem_allocated += blksize; block->freeptr = block->endptr = ((char *) block) + blksize; /* Update pointers since block has likely been moved */ - chunk = (AllocChunk) (((char *) block) + ALLOC_BLOCKHDRSZ); - pointer = AllocChunkGetPointer(chunk); + chunk = (MemoryChunk *) (((char *) block) + ALLOC_BLOCKHDRSZ); + pointer = MemoryChunkGetPointer(chunk); if (block->prev) block->prev->next = block; else set->blocks = block; if (block->next) block->next->prev = block; - chunk->size = chksize; #ifdef MEMORY_CONTEXT_CHECKING #ifdef RANDOMIZE_ALLOCATED_MEMORY @@ -1172,9 +1161,8 @@ AllocSetRealloc(MemoryContext context, void *pointer, Size size) #endif chunk->requested_size = size; - /* set mark to catch clobber of "unused" space */ - if (size < chunk->size) + if (size < chksize) set_sentinel(pointer, size); #else /* !MEMORY_CONTEXT_CHECKING */ @@ -1195,12 +1183,24 @@ AllocSetRealloc(MemoryContext context, void *pointer, Size size) return pointer; } + block = MemoryChunkGetBlock(chunk); + oldsize = GetChunkSizeFromFreeListIdx(MemoryChunkGetValue(chunk)); + set = block->aset; + +#ifdef MEMORY_CONTEXT_CHECKING + /* Test for someone scribbling on unused space in chunk */ + if (chunk->requested_size < oldsize) + if (!sentinel_ok(pointer, chunk->requested_size)) + elog(WARNING, "detected write past chunk end in %s %p", + set->header.name, chunk); +#endif + /* * Chunk sizes are aligned to power of 2 in AllocSetAlloc(). Maybe the * allocated area already is >= the new size. (In particular, we will * fall out here if the requested size is a decrease.) */ - else if (oldsize >= size) + if (oldsize >= size) { #ifdef MEMORY_CONTEXT_CHECKING Size oldrequest = chunk->requested_size; @@ -1289,34 +1289,59 @@ AllocSetRealloc(MemoryContext context, void *pointer, Size size) memcpy(newPointer, pointer, oldsize); /* free old chunk */ - AllocSetFree((MemoryContext) set, pointer); + AllocSetFree(pointer); return newPointer; } } +/* + * AllocSetGetChunkContext + * Return the MemoryContext that 'pointer' belongs to. + */ +MemoryContext +AllocSetGetChunkContext(void *pointer) +{ + MemoryChunk *chunk = PointerGetMemoryChunk(pointer); + AllocBlock block; + AllocSet set; + + if (MemoryChunkIsExternal(chunk)) + block = ExternalChunkGetBlock(chunk); + else + block = (AllocBlock) MemoryChunkGetBlock(chunk); + + set = block->aset; + + return &set->header; +} + /* * AllocSetGetChunkSpace * Given a currently-allocated chunk, determine the total space * it occupies (including all memory-allocation overhead). */ -static Size -AllocSetGetChunkSpace(MemoryContext context, void *pointer) +Size +AllocSetGetChunkSpace(void *pointer) { - AllocChunk chunk = AllocPointerGetChunk(pointer); - Size result; + MemoryChunk *chunk = PointerGetMemoryChunk(pointer); - VALGRIND_MAKE_MEM_DEFINED(chunk, ALLOCCHUNK_PRIVATE_LEN); - result = chunk->size + ALLOC_CHUNKHDRSZ; - VALGRIND_MAKE_MEM_NOACCESS(chunk, ALLOCCHUNK_PRIVATE_LEN); - return result; + if (MemoryChunkIsExternal(chunk)) + { + AllocBlock block = ExternalChunkGetBlock(chunk); + + return block->endptr - (char *) chunk; + } + + return GetChunkSizeFromFreeListIdx(MemoryChunkGetValue(chunk)) + + ALLOC_CHUNKHDRSZ; } /* * AllocSetIsEmpty * Is an allocset empty of any allocated space? */ -static bool +bool AllocSetIsEmpty(MemoryContext context) { /* @@ -1339,7 +1364,7 @@ AllocSetIsEmpty(MemoryContext context) * totals: if not NULL, add stats about this context into *totals. * print_to_stderr: print stats to stderr if true, elog otherwise. */ -static void +void AllocSetStats(MemoryContext context, MemoryStatsPrintFunc printfunc, void *passthru, MemoryContextCounters *totals, bool print_to_stderr) @@ -1363,13 +1388,21 @@ AllocSetStats(MemoryContext context, } for (fidx = 0; fidx < ALLOCSET_NUM_FREELISTS; fidx++) { - AllocChunk chunk; + MemoryChunk *chunk = set->freelist[fidx]; - for (chunk = set->freelist[fidx]; chunk != NULL; - chunk = (AllocChunk) chunk->aset) + while (chunk != NULL) { + Size chksz = GetChunkSizeFromFreeListIdx(MemoryChunkGetValue(chunk)); + AllocFreeListLink *link = GetFreeListLink(chunk); + + Assert(GetChunkSizeFromFreeListIdx(fidx) == chksz); + freechunks++; - freespace += chunk->size + ALLOC_CHUNKHDRSZ; + freespace += chksz + ALLOC_CHUNKHDRSZ; + + VALGRIND_MAKE_MEM_DEFINED(link, sizeof(AllocFreeListLink)); + chunk = link->next; + VALGRIND_MAKE_MEM_NOACCESS(link, sizeof(AllocFreeListLink)); } } @@ -1404,7 +1437,7 @@ AllocSetStats(MemoryContext context, * find yourself in an infinite loop when trouble occurs, because this * routine will be entered again when elog cleanup tries to release memory! */ -static void +void AllocSetCheck(MemoryContext context) { AllocSet set = (AllocSet) context; @@ -1421,6 +1454,7 @@ AllocSetCheck(MemoryContext context) long blk_used = block->freeptr - bpoz; long blk_data = 0; long nchunks = 0; + bool has_external_chunk = false; if (set->keeper == block) total_allocated += block->endptr - ((char *) set); @@ -1452,46 +1486,51 @@ AllocSetCheck(MemoryContext context) */ while (bpoz < block->freeptr) { - AllocChunk chunk = (AllocChunk) bpoz; + MemoryChunk *chunk = (MemoryChunk *) bpoz; Size chsize, dsize; /* Allow access to private part of chunk header. */ VALGRIND_MAKE_MEM_DEFINED(chunk, ALLOCCHUNK_PRIVATE_LEN); - chsize = chunk->size; /* aligned chunk size */ + if (MemoryChunkIsExternal(chunk)) + { + chsize = block->endptr - (char *) MemoryChunkGetPointer(chunk); /* aligned chunk size */ + has_external_chunk = true; + + /* make sure this chunk consumes the entire block */ + if (chsize + ALLOC_CHUNKHDRSZ != blk_used) + elog(WARNING, "problem in alloc set %s: bad single-chunk %p in block %p", + name, chunk, block); + } + else + { + chsize = GetChunkSizeFromFreeListIdx(MemoryChunkGetValue(chunk)); /* aligned chunk size */ + + /* + * Check the stored block offset correctly references this + * block. + */ + if (block != MemoryChunkGetBlock(chunk)) + elog(WARNING, "problem in alloc set %s: bad block offset for chunk %p in block %p", + name, chunk, block); + } dsize = chunk->requested_size; /* real data */ - /* - * Check chunk size - */ - if (dsize > chsize) + /* an allocated chunk's requested size must be <= the chsize */ + if (dsize != InvalidAllocSize && dsize > chsize) elog(WARNING, "problem in alloc set %s: req size > alloc size for chunk %p in block %p", name, chunk, block); + + /* chsize must not be smaller than the first freelist's size */ if (chsize < (1 << ALLOC_MINBITS)) elog(WARNING, "problem in alloc set %s: bad size %zu for chunk %p in block %p", name, chsize, chunk, block); - /* single-chunk block? */ - if (chsize > set->allocChunkLimit && - chsize + ALLOC_CHUNKHDRSZ != blk_used) - elog(WARNING, "problem in alloc set %s: bad single-chunk %p in block %p", - name, chunk, block); - - /* - * If chunk is allocated, check for correct aset pointer. (If it's - * free, the aset is the freelist pointer, which we can't check as - * easily...) Note this is an incomplete test, since palloc(0) - * produces an allocated chunk with requested_size == 0. - */ - if (dsize > 0 && chunk->aset != (void *) set) - elog(WARNING, "problem in alloc set %s: bogus aset link in block %p, chunk %p", - name, block, chunk); - /* * Check for overwrite of padding space in an allocated chunk. */ - if (chunk->aset == (void *) set && dsize < chsize && + if (dsize != InvalidAllocSize && dsize < chsize && !sentinel_ok(chunk, ALLOC_CHUNKHDRSZ + dsize)) elog(WARNING, "problem in alloc set %s: detected write past chunk end in block %p, chunk %p", name, block, chunk); @@ -1500,7 +1539,7 @@ AllocSetCheck(MemoryContext context) * If chunk is allocated, disallow external access to private part * of chunk header. */ - if (chunk->aset == (void *) set) + if (dsize != InvalidAllocSize) VALGRIND_MAKE_MEM_NOACCESS(chunk, ALLOCCHUNK_PRIVATE_LEN); blk_data += chsize; @@ -1512,6 +1551,10 @@ AllocSetCheck(MemoryContext context) if ((blk_data + (nchunks * ALLOC_CHUNKHDRSZ)) != blk_used) elog(WARNING, "problem in alloc set %s: found inconsistent memory block %p", name, block); + + if (has_external_chunk && nchunks > 1) + elog(WARNING, "problem in alloc set %s: external chunk on non-dedicated block %p", + name, block); } Assert(total_allocated == context->mem_allocated); diff --git a/src/backend/utils/mmgr/generation.c b/src/backend/utils/mmgr/generation.c index e530e272e05..b39894ec946 100644 --- a/src/backend/utils/mmgr/generation.c +++ b/src/backend/utils/mmgr/generation.c @@ -39,15 +39,16 @@ #include "port/pg_bitutils.h" #include "utils/memdebug.h" #include "utils/memutils.h" +#include "utils/memutils_memorychunk.h" +#include "utils/memutils_internal.h" #define Generation_BLOCKHDRSZ MAXALIGN(sizeof(GenerationBlock)) -#define Generation_CHUNKHDRSZ sizeof(GenerationChunk) +#define Generation_CHUNKHDRSZ sizeof(MemoryChunk) #define Generation_CHUNK_FRACTION 8 typedef struct GenerationBlock GenerationBlock; /* forward reference */ -typedef struct GenerationChunk GenerationChunk; typedef void *GenerationPointer; @@ -77,11 +78,11 @@ typedef struct GenerationContext /* * GenerationBlock * GenerationBlock is the unit of memory that is obtained by generation.c - * from malloc(). It contains zero or more GenerationChunks, which are - * the units requested by palloc() and freed by pfree(). GenerationChunks - * cannot be returned to malloc() individually, instead pfree() - * updates the free counter of the block and when all chunks in a block - * are free the whole block can be returned to malloc(). + * from malloc(). It contains zero or more MemoryChunks, which are the + * units requested by palloc() and freed by pfree(). MemoryChunks cannot + * be returned to malloc() individually, instead pfree() updates the free + * counter of the block and when all chunks in a block are free the whole + * block can be returned to malloc(). * * GenerationBlock is the header data for a block --- the usable space * within the block begins at the next alignment boundary. @@ -89,6 +90,7 @@ typedef struct GenerationContext struct GenerationBlock { dlist_node node; /* doubly-linked list of blocks */ + GenerationContext *context; /* pointer back to the owning context */ Size blksize; /* allocated size of this block */ int nchunks; /* number of chunks in the block */ int nfree; /* number of free chunks */ @@ -97,104 +99,36 @@ struct GenerationBlock }; /* - * GenerationChunk - * The prefix of each piece of memory in a GenerationBlock - * - * Note: to meet the memory context APIs, the payload area of the chunk must - * be maxaligned, and the "context" link must be immediately adjacent to the - * payload area (cf. GetMemoryChunkContext). We simplify matters for this - * module by requiring sizeof(GenerationChunk) to be maxaligned, and then - * we can ensure things work by adding any required alignment padding before - * the pointer fields. There is a static assertion below that the alignment - * is done correctly. - */ -struct GenerationChunk -{ - /* size is always the size of the usable space in the chunk */ - Size size; -#ifdef MEMORY_CONTEXT_CHECKING - /* when debugging memory usage, also store actual requested size */ - /* this is zero in a free chunk */ - Size requested_size; - -#define GENERATIONCHUNK_RAWSIZE (SIZEOF_SIZE_T * 2 + SIZEOF_VOID_P * 2) -#else -#define GENERATIONCHUNK_RAWSIZE (SIZEOF_SIZE_T + SIZEOF_VOID_P * 2) -#endif /* MEMORY_CONTEXT_CHECKING */ - - /* ensure proper alignment by adding padding if needed */ -#if (GENERATIONCHUNK_RAWSIZE % MAXIMUM_ALIGNOF) != 0 - char padding[MAXIMUM_ALIGNOF - GENERATIONCHUNK_RAWSIZE % MAXIMUM_ALIGNOF]; -#endif - - GenerationBlock *block; /* block owning this chunk */ - GenerationContext *context; /* owning context, or NULL if freed chunk */ - /* there must not be any padding to reach a MAXALIGN boundary here! */ -}; - -/* - * Only the "context" field should be accessed outside this module. + * Only the "hdrmask" field should be accessed outside this module. * We keep the rest of an allocated chunk's header marked NOACCESS when using * valgrind. But note that freed chunk headers are kept accessible, for * simplicity. */ -#define GENERATIONCHUNK_PRIVATE_LEN offsetof(GenerationChunk, context) - +#define GENERATIONCHUNK_PRIVATE_LEN offsetof(MemoryChunk, hdrmask) /* * GenerationIsValid * True iff set is valid allocation set. */ #define GenerationIsValid(set) PointerIsValid(set) -#define GenerationPointerGetChunk(ptr) \ - ((GenerationChunk *)(((char *)(ptr)) - Generation_CHUNKHDRSZ)) -#define GenerationChunkGetPointer(chk) \ - ((GenerationPointer *)(((char *)(chk)) + Generation_CHUNKHDRSZ)) +/* + * We always store external chunks on a dedicated block. This makes fetching + * the block from an external chunk easy since it's always the first and only + * chunk on the block. + */ +#define ExternalChunkGetBlock(chunk) \ + (GenerationBlock *) ((char *) chunk - Generation_BLOCKHDRSZ) /* Inlined helper functions */ -static inline void GenerationBlockInit(GenerationBlock *block, Size blksize); +static inline void GenerationBlockInit(GenerationContext *context, + GenerationBlock *block, + Size blksize); static inline bool GenerationBlockIsEmpty(GenerationBlock *block); static inline void GenerationBlockMarkEmpty(GenerationBlock *block); static inline Size GenerationBlockFreeBytes(GenerationBlock *block); static inline void GenerationBlockFree(GenerationContext *set, GenerationBlock *block); -/* - * These functions implement the MemoryContext API for Generation contexts. - */ -static void *GenerationAlloc(MemoryContext context, Size size); -static void GenerationFree(MemoryContext context, void *pointer); -static void *GenerationRealloc(MemoryContext context, void *pointer, Size size); -static void GenerationReset(MemoryContext context); -static void GenerationDelete(MemoryContext context); -static Size GenerationGetChunkSpace(MemoryContext context, void *pointer); -static bool GenerationIsEmpty(MemoryContext context); -static void GenerationStats(MemoryContext context, - MemoryStatsPrintFunc printfunc, void *passthru, - MemoryContextCounters *totals, - bool print_to_stderr); - -#ifdef MEMORY_CONTEXT_CHECKING -static void GenerationCheck(MemoryContext context); -#endif - -/* - * This is the virtual function table for Generation contexts. - */ -static const MemoryContextMethods GenerationMethods = { - GenerationAlloc, - GenerationFree, - GenerationRealloc, - GenerationReset, - GenerationDelete, - GenerationGetChunkSpace, - GenerationIsEmpty, - GenerationStats -#ifdef MEMORY_CONTEXT_CHECKING - ,GenerationCheck -#endif -}; - /* * Public routines @@ -223,17 +157,20 @@ GenerationContextCreate(MemoryContext parent, GenerationContext *set; GenerationBlock *block; - /* Assert we padded GenerationChunk properly */ + /* ensure MemoryChunk's size is properly maxaligned */ StaticAssertStmt(Generation_CHUNKHDRSZ == MAXALIGN(Generation_CHUNKHDRSZ), - "sizeof(GenerationChunk) is not maxaligned"); - StaticAssertStmt(offsetof(GenerationChunk, context) + sizeof(MemoryContext) == - Generation_CHUNKHDRSZ, - "padding calculation in GenerationChunk is wrong"); + "sizeof(MemoryChunk) is not maxaligned"); /* * First, validate allocation parameters. Asserts seem sufficient because * nobody varies their parameters at runtime. We somewhat arbitrarily - * enforce a minimum 1K block size. + * enforce a minimum 1K block size. We restrict the maximum block size to + * MEMORYCHUNK_MAX_BLOCKOFFSET as MemoryChunks are limited to this in + * regards to addressing the offset between the chunk and the block that + * the chunk is stored on. We would be unable to store the offset between + * the chunk and block for any chunks that were beyond + * MEMORYCHUNK_MAX_BLOCKOFFSET bytes into the block if the block was to be + * larger than this. */ Assert(initBlockSize == MAXALIGN(initBlockSize) && initBlockSize >= 1024); @@ -244,6 +181,7 @@ GenerationContextCreate(MemoryContext parent, (minContextSize == MAXALIGN(minContextSize) && minContextSize >= 1024 && minContextSize <= maxBlockSize)); + Assert(maxBlockSize <= MEMORYCHUNK_MAX_BLOCKOFFSET); /* Determine size of initial block */ allocSize = MAXALIGN(sizeof(GenerationContext)) + @@ -278,7 +216,7 @@ GenerationContextCreate(MemoryContext parent, block = (GenerationBlock *) (((char *) set) + MAXALIGN(sizeof(GenerationContext))); /* determine the block size and initialize it */ firstBlockSize = allocSize - MAXALIGN(sizeof(GenerationContext)); - GenerationBlockInit(block, firstBlockSize); + GenerationBlockInit(set, block, firstBlockSize); /* add it to the doubly-linked list of blocks */ dlist_push_head(&set->blocks, &block->node); @@ -300,9 +238,14 @@ GenerationContextCreate(MemoryContext parent, /* * Compute the allocation chunk size limit for this context. * - * Follows similar ideas as AllocSet, see aset.c for details ... + * Limit the maximum size a non-dedicated chunk can be so that we can fit + * at least Generation_CHUNK_FRACTION of chunks this big onto the maximum + * sized block. We must further limit this value so that it's no more + * than MEMORYCHUNK_MAX_VALUE. We're unable to have non-external chunks + * larger than that value as we store the chunk size in the MemoryChunk + * 'value' field in the call to MemoryChunkSetHdrMask(). */ - set->allocChunkLimit = maxBlockSize; + set->allocChunkLimit = Min(maxBlockSize, MEMORYCHUNK_MAX_VALUE); while ((Size) (set->allocChunkLimit + Generation_CHUNKHDRSZ) > (Size) ((Size) (maxBlockSize - Generation_BLOCKHDRSZ) / Generation_CHUNK_FRACTION)) set->allocChunkLimit >>= 1; @@ -310,7 +253,7 @@ GenerationContextCreate(MemoryContext parent, /* Finally, do the type-independent part of context creation */ MemoryContextCreate((MemoryContext) set, T_GenerationContext, - &GenerationMethods, + MCTX_GENERATION_ID, parent, name); @@ -326,7 +269,7 @@ GenerationContextCreate(MemoryContext parent, * The code simply frees all the blocks in the context - we don't keep any * keeper blocks or anything like that. */ -static void +void GenerationReset(MemoryContext context) { GenerationContext *set = (GenerationContext *) context; @@ -371,7 +314,7 @@ GenerationReset(MemoryContext context) * GenerationDelete * Free all memory which is allocated in the given context. */ -static void +void GenerationDelete(MemoryContext context) { /* Reset to release all releasable GenerationBlocks */ @@ -393,12 +336,12 @@ GenerationDelete(MemoryContext context) * is marked, as mcxt.c will set it to UNDEFINED. In some paths we will * return space that is marked NOACCESS - GenerationRealloc has to beware! */ -static void * +void * GenerationAlloc(MemoryContext context, Size size) { GenerationContext *set = (GenerationContext *) context; GenerationBlock *block; - GenerationChunk *chunk; + MemoryChunk *chunk; Size chunk_size = MAXALIGN(size); Size required_size = chunk_size + Generation_CHUNKHDRSZ; @@ -414,6 +357,7 @@ GenerationAlloc(MemoryContext context, Size size) context->mem_allocated += blksize; /* block with a single (used) chunk */ + block->context = set; block->blksize = blksize; block->nchunks = 1; block->nfree = 0; @@ -421,33 +365,33 @@ GenerationAlloc(MemoryContext context, Size size) /* the block is completely full */ block->freeptr = block->endptr = ((char *) block) + blksize; - chunk = (GenerationChunk *) (((char *) block) + Generation_BLOCKHDRSZ); - chunk->block = block; - chunk->context = set; - chunk->size = chunk_size; + chunk = (MemoryChunk *) (((char *) block) + Generation_BLOCKHDRSZ); + + /* mark the MemoryChunk as externally managed */ + MemoryChunkSetHdrMaskExternal(chunk, MCTX_GENERATION_ID); #ifdef MEMORY_CONTEXT_CHECKING chunk->requested_size = size; /* set mark to catch clobber of "unused" space */ if (size < chunk_size) - set_sentinel(GenerationChunkGetPointer(chunk), size); + set_sentinel(MemoryChunkGetPointer(chunk), size); #endif #ifdef RANDOMIZE_ALLOCATED_MEMORY /* fill the allocated space with junk */ - randomize_mem((char *) GenerationChunkGetPointer(chunk), size); + randomize_mem((char *) MemoryChunkGetPointer(chunk), size); #endif /* add the block to the list of allocated blocks */ dlist_push_head(&set->blocks, &block->node); /* Ensure any padding bytes are marked NOACCESS. */ - VALGRIND_MAKE_MEM_NOACCESS((char *) GenerationChunkGetPointer(chunk) + size, + VALGRIND_MAKE_MEM_NOACCESS((char *) MemoryChunkGetPointer(chunk) + size, chunk_size - size); /* Disallow external access to private part of chunk header. */ VALGRIND_MAKE_MEM_NOACCESS(chunk, GENERATIONCHUNK_PRIVATE_LEN); - return GenerationChunkGetPointer(chunk); + return MemoryChunkGetPointer(chunk); } /* @@ -516,7 +460,7 @@ GenerationAlloc(MemoryContext context, Size size) context->mem_allocated += blksize; /* initialize the new block */ - GenerationBlockInit(block, blksize); + GenerationBlockInit(set, block, blksize); /* add it to the doubly-linked list of blocks */ dlist_push_head(&set->blocks, &block->node); @@ -533,7 +477,7 @@ GenerationAlloc(MemoryContext context, Size size) Assert(block != NULL); Assert((block->endptr - block->freeptr) >= Generation_CHUNKHDRSZ + chunk_size); - chunk = (GenerationChunk *) block->freeptr; + chunk = (MemoryChunk *) block->freeptr; /* Prepare to initialize the chunk header. */ VALGRIND_MAKE_MEM_UNDEFINED(chunk, Generation_CHUNKHDRSZ); @@ -543,29 +487,26 @@ GenerationAlloc(MemoryContext context, Size size) Assert(block->freeptr <= block->endptr); - chunk->block = block; - chunk->context = set; - chunk->size = chunk_size; - + MemoryChunkSetHdrMask(chunk, block, chunk_size, MCTX_GENERATION_ID); #ifdef MEMORY_CONTEXT_CHECKING chunk->requested_size = size; /* set mark to catch clobber of "unused" space */ - if (size < chunk->size) - set_sentinel(GenerationChunkGetPointer(chunk), size); + if (size < chunk_size) + set_sentinel(MemoryChunkGetPointer(chunk), size); #endif #ifdef RANDOMIZE_ALLOCATED_MEMORY /* fill the allocated space with junk */ - randomize_mem((char *) GenerationChunkGetPointer(chunk), size); + randomize_mem((char *) MemoryChunkGetPointer(chunk), size); #endif /* Ensure any padding bytes are marked NOACCESS. */ - VALGRIND_MAKE_MEM_NOACCESS((char *) GenerationChunkGetPointer(chunk) + size, + VALGRIND_MAKE_MEM_NOACCESS((char *) MemoryChunkGetPointer(chunk) + size, chunk_size - size); /* Disallow external access to private part of chunk header. */ VALGRIND_MAKE_MEM_NOACCESS(chunk, GENERATIONCHUNK_PRIVATE_LEN); - return GenerationChunkGetPointer(chunk); + return MemoryChunkGetPointer(chunk); } /* @@ -574,8 +515,10 @@ GenerationAlloc(MemoryContext context, Size size) * mem_allocated field. */ static inline void -GenerationBlockInit(GenerationBlock *block, Size blksize) +GenerationBlockInit(GenerationContext *context, GenerationBlock *block, + Size blksize) { + block->context = context; block->blksize = blksize; block->nchunks = 0; block->nfree = 0; @@ -661,36 +604,49 @@ GenerationBlockFree(GenerationContext *set, GenerationBlock *block) * Update number of chunks in the block, and if all chunks in the block * are now free then discard the block. */ -static void -GenerationFree(MemoryContext context, void *pointer) +void +GenerationFree(void *pointer) { - GenerationContext *set = (GenerationContext *) context; - GenerationChunk *chunk = GenerationPointerGetChunk(pointer); + MemoryChunk *chunk = PointerGetMemoryChunk(pointer); GenerationBlock *block; + GenerationContext *set; +#if defined(MEMORY_CONTEXT_CHECKING) || defined(CLOBBER_FREED_MEMORY) + Size chunksize; +#endif + + if (MemoryChunkIsExternal(chunk)) + { + block = ExternalChunkGetBlock(chunk); +#if defined(MEMORY_CONTEXT_CHECKING) || defined(CLOBBER_FREED_MEMORY) + chunksize = block->endptr - (char *) pointer; +#endif + } + else + { + block = MemoryChunkGetBlock(chunk); +#if defined(MEMORY_CONTEXT_CHECKING) || defined(CLOBBER_FREED_MEMORY) + chunksize = MemoryChunkGetValue(chunk); +#endif + } /* Allow access to private part of chunk header. */ VALGRIND_MAKE_MEM_DEFINED(chunk, GENERATIONCHUNK_PRIVATE_LEN); - block = chunk->block; - #ifdef MEMORY_CONTEXT_CHECKING /* Test for someone scribbling on unused space in chunk */ - if (chunk->requested_size < chunk->size) + if (chunk->requested_size < chunksize) if (!sentinel_ok(pointer, chunk->requested_size)) elog(WARNING, "detected write past chunk end in %s %p", - ((MemoryContext) set)->name, chunk); + ((MemoryContext) block->context)->name, chunk); #endif #ifdef CLOBBER_FREED_MEMORY - wipe_mem(pointer, chunk->size); + wipe_mem(pointer, chunksize); #endif - /* Reset context to NULL in freed chunks */ - chunk->context = NULL; - #ifdef MEMORY_CONTEXT_CHECKING - /* Reset requested_size to 0 in freed chunks */ - chunk->requested_size = 0; + /* Reset requested_size to InvalidAllocSize in freed chunks */ + chunk->requested_size = InvalidAllocSize; #endif block->nfree += 1; @@ -702,6 +658,8 @@ GenerationFree(MemoryContext context, void *pointer) if (block->nfree < block->nchunks) return; + set = block->context; + /* Don't try to free the keeper block, just mark it empty */ if (block == set->keeper) { @@ -732,7 +690,7 @@ GenerationFree(MemoryContext context, void *pointer) */ dlist_delete(&block->node); - context->mem_allocated -= block->blksize; + set->header.mem_allocated -= block->blksize; free(block); } @@ -742,18 +700,30 @@ GenerationFree(MemoryContext context, void *pointer) * and discard the old one. The only exception is when the new size fits * into the old chunk - in that case we just update chunk header. */ -static void * -GenerationRealloc(MemoryContext context, void *pointer, Size size) +void * +GenerationRealloc(void *pointer, Size size) { - GenerationContext *set = (GenerationContext *) context; - GenerationChunk *chunk = GenerationPointerGetChunk(pointer); + MemoryChunk *chunk = PointerGetMemoryChunk(pointer); + GenerationContext *set; + GenerationBlock *block; GenerationPointer newPointer; Size oldsize; /* Allow access to private part of chunk header. */ VALGRIND_MAKE_MEM_DEFINED(chunk, GENERATIONCHUNK_PRIVATE_LEN); - oldsize = chunk->size; + if (MemoryChunkIsExternal(chunk)) + { + block = ExternalChunkGetBlock(chunk); + oldsize = block->endptr - (char *) pointer; + } + else + { + block = MemoryChunkGetBlock(chunk); + oldsize = MemoryChunkGetValue(chunk); + } + + set = block->context; #ifdef MEMORY_CONTEXT_CHECKING /* Test for someone scribbling on unused space in chunk */ @@ -848,33 +818,57 @@ GenerationRealloc(MemoryContext context, void *pointer, Size size) memcpy(newPointer, pointer, oldsize); /* free old chunk */ - GenerationFree((MemoryContext) set, pointer); + GenerationFree(pointer); return newPointer; } +/* + * GenerationGetChunkContext + * Return the MemoryContext that 'pointer' belongs to. + */ +MemoryContext +GenerationGetChunkContext(void *pointer) +{ + MemoryChunk *chunk = PointerGetMemoryChunk(pointer); + GenerationBlock *block; + + if (MemoryChunkIsExternal(chunk)) + block = ExternalChunkGetBlock(chunk); + else + block = (GenerationBlock *) MemoryChunkGetBlock(chunk); + + return &block->context->header; +} + /* * GenerationGetChunkSpace * Given a currently-allocated chunk, determine the total space * it occupies (including all memory-allocation overhead). */ -static Size -GenerationGetChunkSpace(MemoryContext context, void *pointer) +Size +GenerationGetChunkSpace(void *pointer) { - GenerationChunk *chunk = GenerationPointerGetChunk(pointer); - Size result; + MemoryChunk *chunk = PointerGetMemoryChunk(pointer); + Size chunksize; - VALGRIND_MAKE_MEM_DEFINED(chunk, GENERATIONCHUNK_PRIVATE_LEN); - result = chunk->size + Generation_CHUNKHDRSZ; - VALGRIND_MAKE_MEM_NOACCESS(chunk, GENERATIONCHUNK_PRIVATE_LEN); - return result; + if (MemoryChunkIsExternal(chunk)) + { + GenerationBlock *block = ExternalChunkGetBlock(chunk); + + chunksize = block->endptr - (char *) pointer; + } + else + chunksize = MemoryChunkGetValue(chunk); + + return Generation_CHUNKHDRSZ + chunksize; } /* * GenerationIsEmpty * Is a GenerationContext empty of any allocated space? */ -static bool +bool GenerationIsEmpty(MemoryContext context) { GenerationContext *set = (GenerationContext *) context; @@ -903,7 +897,7 @@ GenerationIsEmpty(MemoryContext context) * XXX freespace only accounts for empty space at the end of the block, not * space of freed chunks (which is unknown). */ -static void +void GenerationStats(MemoryContext context, MemoryStatsPrintFunc printfunc, void *passthru, MemoryContextCounters *totals, bool print_to_stderr) @@ -961,7 +955,7 @@ GenerationStats(MemoryContext context, * find yourself in an infinite loop when trouble occurs, because this * routine will be entered again when elog cleanup tries to release memory! */ -static void +void GenerationCheck(MemoryContext context) { GenerationContext *gen = (GenerationContext *) context; @@ -976,6 +970,7 @@ GenerationCheck(MemoryContext context) int nfree, nchunks; char *ptr; + bool has_external_chunk = false; total_allocated += block->blksize; @@ -987,6 +982,11 @@ GenerationCheck(MemoryContext context) elog(WARNING, "problem in Generation %s: number of free chunks %d in block %p exceeds %d allocated", name, block->nfree, block, block->nchunks); + /* check block belongs to the correct context */ + if (block->context != gen) + elog(WARNING, "problem in Generation %s: bogus context link in block %p", + name, block); + /* Now walk through the chunks and count them. */ nfree = 0; nchunks = 0; @@ -994,42 +994,47 @@ GenerationCheck(MemoryContext context) while (ptr < block->freeptr) { - GenerationChunk *chunk = (GenerationChunk *) ptr; + MemoryChunk *chunk = (MemoryChunk *) ptr; + GenerationBlock *chunkblock; + Size chunksize; /* Allow access to private part of chunk header. */ VALGRIND_MAKE_MEM_DEFINED(chunk, GENERATIONCHUNK_PRIVATE_LEN); + if (MemoryChunkIsExternal(chunk)) + { + chunkblock = ExternalChunkGetBlock(chunk); + chunksize = block->endptr - (char *) MemoryChunkGetPointer(chunk); + has_external_chunk = true; + } + else + { + chunkblock = MemoryChunkGetBlock(chunk); + chunksize = MemoryChunkGetValue(chunk); + } + /* move to the next chunk */ - ptr += (chunk->size + Generation_CHUNKHDRSZ); + ptr += (chunksize + Generation_CHUNKHDRSZ); nchunks += 1; /* chunks have both block and context pointers, so check both */ - if (chunk->block != block) + if (chunkblock != block) elog(WARNING, "problem in Generation %s: bogus block link in block %p, chunk %p", name, block, chunk); - /* - * Check for valid context pointer. Note this is an incomplete - * test, since palloc(0) produces an allocated chunk with - * requested_size == 0. - */ - if ((chunk->requested_size > 0 && chunk->context != gen) || - (chunk->context != gen && chunk->context != NULL)) - elog(WARNING, "problem in Generation %s: bogus context link in block %p, chunk %p", - name, block, chunk); - - /* now make sure the chunk size is correct */ - if (chunk->size < chunk->requested_size || - chunk->size != MAXALIGN(chunk->size)) - elog(WARNING, "problem in Generation %s: bogus chunk size in block %p, chunk %p", - name, block, chunk); /* is chunk allocated? */ - if (chunk->context != NULL) + if (chunk->requested_size != InvalidAllocSize) { - /* check sentinel, but only in allocated blocks */ - if (chunk->requested_size < chunk->size && + /* now make sure the chunk size is correct */ + if (chunksize < chunk->requested_size || + chunksize != MAXALIGN(chunksize)) + elog(WARNING, "problem in Generation %s: bogus chunk size in block %p, chunk %p", + name, block, chunk); + + /* check sentinel */ + if (chunk->requested_size < chunksize && !sentinel_ok(chunk, Generation_CHUNKHDRSZ + chunk->requested_size)) elog(WARNING, "problem in Generation %s: detected write past chunk end in block %p, chunk %p", name, block, chunk); @@ -1041,7 +1046,7 @@ GenerationCheck(MemoryContext context) * If chunk is allocated, disallow external access to private part * of chunk header. */ - if (chunk->context != NULL) + if (chunk->requested_size != InvalidAllocSize) VALGRIND_MAKE_MEM_NOACCESS(chunk, GENERATIONCHUNK_PRIVATE_LEN); } @@ -1056,6 +1061,11 @@ GenerationCheck(MemoryContext context) if (nfree != block->nfree) elog(WARNING, "problem in Generation %s: number of free chunks %d in block %p does not match header %d", name, nfree, block, block->nfree); + + if (has_external_chunk && nchunks > 1) + elog(WARNING, "problem in Generation %s: external chunk on non-dedicated block %p", + name, block); + } Assert(total_allocated == context->mem_allocated); diff --git a/src/backend/utils/mmgr/mcxt.c b/src/backend/utils/mmgr/mcxt.c index e12be1b9bd8..64bcc7ef325 100644 --- a/src/backend/utils/mmgr/mcxt.c +++ b/src/backend/utils/mmgr/mcxt.c @@ -29,12 +29,57 @@ #include "utils/fmgrprotos.h" #include "utils/memdebug.h" #include "utils/memutils.h" +#include "utils/memutils_internal.h" /***************************************************************************** * GLOBAL MEMORY * *****************************************************************************/ +static const MemoryContextMethods mcxt_methods[] = { + /* aset.c */ + [MCTX_ASET_ID].alloc = AllocSetAlloc, + [MCTX_ASET_ID].free_p = AllocSetFree, + [MCTX_ASET_ID].realloc = AllocSetRealloc, + [MCTX_ASET_ID].reset = AllocSetReset, + [MCTX_ASET_ID].delete_context = AllocSetDelete, + [MCTX_ASET_ID].get_chunk_context = AllocSetGetChunkContext, + [MCTX_ASET_ID].get_chunk_space = AllocSetGetChunkSpace, + [MCTX_ASET_ID].is_empty = AllocSetIsEmpty, + [MCTX_ASET_ID].stats = AllocSetStats, +#ifdef MEMORY_CONTEXT_CHECKING + [MCTX_ASET_ID].check = AllocSetCheck, +#endif + + /* generation.c */ + [MCTX_GENERATION_ID].alloc = GenerationAlloc, + [MCTX_GENERATION_ID].free_p = GenerationFree, + [MCTX_GENERATION_ID].realloc = GenerationRealloc, + [MCTX_GENERATION_ID].reset = GenerationReset, + [MCTX_GENERATION_ID].delete_context = GenerationDelete, + [MCTX_GENERATION_ID].get_chunk_context = GenerationGetChunkContext, + [MCTX_GENERATION_ID].get_chunk_space = GenerationGetChunkSpace, + [MCTX_GENERATION_ID].is_empty = GenerationIsEmpty, + [MCTX_GENERATION_ID].stats = GenerationStats, +#ifdef MEMORY_CONTEXT_CHECKING + [MCTX_GENERATION_ID].check = GenerationCheck, +#endif + + /* slab.c */ + [MCTX_SLAB_ID].alloc = SlabAlloc, + [MCTX_SLAB_ID].free_p = SlabFree, + [MCTX_SLAB_ID].realloc = SlabRealloc, + [MCTX_SLAB_ID].reset = SlabReset, + [MCTX_SLAB_ID].delete_context = SlabDelete, + [MCTX_SLAB_ID].get_chunk_context = SlabGetChunkContext, + [MCTX_SLAB_ID].get_chunk_space = SlabGetChunkSpace, + [MCTX_SLAB_ID].is_empty = SlabIsEmpty, + [MCTX_SLAB_ID].stats = SlabStats +#ifdef MEMORY_CONTEXT_CHECKING + ,[MCTX_SLAB_ID].check = SlabCheck +#endif +}; + /* * CurrentMemoryContext * Default memory context for allocations. @@ -73,6 +118,13 @@ static void MemoryContextStatsPrint(MemoryContext context, void *passthru, #define AssertNotInCriticalSection(context) \ Assert(CritSectionCount == 0 || (context)->allowInCritSection) +/* + * Call the given function in the MemoryContextMethods for the memory context + * type that 'pointer' belongs to. + */ +#define MCXT_METHOD(pointer, method) \ + mcxt_methods[GetMemoryChunkMethodID(pointer)].method + /***************************************************************************** * EXPORTED ROUTINES * @@ -422,6 +474,17 @@ MemoryContextAllowInCriticalSection(MemoryContext context, bool allow) context->allowInCritSection = allow; } +/* + * GetMemoryChunkContext + * Given a currently-allocated chunk, determine the MemoryContext that + * the chunk belongs to. + */ +MemoryContext +GetMemoryChunkContext(void *pointer) +{ + return MCXT_METHOD(pointer, get_chunk_context) (pointer); +} + /* * GetMemoryChunkSpace * Given a currently-allocated chunk, determine the total space @@ -433,9 +496,7 @@ MemoryContextAllowInCriticalSection(MemoryContext context, bool allow) Size GetMemoryChunkSpace(void *pointer) { - MemoryContext context = GetMemoryChunkContext(pointer); - - return context->methods->get_chunk_space(context, pointer); + return MCXT_METHOD(pointer, get_chunk_space) (pointer); } /* @@ -804,7 +865,7 @@ MemoryContextContains(MemoryContext context, void *pointer) * * node: the as-yet-uninitialized common part of the context header node. * tag: NodeTag code identifying the memory context type. - * methods: context-type-specific methods (usually statically allocated). + * method_id: MemoryContextMethodID of the context-type being created. * parent: parent context, or NULL if this will be a top-level context. * name: name of context (must be statically allocated). * @@ -814,7 +875,7 @@ MemoryContextContains(MemoryContext context, void *pointer) void MemoryContextCreate(MemoryContext node, NodeTag tag, - const MemoryContextMethods *methods, + MemoryContextMethodID method_id, MemoryContext parent, const char *name) { @@ -824,7 +885,7 @@ MemoryContextCreate(MemoryContext node, /* Initialize all standard fields of memory context header */ node->type = tag; node->isReset = true; - node->methods = methods; + node->methods = &mcxt_methods[method_id]; node->parent = parent; node->firstchild = NULL; node->mem_allocated = 0; @@ -1174,9 +1235,11 @@ palloc_extended(Size size, int flags) void pfree(void *pointer) { +#ifdef USE_VALGRIND MemoryContext context = GetMemoryChunkContext(pointer); +#endif - context->methods->free_p(context, pointer); + MCXT_METHOD(pointer, free_p) (pointer); VALGRIND_MEMPOOL_FREE(context, pointer); } @@ -1187,7 +1250,9 @@ pfree(void *pointer) void * repalloc(void *pointer, Size size) { +#if defined(USE_ASSERT_CHECKING) || defined(USE_VALGRIND) MemoryContext context = GetMemoryChunkContext(pointer); +#endif void *ret; if (!AllocSizeIsValid(size)) @@ -1198,15 +1263,17 @@ repalloc(void *pointer, Size size) /* isReset must be false already */ Assert(!context->isReset); - ret = context->methods->realloc(context, pointer, size); + ret = MCXT_METHOD(pointer, realloc) (pointer, size); if (unlikely(ret == NULL)) { + MemoryContext cxt = GetMemoryChunkContext(pointer); + MemoryContextStats(TopMemoryContext); ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"), errdetail("Failed on request of size %zu in memory context \"%s\".", - size, context->name))); + size, cxt->name))); } VALGRIND_MEMPOOL_CHANGE(context, pointer, ret, size); @@ -1257,7 +1324,9 @@ MemoryContextAllocHuge(MemoryContext context, Size size) void * repalloc_huge(void *pointer, Size size) { +#if defined(USE_ASSERT_CHECKING) || defined(USE_VALGRIND) MemoryContext context = GetMemoryChunkContext(pointer); +#endif void *ret; if (!AllocHugeSizeIsValid(size)) @@ -1268,15 +1337,17 @@ repalloc_huge(void *pointer, Size size) /* isReset must be false already */ Assert(!context->isReset); - ret = context->methods->realloc(context, pointer, size); + ret = MCXT_METHOD(pointer, realloc) (pointer, size); if (unlikely(ret == NULL)) { + MemoryContext cxt = GetMemoryChunkContext(pointer); + MemoryContextStats(TopMemoryContext); ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"), errdetail("Failed on request of size %zu in memory context \"%s\".", - size, context->name))); + size, cxt->name))); } VALGRIND_MEMPOOL_CHANGE(context, pointer, ret, size); diff --git a/src/backend/utils/mmgr/slab.c b/src/backend/utils/mmgr/slab.c index 67d97b22e56..ae1a735b8cb 100644 --- a/src/backend/utils/mmgr/slab.c +++ b/src/backend/utils/mmgr/slab.c @@ -55,6 +55,8 @@ #include "lib/ilist.h" #include "utils/memdebug.h" #include "utils/memutils.h" +#include "utils/memutils_memorychunk.h" +#include "utils/memutils_internal.h" /* * SlabContext is a specialized implementation of MemoryContext. @@ -90,75 +92,23 @@ typedef struct SlabBlock dlist_node node; /* doubly-linked list */ int nfree; /* number of free chunks */ int firstFreeChunk; /* index of the first free chunk in the block */ + SlabContext *slab; /* owning context */ } SlabBlock; -/* - * SlabChunk - * The prefix of each piece of memory in a SlabBlock - * - * Note: to meet the memory context APIs, the payload area of the chunk must - * be maxaligned, and the "slab" link must be immediately adjacent to the - * payload area (cf. GetMemoryChunkContext). Since we support no machines on - * which MAXALIGN is more than twice sizeof(void *), this happens without any - * special hacking in this struct declaration. But there is a static - * assertion below that the alignment is done correctly. - */ -typedef struct SlabChunk -{ - SlabBlock *block; /* block owning this chunk */ - SlabContext *slab; /* owning context */ - /* there must not be any padding to reach a MAXALIGN boundary here! */ -} SlabChunk; - +#define Slab_CHUNKHDRSZ sizeof(MemoryChunk) #define SlabPointerGetChunk(ptr) \ - ((SlabChunk *)(((char *)(ptr)) - sizeof(SlabChunk))) + ((MemoryChunk *)(((char *)(ptr)) - sizeof(MemoryChunk))) #define SlabChunkGetPointer(chk) \ - ((void *)(((char *)(chk)) + sizeof(SlabChunk))) + ((void *)(((char *)(chk)) + sizeof(MemoryChunk))) #define SlabBlockGetChunk(slab, block, idx) \ - ((SlabChunk *) ((char *) (block) + sizeof(SlabBlock) \ + ((MemoryChunk *) ((char *) (block) + sizeof(SlabBlock) \ + (idx * slab->fullChunkSize))) #define SlabBlockStart(block) \ ((char *) block + sizeof(SlabBlock)) #define SlabChunkIndex(slab, block, chunk) \ (((char *) chunk - SlabBlockStart(block)) / slab->fullChunkSize) -/* - * These functions implement the MemoryContext API for Slab contexts. - */ -static void *SlabAlloc(MemoryContext context, Size size); -static void SlabFree(MemoryContext context, void *pointer); -static void *SlabRealloc(MemoryContext context, void *pointer, Size size); -static void SlabReset(MemoryContext context); -static void SlabDelete(MemoryContext context); -static Size SlabGetChunkSpace(MemoryContext context, void *pointer); -static bool SlabIsEmpty(MemoryContext context); -static void SlabStats(MemoryContext context, - MemoryStatsPrintFunc printfunc, void *passthru, - MemoryContextCounters *totals, - bool print_to_stderr); -#ifdef MEMORY_CONTEXT_CHECKING -static void SlabCheck(MemoryContext context); -#endif - -/* - * This is the virtual function table for Slab contexts. - */ -static const MemoryContextMethods SlabMethods = { - SlabAlloc, - SlabFree, - SlabRealloc, - SlabReset, - SlabDelete, - SlabGetChunkSpace, - SlabIsEmpty, - SlabStats -#ifdef MEMORY_CONTEXT_CHECKING - ,SlabCheck -#endif -}; - - /* * SlabContextCreate * Create a new Slab context. @@ -168,8 +118,7 @@ static const MemoryContextMethods SlabMethods = { * blockSize: allocation block size * chunkSize: allocation chunk size * - * The chunkSize may not exceed: - * MAXALIGN_DOWN(SIZE_MAX) - MAXALIGN(sizeof(SlabBlock)) - sizeof(SlabChunk) + * The MAXALIGN(chunkSize) may not exceed MEMORYCHUNK_MAX_VALUE */ MemoryContext SlabContextCreate(MemoryContext parent, @@ -184,19 +133,17 @@ SlabContextCreate(MemoryContext parent, SlabContext *slab; int i; - /* Assert we padded SlabChunk properly */ - StaticAssertStmt(sizeof(SlabChunk) == MAXALIGN(sizeof(SlabChunk)), - "sizeof(SlabChunk) is not maxaligned"); - StaticAssertStmt(offsetof(SlabChunk, slab) + sizeof(MemoryContext) == - sizeof(SlabChunk), - "padding calculation in SlabChunk is wrong"); + /* ensure MemoryChunk's size is properly maxaligned */ + StaticAssertStmt(Slab_CHUNKHDRSZ == MAXALIGN(Slab_CHUNKHDRSZ), + "sizeof(MemoryChunk) is not maxaligned"); + Assert(MAXALIGN(chunkSize) <= MEMORYCHUNK_MAX_VALUE); /* Make sure the linked list node fits inside a freed chunk */ if (chunkSize < sizeof(int)) chunkSize = sizeof(int); /* chunk, including SLAB header (both addresses nicely aligned) */ - fullChunkSize = sizeof(SlabChunk) + MAXALIGN(chunkSize); + fullChunkSize = Slab_CHUNKHDRSZ + MAXALIGN(chunkSize); /* Make sure the block can store at least one chunk. */ if (blockSize < fullChunkSize + sizeof(SlabBlock)) @@ -265,7 +212,7 @@ SlabContextCreate(MemoryContext parent, /* Finally, do the type-independent part of context creation */ MemoryContextCreate((MemoryContext) slab, T_SlabContext, - &SlabMethods, + MCTX_SLAB_ID, parent, name); @@ -279,7 +226,7 @@ SlabContextCreate(MemoryContext parent, * The code simply frees all the blocks in the context - we don't keep any * keeper blocks or anything like that. */ -static void +void SlabReset(MemoryContext context) { int i; @@ -322,7 +269,7 @@ SlabReset(MemoryContext context) * SlabDelete * Free all memory which is allocated in the given context. */ -static void +void SlabDelete(MemoryContext context) { /* Reset to release all the SlabBlocks */ @@ -336,12 +283,12 @@ SlabDelete(MemoryContext context) * Returns pointer to allocated memory of given size or NULL if * request could not be completed; memory is added to the slab. */ -static void * +void * SlabAlloc(MemoryContext context, Size size) { SlabContext *slab = castNode(SlabContext, context); SlabBlock *block; - SlabChunk *chunk; + MemoryChunk *chunk; int idx; Assert(slab); @@ -370,6 +317,7 @@ SlabAlloc(MemoryContext context, Size size) block->nfree = slab->chunksPerBlock; block->firstFreeChunk = 0; + block->slab = slab; /* * Put all the chunks on a freelist. Walk the chunks and point each @@ -378,7 +326,7 @@ SlabAlloc(MemoryContext context, Size size) for (idx = 0; idx < slab->chunksPerBlock; idx++) { chunk = SlabBlockGetChunk(slab, block, idx); - *(int32 *) SlabChunkGetPointer(chunk) = (idx + 1); + *(int32 *) MemoryChunkGetPointer(chunk) = (idx + 1); } /* @@ -426,8 +374,8 @@ SlabAlloc(MemoryContext context, Size size) * Remove the chunk from the freelist head. The index of the next free * chunk is stored in the chunk itself. */ - VALGRIND_MAKE_MEM_DEFINED(SlabChunkGetPointer(chunk), sizeof(int32)); - block->firstFreeChunk = *(int32 *) SlabChunkGetPointer(chunk); + VALGRIND_MAKE_MEM_DEFINED(MemoryChunkGetPointer(chunk), sizeof(int32)); + block->firstFreeChunk = *(int32 *) MemoryChunkGetPointer(chunk); Assert(block->firstFreeChunk >= 0); Assert(block->firstFreeChunk <= slab->chunksPerBlock); @@ -464,47 +412,47 @@ SlabAlloc(MemoryContext context, Size size) slab->minFreeChunks = 0; /* Prepare to initialize the chunk header. */ - VALGRIND_MAKE_MEM_UNDEFINED(chunk, sizeof(SlabChunk)); - - chunk->block = block; - chunk->slab = slab; + VALGRIND_MAKE_MEM_UNDEFINED(chunk, Slab_CHUNKHDRSZ); + MemoryChunkSetHdrMask(chunk, block, MAXALIGN(slab->chunkSize), + MCTX_SLAB_ID); #ifdef MEMORY_CONTEXT_CHECKING /* slab mark to catch clobber of "unused" space */ - if (slab->chunkSize < (slab->fullChunkSize - sizeof(SlabChunk))) + if (slab->chunkSize < (slab->fullChunkSize - Slab_CHUNKHDRSZ)) { - set_sentinel(SlabChunkGetPointer(chunk), size); + set_sentinel(MemoryChunkGetPointer(chunk), size); VALGRIND_MAKE_MEM_NOACCESS(((char *) chunk) + - sizeof(SlabChunk) + slab->chunkSize, + Slab_CHUNKHDRSZ + slab->chunkSize, slab->fullChunkSize - - (slab->chunkSize + sizeof(SlabChunk))); + (slab->chunkSize + Slab_CHUNKHDRSZ)); } #endif + #ifdef RANDOMIZE_ALLOCATED_MEMORY /* fill the allocated space with junk */ - randomize_mem((char *) SlabChunkGetPointer(chunk), size); + randomize_mem((char *) MemoryChunkGetPointer(chunk), size); #endif Assert(slab->nblocks * slab->blockSize == context->mem_allocated); - return SlabChunkGetPointer(chunk); + return MemoryChunkGetPointer(chunk); } /* * SlabFree * Frees allocated memory; memory is removed from the slab. */ -static void -SlabFree(MemoryContext context, void *pointer) +void +SlabFree(void *pointer) { int idx; - SlabContext *slab = castNode(SlabContext, context); - SlabChunk *chunk = SlabPointerGetChunk(pointer); - SlabBlock *block = chunk->block; + MemoryChunk *chunk = PointerGetMemoryChunk(pointer); + SlabBlock *block = MemoryChunkGetBlock(chunk); + SlabContext *slab = block->slab; #ifdef MEMORY_CONTEXT_CHECKING /* Test for someone scribbling on unused space in chunk */ - if (slab->chunkSize < (slab->fullChunkSize - sizeof(SlabChunk))) + if (slab->chunkSize < (slab->fullChunkSize - Slab_CHUNKHDRSZ)) if (!sentinel_ok(pointer, slab->chunkSize)) elog(WARNING, "detected write past chunk end in %s %p", slab->header.name, chunk); @@ -560,13 +508,13 @@ SlabFree(MemoryContext context, void *pointer) { free(block); slab->nblocks--; - context->mem_allocated -= slab->blockSize; + slab->header.mem_allocated -= slab->blockSize; } else dlist_push_head(&slab->freelist[block->nfree], &block->node); Assert(slab->nblocks >= 0); - Assert(slab->nblocks * slab->blockSize == context->mem_allocated); + Assert(slab->nblocks * slab->blockSize == slab->header.mem_allocated); } /* @@ -582,13 +530,14 @@ SlabFree(MemoryContext context, void *pointer) * rather pointless - Slab is meant for chunks of constant size, and moreover * realloc is usually used to enlarge the chunk. */ -static void * -SlabRealloc(MemoryContext context, void *pointer, Size size) +void * +SlabRealloc(void *pointer, Size size) { - SlabContext *slab = castNode(SlabContext, context); + MemoryChunk *chunk = PointerGetMemoryChunk(pointer); + SlabBlock *block = MemoryChunkGetBlock(chunk); + SlabContext *slab = block->slab; Assert(slab); - /* can't do actual realloc with slab, but let's try to be gentle */ if (size == slab->chunkSize) return pointer; @@ -597,15 +546,33 @@ SlabRealloc(MemoryContext context, void *pointer, Size size) return NULL; /* keep compiler quiet */ } +/* + * SlabGetChunkContext + * Return the MemoryContext that 'pointer' belongs to. + */ +MemoryContext +SlabGetChunkContext(void *pointer) +{ + MemoryChunk *chunk = PointerGetMemoryChunk(pointer); + SlabBlock *block = MemoryChunkGetBlock(chunk); + SlabContext *slab = block->slab; + + Assert(slab != NULL); + + return &slab->header; +} + /* * SlabGetChunkSpace * Given a currently-allocated chunk, determine the total space * it occupies (including all memory-allocation overhead). */ -static Size -SlabGetChunkSpace(MemoryContext context, void *pointer) +Size +SlabGetChunkSpace(void *pointer) { - SlabContext *slab = castNode(SlabContext, context); + MemoryChunk *chunk = PointerGetMemoryChunk(pointer); + SlabBlock *block = MemoryChunkGetBlock(chunk); + SlabContext *slab = block->slab; Assert(slab); @@ -616,7 +583,7 @@ SlabGetChunkSpace(MemoryContext context, void *pointer) * SlabIsEmpty * Is an Slab empty of any allocated space? */ -static bool +bool SlabIsEmpty(MemoryContext context) { SlabContext *slab = castNode(SlabContext, context); @@ -635,7 +602,7 @@ SlabIsEmpty(MemoryContext context) * totals: if not NULL, add stats about this context into *totals. * print_to_stderr: print stats to stderr if true, elog otherwise. */ -static void +void SlabStats(MemoryContext context, MemoryStatsPrintFunc printfunc, void *passthru, MemoryContextCounters *totals, @@ -697,7 +664,7 @@ SlabStats(MemoryContext context, * find yourself in an infinite loop when trouble occurs, because this * routine will be entered again when elog cleanup tries to release memory! */ -static void +void SlabCheck(MemoryContext context) { int i; @@ -728,6 +695,11 @@ SlabCheck(MemoryContext context) elog(WARNING, "problem in slab %s: number of free chunks %d in block %p does not match freelist %d", name, block->nfree, block, i); + /* make sure the slab pointer correctly points to this context */ + if (block->slab != slab) + elog(WARNING, "problem in slab %s: bogus slab link in block %p", + name, block); + /* reset the bitmap of free chunks for this block */ memset(slab->freechunks, 0, (slab->chunksPerBlock * sizeof(bool))); idx = block->firstFreeChunk; @@ -742,7 +714,7 @@ SlabCheck(MemoryContext context) nfree = 0; while (idx < slab->chunksPerBlock) { - SlabChunk *chunk; + MemoryChunk *chunk; /* count the chunk as free, add it to the bitmap */ nfree++; @@ -750,8 +722,8 @@ SlabCheck(MemoryContext context) /* read index of the next free chunk */ chunk = SlabBlockGetChunk(slab, block, idx); - VALGRIND_MAKE_MEM_DEFINED(SlabChunkGetPointer(chunk), sizeof(int32)); - idx = *(int32 *) SlabChunkGetPointer(chunk); + VALGRIND_MAKE_MEM_DEFINED(MemoryChunkGetPointer(chunk), sizeof(int32)); + idx = *(int32 *) MemoryChunkGetPointer(chunk); } for (j = 0; j < slab->chunksPerBlock; j++) @@ -759,19 +731,19 @@ SlabCheck(MemoryContext context) /* non-zero bit in the bitmap means chunk the chunk is used */ if (!slab->freechunks[j]) { - SlabChunk *chunk = SlabBlockGetChunk(slab, block, j); + MemoryChunk *chunk = SlabBlockGetChunk(slab, block, j); + SlabBlock *chunkblock = (SlabBlock *) MemoryChunkGetBlock(chunk); - /* chunks have both block and slab pointers, so check both */ - if (chunk->block != block) + /* + * check the chunk's blockoffset correctly points back to + * the block + */ + if (chunkblock != block) elog(WARNING, "problem in slab %s: bogus block link in block %p, chunk %p", name, block, chunk); - if (chunk->slab != slab) - elog(WARNING, "problem in slab %s: bogus slab link in block %p, chunk %p", - name, block, chunk); - /* there might be sentinel (thanks to alignment) */ - if (slab->chunkSize < (slab->fullChunkSize - sizeof(SlabChunk))) + if (slab->chunkSize < (slab->fullChunkSize - Slab_CHUNKHDRSZ)) if (!sentinel_ok(chunk, slab->chunkSize)) elog(WARNING, "problem in slab %s: detected write past chunk end in block %p, chunk %p", name, block, chunk); diff --git a/src/include/nodes/memnodes.h b/src/include/nodes/memnodes.h index 16cd56da6db..63d07358cdb 100644 --- a/src/include/nodes/memnodes.h +++ b/src/include/nodes/memnodes.h @@ -59,11 +59,12 @@ typedef struct MemoryContextMethods { void *(*alloc) (MemoryContext context, Size size); /* call this free_p in case someone #define's free() */ - void (*free_p) (MemoryContext context, void *pointer); - void *(*realloc) (MemoryContext context, void *pointer, Size size); + void (*free_p) (void *pointer); + void *(*realloc) (void *pointer, Size size); void (*reset) (MemoryContext context); void (*delete_context) (MemoryContext context); - Size (*get_chunk_space) (MemoryContext context, void *pointer); + MemoryContext (*get_chunk_context) (void *pointer); + Size (*get_chunk_space) (void *pointer); bool (*is_empty) (MemoryContext context); void (*stats) (MemoryContext context, MemoryStatsPrintFunc printfunc, void *passthru, diff --git a/src/include/utils/memutils.h b/src/include/utils/memutils.h index 495d1af2010..52bc41ec53b 100644 --- a/src/include/utils/memutils.h +++ b/src/include/utils/memutils.h @@ -41,8 +41,11 @@ #define AllocSizeIsValid(size) ((Size) (size) <= MaxAllocSize) +/* Must be less than SIZE_MAX */ #define MaxAllocHugeSize (SIZE_MAX / 2) +#define InvalidAllocSize SIZE_MAX + #define AllocHugeSizeIsValid(size) ((Size) (size) <= MaxAllocHugeSize) @@ -79,6 +82,7 @@ extern void MemoryContextDeleteChildren(MemoryContext context); extern void MemoryContextSetIdentifier(MemoryContext context, const char *id); extern void MemoryContextSetParent(MemoryContext context, MemoryContext new_parent); +extern MemoryContext GetMemoryChunkContext(void *pointer); extern Size GetMemoryChunkSpace(void *pointer); extern MemoryContext MemoryContextGetParent(MemoryContext context); extern bool MemoryContextIsEmpty(MemoryContext context); @@ -98,53 +102,6 @@ extern bool MemoryContextContains(MemoryContext context, void *pointer); #define MemoryContextCopyAndSetIdentifier(cxt, id) \ MemoryContextSetIdentifier(cxt, MemoryContextStrdup(cxt, id)) -/* - * GetMemoryChunkContext - * Given a currently-allocated chunk, determine the context - * it belongs to. - * - * All chunks allocated by any memory context manager are required to be - * preceded by the corresponding MemoryContext stored, without padding, in the - * preceding sizeof(void*) bytes. A currently-allocated chunk must contain a - * backpointer to its owning context. The backpointer is used by pfree() and - * repalloc() to find the context to call. - */ -#ifndef FRONTEND -static inline MemoryContext -GetMemoryChunkContext(void *pointer) -{ - MemoryContext context; - - /* - * Try to detect bogus pointers handed to us, poorly though we can. - * Presumably, a pointer that isn't MAXALIGNED isn't pointing at an - * allocated chunk. - */ - Assert(pointer != NULL); - Assert(pointer == (void *) MAXALIGN(pointer)); - - /* - * OK, it's probably safe to look at the context. - */ - context = *(MemoryContext *) (((char *) pointer) - sizeof(void *)); - - AssertArg(MemoryContextIsValid(context)); - - return context; -} -#endif - -/* - * This routine handles the context-type-independent part of memory - * context creation. It's intended to be called from context-type- - * specific creation routines, and noplace else. - */ -extern void MemoryContextCreate(MemoryContext node, - NodeTag tag, - const MemoryContextMethods *methods, - MemoryContext parent, - const char *name); - extern void HandleLogMemoryContextInterrupt(void); extern void ProcessLogMemoryContextInterrupt(void); diff --git a/src/include/utils/memutils_internal.h b/src/include/utils/memutils_internal.h new file mode 100644 index 00000000000..9a9f52ef164 --- /dev/null +++ b/src/include/utils/memutils_internal.h @@ -0,0 +1,127 @@ +/*------------------------------------------------------------------------- + * + * memutils_internal.h + * This file contains declarations for memory allocation utility + * functions for internal use. + * + * + * Portions Copyright (c) 2022, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/utils/memutils_internal.h + * + *------------------------------------------------------------------------- + */ + +#ifndef MEMUTILS_INTERNAL_H +#define MEMUTILS_INTERNAL_H + +#include "utils/memutils.h" + +/* These functions implement the MemoryContext API for AllocSet context. */ +extern void *AllocSetAlloc(MemoryContext context, Size size); +extern void AllocSetFree(void *pointer); +extern void *AllocSetRealloc(void *pointer, Size size); +extern void AllocSetReset(MemoryContext context); +extern void AllocSetDelete(MemoryContext context); +extern MemoryContext AllocSetGetChunkContext(void *pointer); +extern Size AllocSetGetChunkSpace(void *pointer); +extern bool AllocSetIsEmpty(MemoryContext context); +extern void AllocSetStats(MemoryContext context, + MemoryStatsPrintFunc printfunc, void *passthru, + MemoryContextCounters *totals, + bool print_to_stderr); +#ifdef MEMORY_CONTEXT_CHECKING +extern void AllocSetCheck(MemoryContext context); +#endif + +/* These functions implement the MemoryContext API for Generation context. */ +extern void *GenerationAlloc(MemoryContext context, Size size); +extern void GenerationFree(void *pointer); +extern void *GenerationRealloc(void *pointer, Size size); +extern void GenerationReset(MemoryContext context); +extern void GenerationDelete(MemoryContext context); +extern MemoryContext GenerationGetChunkContext(void *pointer); +extern Size GenerationGetChunkSpace(void *pointer); +extern bool GenerationIsEmpty(MemoryContext context); +extern void GenerationStats(MemoryContext context, + MemoryStatsPrintFunc printfunc, void *passthru, + MemoryContextCounters *totals, + bool print_to_stderr); +#ifdef MEMORY_CONTEXT_CHECKING +extern void GenerationCheck(MemoryContext context); +#endif + + +/* These functions implement the MemoryContext API for Slab context. */ +extern void *SlabAlloc(MemoryContext context, Size size); +extern void SlabFree(void *pointer); +extern void *SlabRealloc(void *pointer, Size size); +extern void SlabReset(MemoryContext context); +extern void SlabDelete(MemoryContext context); +extern MemoryContext SlabGetChunkContext(void *pointer); +extern Size SlabGetChunkSpace(void *pointer); +extern bool SlabIsEmpty(MemoryContext context); +extern void SlabStats(MemoryContext context, + MemoryStatsPrintFunc printfunc, void *passthru, + MemoryContextCounters *totals, + bool print_to_stderr); +#ifdef MEMORY_CONTEXT_CHECKING +extern void SlabCheck(MemoryContext context); +#endif + +/* + * MemoryContextMethodID + * A unique identifier for each MemoryContext implementation which + * indicates the index into the mcxt_methods[] array. See mcxt.c. + */ +typedef enum MemoryContextMethodID +{ + MCTX_ASET_ID, + MCTX_GENERATION_ID, + MCTX_SLAB_ID, +} MemoryContextMethodID; + +/* + * The number of bits that 8-byte memory chunk headers can use to encode the + * MemoryContextMethodID. + */ +#define MEMORY_CONTEXT_METHODID_BITS 3 +#define MEMORY_CONTEXT_METHODID_MASK \ + UINT64CONST((1 << MEMORY_CONTEXT_METHODID_BITS) - 1) + +/* + * This routine handles the context-type-independent part of memory + * context creation. It's intended to be called from context-type- + * specific creation routines, and noplace else. + */ +extern void MemoryContextCreate(MemoryContext node, + NodeTag tag, + MemoryContextMethodID method_id, + MemoryContext parent, + const char *name); + +/* + * GetMemoryChunkMethodID + * Return the MemoryContextMethodID from the uint64 chunk header which + * directly precedes 'pointer'. + */ +static inline MemoryContextMethodID +GetMemoryChunkMethodID(void *pointer) +{ + uint64 header; + + /* + * Try to detect bogus pointers handed to us, poorly though we can. + * Presumably, a pointer that isn't MAXALIGNED isn't pointing at an + * allocated chunk. + */ + Assert(pointer != NULL); + Assert(pointer == (void *) MAXALIGN(pointer)); + + header = *((uint64 *) ((char *) pointer - sizeof(uint64))); + + return (MemoryContextMethodID) (header & MEMORY_CONTEXT_METHODID_MASK); +} + +#endif /* MEMUTILS_INTERNAL_H */ diff --git a/src/include/utils/memutils_memorychunk.h b/src/include/utils/memutils_memorychunk.h new file mode 100644 index 00000000000..685c177b681 --- /dev/null +++ b/src/include/utils/memutils_memorychunk.h @@ -0,0 +1,237 @@ +/*------------------------------------------------------------------------- + * + * memutils_memorychunk.h + * Here we define a struct named MemoryChunk which implementations of + * MemoryContexts may use as a header for chunks of memory they allocate. + * + * MemoryChunk provides a lightweight header that a MemoryContext can use to + * store a reference back to the block the which the given chunk is allocated + * on and also an additional 30-bits to store another value such as the size + * of the allocated chunk. + * + * Although MemoryChunks are used by each of our MemoryContexts, future + * implementations may choose to implement their own method for storing chunk + * headers. The only requirement is that the header ends with an 8-byte value + * which the least significant 3-bits of are set to the MemoryContextMethodID + * of the given context. + * + * By default, a MemoryChunk is 8 bytes in size, however, when + * MEMORY_CONTEXT_CHECKING is defined the header becomes 16 bytes in size due + * to the additional requested_size field. The MemoryContext may use this + * field for whatever they wish, but it is intended to be used for additional + * checks which are only done in MEMORY_CONTEXT_CHECKING builds. + * + * The MemoryChunk contains a uint64 field named 'hdrmask'. This field is + * used to encode 4 separate pieces of information. Starting with the least + * significant bits of 'hdrmask', the bit space is reserved as follows: + * + * 1. 3-bits to indicate the MemoryContextMethodID as defined by + * MEMORY_CONTEXT_METHODID_MASK + * 2. 1-bit to denote an "external" chunk (see below) + * 3. 30-bits reserved for the MemoryContext to use for anything it + * requires. Most MemoryContext likely want to store the size of the + * chunk here. + * 4. 30-bits for the number of bytes that must be subtracted from the chunk + * to obtain the address of the block that the chunk is stored on. + * + * In some cases, for example when memory allocations become large, it's + * possible fields 3 and 4 above are not large enough to store the values + * required for the chunk. In this case, the MemoryContext can choose to mark + * the chunk as "external" by calling the MemoryChunkSetExternal() function. + * When this is done, fields 3 and 4 are unavailable for use by the + * MemoryContext and it's up to the MemoryContext itself to devise its own + * method for getting the reference to the block. + * + * Interface: + * + * MemoryChunkSetHdrMask: + * Used to set up a non-external MemoryChunk. + * + * MemoryChunkSetHdrMaskExternal: + * Used to set up an externally managed MemoryChunk. + * + * MemoryChunkIsExternal: + * Determine if the given MemoryChunk is externally managed, i.e. + * MemoryChunkSetHdrMaskExternal() was called on the chunk. + * + * MemoryChunkGetValue: + * For non-external chunks, return the stored 30-bit value as it was set + * in the call to MemoryChunkSetHdrMask(). + * + * MemoryChunkGetBlock: + * For non-external chunks, return a pointer to the block as it was set + * in the call to MemoryChunkSetHdrMask(). + * + * Also exports: + * MEMORYCHUNK_MAX_VALUE + * MEMORYCHUNK_MAX_BLOCKOFFSET + * PointerGetMemoryChunk + * MemoryChunkGetPointer + * + * Portions Copyright (c) 2022, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/utils/memutils_memorychunk.h + * + *------------------------------------------------------------------------- + */ + +#ifndef MEMUTILS_MEMORYCHUNK_H +#define MEMUTILS_MEMORYCHUNK_H + +#include "utils/memutils_internal.h" + + /* + * The maximum allowed value that MemoryContexts can store in the value + * field. Must be 1 less than a power of 2. + */ +#define MEMORYCHUNK_MAX_VALUE UINT64CONST(0x3FFFFFFF) + +/* + * The maximum distance in bytes that a MemoryChunk can be offset from the + * block that is storing the chunk. Must be 1 less than a power of 2. + */ +#define MEMORYCHUNK_MAX_BLOCKOFFSET UINT64CONST(0x3FFFFFFF) + +/* define the least significant base-0 bit of each portion of the hdrmask */ +#define MEMORYCHUNK_EXTERNAL_BASEBIT MEMORY_CONTEXT_METHODID_BITS +#define MEMORYCHUNK_VALUE_BASEBIT (MEMORYCHUNK_EXTERNAL_BASEBIT + 1) +#define MEMORYCHUNK_BLOCKOFFSET_BASEBIT (MEMORYCHUNK_VALUE_BASEBIT + 30) + +/* + * A magic number for storing in the free bits of an external chunk. This + * must mask out the bits used for storing the MemoryContextMethodID and the + * external bit. + */ +#define MEMORYCHUNK_MAGIC (UINT64CONST(0xB1A8DB858EB6EFBA) >> \ + MEMORYCHUNK_VALUE_BASEBIT << \ + MEMORYCHUNK_VALUE_BASEBIT) + +typedef struct MemoryChunk +{ +#ifdef MEMORY_CONTEXT_CHECKING + Size requested_size; +#endif + + /* bitfield for storing details about the chunk */ + uint64 hdrmask; /* must be last */ +} MemoryChunk; + +/* Get the MemoryChunk from the pointer */ +#define PointerGetMemoryChunk(p) \ + ((MemoryChunk *) ((char *) (p) - sizeof(MemoryChunk))) +/* Get the pointer from the MemoryChunk */ +#define MemoryChunkGetPointer(c) \ + ((void *) ((char *) (c) + sizeof(MemoryChunk))) + +/* private macros for making the inline functions below more simple */ +#define HdrMaskIsExternal(hdrmask) \ + ((hdrmask) & (((uint64) 1) << MEMORYCHUNK_EXTERNAL_BASEBIT)) +#define HdrMaskGetValue(hdrmask) \ + (((hdrmask) >> MEMORYCHUNK_VALUE_BASEBIT) & MEMORYCHUNK_MAX_VALUE) + +/* + * We should have used up all the bits here, so the compiler is likely to + * optimize out the & MEMORYCHUNK_MAX_BLOCKOFFSET. + */ +#define HdrMaskBlockOffset(hdrmask) \ + (((hdrmask) >> MEMORYCHUNK_BLOCKOFFSET_BASEBIT) & MEMORYCHUNK_MAX_BLOCKOFFSET) + +/* For external chunks only, check the magic number matches */ +#define HdrMaskCheckMagic(hdrmask) \ + (MEMORYCHUNK_MAGIC == \ + ((hdrmask) >> MEMORYCHUNK_VALUE_BASEBIT << MEMORYCHUNK_VALUE_BASEBIT)) +/* + * MemoryChunkSetHdrMask + * Store the given 'block', 'chunk_size' and 'methodid' in the given + * MemoryChunk. + * + * The number of bytes between 'block' and 'chunk' must be <= + * MEMORYCHUNK_MAX_BLOCKOFFSET. + * 'value' must be <= MEMORYCHUNK_MAX_VALUE. + */ +static inline void +MemoryChunkSetHdrMask(MemoryChunk *chunk, void *block, + Size value, MemoryContextMethodID methodid) +{ + Size blockoffset = (char *) chunk - (char *) block; + + Assert((char *) chunk > (char *) block); + Assert(blockoffset <= MEMORYCHUNK_MAX_BLOCKOFFSET); + Assert(value <= MEMORYCHUNK_MAX_VALUE); + Assert(methodid <= MEMORY_CONTEXT_METHODID_MASK); + + chunk->hdrmask = (((uint64) blockoffset) << MEMORYCHUNK_BLOCKOFFSET_BASEBIT) | + (((uint64) value) << MEMORYCHUNK_VALUE_BASEBIT) | + methodid; +} + +/* + * MemoryChunkSetHdrMaskExternal + * Set 'chunk' as an externally managed chunk. Here we only record the + * MemoryContextMethodID and set the external chunk bit. + */ +static inline void +MemoryChunkSetHdrMaskExternal(MemoryChunk *chunk, + MemoryContextMethodID methodid) +{ + Assert(methodid <= MEMORY_CONTEXT_METHODID_MASK); + + chunk->hdrmask = MEMORYCHUNK_MAGIC | (((uint64) 1) << MEMORYCHUNK_EXTERNAL_BASEBIT) | + methodid; +} + +/* + * MemoryChunkIsExternal + * Return true if 'chunk' is marked as external. + */ +static inline bool +MemoryChunkIsExternal(MemoryChunk *chunk) +{ + /* + * External chunks should always store MEMORYCHUNK_MAGIC in the upper + * portion of the hdrmask, check that nothing has stomped on that. + */ + Assert(!HdrMaskIsExternal(chunk->hdrmask) || + HdrMaskCheckMagic(chunk->hdrmask)); + + return HdrMaskIsExternal(chunk->hdrmask); +} + +/* + * MemoryChunkGetValue + * For non-external chunks, returns the value field as it was set in + * MemoryChunkSetHdrMask. + */ +static inline Size +MemoryChunkGetValue(MemoryChunk *chunk) +{ + Assert(!HdrMaskIsExternal(chunk->hdrmask)); + + return HdrMaskGetValue(chunk->hdrmask); +} + +/* + * MemoryChunkGetBlock + * For non-external chunks, returns the pointer to the block as was set + * in MemoryChunkSetHdrMask. + */ +static inline void * +MemoryChunkGetBlock(MemoryChunk *chunk) +{ + Assert(!HdrMaskIsExternal(chunk->hdrmask)); + + return (void *) ((char *) chunk - HdrMaskBlockOffset(chunk->hdrmask)); +} + +/* cleanup all internal definitions */ +#undef MEMORYCHUNK_EXTERNAL_BASEBIT +#undef MEMORYCHUNK_VALUE_BASEBIT +#undef MEMORYCHUNK_BLOCKOFFSET_BASEBIT +#undef MEMORYCHUNK_MAGIC +#undef HdrMaskIsExternal +#undef HdrMaskGetValue +#undef HdrMaskBlockOffset +#undef HdrMaskCheckMagic + +#endif /* MEMUTILS_MEMORYCHUNK_H */ diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index a4a4e356e51..8ad112c44d0 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -56,7 +56,7 @@ AggregateInstrumentation AlenState Alias AllocBlock -AllocChunk +AllocFreeListLink AllocPointer AllocSet AllocSetContext @@ -947,7 +947,6 @@ GatherState Gene GeneratePruningStepsContext GenerationBlock -GenerationChunk GenerationContext GenerationPointer GenericCosts @@ -1522,12 +1521,14 @@ MemoizeKey MemoizePath MemoizeState MemoizeTuple +MemoryChunk MemoryContext MemoryContextCallback MemoryContextCallbackFunction MemoryContextCounters MemoryContextData MemoryContextMethods +MemoryContextMethodID MemoryStatsPrintFunc MergeAction MergeActionState @@ -2530,7 +2531,6 @@ SingleBoundSortItem Size SkipPages SlabBlock -SlabChunk SlabContext SlabSlot SlotNumber