database/postgres
1
0
Fork 0
mirror of https://github.com/postgres/postgres.git synced 2025-08-21 10:42:50 +03:00
Code Activity

Add a new WAL summarizer process.

Browse Source 
When active, this process writes WAL summary files to
$PGDATA/pg_wal/summaries. Each summary file contains information for a
certain range of LSNs on a certain TLI. For each relation, it stores a
"limit block" which is 0 if a relation is created or destroyed within
a certain range of WAL records, or otherwise the shortest length to
which the relation was truncated during that range of WAL records, or
otherwise InvalidBlockNumber. In addition, it stores a list of blocks
which have been modified during that range of WAL records, but
excluding blocks which were removed by truncation after they were
modified and never subsequently modified again.

In other words, it tells us which blocks need to copied in case of an
incremental backup covering that range of WAL records. But this
doesn't yet add the capability to actually perform an incremental
backup; the next patch will do that.

A new parameter summarize_wal enables or disables this new background
process.  The background process also automatically deletes summary
files that are older than wal_summarize_keep_time, if that parameter
has a non-zero value and the summarizer is configured to run.

Patch by me, with some design help from Dilip Kumar and Andres Freund.
Reviewed by Matthias van de Meent, Dilip Kumar, Jakub Wartak, Peter
Eisentraut, and Álvaro Herrera.

Discussion: http://postgr.es/m/CA+TgmoYOYZfMCyOXFyC-P+-mdrZqm5pP2N7S-r0z3_402h9rsA@mail.gmail.com
This commit is contained in:
Robert Haas
2023-12-20 08:41:09 -05:00
parent 00498b7185
commit 174c480508
30 changed files with 3743 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

169
src/backend/backup/walsummaryfuncs.c Normal file
View File

@@ -0,0 +1,169 @@
/*-------------------------------------------------------------------------
*
* walsummaryfuncs.c
* SQL-callable functions for accessing WAL summary data.
*
* Portions Copyright (c) 2010-2022, PostgreSQL Global Development Group
*
* src/backend/backup/walsummaryfuncs.c
*
*-------------------------------------------------------------------------
*/
#include "postgres.h"
#include "backup/walsummary.h"
#include "common/blkreftable.h"
#include "funcapi.h"
#include "miscadmin.h"
#include "utils/fmgrprotos.h"
#include "utils/pg_lsn.h"
#define NUM_WS_ATTS 3
#define NUM_SUMMARY_ATTS 6
#define MAX_BLOCKS_PER_CALL 256
/*
* List the WAL summary files available in pg_wal/summaries.
*/
Datum
pg_available_wal_summaries(PG_FUNCTION_ARGS)
{
ReturnSetInfo *rsi;
List *wslist;
ListCell *lc;
Datum values[NUM_WS_ATTS];
bool nulls[NUM_WS_ATTS];
InitMaterializedSRF(fcinfo, 0);
rsi = (ReturnSetInfo *) fcinfo->resultinfo;
memset(nulls, 0, sizeof(nulls));
wslist = GetWalSummaries(0, InvalidXLogRecPtr, InvalidXLogRecPtr);
foreach(lc, wslist)
{
WalSummaryFile *ws = (WalSummaryFile *) lfirst(lc);
HeapTuple tuple;
CHECK_FOR_INTERRUPTS();
values[0] = Int64GetDatum((int64) ws->tli);
values[1] = LSNGetDatum(ws->start_lsn);
values[2] = LSNGetDatum(ws->end_lsn);
tuple = heap_form_tuple(rsi->setDesc, values, nulls);
tuplestore_puttuple(rsi->setResult, tuple);
}
return (Datum) 0;
}
/*
* List the contents of a WAL summary file identified by TLI, start LSN,
* and end LSN.
*/
Datum
pg_wal_summary_contents(PG_FUNCTION_ARGS)
{
ReturnSetInfo *rsi;
Datum values[NUM_SUMMARY_ATTS];
bool nulls[NUM_SUMMARY_ATTS];
WalSummaryFile ws;
WalSummaryIO io;
BlockRefTableReader *reader;
int64 raw_tli;
RelFileLocator rlocator;
ForkNumber forknum;
BlockNumber limit_block;
InitMaterializedSRF(fcinfo, 0);
rsi = (ReturnSetInfo *) fcinfo->resultinfo;
memset(nulls, 0, sizeof(nulls));
/*
* Since the timeline could at least in theory be more than 2^31, and
* since we don't have unsigned types at the SQL level, it is passed as a
* 64-bit integer. Test whether it's out of range.
*/
raw_tli = PG_GETARG_INT64(0);
if (raw_tli < 1 || raw_tli > PG_INT32_MAX)
ereport(ERROR,
errcode(ERRCODE_INVALID_PARAMETER_VALUE),
errmsg("invalid timeline %lld", (long long) raw_tli));
/* Prepare to read the specified WAL summry file. */
ws.tli = (TimeLineID) raw_tli;
ws.start_lsn = PG_GETARG_LSN(1);
ws.end_lsn = PG_GETARG_LSN(2);
io.filepos = 0;
io.file = OpenWalSummaryFile(&ws, false);
reader = CreateBlockRefTableReader(ReadWalSummary, &io,
FilePathName(io.file),
ReportWalSummaryError, NULL);
/* Loop over relation forks. */
while (BlockRefTableReaderNextRelation(reader, &rlocator, &forknum,
&limit_block))
{
BlockNumber blocks[MAX_BLOCKS_PER_CALL];
HeapTuple tuple;
CHECK_FOR_INTERRUPTS();
values[0] = ObjectIdGetDatum(rlocator.relNumber);
values[1] = ObjectIdGetDatum(rlocator.spcOid);
values[2] = ObjectIdGetDatum(rlocator.dbOid);
values[3] = Int16GetDatum((int16) forknum);
/* Loop over blocks within the current relation fork. */
while (1)
{
unsigned nblocks;
unsigned i;
CHECK_FOR_INTERRUPTS();
nblocks = BlockRefTableReaderGetBlocks(reader, blocks,
MAX_BLOCKS_PER_CALL);
if (nblocks == 0)
break;
/*
* For each block that we specifically know to have been modified,
* emit a row with that block number and limit_block = false.
*/
values[5] = BoolGetDatum(false);
for (i = 0; i < nblocks; ++i)
{
values[4] = Int64GetDatum((int64) blocks[i]);
tuple = heap_form_tuple(rsi->setDesc, values, nulls);
tuplestore_puttuple(rsi->setResult, tuple);
}
/*
* If the limit block is not InvalidBlockNumber, emit an exta row
* with that block number and limit_block = true.
*
* There is no point in doing this when the limit_block is
* InvalidBlockNumber, because no block with that number or any
* higher number can ever exist.
*/
if (BlockNumberIsValid(limit_block))
{
values[4] = Int64GetDatum((int64) limit_block);
values[5] = BoolGetDatum(true);
tuple = heap_form_tuple(rsi->setDesc, values, nulls);
tuplestore_puttuple(rsi->setResult, tuple);
}
}
}
/* Cleanup */
DestroyBlockRefTableReader(reader);
FileClose(io.file);
return (Datum) 0;
}