database/postgres
1
0
Fork 0
mirror of https://github.com/postgres/postgres.git synced 2025-07-28 23:42:10 +03:00
Code Activity

Add contrib/pg_walinspect.

Browse Source 
Provides similar functionality to pg_waldump, but from a SQL interface
rather than a separate utility.

Author: Bharath Rupireddy
Reviewed-by: Greg Stark, Kyotaro Horiguchi, Andres Freund, Ashutosh Sharma, Nitin Jadhav, RKN Sai Krishna
Discussion: https://postgr.es/m/CALj2ACUGUYXsEQdKhEdsBzhGEyF3xggvLdD8C0VT72TNEfOiog%40mail.gmail.com
This commit is contained in:
Jeff Davis
2022-04-08 00:02:10 -07:00
parent 708007dced
commit 2258e76f90
25 changed files with 1675 additions and 204 deletions
Download Patch File Download Diff File Expand all files Collapse all files

130
src/backend/access/rmgrdesc/xlogdesc.c
View File

@ -200,3 +200,133 @@ xlog_identify(uint8 info)
return id;
}
/*
* Returns a string giving information about all the blocks in an
* XLogRecord.
*/
void
XLogRecGetBlockRefInfo(XLogReaderState *record, bool pretty,
bool detailed_format, StringInfo buf,
uint32 *fpi_len)
{
int block_id;
Assert(record != NULL);
if (detailed_format && pretty)
appendStringInfoChar(buf, '\n');
for (block_id = 0; block_id <= XLogRecMaxBlockId(record); block_id++)
{
RelFileNode rnode = {InvalidOid, InvalidOid, InvalidOid};
ForkNumber forknum = InvalidForkNumber;
BlockNumber blk = InvalidBlockNumber;
if (!XLogRecHasBlockRef(record, block_id))
continue;
XLogRecGetBlockTag(record, block_id, &rnode, &forknum, &blk);
if (detailed_format)
{
/* Get block references in detailed format. */
if (pretty)
appendStringInfoChar(buf, '\t');
else if (block_id > 0)
appendStringInfoChar(buf, ' ');
appendStringInfo(buf,
"blkref #%d: rel %u/%u/%u fork %s blk %u",
block_id,
rnode.spcNode, rnode.dbNode, rnode.relNode,
forkNames[forknum],
blk);
if (XLogRecHasBlockImage(record, block_id))
{
uint8 bimg_info = XLogRecGetBlock(record, block_id)->bimg_info;
/* Calculate the amount of FPI data in the record. */
if (fpi_len)
*fpi_len += XLogRecGetBlock(record, block_id)->bimg_len;
if (BKPIMAGE_COMPRESSED(bimg_info))
{
const char *method;
if ((bimg_info & BKPIMAGE_COMPRESS_PGLZ) != 0)
method = "pglz";
else if ((bimg_info & BKPIMAGE_COMPRESS_LZ4) != 0)
method = "lz4";
else if ((bimg_info & BKPIMAGE_COMPRESS_ZSTD) != 0)
method = "zstd";
else
method = "unknown";
appendStringInfo(buf,
" (FPW%s); hole: offset: %u, length: %u, "
"compression saved: %u, method: %s",
XLogRecBlockImageApply(record, block_id) ?
"" : " for WAL verification",
XLogRecGetBlock(record, block_id)->hole_offset,
XLogRecGetBlock(record, block_id)->hole_length,
BLCKSZ -
XLogRecGetBlock(record, block_id)->hole_length -
XLogRecGetBlock(record, block_id)->bimg_len,
method);
}
else
{
appendStringInfo(buf,
" (FPW%s); hole: offset: %u, length: %u",
XLogRecBlockImageApply(record, block_id) ?
"" : " for WAL verification",
XLogRecGetBlock(record, block_id)->hole_offset,
XLogRecGetBlock(record, block_id)->hole_length);
}
}
if (pretty)
appendStringInfoChar(buf, '\n');
}
else
{
/* Get block references in short format. */
if (forknum != MAIN_FORKNUM)
{
appendStringInfo(buf,
", blkref #%d: rel %u/%u/%u fork %s blk %u",
block_id,
rnode.spcNode, rnode.dbNode, rnode.relNode,
forkNames[forknum],
blk);
}
else
{
appendStringInfo(buf,
", blkref #%d: rel %u/%u/%u blk %u",
block_id,
rnode.spcNode, rnode.dbNode, rnode.relNode,
blk);
}
if (XLogRecHasBlockImage(record, block_id))
{
/* Calculate the amount of FPI data in the record. */
if (fpi_len)
*fpi_len += XLogRecGetBlock(record, block_id)->bimg_len;
if (XLogRecBlockImageApply(record, block_id))
appendStringInfo(buf, " FPW");
else
appendStringInfo(buf, " FPW for WAL verification");
}
}
}
if (!detailed_format && pretty)
appendStringInfoChar(buf, '\n');
}

1
src/backend/access/transam/Makefile
View File

@ -34,6 +34,7 @@ OBJS = \
xlogprefetcher.o \
xlogreader.o \
xlogrecovery.o \
xlogstats.o \
xlogutils.o
include $(top_srcdir)/src/backend/common.mk

9
src/backend/access/transam/xlogreader.c
View File

@ -1320,13 +1320,6 @@ XLogReaderValidatePageHeader(XLogReaderState *state, XLogRecPtr recptr,
return true;
}
#ifdef FRONTEND
/*
* Functions that are currently not needed in the backend, but are better
* implemented inside xlogreader.c because of the internal facilities available
* here.
*/
/*
* Find the first record with an lsn >= RecPtr.
*
@ -1447,8 +1440,6 @@ err:
return InvalidXLogRecPtr;
}
#endif /* FRONTEND */
/*
* Helper function to ease writing of XLogRoutine->page_read callbacks.
* If this function is used, caller must supply a segment_open callback in

93
src/backend/access/transam/xlogstats.c Normal file
View File

@ -0,0 +1,93 @@
/*-------------------------------------------------------------------------
*
* xlogstats.c
* Functions for WAL Statitstics
*
* Copyright (c) 2022, PostgreSQL Global Development Group
*
* IDENTIFICATION
* src/backend/access/transam/xlogstats.c
*
*-------------------------------------------------------------------------
*/
#include "postgres.h"
#include "access/xlogreader.h"
#include "access/xlogstats.h"
/*
* Calculate the size of a record, split into !FPI and FPI parts.
*/
void
XLogRecGetLen(XLogReaderState *record, uint32 *rec_len,
uint32 *fpi_len)
{
int block_id;
/*
* Calculate the amount of FPI data in the record.
*
* XXX: We peek into xlogreader's private decoded backup blocks for the
* bimg_len indicating the length of FPI data.
*/
*fpi_len = 0;
for (block_id = 0; block_id <= XLogRecMaxBlockId(record); block_id++)
{
if (XLogRecHasBlockImage(record, block_id))
*fpi_len += XLogRecGetBlock(record, block_id)->bimg_len;
}
/*
* Calculate the length of the record as the total length - the length of
* all the block images.
*/
*rec_len = XLogRecGetTotalLen(record) - *fpi_len;
}
/*
* Store per-rmgr and per-record statistics for a given record.
*/
void
XLogRecStoreStats(XLogStats *stats, XLogReaderState *record)
{
RmgrId rmid;
uint8 recid;
uint32 rec_len;
uint32 fpi_len;
Assert(stats != NULL && record != NULL);
stats->count++;
rmid = XLogRecGetRmid(record);
XLogRecGetLen(record, &rec_len, &fpi_len);
/* Update per-rmgr statistics */
stats->rmgr_stats[rmid].count++;
stats->rmgr_stats[rmid].rec_len += rec_len;
stats->rmgr_stats[rmid].fpi_len += fpi_len;
/*
* Update per-record statistics, where the record is identified by a
* combination of the RmgrId and the four bits of the xl_info field that
* are the rmgr's domain (resulting in sixteen possible entries per
* RmgrId).
*/
recid = XLogRecGetInfo(record) >> 4;
/*
* XACT records need to be handled differently. Those records use the
* first bit of those four bits for an optional flag variable and the
* following three bits for the opcode. We filter opcode out of xl_info
* and use it as the identifier of the record.
*/
if (rmid == RM_XACT_ID)
recid &= 0x07;
stats->record_stats[rmid][recid].count++;
stats->record_stats[rmid][recid].rec_len += rec_len;
stats->record_stats[rmid][recid].fpi_len += fpi_len;
}

33
src/backend/access/transam/xlogutils.c
View File

@ -80,6 +80,10 @@ typedef struct xl_invalid_page
static HTAB *invalid_page_tab = NULL;
static int
read_local_xlog_page_guts(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr,
char *cur_page, bool wait_for_wal);
/* Report a reference to an invalid page */
static void
@ -870,6 +874,31 @@ wal_segment_close(XLogReaderState *state)
int
read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr, char *cur_page)
{
return read_local_xlog_page_guts(state, targetPagePtr, reqLen,
targetRecPtr, cur_page, true);
}
/*
* Same as read_local_xlog_page except that it doesn't wait for future WAL
* to be available.
*/
int
read_local_xlog_page_no_wait(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr,
char *cur_page)
{
return read_local_xlog_page_guts(state, targetPagePtr, reqLen,
targetRecPtr, cur_page, false);
}
/*
* Implementation of read_local_xlog_page and its no wait version.
*/
static int
read_local_xlog_page_guts(XLogReaderState *state, XLogRecPtr targetPagePtr,
int reqLen, XLogRecPtr targetRecPtr,
char *cur_page, bool wait_for_wal)
{
XLogRecPtr read_upto,
loc;
@ -925,6 +954,10 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr,
if (loc <= read_upto)
break;
/* If asked, let's not wait for future WAL. */
if (!wait_for_wal)
break;
CHECK_FOR_INTERRUPTS();
pg_usleep(1000L);
}

1
src/bin/pg_waldump/.gitignore vendored
View File

@ -23,6 +23,7 @@
/xactdesc.c
/xlogdesc.c
/xlogreader.c
/xlogstat.c
# Generated by test suite
/tmp_check/

8
src/bin/pg_waldump/Makefile
View File

@ -13,7 +13,8 @@ OBJS = \
compat.o \
pg_waldump.o \
rmgrdesc.o \
xlogreader.o
xlogreader.o \
xlogstats.o
override CPPFLAGS := -DFRONTEND $(CPPFLAGS)
@ -29,6 +30,9 @@ pg_waldump: $(OBJS) | submake-libpgport
xlogreader.c: % : $(top_srcdir)/src/backend/access/transam/%
rm -f $@ && $(LN_S) $< .
xlogstats.c: % : $(top_srcdir)/src/backend/access/transam/%
rm -f $@ && $(LN_S) $< .
$(RMGRDESCSOURCES): % : $(top_srcdir)/src/backend/access/rmgrdesc/%
rm -f $@ && $(LN_S) $< .
@ -42,7 +46,7 @@ uninstall:
rm -f '$(DESTDIR)$(bindir)/pg_waldump$(X)'
clean distclean maintainer-clean:
rm -f pg_waldump$(X) $(OBJS) $(RMGRDESCSOURCES) xlogreader.c
rm -f pg_waldump$(X) $(OBJS) $(RMGRDESCSOURCES) xlogreader.c xlogstats.c
rm -rf tmp_check
check:

206
src/bin/pg_waldump/pg_waldump.c
View File

@ -21,11 +21,17 @@
#include "access/xlog_internal.h"
#include "access/xlogreader.h"
#include "access/xlogrecord.h"
#include "access/xlogstats.h"
#include "common/fe_memutils.h"
#include "common/logging.h"
#include "getopt_long.h"
#include "rmgrdesc.h"
/*
* NOTE: For any code change or issue fix here, it is highly recommended to
* give a thought about doing the same in pg_walinspect contrib module as well.
*/
static const char *progname;
static int WalSegSz;
@ -66,24 +72,6 @@ typedef struct XLogDumpConfig
bool filter_by_fpw;
} XLogDumpConfig;
typedef struct Stats
{
uint64 count;
uint64 rec_len;
uint64 fpi_len;
} Stats;
#define MAX_XLINFO_TYPES 16
typedef struct XLogDumpStats
{
uint64 count;
XLogRecPtr startptr;
XLogRecPtr endptr;
Stats rmgr_stats[RM_MAX_ID + 1];
Stats record_stats[RM_MAX_ID + 1][MAX_XLINFO_TYPES];
} XLogDumpStats;
#define fatal_error(...) do { pg_log_fatal(__VA_ARGS__); exit(EXIT_FAILURE); } while(0)
/*
@ -453,81 +441,6 @@ XLogRecordHasFPW(XLogReaderState *record)
return false;
}
/*
* Calculate the size of a record, split into !FPI and FPI parts.
*/
static void
XLogDumpRecordLen(XLogReaderState *record, uint32 *rec_len, uint32 *fpi_len)
{
int block_id;
/*
* Calculate the amount of FPI data in the record.
*
* XXX: We peek into xlogreader's private decoded backup blocks for the
* bimg_len indicating the length of FPI data.
*/
*fpi_len = 0;
for (block_id = 0; block_id <= XLogRecMaxBlockId(record); block_id++)
{
if (XLogRecHasBlockImage(record, block_id))
*fpi_len += XLogRecGetBlock(record, block_id)->bimg_len;
}
/*
* Calculate the length of the record as the total length - the length of
* all the block images.
*/
*rec_len = XLogRecGetTotalLen(record) - *fpi_len;
}
/*
* Store per-rmgr and per-record statistics for a given record.
*/
static void
XLogDumpCountRecord(XLogDumpConfig *config, XLogDumpStats *stats,
XLogReaderState *record)
{
RmgrId rmid;
uint8 recid;
uint32 rec_len;
uint32 fpi_len;
stats->count++;
rmid = XLogRecGetRmid(record);
XLogDumpRecordLen(record, &rec_len, &fpi_len);
/* Update per-rmgr statistics */
stats->rmgr_stats[rmid].count++;
stats->rmgr_stats[rmid].rec_len += rec_len;
stats->rmgr_stats[rmid].fpi_len += fpi_len;
/*
* Update per-record statistics, where the record is identified by a
* combination of the RmgrId and the four bits of the xl_info field that
* are the rmgr's domain (resulting in sixteen possible entries per
* RmgrId).
*/
recid = XLogRecGetInfo(record) >> 4;
/*
* XACT records need to be handled differently. Those records use the
* first bit of those four bits for an optional flag variable and the
* following three bits for the opcode. We filter opcode out of xl_info
* and use it as the identifier of the record.
*/
if (rmid == RM_XACT_ID)
recid &= 0x07;
stats->record_stats[rmid][recid].count++;
stats->record_stats[rmid][recid].rec_len += rec_len;
stats->record_stats[rmid][recid].fpi_len += fpi_len;
}
/*
* Print a record to stdout
*/
@ -538,15 +451,11 @@ XLogDumpDisplayRecord(XLogDumpConfig *config, XLogReaderState *record)
const RmgrDescData *desc = GetRmgrDesc(XLogRecGetRmid(record));
uint32 rec_len;
uint32 fpi_len;
RelFileNode rnode;
ForkNumber forknum;
BlockNumber blk;
int block_id;
uint8 info = XLogRecGetInfo(record);
XLogRecPtr xl_prev = XLogRecGetPrev(record);
StringInfoData s;
XLogDumpRecordLen(record, &rec_len, &fpi_len);
XLogRecGetLen(record, &rec_len, &fpi_len);
printf("rmgr: %-11s len (rec/tot): %6u/%6u, tx: %10u, lsn: %X/%08X, prev %X/%08X, ",
desc->rm_name,
@ -564,93 +473,11 @@ XLogDumpDisplayRecord(XLogDumpConfig *config, XLogReaderState *record)
initStringInfo(&s);
desc->rm_desc(&s, record);
printf("%s", s.data);
resetStringInfo(&s);
XLogRecGetBlockRefInfo(record, true, config->bkp_details, &s, NULL);
printf("%s", s.data);
pfree(s.data);
if (!config->bkp_details)
{
/* print block references (short format) */
for (block_id = 0; block_id <= XLogRecMaxBlockId(record); block_id++)
{
if (!XLogRecHasBlockRef(record, block_id))
continue;
XLogRecGetBlockTag(record, block_id, &rnode, &forknum, &blk);
if (forknum != MAIN_FORKNUM)
printf(", blkref #%d: rel %u/%u/%u fork %s blk %u",
block_id,
rnode.spcNode, rnode.dbNode, rnode.relNode,
forkNames[forknum],
blk);
else
printf(", blkref #%d: rel %u/%u/%u blk %u",
block_id,
rnode.spcNode, rnode.dbNode, rnode.relNode,
blk);
if (XLogRecHasBlockImage(record, block_id))
{
if (XLogRecBlockImageApply(record, block_id))
printf(" FPW");
else
printf(" FPW for WAL verification");
}
}
putchar('\n');
}
else
{
/* print block references (detailed format) */
putchar('\n');
for (block_id = 0; block_id <= XLogRecMaxBlockId(record); block_id++)
{
if (!XLogRecHasBlockRef(record, block_id))
continue;
XLogRecGetBlockTag(record, block_id, &rnode, &forknum, &blk);
printf("\tblkref #%d: rel %u/%u/%u fork %s blk %u",
block_id,
rnode.spcNode, rnode.dbNode, rnode.relNode,
forkNames[forknum],
blk);
if (XLogRecHasBlockImage(record, block_id))
{
uint8 bimg_info = XLogRecGetBlock(record, block_id)->bimg_info;
if (BKPIMAGE_COMPRESSED(bimg_info))
{
const char *method;
if ((bimg_info & BKPIMAGE_COMPRESS_PGLZ) != 0)
method = "pglz";
else if ((bimg_info & BKPIMAGE_COMPRESS_LZ4) != 0)
method = "lz4";
else if ((bimg_info & BKPIMAGE_COMPRESS_ZSTD) != 0)
method = "zstd";
else
method = "unknown";
printf(" (FPW%s); hole: offset: %u, length: %u, "
"compression saved: %u, method: %s",
XLogRecBlockImageApply(record, block_id) ?
"" : " for WAL verification",
XLogRecGetBlock(record, block_id)->hole_offset,
XLogRecGetBlock(record, block_id)->hole_length,
BLCKSZ -
XLogRecGetBlock(record, block_id)->hole_length -
XLogRecGetBlock(record, block_id)->bimg_len,
method);
}
else
{
printf(" (FPW%s); hole: offset: %u, length: %u",
XLogRecBlockImageApply(record, block_id) ?
"" : " for WAL verification",
XLogRecGetBlock(record, block_id)->hole_offset,
XLogRecGetBlock(record, block_id)->hole_length);
}
}
putchar('\n');
}
}
}
/*
@ -698,7 +525,7 @@ XLogDumpStatsRow(const char *name,
* Display summary statistics about the records seen so far.
*/
static void
XLogDumpDisplayStats(XLogDumpConfig *config, XLogDumpStats *stats)
XLogDumpDisplayStats(XLogDumpConfig *config, XLogStats *stats)
{
int ri,
rj;
@ -722,6 +549,9 @@ XLogDumpDisplayStats(XLogDumpConfig *config, XLogDumpStats *stats)
for (ri = 0; ri <= RM_MAX_ID; ri++)
{
if (!RmgrIdIsValid(ri))
continue;
total_count += stats->rmgr_stats[ri].count;
total_rec_len += stats->rmgr_stats[ri].rec_len;
total_fpi_len += stats->rmgr_stats[ri].fpi_len;
@ -867,7 +697,7 @@ main(int argc, char **argv)
XLogReaderState *xlogreader_state;
XLogDumpPrivate private;
XLogDumpConfig config;
XLogDumpStats stats;
XLogStats stats;
XLogRecord *record;
XLogRecPtr first_record;
char *waldir = NULL;
@ -921,7 +751,7 @@ main(int argc, char **argv)
memset(&private, 0, sizeof(XLogDumpPrivate));
memset(&config, 0, sizeof(XLogDumpConfig));
memset(&stats, 0, sizeof(XLogDumpStats));
memset(&stats, 0, sizeof(XLogStats));
private.timeline = 1;
private.startptr = InvalidXLogRecPtr;
@ -1319,7 +1149,7 @@ main(int argc, char **argv)
{
if (config.stats == true)
{
XLogDumpCountRecord(&config, &stats, xlogreader_state);
XLogRecStoreStats(&stats, xlogreader_state);
stats.endptr = xlogreader_state->EndRecPtr;
}
else

2
src/include/access/xlog.h
View File

@ -31,7 +31,7 @@ extern XLogRecPtr XactLastRecEnd;
extern PGDLLIMPORT XLogRecPtr XactLastCommitEnd;
/* these variables are GUC parameters related to XLOG */
extern int wal_segment_size;
extern PGDLLIMPORT int wal_segment_size;
extern int min_wal_size_mb;
extern int max_wal_size_mb;
extern int wal_keep_size_mb;

6
src/include/access/xlog_internal.h
View File

@ -320,7 +320,7 @@ typedef struct RmgrData
struct XLogRecordBuffer *buf);
} RmgrData;
extern RmgrData RmgrTable[];
extern PGDLLIMPORT RmgrData RmgrTable[];
extern void RmgrStartup(void);
extern void RmgrCleanup(void);
extern void RmgrNotFound(RmgrId rmid);
@ -350,6 +350,10 @@ extern XLogRecPtr RequestXLogSwitch(bool mark_unimportant);
extern void GetOldestRestartPoint(XLogRecPtr *oldrecptr, TimeLineID *oldtli);
extern void XLogRecGetBlockRefInfo(XLogReaderState *record, bool pretty,
bool detailed_format, StringInfo buf,
uint32 *fpi_len);
/*
* Exported for the functions in timeline.c and xlogarchive.c. Only valid
* in the startup process.

2
src/include/access/xlogreader.h
View File

@ -344,9 +344,7 @@ extern void XLogReaderSetDecodeBuffer(XLogReaderState *state,
/* Position the XLogReader to given record */
extern void XLogBeginRead(XLogReaderState *state, XLogRecPtr RecPtr);
#ifdef FRONTEND
extern XLogRecPtr XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr);
#endif /* FRONTEND */
/* Return values from XLogPageReadCB. */
typedef enum XLogPageReadResult

40
src/include/access/xlogstats.h Normal file
View File

@ -0,0 +1,40 @@
/*-------------------------------------------------------------------------
*
* xlogstats.h
* Definitions for WAL Statitstics
*
* Copyright (c) 2022, PostgreSQL Global Development Group
*
* IDENTIFICATION
* src/include/access/xlogstats.h
*
*-------------------------------------------------------------------------
*/
#ifndef XLOGSTATS_H
#define XLOGSTATS_H
#define MAX_XLINFO_TYPES 16
typedef struct XLogRecStats
{
uint64 count;
uint64 rec_len;
uint64 fpi_len;
} XLogRecStats;
typedef struct XLogStats
{
uint64 count;
#ifdef FRONTEND
XLogRecPtr startptr;
XLogRecPtr endptr;
#endif
XLogRecStats rmgr_stats[RM_MAX_ID + 1];
XLogRecStats record_stats[RM_MAX_ID + 1][MAX_XLINFO_TYPES];
} XLogStats;
extern void XLogRecGetLen(XLogReaderState *record, uint32 *rec_len,
uint32 *fpi_len);
extern void XLogRecStoreStats(XLogStats *stats, XLogReaderState *record);
#endif /* XLOGSTATS_H */

4
src/include/access/xlogutils.h
View File

@ -93,6 +93,10 @@ extern void FreeFakeRelcacheEntry(Relation fakerel);
extern int read_local_xlog_page(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr, char *cur_page);
extern int read_local_xlog_page_no_wait(XLogReaderState *state,
XLogRecPtr targetPagePtr, int reqLen,
XLogRecPtr targetRecPtr,
char *cur_page);
extern void wal_segment_open(XLogReaderState *state,
XLogSegNo nextSegNo,
TimeLineID *tli_p);