database/postgres
1
0
Fork 0
mirror of https://github.com/postgres/postgres.git synced 2025-12-16 16:42:29 +03:00
Code Activity

pg_rewind: Refactor the abstraction to fetch from local/libpq source.

Browse Source 
This makes the abstraction of a "source" server more clear, by introducing
a common abstract class, borrowing the object-oriented programming term,
that represents all the operations that can be done on the source server.
There are two implementations of it, one for fetching via libpq, and
another to fetch from a local directory. This adds some code, but makes it
easier to understand what's going on.

The copy_executeFileMap() and libpq_executeFileMap() functions contained
basically the same logic, just calling different functions to fetch the
source files. Refactor so that the common logic is in one place, in a new
function called perform_rewind().

Reviewed-by: Kyotaro Horiguchi, Soumyadeep Chakraborty
Discussion: https://www.postgresql.org/message-id/0c5b3783-af52-3ee5-f8fa-6e794061f70d%40iki.fi
This commit is contained in:
Heikki Linnakangas
2020-11-04 11:21:18 +02:00
parent f81e97d047
commit 37d2ff3803
12 changed files with 700 additions and 634 deletions
Download Patch File Download Diff File Expand all files Collapse all files

208
src/bin/pg_rewind/pg_rewind.c
View File

@@ -23,20 +23,25 @@
#include "common/restricted_token.h"
#include "common/string.h"
#include "fe_utils/recovery_gen.h"
#include "fetch.h"
#include "file_ops.h"
#include "filemap.h"
#include "getopt_long.h"
#include "pg_rewind.h"
#include "rewind_source.h"
#include "storage/bufpage.h"
static void usage(const char *progname);
static void perform_rewind(filemap_t *filemap, rewind_source *source,
XLogRecPtr chkptrec,
TimeLineID chkpttli,
XLogRecPtr chkptredo);
static void createBackupLabel(XLogRecPtr startpoint, TimeLineID starttli,
XLogRecPtr checkpointloc);
static void digestControlFile(ControlFileData *ControlFile, char *source,
size_t size);
static void digestControlFile(ControlFileData *ControlFile,
const char *content, size_t size);
static void getRestoreCommand(const char *argv0);
static void sanityChecks(void);
static void findCommonAncestorTimeline(XLogRecPtr *recptr, int *tliIndex);
@@ -69,6 +74,8 @@ int targetNentries;
uint64 fetch_size;
uint64 fetch_done;
static PGconn *conn;
static rewind_source *source;
static void
usage(const char *progname)
@@ -125,9 +132,6 @@ main(int argc, char **argv)
char *buffer;
bool no_ensure_shutdown = false;
bool rewind_needed;
XLogRecPtr endrec;
TimeLineID endtli;
ControlFileData ControlFile_new;
bool writerecoveryconf = false;
filemap_t *filemap;
@@ -269,19 +273,29 @@ main(int argc, char **argv)
atexit(disconnect_atexit);
/* Connect to remote server */
if (connstr_source)
libpqConnect(connstr_source);
/*
* Ok, we have all the options and we're ready to start. Read in all the
* information we need from both clusters.
* Ok, we have all the options and we're ready to start. First, connect to
* remote server.
*/
buffer = slurpFile(datadir_target, "global/pg_control", &size);
digestControlFile(&ControlFile_target, buffer, size);
pg_free(buffer);
if (connstr_source)
{
conn = PQconnectdb(connstr_source);
if (PQstatus(conn) == CONNECTION_BAD)
pg_fatal("could not connect to server: %s",
PQerrorMessage(conn));
if (showprogress)
pg_log_info("connected to server");
source = init_libpq_source(conn);
}
else
source = init_local_source(datadir_source);
/*
* Check the status of the target instance.
*
* If the target instance was not cleanly shut down, start and stop the
* target cluster once in single-user mode to enforce recovery to finish,
* ensuring that the cluster can be used by pg_rewind. Note that if
@@ -289,6 +303,10 @@ main(int argc, char **argv)
* need to make sure by themselves that the target cluster is in a clean
* state.
*/
buffer = slurpFile(datadir_target, "global/pg_control", &size);
digestControlFile(&ControlFile_target, buffer, size);
pg_free(buffer);
if (!no_ensure_shutdown &&
ControlFile_target.state != DB_SHUTDOWNED &&
ControlFile_target.state != DB_SHUTDOWNED_IN_RECOVERY)
@@ -300,17 +318,20 @@ main(int argc, char **argv)
pg_free(buffer);
}
buffer = fetchFile("global/pg_control", &size);
buffer = source->fetch_file(source, "global/pg_control", &size);
digestControlFile(&ControlFile_source, buffer, size);
pg_free(buffer);
sanityChecks();
/*
* Find the common ancestor timeline between the clusters.
*
* If both clusters are already on the same timeline, there's nothing to
* do.
*/
if (ControlFile_target.checkPointCopy.ThisTimeLineID == ControlFile_source.checkPointCopy.ThisTimeLineID)
if (ControlFile_target.checkPointCopy.ThisTimeLineID ==
ControlFile_source.checkPointCopy.ThisTimeLineID)
{
pg_log_info("source and target cluster are on the same timeline");
rewind_needed = false;
@@ -373,11 +394,11 @@ main(int argc, char **argv)
filehash_init();
/*
* Collect information about all files in the target and source systems.
* Collect information about all files in the both data directories.
*/
if (showprogress)
pg_log_info("reading source file list");
fetchSourceFileList();
source->traverse_files(source, &process_source_file);
if (showprogress)
pg_log_info("reading target file list");
@@ -421,11 +442,124 @@ main(int argc, char **argv)
}
/*
* This is the point of no return. Once we start copying things, we have
* modified the target directory and there is no turning back!
* We have now collected all the information we need from both systems,
* and we are ready to start modifying the target directory.
*
* This is the point of no return. Once we start copying things, there is
* no turning back!
*/
perform_rewind(filemap, source, chkptrec, chkpttli, chkptredo);
execute_file_actions(filemap);
if (showprogress)
pg_log_info("syncing target data directory");
sync_target_dir();
/* Also update the standby configuration, if requested. */
if (writerecoveryconf && !dry_run)
WriteRecoveryConfig(conn, datadir_target,
GenerateRecoveryConfig(conn, NULL));
/* don't need the source connection anymore */
source->destroy(source);
if (conn)
{
PQfinish(conn);
conn = NULL;
}
pg_log_info("Done!");
return 0;
}
/*
* Perform the rewind.
*
* We have already collected all the information we need from the
* target and the source.
*/
static void
perform_rewind(filemap_t *filemap, rewind_source *source,
XLogRecPtr chkptrec,
TimeLineID chkpttli,
XLogRecPtr chkptredo)
{
XLogRecPtr endrec;
TimeLineID endtli;
ControlFileData ControlFile_new;
/*
* Execute the actions in the file map, fetching data from the source
* system as needed.
*/
for (int i = 0; i < filemap->nentries; i++)
{
file_entry_t *entry = filemap->entries[i];
/*
* If this is a relation file, copy the modified blocks.
*
* This is in addition to any other changes.
*/
if (entry->target_pages_to_overwrite.bitmapsize > 0)
{
datapagemap_iterator_t *iter;
BlockNumber blkno;
off_t offset;
iter = datapagemap_iterate(&entry->target_pages_to_overwrite);
while (datapagemap_next(iter, &blkno))
{
offset = blkno * BLCKSZ;
source->queue_fetch_range(source, entry->path, offset, BLCKSZ);
}
pg_free(iter);
}
switch (entry->action)
{
case FILE_ACTION_NONE:
/* nothing else to do */
break;
case FILE_ACTION_COPY:
/* Truncate the old file out of the way, if any */
open_target_file(entry->path, true);
source->queue_fetch_range(source, entry->path,
0, entry->source_size);
break;
case FILE_ACTION_TRUNCATE:
truncate_target_file(entry->path, entry->source_size);
break;
case FILE_ACTION_COPY_TAIL:
source->queue_fetch_range(source, entry->path,
entry->target_size,
entry->source_size - entry->target_size);
break;
case FILE_ACTION_REMOVE:
remove_target(entry);
break;
case FILE_ACTION_CREATE:
create_target(entry);
break;
case FILE_ACTION_UNDECIDED:
pg_fatal("no action decided for \"%s\"", entry->path);
break;
}
}
/*
* We've now copied the list of file ranges that we need to fetch to the
* temporary table. Now, actually fetch all of those ranges.
*/
source->finish_fetch(source);
close_target_file();
progress_report(true);
@@ -437,15 +571,15 @@ main(int argc, char **argv)
* Update control file of target. Make it ready to perform archive
* recovery when restarting.
*
* minRecoveryPoint is set to the current WAL insert location in the
* source server. Like in an online backup, it's important that we recover
* all the WAL that was generated while we copied the files over.
* Like in an online backup, it's important that we replay all the WAL
* that was generated while we copied the files over. To enforce that, set
* 'minRecoveryPoint' in the control file.
*/
memcpy(&ControlFile_new, &ControlFile_source, sizeof(ControlFileData));
if (connstr_source)
{
endrec = libpqGetCurrentXlogInsertLocation();
endrec = source->get_current_wal_insert_lsn(source);
endtli = ControlFile_source.checkPointCopy.ThisTimeLineID;
}
else
@@ -458,18 +592,6 @@ main(int argc, char **argv)
ControlFile_new.state = DB_IN_ARCHIVE_RECOVERY;
if (!dry_run)
update_controlfile(datadir_target, &ControlFile_new, do_sync);
if (showprogress)
pg_log_info("syncing target data directory");
sync_target_dir();
if (writerecoveryconf && !dry_run)
WriteRecoveryConfig(conn, datadir_target,
GenerateRecoveryConfig(conn, NULL));
pg_log_info("Done!");
return 0;
}
static void
@@ -629,7 +751,7 @@ getTimelineHistory(ControlFileData *controlFile, int *nentries)
/* Get history file from appropriate source */
if (controlFile == &ControlFile_source)
histfile = fetchFile(path, NULL);
histfile = source->fetch_file(source, path, NULL);
else if (controlFile == &ControlFile_target)
histfile = slurpFile(datadir_target, path, NULL);
else
@@ -785,16 +907,18 @@ checkControlFile(ControlFileData *ControlFile)
}
/*
* Verify control file contents in the buffer src, and copy it to *ControlFile.
* Verify control file contents in the buffer 'content', and copy it to
* *ControlFile.
*/
static void
digestControlFile(ControlFileData *ControlFile, char *src, size_t size)
digestControlFile(ControlFileData *ControlFile, const char *content,
size_t size)
{
if (size != PG_CONTROL_FILE_SIZE)
pg_fatal("unexpected control file size %d, expected %d",
(int) size, PG_CONTROL_FILE_SIZE);
memcpy(ControlFile, src, sizeof(ControlFileData));
memcpy(ControlFile, content, sizeof(ControlFileData));
/* set and validate WalSegSz */
WalSegSz = ControlFile->xlog_seg_size;