database/postgres
1
0
Fork 0
mirror of https://github.com/postgres/postgres.git synced 2025-07-28 23:42:10 +03:00
Code Activity

pg_upgrade: Allow use of file cloning

Browse Source 
Add another transfer mode --clone to pg_upgrade (besides the existing
--link and the default copy), using special file cloning calls.  This
makes the file transfer faster and more space efficient, achieving
speed similar to --link mode without the associated drawbacks.

On Linux, file cloning is supported on Btrfs and XFS (if formatted with
reflink support).  On macOS, file cloning is supported on APFS.

Reviewed-by: Michael Paquier <michael@paquier.xyz>
This commit is contained in:
Peter Eisentraut
2018-11-07 18:05:54 +01:00
parent 5f32b29c18
commit 3a769d8239
9 changed files with 181 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
src/bin/pg_upgrade/check.c
View File

@ -149,8 +149,17 @@ check_new_cluster(void)
check_loadable_libraries();
if (user_opts.transfer_mode == TRANSFER_MODE_LINK)
check_hard_link();
switch (user_opts.transfer_mode)
{
case TRANSFER_MODE_CLONE:
check_file_clone();
break;
case TRANSFER_MODE_COPY:
break;
case TRANSFER_MODE_LINK:
check_hard_link();
break;
}
check_is_install_user(&new_cluster);

90
src/bin/pg_upgrade/file.c
View File

@ -18,6 +18,13 @@
#include <sys/stat.h>
#include <fcntl.h>
#ifdef HAVE_COPYFILE
#include <copyfile.h>
#endif
#ifdef __linux__
#include <sys/ioctl.h>
#include <linux/fs.h>
#endif
#ifdef WIN32
@ -25,6 +32,47 @@ static int win32_pghardlink(const char *src, const char *dst);
#endif
/*
* cloneFile()
*
* Clones/reflinks a relation file from src to dst.
*
* schemaName/relName are relation's SQL name (used for error messages only).
*/
void
cloneFile(const char *src, const char *dst,
const char *schemaName, const char *relName)
{
#if defined(HAVE_COPYFILE) && defined(COPYFILE_CLONE_FORCE)
if (copyfile(src, dst, NULL, COPYFILE_CLONE_FORCE) < 0)
pg_fatal("error while cloning relation \"%s.%s\" (\"%s\" to \"%s\"): %s\n",
schemaName, relName, src, dst, strerror(errno));
#elif defined(__linux__) && defined(FICLONE)
int src_fd;
int dest_fd;
if ((src_fd = open(src, O_RDONLY | PG_BINARY, 0)) < 0)
pg_fatal("error while cloning relation \"%s.%s\": could not open file \"%s\": %s\n",
schemaName, relName, src, strerror(errno));
if ((dest_fd = open(dst, O_RDWR | O_CREAT | O_EXCL | PG_BINARY,
pg_file_create_mode)) < 0)
pg_fatal("error while cloning relation \"%s.%s\": could not create file \"%s\": %s\n",
schemaName, relName, dst, strerror(errno));
if (ioctl(dest_fd, FICLONE, src_fd) < 0)
{
unlink(dst);
pg_fatal("error while cloning relation \"%s.%s\" (\"%s\" to \"%s\"): %s\n",
schemaName, relName, src, dst, strerror(errno));
}
close(src_fd);
close(dest_fd);
#endif
}
/*
* copyFile()
*
@ -270,6 +318,48 @@ rewriteVisibilityMap(const char *fromfile, const char *tofile,
close(src_fd);
}
void
check_file_clone(void)
{
char existing_file[MAXPGPATH];
char new_link_file[MAXPGPATH];
snprintf(existing_file, sizeof(existing_file), "%s/PG_VERSION", old_cluster.pgdata);
snprintf(new_link_file, sizeof(new_link_file), "%s/PG_VERSION.clonetest", new_cluster.pgdata);
unlink(new_link_file); /* might fail */
#if defined(HAVE_COPYFILE) && defined(COPYFILE_CLONE_FORCE)
if (copyfile(existing_file, new_link_file, NULL, COPYFILE_CLONE_FORCE) < 0)
pg_fatal("could not clone file between old and new data directories: %s\n",
strerror(errno));
#elif defined(__linux__) && defined(FICLONE)
{
int src_fd;
int dest_fd;
if ((src_fd = open(existing_file, O_RDONLY | PG_BINARY, 0)) < 0)
pg_fatal("could not open file \"%s\": %s\n",
existing_file, strerror(errno));
if ((dest_fd = open(new_link_file, O_RDWR | O_CREAT | O_EXCL | PG_BINARY,
pg_file_create_mode)) < 0)
pg_fatal("could not create file \"%s\": %s\n",
new_link_file, strerror(errno));
if (ioctl(dest_fd, FICLONE, src_fd) < 0)
pg_fatal("could not clone file between old and new data directories: %s\n",
strerror(errno));
close(src_fd);
close(dest_fd);
}
#else
pg_fatal("file cloning not supported on this platform\n");
#endif
unlink(new_link_file);
}
void
check_hard_link(void)
{

7
src/bin/pg_upgrade/option.c
View File

@ -53,6 +53,8 @@ parseCommandLine(int argc, char *argv[])
{"retain", no_argument, NULL, 'r'},
{"jobs", required_argument, NULL, 'j'},
{"verbose", no_argument, NULL, 'v'},
{"clone", no_argument, NULL, 1},
{NULL, 0, NULL, 0}
};
int option; /* Command line option */
@ -203,6 +205,10 @@ parseCommandLine(int argc, char *argv[])
log_opts.verbose = true;
break;
case 1:
user_opts.transfer_mode = TRANSFER_MODE_CLONE;
break;
default:
pg_fatal("Try \"%s --help\" for more information.\n",
os_info.progname);
@ -293,6 +299,7 @@ usage(void)
printf(_(" -U, --username=NAME cluster superuser (default \"%s\")\n"), os_info.user);
printf(_(" -v, --verbose enable verbose internal logging\n"));
printf(_(" -V, --version display version information, then exit\n"));
printf(_(" --clone clone instead of copying files to new cluster\n"));
printf(_(" -?, --help show this help, then exit\n"));
printf(_("\n"
"Before running pg_upgrade you must:\n"

6
src/bin/pg_upgrade/pg_upgrade.h
View File

@ -230,10 +230,11 @@ typedef struct
} ControlData;
/*
* Enumeration to denote link modes
* Enumeration to denote transfer modes
*/
typedef enum
{
TRANSFER_MODE_CLONE,
TRANSFER_MODE_COPY,
TRANSFER_MODE_LINK
} transferMode;
@ -372,12 +373,15 @@ bool pid_lock_file_exists(const char *datadir);
/* file.c */
void cloneFile(const char *src, const char *dst,
const char *schemaName, const char *relName);
void copyFile(const char *src, const char *dst,
const char *schemaName, const char *relName);
void linkFile(const char *src, const char *dst,
const char *schemaName, const char *relName);
void rewriteVisibilityMap(const char *fromfile, const char *tofile,
const char *schemaName, const char *relName);
void check_file_clone(void);
void check_hard_link(void);
/* fopen_priv() is no longer different from fopen() */

44
src/bin/pg_upgrade/relfilenode.c
View File

@ -30,10 +30,18 @@ void
transfer_all_new_tablespaces(DbInfoArr *old_db_arr, DbInfoArr *new_db_arr,
char *old_pgdata, char *new_pgdata)
{
if (user_opts.transfer_mode == TRANSFER_MODE_LINK)
pg_log(PG_REPORT, "Linking user relation files\n");
else
pg_log(PG_REPORT, "Copying user relation files\n");
switch (user_opts.transfer_mode)
{
case TRANSFER_MODE_CLONE:
pg_log(PG_REPORT, "Cloning user relation files\n");
break;
case TRANSFER_MODE_COPY:
pg_log(PG_REPORT, "Copying user relation files\n");
break;
case TRANSFER_MODE_LINK:
pg_log(PG_REPORT, "Linking user relation files\n");
break;
}
/*
* Transferring files by tablespace is tricky because a single database
@ -250,17 +258,23 @@ transfer_relfile(FileNameMap *map, const char *type_suffix, bool vm_must_add_fro
old_file, new_file);
rewriteVisibilityMap(old_file, new_file, map->nspname, map->relname);
}
else if (user_opts.transfer_mode == TRANSFER_MODE_COPY)
{
pg_log(PG_VERBOSE, "copying \"%s\" to \"%s\"\n",
old_file, new_file);
copyFile(old_file, new_file, map->nspname, map->relname);
}
else
{
pg_log(PG_VERBOSE, "linking \"%s\" to \"%s\"\n",
old_file, new_file);
linkFile(old_file, new_file, map->nspname, map->relname);
}
switch (user_opts.transfer_mode)
{
case TRANSFER_MODE_CLONE:
pg_log(PG_VERBOSE, "cloning \"%s\" to \"%s\"\n",
old_file, new_file);
cloneFile(old_file, new_file, map->nspname, map->relname);
break;
case TRANSFER_MODE_COPY:
pg_log(PG_VERBOSE, "copying \"%s\" to \"%s\"\n",
old_file, new_file);
copyFile(old_file, new_file, map->nspname, map->relname);
break;
case TRANSFER_MODE_LINK:
pg_log(PG_VERBOSE, "linking \"%s\" to \"%s\"\n",
old_file, new_file);
linkFile(old_file, new_file, map->nspname, map->relname);
}
}
}

3
src/include/pg_config.h.in
View File

@ -114,6 +114,9 @@
/* Define to 1 if your compiler handles computed gotos. */
#undef HAVE_COMPUTED_GOTO
/* Define to 1 if you have the `copyfile' function. */
#undef HAVE_COPYFILE
/* Define to 1 if you have the <crtdefs.h> header file. */
#undef HAVE_CRTDEFS_H