From 21c715b25c2fd484e8b4e8d53b64a9b2d59ad7d6 Mon Sep 17 00:00:00 2001 From: David Steele Date: Thu, 22 Jan 2015 20:11:33 -0500 Subject: [PATCH] Restore tablespace remaps are working. --- bin/pg_backrest.pl | 2 +- lib/BackRest/Config.pm | 19 +++++-- lib/BackRest/Manifest.pm | 14 +++++ lib/BackRest/Restore.pm | 80 ++++++++++++----------------- test/lib/BackRestTest/BackupTest.pm | 60 +++++++++++----------- test/lib/BackRestTest/CommonTest.pm | 22 ++++++-- 6 files changed, 112 insertions(+), 85 deletions(-) diff --git a/bin/pg_backrest.pl b/bin/pg_backrest.pl index f2e3cd2a8..74e888b97 100755 --- a/bin/pg_backrest.pl +++ b/bin/pg_backrest.pl @@ -435,7 +435,7 @@ if (operation_get() eq OP_RESTORE) ( config_key_load(CONFIG_SECTION_STANZA, CONFIG_KEY_PATH), param_get(PARAM_SET), - undef, #param_get(PARAM_REMAP), + config_section_load(CONFIG_SECTION_TABLESPACE_MAP), $oFile, undef, #param_get(PARAM_THREAD), param_get(PARAM_DELTA), diff --git a/lib/BackRest/Config.pm b/lib/BackRest/Config.pm index 06c6680c2..8e39a4901 100644 --- a/lib/BackRest/Config.pm +++ b/lib/BackRest/Config.pm @@ -17,7 +17,7 @@ use BackRest::Utility; use Exporter qw(import); -our @EXPORT = qw(config_load config_key_load operation_get operation_set param_get +our @EXPORT = qw(config_load config_key_load config_section_load operation_get operation_set param_get FILE_MANIFEST FILE_VERSION FILE_POSTMASTER_PID PATH_LATEST @@ -30,7 +30,8 @@ our @EXPORT = qw(config_load config_key_load operation_get operation_set param_g PARAM_VERSION PARAM_HELP PARAM_TEST PARAM_TEST_DELAY PARAM_TEST_NO_FORK CONFIG_SECTION_COMMAND CONFIG_SECTION_COMMAND_OPTION CONFIG_SECTION_LOG CONFIG_SECTION_BACKUP - CONFIG_SECTION_RESTORE CONFIG_SECTION_ARCHIVE CONFIG_SECTION_RETENTION CONFIG_SECTION_STANZA + CONFIG_SECTION_RESTORE CONFIG_SECTION_TABLESPACE_MAP CONFIG_SECTION_ARCHIVE CONFIG_SECTION_RETENTION + CONFIG_SECTION_STANZA CONFIG_KEY_USER CONFIG_KEY_HOST CONFIG_KEY_PATH @@ -108,7 +109,7 @@ use constant CONFIG_SECTION_LOG => 'log', CONFIG_SECTION_BACKUP => 'backup', CONFIG_SECTION_RESTORE => 'restore', - CONFIG_SECTION_RESTORE_REMAP => 'restore:remap', + CONFIG_SECTION_TABLESPACE_MAP => 'tablespace:map', CONFIG_SECTION_ARCHIVE => 'archive', CONFIG_SECTION_RETENTION => 'retention', CONFIG_SECTION_STANZA => 'stanza', @@ -266,6 +267,18 @@ sub config_load test_set(param_get(PARAM_TEST), param_get(PARAM_TEST_DELAY)); } +#################################################################################################################################### +# CONFIG_STANZA_SECTION_LOAD - Get an entire stanza section +#################################################################################################################################### +sub config_section_load +{ + my $strSection = shift; + + $strSection = param_get(PARAM_STANZA) . ':' . $strSection; + + return $oConfig{$strSection}; +} + #################################################################################################################################### # CONFIG_KEY_LOAD - Get a value from the config and be sure that it is defined (unless bRequired is false) #################################################################################################################################### diff --git a/lib/BackRest/Manifest.pm b/lib/BackRest/Manifest.pm index c2dadd7fd..fe9d4be9e 100644 --- a/lib/BackRest/Manifest.pm +++ b/lib/BackRest/Manifest.pm @@ -343,6 +343,12 @@ sub valid my $strPath = (split(':', $strSection))[0]; my $strType = (split(':', $strSection))[1]; + if ($strPath eq 'tablespace') + { + $strPath = (split(':', $strSection))[1]; + $strType = (split(':', $strSection))[2]; + } + if (($strType eq 'path' || $strType eq 'file' || $strType eq 'link') && ($strSubKey eq MANIFEST_SUBKEY_USER || $strSubKey eq MANIFEST_SUBKEY_GROUP)) @@ -398,6 +404,14 @@ sub valid return true; } } + elsif ($strSection eq MANIFEST_SECTION_BACKUP_TABLESPACE) + { + if ($strSubKey eq 'link' || + $strSubKey eq 'path') + { + return true; + } + } elsif ($strSection eq MANIFEST_SECTION_BACKUP_PATH) { if ($strKey eq 'base' || $strKey =~ /^tablespace\:.*$/) diff --git a/lib/BackRest/Restore.pm b/lib/BackRest/Restore.pm index cda316918..773cf7412 100644 --- a/lib/BackRest/Restore.pm +++ b/lib/BackRest/Restore.pm @@ -187,50 +187,35 @@ sub manifest_load ' - this indicates some sort of corruption (at the very least paths have been renamed.'); } - if ($self->{strDbClusterPath} ne $oManifest->set(MANIFEST_SECTION_BACKUP_PATH, MANIFEST_KEY_BASE)) + if ($self->{strDbClusterPath} ne $oManifest->get(MANIFEST_SECTION_BACKUP_PATH, MANIFEST_KEY_BASE)) { &log(INFO, 'base path remapped to ' . $self->{strDbClusterPath}); $oManifest->set(MANIFEST_SECTION_BACKUP_PATH, MANIFEST_KEY_BASE, undef, $self->{strDbClusterPath}); } - # # If tablespaces have been remapped, update the manifest - # if (defined($self->{oRemapRef})) - # { - # foreach my $strPathKey (sort(keys $self->{oRemapRef})) - # { - # my $strRemapPath = ${$self->{oRemapRef}}{$strPathKey}; - # - # if ($strPathKey eq 'base') - # { - # &log(INFO, "remapping base to ${strRemapPath}"); - # $oManifest->set(MANIFEST_SECTION_BACKUP_PATH, $strPathKey, undef, $strRemapPath); - # } - # else - # { - # # If the tablespace begins with prefix 'tablespace:' then strip the prefix. This only needs to be used in - # # the case that there is a tablespace called 'base' - # if (index($strPathKey, 'tablespace:') == 0) - # { - # $strPathKey = substr($strPathKey, length('tablespace:')); - # } - # - # # Make sure that the tablespace exists in the manifest - # if (!$oManifest->test(MANIFEST_SECTION_BACKUP_TABLESPACE, $strPathKey)) - # { - # confess &log(ERROR, "cannot remap invalid tablespace ${strPathKey} to ${strRemapPath}"); - # } - # - # # Remap the tablespace in the manifest - # &log(INFO, "remapping tablespace to ${strRemapPath}"); - # - # my $strTablespaceLink = $oManifest->get(MANIFEST_SECTION_BACKUP_TABLESPACE, $strPathKey, MANIFEST_SUBKEY_LINK); - # - # $oManifest->set(MANIFEST_SECTION_BACKUP_PATH, "tablespace:${strPathKey}", undef, $strRemapPath); - # $oManifest->set(MANIFEST_SECTION_BACKUP_TABLESPACE, $strPathKey, MANIFEST_SUBKEY_PATH, $strRemapPath); - # $oManifest->set('base:link', "pg_tblspc/${strTablespaceLink}", MANIFEST_SUBKEY_DESTINATION, $strRemapPath); - # } - # } - # } + # If tablespaces have been remapped, update the manifest + if (defined($self->{oRemapRef})) + { + foreach my $strPathKey (sort(keys $self->{oRemapRef})) + { + my $strRemapPath = ${$self->{oRemapRef}}{$strPathKey}; + + # Make sure that the tablespace exists in the manifest + if (!$oManifest->test(MANIFEST_SECTION_BACKUP_TABLESPACE, $strPathKey)) + { + confess &log(ERROR, "cannot remap invalid tablespace ${strPathKey} to ${strRemapPath}"); + } + + # Remap the tablespace in the manifest + &log(INFO, "remapping tablespace ${strPathKey} to ${strRemapPath}"); + + my $strTablespaceLink = $oManifest->get(MANIFEST_SECTION_BACKUP_TABLESPACE, $strPathKey, MANIFEST_SUBKEY_LINK); + + $oManifest->set(MANIFEST_SECTION_BACKUP_PATH, "tablespace:${strPathKey}", undef, $strRemapPath); + $oManifest->set(MANIFEST_SECTION_BACKUP_TABLESPACE, $strPathKey, MANIFEST_SUBKEY_PATH, $strRemapPath); + $oManifest->set('base:link', "pg_tblspc/${strTablespaceLink}", MANIFEST_SUBKEY_DESTINATION, $strRemapPath); + } + } $self->manifest_ownership_check($oManifest); @@ -411,15 +396,18 @@ sub build # Create all links in the manifest that do not already exist $strSection = "${strSectionPathKey}:link"; - foreach my $strName ($oManifest->keys($strSection)) + if ($oManifest->test($strSection)) { - my $strLink = "${strSectionPath}/${strName}"; - - if (!$self->{oFile}->exists(PATH_DB_ABSOLUTE, $strLink)) + foreach my $strName ($oManifest->keys($strSection)) { - $self->{oFile}->link_create(PATH_DB_ABSOLUTE, - $oManifest->get($strSection, $strName, MANIFEST_SUBKEY_DESTINATION), - PATH_DB_ABSOLUTE, $strLink); + my $strLink = "${strSectionPath}/${strName}"; + + if (!$self->{oFile}->exists(PATH_DB_ABSOLUTE, $strLink)) + { + $self->{oFile}->link_create(PATH_DB_ABSOLUTE, + $oManifest->get($strSection, $strName, MANIFEST_SUBKEY_DESTINATION), + PATH_DB_ABSOLUTE, $strLink); + } } } } diff --git a/test/lib/BackRestTest/BackupTest.pm b/test/lib/BackRestTest/BackupTest.pm index 530a18ff0..2aa1ad789 100755 --- a/test/lib/BackRestTest/BackupTest.pm +++ b/test/lib/BackRestTest/BackupTest.pm @@ -184,12 +184,14 @@ sub BackRestTestBackup_Create # Create the db paths BackRestTestCommon_PathCreate(BackRestTestCommon_DbPathGet()); BackRestTestCommon_PathCreate(BackRestTestCommon_DbCommonPathGet()); - BackRestTestCommon_PathCreate(BackRestTestCommon_DbCommonPathGet() . '2'); + BackRestTestCommon_PathCreate(BackRestTestCommon_DbCommonPathGet(2)); # Create tablespace paths BackRestTestCommon_PathCreate(BackRestTestCommon_DbTablespacePathGet()); - BackRestTestCommon_PathCreate(BackRestTestCommon_DbTablespacePathGet() . '/ts1'); - BackRestTestCommon_PathCreate(BackRestTestCommon_DbTablespacePathGet() . '/ts2'); + BackRestTestCommon_PathCreate(BackRestTestCommon_DbTablespacePathGet(1)); + BackRestTestCommon_PathCreate(BackRestTestCommon_DbTablespacePathGet(1, 2)); + BackRestTestCommon_PathCreate(BackRestTestCommon_DbTablespacePathGet(2)); + BackRestTestCommon_PathCreate(BackRestTestCommon_DbTablespacePathGet(2, 2)); # Create the archive directory BackRestTestCommon_PathCreate(BackRestTestCommon_ArchivePathGet()); @@ -319,13 +321,13 @@ sub BackRestTestBackup_ManifestTablespaceCreate my $strMode = shift; # Create final file location - my $strPath = BackRestTestCommon_DbPathGet() . "/ts${iOid}"; + my $strPath = BackRestTestCommon_DbTablespacePathGet($iOid); # Create the path - if (!(-e $strPath)) - { - BackRestTestCommon_PathCreate($strPath, $strMode); - } + # if (!(-e $strPath)) + # { + # BackRestTestCommon_PathCreate($strPath, $strMode); + # } # Stat the path my $oStat = lstat($strPath); @@ -1181,14 +1183,6 @@ sub BackRestTestBackup_Test # Create tablespace path BackRestTestBackup_ManifestPathCreate(\%oManifest, 'base', 'pg_tblspc'); - # for (my $iTablespaceIdx = 1; $iTablespaceIdx <= $iTablespaceTotal; $iTablespaceIdx++) - # { - # BackRestTestBackup_ManifestTablespaceCreate(\%oManifest, $iTablespaceIdx); - # - # BackRestTestBackup_ManifestFileCreate(\%oManifest, "tablespace:${iTablespaceIdx}", 'tablespace1.txt', 'TBLSPC', - # $bChecksum ? '44ad0bf042936c576c75891d0e5ded8e2b60fb54' : undef, $lTime); - # } - # Create db config BackRestTestCommon_ConfigCreate('db', # local $bRemote ? BACKUP : undef, # remote @@ -1263,25 +1257,19 @@ sub BackRestTestBackup_Test $strType = 'incr'; BackRestTestBackup_ManifestReference(\%oManifest, $strFullBackup); - # Actually do add tablespace here + # Add tablespace 1 + BackRestTestBackup_ManifestTablespaceCreate(\%oManifest, 1); + + BackRestTestBackup_ManifestFileCreate(\%oManifest, "tablespace:1", 'tablespace1.txt', 'TBLSPC', + $bChecksum ? '44ad0bf042936c576c75891d0e5ded8e2b60fb54' : undef, $lTime); + my $strBackup = BackRestTestBackup_Backup($strType, $strStanza, $bRemote, $oFile, \%oManifest, 'add tablespace 1'); - # Restore - - #----------------------------------------------------------------------------------------------------------------------- - $bDelta = false; - - # Remap the base path - my %oRemapHash; - $oRemapHash{base} = BackRestTestCommon_DbCommonPathGet . '2'; - - BackRestTestBackup_Restore($oFile, $strFullBackup, $strStanza, \%oManifest, \%oRemapHash, $bDelta, $bForce, - 'remap base path'); - # Resume Incr Backup #----------------------------------------------------------------------------------------------------------------------- $strType = 'incr'; - BackRestTestBackup_ManifestReference(\%oManifest, $strFullBackup); +# BackRestTestBackup_ManifestReference(\%oManifest, $strFullBackup); $strTmpPath = BackRestTestCommon_BackupPathGet() . "/temp/${strStanza}.tmp"; @@ -1294,7 +1282,7 @@ sub BackRestTestBackup_Test # Resume Diff Backup #----------------------------------------------------------------------------------------------------------------------- $strType = 'diff'; - BackRestTestBackup_ManifestReference(\%oManifest, $strFullBackup); +# BackRestTestBackup_ManifestReference(\%oManifest, $strFullBackup); $strTmpPath = BackRestTestCommon_BackupPathGet() . "/temp/${strStanza}.tmp"; @@ -1304,6 +1292,18 @@ sub BackRestTestBackup_Test $strBackup = BackRestTestBackup_Backup($strType, $strStanza, $bRemote, $oFile, \%oManifest, 'resume - fail', TEST_BACKUP_NORESUME); + # Restore - + #----------------------------------------------------------------------------------------------------------------------- + $bDelta = false; + + # Remap the base path + my %oRemapHash; + $oRemapHash{base} = BackRestTestCommon_DbCommonPathGet(2); +# $oRemapHash{1} = BackRestTestCommon_DbTablespacePathGet(1, 2); + + BackRestTestBackup_Restore($oFile, $strFullBackup, $strStanza, \%oManifest, \%oRemapHash, $bDelta, $bForce, + 'remap base path'); + # Incr Backup #----------------------------------------------------------------------------------------------------------------------- $strType = 'incr'; diff --git a/test/lib/BackRestTest/CommonTest.pm b/test/lib/BackRestTest/CommonTest.pm index ae57d7595..56f4e3720 100755 --- a/test/lib/BackRestTest/CommonTest.pm +++ b/test/lib/BackRestTest/CommonTest.pm @@ -23,6 +23,7 @@ use lib dirname($0) . '/../lib'; use BackRest::Utility; use BackRest::Remote; use BackRest::File; +use BackRest::Manifest; use Exporter qw(import); our @EXPORT = qw(BackRestTestCommon_Setup BackRestTestCommon_ExecuteBegin BackRestTestCommon_ExecuteEnd @@ -431,14 +432,20 @@ sub BackRestTestCommon_ConfigRemap foreach my $strRemap (sort(keys $oRemapHashRef)) { + my $strRemapPath = ${$oRemapHashRef}{$strRemap}; + if ($strRemap eq 'base') { - $oConfig{$strStanza}{path} = ${$oRemapHashRef}{$strRemap}; - ${$oManifestRef}{'backup:path'}{base} = ${$oRemapHashRef}{$strRemap}; + $oConfig{$strStanza}{path} = $strRemapPath; + ${$oManifestRef}{'backup:path'}{base} = $strRemapPath; } else { - confess " not coded yet"; + $oConfig{"${strStanza}:tablespace:map"}{$strRemap} = $strRemapPath; + + ${$oManifestRef}{'backup:path'}{"tablespace:${strRemap}"} = $strRemapPath; + ${$oManifestRef}{'backup:tablespace'}{$strRemap}{'path'} = $strRemapPath; + ${$oManifestRef}{'base:link'}{"pg_tblspc/${strRemap}"}{'link_destination'} = $strRemapPath; } } @@ -630,12 +637,17 @@ sub BackRestTestCommon_DbPathGet sub BackRestTestCommon_DbCommonPathGet { - return $strCommonDbCommonPath; + my $iIndex = shift; + + return $strCommonDbCommonPath . (defined($iIndex) ? "-${iIndex}" : ''); } sub BackRestTestCommon_DbTablespacePathGet { - return $strCommonDbTablespacePath; + my $iTablespace = shift; + my $iIndex = shift; + + return $strCommonDbTablespacePath . (defined($iTablespace) ? "/ts${iTablespace}" . (defined($iIndex) ? "-${iIndex}" : '') : ''); } sub BackRestTestCommon_DbPortGet