database/postgres
1
0
Fork 0
mirror of https://github.com/postgres/postgres.git synced 2025-09-02 04:21:28 +03:00
Code Activity

Adopt the GNU convention for handling tar-archive members exceeding 8GB.

Browse Source 
The POSIX standard for tar headers requires archive member sizes to be
printed in octal with at most 11 digits, limiting the representable file
size to 8GB.  However, GNU tar and apparently most other modern tars
support a convention in which oversized values can be stored in base-256,
allowing any practical file to be a tar member.  Adopt this convention
to remove two limitations:
* pg_dump with -Ft output format failed if the contents of any one table
exceeded 8GB.
* pg_basebackup failed if the data directory contained any file exceeding
8GB.  (This would be a fatal problem for installations configured with a
table segment size of 8GB or more, and it has also been seen to fail when
large core dump files exist in the data directory.)

File sizes under 8GB are still printed in octal, so that no compatibility
issues are created except in cases that would have failed entirely before.

In addition, this patch fixes several bugs in the same area:

* In 9.3 and later, we'd defined tarCreateHeader's file-size argument as
size_t, which meant that on 32-bit machines it would write a corrupt tar
header for file sizes between 4GB and 8GB, even though no error was raised.
This broke both "pg_dump -Ft" and pg_basebackup for such cases.

* pg_restore from a tar archive would fail on tables of size between 4GB
and 8GB, on machines where either "size_t" or "unsigned long" is 32 bits.
This happened even with an archive file not affected by the previous bug.

* pg_basebackup would fail if there were files of size between 4GB and 8GB,
even on 64-bit machines.

* In 9.3 and later, "pg_basebackup -Ft" failed entirely, for any file size,
on 64-bit big-endian machines.

In view of these potential data-loss bugs, back-patch to all supported
branches, even though removal of the documented 8GB limit might otherwise
be considered a new feature rather than a bug fix.
This commit is contained in:
Tom Lane
2015-11-21 20:21:32 -05:00
parent c49279dfd5
commit b054ca0318
7 changed files with 267 additions and 293 deletions
Download Patch File Download Diff File Expand all files Collapse all files

149
src/bin/pg_dump/pg_backup_tar.c
View File

@@ -30,6 +30,7 @@
#include "pg_backup_tar.h"
#include "dumpmem.h"
#include "dumputils.h"
#include "pgtar.h"
#include <sys/stat.h>
#include <ctype.h>
@@ -75,13 +76,6 @@ typedef struct
ArchiveHandle *AH;
} TAR_MEMBER;
/*
* Maximum file size for a tar member: The limit inherent in the
* format is 2^33-1 bytes (nearly 8 GB). But we don't want to exceed
* what we can represent in pgoff_t.
*/
#define MAX_TAR_MEMBER_FILELEN (((int64) 1 << Min(33, sizeof(pgoff_t)*8 - 1)) - 1)
typedef struct
{
int hasSeek;
@@ -115,7 +109,6 @@ static char *tarGets(char *buf, size_t len, TAR_MEMBER *th);
static int tarPrintf(ArchiveHandle *AH, TAR_MEMBER *th, const char *fmt,...) __attribute__((format(PG_PRINTF_ATTRIBUTE, 3, 4)));
static void _tarAddFile(ArchiveHandle *AH, TAR_MEMBER *th);
static int _tarChecksum(char *th);
static TAR_MEMBER *_tarPositionTo(ArchiveHandle *AH, const char *filename);
static size_t tarRead(void *buf, size_t len, TAR_MEMBER *th);
static size_t tarWrite(const void *buf, size_t len, TAR_MEMBER *th);
@@ -1021,31 +1014,13 @@ tarPrintf(ArchiveHandle *AH, TAR_MEMBER *th, const char *fmt,...)
return cnt;
}
static int
_tarChecksum(char *header)
{
int i,
sum;
/*
* Per POSIX, the checksum is the simple sum of all bytes in the header,
* treating the bytes as unsigned, and treating the checksum field (at
* offset 148) as though it contained 8 spaces.
*/
sum = 8 * ' '; /* presumed value for checksum field */
for (i = 0; i < 512; i++)
if (i < 148 || i >= 156)
sum += 0xFF & header[i];
return sum;
}
bool
isValidTarHeader(char *header)
{
int sum;
int chk = _tarChecksum(header);
int chk = tarChecksum(header);
sscanf(&header[148], "%8o", &sum);
sum = read_tar_number(&header[148], 8);
if (sum != chk)
return false;
@@ -1084,13 +1059,6 @@ _tarAddFile(ArchiveHandle *AH, TAR_MEMBER *th)
th->fileLen = ftello(tmp);
fseeko(tmp, 0, SEEK_SET);
/*
* Some compilers will throw a warning knowing this test can never be true
* because pgoff_t can't exceed the compared maximum on their platform.
*/
if (th->fileLen > MAX_TAR_MEMBER_FILELEN)
exit_horribly(modulename, "archive member too large for tar format\n");
_tarWriteHeader(th);
while ((cnt = fread(buf, 1, sizeof(buf), tmp)) > 0)
@@ -1216,11 +1184,10 @@ _tarGetHeader(ArchiveHandle *AH, TAR_MEMBER *th)
{
lclContext *ctx = (lclContext *) AH->formatData;
char h[512];
char tag[100];
char tag[100 + 1];
int sum,
chk;
size_t len;
unsigned long ullen;
pgoff_t len;
pgoff_t hPos;
bool gotBlock = false;
@@ -1256,8 +1223,8 @@ _tarGetHeader(ArchiveHandle *AH, TAR_MEMBER *th)
(unsigned long) len);
/* Calc checksum */
chk = _tarChecksum(h);
sscanf(&h[148], "%8o", &sum);
chk = tarChecksum(h);
sum = read_tar_number(&h[148], 8);
/*
* If the checksum failed, see if it is a null block. If so, silently
@@ -1280,27 +1247,31 @@ _tarGetHeader(ArchiveHandle *AH, TAR_MEMBER *th)
}
}
sscanf(&h[0], "%99s", tag);
sscanf(&h[124], "%12lo", &ullen);
len = (size_t) ullen;
/* Name field is 100 bytes, might not be null-terminated */
strlcpy(tag, &h[0], 100 + 1);
len = read_tar_number(&h[124], 12);
{
char buf[100];
char posbuf[32];
char lenbuf[32];
snprintf(buf, sizeof(buf), INT64_FORMAT, (int64) hPos);
ahlog(AH, 3, "TOC Entry %s at %s (length %lu, checksum %d)\n",
tag, buf, (unsigned long) len, sum);
snprintf(posbuf, sizeof(posbuf), UINT64_FORMAT, (uint64) hPos);
snprintf(lenbuf, sizeof(lenbuf), UINT64_FORMAT, (uint64) len);
ahlog(AH, 3, "TOC Entry %s at %s (length %s, checksum %d)\n",
tag, posbuf, lenbuf, sum);
}
if (chk != sum)
{
char buf[100];
char posbuf[32];
snprintf(buf, sizeof(buf), INT64_FORMAT, (int64) ftello(ctx->tarFH));
snprintf(posbuf, sizeof(posbuf), UINT64_FORMAT,
(uint64) ftello(ctx->tarFH));
exit_horribly(modulename,
"corrupt tar header found in %s "
"(expected %d, computed %d) file position %s\n",
tag, sum, chk, buf);
tag, sum, chk, posbuf);
}
th->targetFile = pg_strdup(tag);
@@ -1310,86 +1281,16 @@ _tarGetHeader(ArchiveHandle *AH, TAR_MEMBER *th)
}
/*
* Utility routine to print possibly larger than 32 bit integers in a
* portable fashion. Filled with zeros.
*/
static void
print_val(char *s, uint64 val, unsigned int base, size_t len)
{
int i;
for (i = len; i > 0; i--)
{
int digit = val % base;
s[i - 1] = '0' + digit;
val = val / base;
}
}
static void
_tarWriteHeader(TAR_MEMBER *th)
{
char h[512];
int lastSum = 0;
int sum;
memset(h, 0, sizeof(h));
/* Name 100 */
sprintf(&h[0], "%.99s", th->targetFile);
/* Mode 8 */
sprintf(&h[100], "100600 ");
/* User ID 8 */
sprintf(&h[108], "004000 ");
/* Group 8 */
sprintf(&h[116], "002000 ");
/* File size 12 - 11 digits, 1 space, no NUL */
print_val(&h[124], th->fileLen, 8, 11);
sprintf(&h[135], " ");
/* Mod Time 12 */
sprintf(&h[136], "%011o ", (int) time(NULL));
/* Checksum 8 */
sprintf(&h[148], "%06o ", lastSum);
/* Type - regular file */
sprintf(&h[156], "0");
/* Link tag 100 (NULL) */
/* Magic 6 + Version 2 */
sprintf(&h[257], "ustar00");
#if 0
/* User 32 */
sprintf(&h[265], "%.31s", ""); /* How do I get username reliably? Do
* I need to? */
/* Group 32 */
sprintf(&h[297], "%.31s", ""); /* How do I get group reliably? Do I
* need to? */
/* Maj Dev 8 */
sprintf(&h[329], "%6o ", 0);
/* Min Dev 8 */
sprintf(&h[337], "%6o ", 0);
#endif
while ((sum = _tarChecksum(h)) != lastSum)
{
sprintf(&h[148], "%06o ", sum);
lastSum = sum;
}
tarCreateHeader(h, th->targetFile, NULL, th->fileLen,
0600, 04000, 02000, time(NULL),
true /* backwards compatible format */);
/* Now write the completed header. */
if (fwrite(h, 1, 512, th->tarFH) != 512)
exit_horribly(modulename, "could not write to output file: %s\n", strerror(errno));
}