database/postgres
1
0
Fork 0
mirror of https://github.com/postgres/postgres.git synced 2025-06-11 20:28:21 +03:00
Code Activity

Fix pg_dump on win32 to properly dump files larger than 2Gb when using

Browse Source 
binary dump formats.
This commit is contained in:
Magnus Hagander
2007-02-19 15:05:06 +00:00
parent bc959b7bd2
commit 74096ed1fd
6 changed files with 59 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
src/bin/pg_dump/pg_backup_archiver.c
View File

@ -15,7 +15,7 @@
* *
* *
* IDENTIFICATION * IDENTIFICATION
* $PostgreSQL: pgsql/src/bin/pg_dump/pg_backup_archiver.c,v 1.141 2007/02/01 19:10:28 momjian Exp $ * $PostgreSQL: pgsql/src/bin/pg_dump/pg_backup_archiver.c,v 1.142 2007/02/19 15:05:06 mha Exp $
* *
*------------------------------------------------------------------------- *-------------------------------------------------------------------------
*/ */
@ -1311,24 +1311,24 @@ TocIDRequired(ArchiveHandle *AH, DumpId id, RestoreOptions *ropt)
} }
size_t size_t
WriteOffset(ArchiveHandle *AH, off_t o, int wasSet) WriteOffset(ArchiveHandle *AH, pgoff_t o, int wasSet)
{ {
int off; int off;
/* Save the flag */ /* Save the flag */
(*AH->WriteBytePtr) (AH, wasSet); (*AH->WriteBytePtr) (AH, wasSet);
/* Write out off_t smallest byte first, prevents endian mismatch */ /* Write out pgoff_t smallest byte first, prevents endian mismatch */
for (off = 0; off < sizeof(off_t); off++) for (off = 0; off < sizeof(pgoff_t); off++)
{ {
(*AH->WriteBytePtr) (AH, o & 0xFF); (*AH->WriteBytePtr) (AH, o & 0xFF);
o >>= 8; o >>= 8;
} }
return sizeof(off_t) + 1; return sizeof(pgoff_t) + 1;
} }
int int
ReadOffset(ArchiveHandle *AH, off_t *o) ReadOffset(ArchiveHandle *AH, pgoff_t *o)
{ {
int i; int i;
int off; int off;
@ -1348,8 +1348,8 @@ ReadOffset(ArchiveHandle *AH, off_t *o)
else if (i == 0) else if (i == 0)
return K_OFFSET_NO_DATA; return K_OFFSET_NO_DATA;
/* Cast to off_t because it was written as an int. */ /* Cast to pgoff_t because it was written as an int. */
*o = (off_t) i; *o = (pgoff_t) i;
return K_OFFSET_POS_SET; return K_OFFSET_POS_SET;
} }
@ -1379,8 +1379,8 @@ ReadOffset(ArchiveHandle *AH, off_t *o)
*/ */
for (off = 0; off < AH->offSize; off++) for (off = 0; off < AH->offSize; off++)
{ {
if (off < sizeof(off_t)) if (off < sizeof(pgoff_t))
*o |= ((off_t) ((*AH->ReadBytePtr) (AH))) << (off * 8); *o |= ((pgoff_t) ((*AH->ReadBytePtr) (AH))) << (off * 8);
else else
{ {
if ((*AH->ReadBytePtr) (AH) != 0) if ((*AH->ReadBytePtr) (AH) != 0)
@ -1647,7 +1647,7 @@ _allocAH(const char *FileSpec, const ArchiveFormat fmt,
AH->createDate = time(NULL); AH->createDate = time(NULL);
AH->intSize = sizeof(int); AH->intSize = sizeof(int);
AH->offSize = sizeof(off_t); AH->offSize = sizeof(pgoff_t);
if (FileSpec) if (FileSpec)
{ {
AH->fSpec = strdup(FileSpec); AH->fSpec = strdup(FileSpec);
@ -2768,11 +2768,11 @@ checkSeek(FILE *fp)
if (fseeko(fp, 0, SEEK_CUR) != 0) if (fseeko(fp, 0, SEEK_CUR) != 0)
return false; return false;
else if (sizeof(off_t) > sizeof(long)) else if (sizeof(pgoff_t) > sizeof(long))
/* /*
* At this point, off_t is too large for long, so we return based on * At this point, pgoff_t is too large for long, so we return based on
* whether an off_t version of fseek is available. * whether an pgoff_t version of fseek is available.
*/ */
#ifdef HAVE_FSEEKO #ifdef HAVE_FSEEKO
return true; return true;

8
src/bin/pg_dump/pg_backup_archiver.h
View File

@ -17,7 +17,7 @@
* *
* *
* IDENTIFICATION * IDENTIFICATION
* $PostgreSQL: pgsql/src/bin/pg_dump/pg_backup_archiver.h,v 1.74 2007/01/25 03:30:43 momjian Exp $ * $PostgreSQL: pgsql/src/bin/pg_dump/pg_backup_archiver.h,v 1.75 2007/02/19 15:05:06 mha Exp $
* *
*------------------------------------------------------------------------- *-------------------------------------------------------------------------
*/ */
@ -199,7 +199,7 @@ typedef struct _archiveHandle
* format */ * format */
size_t lookaheadSize; /* Size of allocated buffer */ size_t lookaheadSize; /* Size of allocated buffer */
size_t lookaheadLen; /* Length of data in lookahead */ size_t lookaheadLen; /* Length of data in lookahead */
off_t lookaheadPos; /* Current read position in lookahead buffer */ pgoff_t lookaheadPos; /* Current read position in lookahead buffer */
ArchiveEntryPtr ArchiveEntryPtr; /* Called for each metadata object */ ArchiveEntryPtr ArchiveEntryPtr; /* Called for each metadata object */
StartDataPtr StartDataPtr; /* Called when table data is about to be StartDataPtr StartDataPtr; /* Called when table data is about to be
@ -332,8 +332,8 @@ extern int ReadInt(ArchiveHandle *AH);
extern char *ReadStr(ArchiveHandle *AH); extern char *ReadStr(ArchiveHandle *AH);
extern size_t WriteStr(ArchiveHandle *AH, const char *s); extern size_t WriteStr(ArchiveHandle *AH, const char *s);
int ReadOffset(ArchiveHandle *, off_t *); int ReadOffset(ArchiveHandle *, pgoff_t *);
size_t WriteOffset(ArchiveHandle *, off_t, int); size_t WriteOffset(ArchiveHandle *, pgoff_t, int);
extern void StartRestoreBlobs(ArchiveHandle *AH); extern void StartRestoreBlobs(ArchiveHandle *AH);
extern void StartRestoreBlob(ArchiveHandle *AH, Oid oid); extern void StartRestoreBlob(ArchiveHandle *AH, Oid oid);

18
src/bin/pg_dump/pg_backup_custom.c
View File

@ -19,7 +19,7 @@
* *
* *
* IDENTIFICATION * IDENTIFICATION
* $PostgreSQL: pgsql/src/bin/pg_dump/pg_backup_custom.c,v 1.36 2006/10/04 00:30:05 momjian Exp $ * $PostgreSQL: pgsql/src/bin/pg_dump/pg_backup_custom.c,v 1.37 2007/02/19 15:05:06 mha Exp $
* *
*------------------------------------------------------------------------- *-------------------------------------------------------------------------
*/ */
@ -70,14 +70,14 @@ typedef struct
char *zlibIn; char *zlibIn;
size_t inSize; size_t inSize;
int hasSeek; int hasSeek;
off_t filePos; pgoff_t filePos;
off_t dataStart; pgoff_t dataStart;
} lclContext; } lclContext;
typedef struct typedef struct
{ {
int dataState; int dataState;
off_t dataPos; pgoff_t dataPos;
} lclTocEntry; } lclTocEntry;
@ -88,7 +88,7 @@ typedef struct
static void _readBlockHeader(ArchiveHandle *AH, int *type, int *id); static void _readBlockHeader(ArchiveHandle *AH, int *type, int *id);
static void _StartDataCompressor(ArchiveHandle *AH, TocEntry *te); static void _StartDataCompressor(ArchiveHandle *AH, TocEntry *te);
static void _EndDataCompressor(ArchiveHandle *AH, TocEntry *te); static void _EndDataCompressor(ArchiveHandle *AH, TocEntry *te);
static off_t _getFilePos(ArchiveHandle *AH, lclContext *ctx); static pgoff_t _getFilePos(ArchiveHandle *AH, lclContext *ctx);
static int _DoDeflate(ArchiveHandle *AH, lclContext *ctx, int flush); static int _DoDeflate(ArchiveHandle *AH, lclContext *ctx, int flush);
static char *modulename = gettext_noop("custom archiver"); static char *modulename = gettext_noop("custom archiver");
@ -791,7 +791,7 @@ static void
_CloseArchive(ArchiveHandle *AH) _CloseArchive(ArchiveHandle *AH)
{ {
lclContext *ctx = (lclContext *) AH->formatData; lclContext *ctx = (lclContext *) AH->formatData;
off_t tpos; pgoff_t tpos;
if (AH->mode == archModeWrite) if (AH->mode == archModeWrite)
{ {
@ -827,10 +827,10 @@ _CloseArchive(ArchiveHandle *AH)
/* /*
* Get the current position in the archive file. * Get the current position in the archive file.
*/ */
static off_t static pgoff_t
_getFilePos(ArchiveHandle *AH, lclContext *ctx) _getFilePos(ArchiveHandle *AH, lclContext *ctx)
{ {
off_t pos; pgoff_t pos;
if (ctx->hasSeek) if (ctx->hasSeek)
{ {
@ -841,7 +841,7 @@ _getFilePos(ArchiveHandle *AH, lclContext *ctx)
/* /*
* Prior to 1.7 (pg7.3) we relied on the internally maintained * Prior to 1.7 (pg7.3) we relied on the internally maintained
* pointer. Now we rely on off_t always. pos = ctx->filePos; * pointer. Now we rely on pgoff_t always. pos = ctx->filePos;
*/ */
} }
} }

4
src/bin/pg_dump/pg_backup_files.c
View File

@ -20,7 +20,7 @@
* *
* *
* IDENTIFICATION * IDENTIFICATION
* $PostgreSQL: pgsql/src/bin/pg_dump/pg_backup_files.c,v 1.30 2007/02/08 11:10:27 petere Exp $ * $PostgreSQL: pgsql/src/bin/pg_dump/pg_backup_files.c,v 1.31 2007/02/19 15:05:06 mha Exp $
* *
*------------------------------------------------------------------------- *-------------------------------------------------------------------------
*/ */
@ -51,7 +51,7 @@ static void _EndBlobs(ArchiveHandle *AH, TocEntry *te);
typedef struct typedef struct
{ {
int hasSeek; int hasSeek;
off_t filePos; pgoff_t filePos;
FILE *blobToc; FILE *blobToc;
} lclContext; } lclContext;

22
src/bin/pg_dump/pg_backup_tar.c
View File

@ -16,7 +16,7 @@
* *
* *
* IDENTIFICATION * IDENTIFICATION
* $PostgreSQL: pgsql/src/bin/pg_dump/pg_backup_tar.c,v 1.56 2006/11/01 15:59:26 tgl Exp $ * $PostgreSQL: pgsql/src/bin/pg_dump/pg_backup_tar.c,v 1.57 2007/02/19 15:05:06 mha Exp $
* *
*------------------------------------------------------------------------- *-------------------------------------------------------------------------
*/ */
@ -67,30 +67,30 @@ typedef struct
FILE *tmpFH; FILE *tmpFH;
char *targetFile; char *targetFile;
char mode; char mode;
off_t pos; pgoff_t pos;
off_t fileLen; pgoff_t fileLen;
ArchiveHandle *AH; ArchiveHandle *AH;
} TAR_MEMBER; } TAR_MEMBER;
/* /*
* Maximum file size for a tar member: The limit inherent in the * Maximum file size for a tar member: The limit inherent in the
* format is 2^33-1 bytes (nearly 8 GB). But we don't want to exceed * format is 2^33-1 bytes (nearly 8 GB). But we don't want to exceed
* what we can represent by an off_t. * what we can represent by an pgoff_t.
*/ */
#ifdef INT64_IS_BUSTED #ifdef INT64_IS_BUSTED
#define MAX_TAR_MEMBER_FILELEN INT_MAX #define MAX_TAR_MEMBER_FILELEN INT_MAX
#else #else
#define MAX_TAR_MEMBER_FILELEN (((int64) 1 << Min(33, sizeof(off_t)*8 - 1)) - 1) #define MAX_TAR_MEMBER_FILELEN (((int64) 1 << Min(33, sizeof(pgoff_t)*8 - 1)) - 1)
#endif #endif
typedef struct typedef struct
{ {
int hasSeek; int hasSeek;
off_t filePos; pgoff_t filePos;
TAR_MEMBER *blobToc; TAR_MEMBER *blobToc;
FILE *tarFH; FILE *tarFH;
off_t tarFHpos; pgoff_t tarFHpos;
off_t tarNextMember; pgoff_t tarNextMember;
TAR_MEMBER *FH; TAR_MEMBER *FH;
int isSpecialScript; int isSpecialScript;
TAR_MEMBER *scriptTH; TAR_MEMBER *scriptTH;
@ -1048,7 +1048,7 @@ _tarAddFile(ArchiveHandle *AH, TAR_MEMBER *th)
FILE *tmp = th->tmpFH; /* Grab it for convenience */ FILE *tmp = th->tmpFH; /* Grab it for convenience */
char buf[32768]; char buf[32768];
size_t cnt; size_t cnt;
off_t len = 0; pgoff_t len = 0;
size_t res; size_t res;
size_t i, size_t i,
pad; pad;
@ -1061,7 +1061,7 @@ _tarAddFile(ArchiveHandle *AH, TAR_MEMBER *th)
/* /*
* Some compilers with throw a warning knowing this test can never be true * Some compilers with throw a warning knowing this test can never be true
* because off_t can't exceed the compared maximum. * because pgoff_t can't exceed the compared maximum.
*/ */
if (th->fileLen > MAX_TAR_MEMBER_FILELEN) if (th->fileLen > MAX_TAR_MEMBER_FILELEN)
die_horribly(AH, modulename, "archive member too large for tar format\n"); die_horribly(AH, modulename, "archive member too large for tar format\n");
@ -1197,7 +1197,7 @@ _tarGetHeader(ArchiveHandle *AH, TAR_MEMBER *th)
chk; chk;
size_t len; size_t len;
unsigned long ullen; unsigned long ullen;
off_t hPos; pgoff_t hPos;
bool gotBlock = false; bool gotBlock = false;
while (!gotBlock) while (!gotBlock)

20
src/bin/pg_dump/pg_dump.h
View File

@ -6,7 +6,7 @@
* Portions Copyright (c) 1996-2007, PostgreSQL Global Development Group * Portions Copyright (c) 1996-2007, PostgreSQL Global Development Group
* Portions Copyright (c) 1994, Regents of the University of California * Portions Copyright (c) 1994, Regents of the University of California
* *
* $PostgreSQL: pgsql/src/bin/pg_dump/pg_dump.h,v 1.132 2007/01/23 17:54:50 tgl Exp $ * $PostgreSQL: pgsql/src/bin/pg_dump/pg_dump.h,v 1.133 2007/02/19 15:05:06 mha Exp $
* *
*------------------------------------------------------------------------- *-------------------------------------------------------------------------
*/ */
@ -16,6 +16,24 @@
#include "postgres_fe.h" #include "postgres_fe.h"
/*
* WIN32 does not provide 64-bit off_t, but does provide the functions operating
* with 64-bit offsets.
*/
#ifdef WIN32
#define pgoff_t __int64
#undef fseeko
#undef ftello
#ifdef WIN32_ONLY_COMPILER
#define fseeko(stream, offset, origin) _fseeki64(stream, offset, origin)
#define ftello(stream) _ftelli64(stream)
#else
#define fseeko(stream, offset, origin) fseeko64(stream, offset, origin)
#define ftello(stream) ftello64(stream)
#endif
#else
#define pgoff_t off_t
#endif
/* /*
* pg_dump uses two different mechanisms for identifying database objects: * pg_dump uses two different mechanisms for identifying database objects: