Speed improvement for large object restore.

Mario Weilguni

src/bin/pg_dump/pg_backup_archiver.c

@@ -15,7 +15,7 @@
  *
  *
  * IDENTIFICATION
- *		$Header: /cvsroot/pgsql/src/bin/pg_dump/pg_backup_archiver.c,v 1.42 2002/02/11 00:18:20 tgl Exp $
+ *		$Header: /cvsroot/pgsql/src/bin/pg_dump/pg_backup_archiver.c,v 1.43 2002/04/24 02:21:04 momjian Exp $
  *
  * Modifications - 28-Jun-2000 - pjw@rhyme.com.au
  *
@@ -819,6 +819,9 @@ StartRestoreBlob(ArchiveHandle *AH, Oid oid)
 		AH->createdBlobXref = 1;
 	}
 
+	/* Initialize the LO Buffer */
+	AH->lo_buf_used = 0;
+
 	/*
 	 * Start long-running TXs if necessary
 	 */
@@ -848,6 +851,19 @@ StartRestoreBlob(ArchiveHandle *AH, Oid oid)
 void
 EndRestoreBlob(ArchiveHandle *AH, Oid oid)
 {
+        if(AH->lo_buf_used > 0) {
+	  /* Write remaining bytes from the LO buffer */
+  	  int res;
+          res = lo_write(AH->connection, AH->loFd, (void *) AH->lo_buf, AH->lo_buf_used);
+
+	  ahlog(AH, 5, "wrote remaining %d bytes of large object data (result = %d)\n",
+	  	  (int)AH->lo_buf_used, res);
+	  if (res != AH->lo_buf_used)
+		die_horribly(AH, modulename, "could not write to large object (result: %d, expected: %d)\n",
+			 res, AH->lo_buf_used);
+          AH->lo_buf_used = 0;
+        }
+
 	lo_close(AH->connection, AH->loFd);
 	AH->writingBlob = 0;
 
@@ -1228,14 +1244,27 @@ ahwrite(const void *ptr, size_t size, size_t nmemb, ArchiveHandle *AH)
 
 	if (AH->writingBlob)
 	{
-		res = lo_write(AH->connection, AH->loFd, (void *) ptr, size * nmemb);
+	        if(AH->lo_buf_used + size * nmemb > AH->lo_buf_size) {
-		ahlog(AH, 5, "wrote %d bytes of large object data (result = %d)\n",
+		  /* Split LO buffer */
-			  (int) (size * nmemb), res);
+		  int remaining = AH->lo_buf_size - AH->lo_buf_used;
-		if (res != size * nmemb)
+		  int slack = nmemb * size - remaining;
-			die_horribly(AH, modulename, "could not write to large object (result: %d, expected: %d)\n",
-						 res, (int) (size * nmemb));
 
-		return res;
+		  memcpy(AH->lo_buf + AH->lo_buf_used, ptr, remaining);
+		  res = lo_write(AH->connection, AH->loFd, AH->lo_buf, AH->lo_buf_size);
+		  ahlog(AH, 5, "wrote %d bytes of large object data (result = %d)\n",
+		  	        AH->lo_buf_size, res);
+		  if (res != AH->lo_buf_size)
+			die_horribly(AH, modulename, "could not write to large object (result: %d, expected: %d)\n",
+						 res, AH->lo_buf_size);
+	          memcpy(AH->lo_buf, ptr + remaining, slack);
+		  AH->lo_buf_used = slack;
+	       } else {
+	         /* LO Buffer is still large enough, buffer it */
+		 memcpy(AH->lo_buf + AH->lo_buf_used, ptr, size * nmemb);
+		 AH->lo_buf_used += size * nmemb;
+	       }
+
+	       return size * nmemb;
 	}
 	else if (AH->gzOut)
 	{

src/bin/pg_dump/pg_backup_archiver.h

@@ -17,7 +17,7 @@
  *
  *
  * IDENTIFICATION
- *		$Header: /cvsroot/pgsql/src/bin/pg_dump/pg_backup_archiver.h,v 1.41 2001/11/05 17:46:30 momjian Exp $
+ *		$Header: /cvsroot/pgsql/src/bin/pg_dump/pg_backup_archiver.h,v 1.42 2002/04/24 02:21:04 momjian Exp $
  *
  * Modifications - 28-Jun-2000 - pjw@rhyme.com.au
  *	-	Initial version.
@@ -41,6 +41,7 @@
 #include <errno.h>
 
 #include "pqexpbuffer.h"
+#define LOBBUFSIZE 32768
 
 #ifdef HAVE_LIBZ
 #include <zlib.h>
@@ -240,6 +241,9 @@ typedef struct _archiveHandle
 
 	RestoreOptions *ropt;		/* Used to check restore options in
 								 * ahwrite etc */
+	void                    *lo_buf;
+	int                     lo_buf_used;
+	int                     lo_buf_size;
 } ArchiveHandle;
 
 typedef struct _tocEntry

src/bin/pg_dump/pg_backup_custom.c

@@ -19,7 +19,7 @@
  *
  *
  * IDENTIFICATION
- *		$Header: /cvsroot/pgsql/src/bin/pg_dump/pg_backup_custom.c,v 1.17 2001/11/27 23:48:12 tgl Exp $
+ *		$Header: /cvsroot/pgsql/src/bin/pg_dump/pg_backup_custom.c,v 1.18 2002/04/24 02:21:04 momjian Exp $
  *
  * Modifications - 28-Jun-2000 - pjw@rhyme.com.au
  *
@@ -153,6 +153,12 @@ InitArchiveFmt_Custom(ArchiveHandle *AH)
 	if (ctx->zp == NULL)
 		die_horribly(AH, modulename, "out of memory\n");
 
+	/* Initialize LO buffering */
+	AH->lo_buf_size = LOBBUFSIZE;
+	AH->lo_buf = (void *)malloc(LOBBUFSIZE);
+	if(AH->lo_buf == NULL)
+                die_horribly(AH, modulename, "out of memory\n");
+
 	/*
 	 * zlibOutSize is the buffer size we tell zlib it can output to.  We
 	 * actually allocate one extra byte because some routines want to

src/bin/pg_dump/pg_backup_files.c

@@ -20,7 +20,7 @@
  *
  *
  * IDENTIFICATION
- *		$Header: /cvsroot/pgsql/src/bin/pg_dump/pg_backup_files.c,v 1.14 2001/10/25 05:49:52 momjian Exp $
+ *		$Header: /cvsroot/pgsql/src/bin/pg_dump/pg_backup_files.c,v 1.15 2002/04/24 02:21:04 momjian Exp $
  *
  * Modifications - 28-Jun-2000 - pjw@rhyme.com.au
  *
@@ -113,6 +113,12 @@ InitArchiveFmt_Files(ArchiveHandle *AH)
 	AH->formatData = (void *) ctx;
 	ctx->filePos = 0;
 
+	/* Initialize LO buffering */
+	AH->lo_buf_size = LOBBUFSIZE;
+	AH->lo_buf = (void *)malloc(LOBBUFSIZE);
+	if(AH->lo_buf == NULL)
+                die_horribly(AH, modulename, "out of memory\n");
+
 	/*
 	 * Now open the TOC file
 	 */

src/bin/pg_dump/pg_backup_null.c

@@ -17,7 +17,7 @@
  *
  *
  * IDENTIFICATION
- *		$Header: /cvsroot/pgsql/src/bin/pg_dump/pg_backup_null.c,v 1.7 2001/06/27 21:21:37 petere Exp $
+ *		$Header: /cvsroot/pgsql/src/bin/pg_dump/pg_backup_null.c,v 1.8 2002/04/24 02:21:04 momjian Exp $
  *
  * Modifications - 09-Jul-2000 - pjw@rhyme.com.au
  *
@@ -64,7 +64,6 @@ InitArchiveFmt_Null(ArchiveHandle *AH)
 	 */
 	if (AH->mode == archModeRead)
 		die_horribly(AH, NULL, "this format cannot be read\n");
-
 }
 
 /*

src/bin/pg_dump/pg_backup_tar.c

@@ -16,7 +16,7 @@
  *
  *
  * IDENTIFICATION
- *		$Header: /cvsroot/pgsql/src/bin/pg_dump/pg_backup_tar.c,v 1.20 2001/10/28 06:25:58 momjian Exp $
+ *		$Header: /cvsroot/pgsql/src/bin/pg_dump/pg_backup_tar.c,v 1.21 2002/04/24 02:21:04 momjian Exp $
  *
  * Modifications - 28-Jun-2000 - pjw@rhyme.com.au
  *
@@ -157,6 +157,12 @@ InitArchiveFmt_Tar(ArchiveHandle *AH)
 	ctx = (lclContext *) malloc(sizeof(lclContext));
 	AH->formatData = (void *) ctx;
 	ctx->filePos = 0;
+	
+	/* Initialize LO buffering */
+	AH->lo_buf_size = LOBBUFSIZE;
+	AH->lo_buf = (void *)malloc(LOBBUFSIZE);
+	if(AH->lo_buf == NULL)
+                die_horribly(AH, modulename, "out of memory\n");
 
 	/*
 	 * Now open the TOC file