Avoid loading large intkey rows when VACUUMing, even if the page-size is changing.

FossilOrigin-Name: 0d2c3776065dc94119899ae4164995193b82fca7ac31868f3141b729d0b65ab9

manifest

@@ -1,5 +1,5 @@
-C Experimental\schanges\sto\svacuum\sto\savoid\sloading\slarge\srecords\sentirely\sinto\smemory.\sCurrently\sonly\sworks\sin\slimited\scases\sonly\s-\sfor\srowid\stables\swhen\sthe\spage-size\sdoes\snot\schange.
+C Avoid\sloading\slarge\sintkey\srows\swhen\sVACUUMing,\seven\sif\sthe\spage-size\sis\schanging.
-D 2020-12-08T20:19:07.385
+D 2020-12-09T16:32:11.593
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -481,7 +481,7 @@ F src/auth.c a3d5bfdba83d25abed1013a8c7a5f204e2e29b0c25242a56bc02bb0c07bf1e06
 F src/backup.c 3014889fa06e20e6adfa0d07b60097eec1f6e5b06671625f476a714d2356513d
 F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33
 F src/btmutex.c 8acc2f464ee76324bf13310df5692a262b801808984c1b79defb2503bbafadb6
-F src/btree.c 6d5cfd0e411e1db8042c94593a4847d85fc5398fa4860689d6f0d74cefa4e534
+F src/btree.c 2dd52a7d6f473ef345ac028f49f3cc14e1207a3596b2d9fd92fe532e57da6328
 F src/btree.h cff4cd182eb0d97802d82f398e4c8447f01bd2f5e501f70386b1097c910091cb
 F src/btreeInt.h ffd66480520d9d70222171b3a026d78b80833b5cea49c89867949f3e023d5f43
 F src/build.c f6449d4e85e998e14d3f537e8ea898dca2fcb83c277db3e60945af9b9177db81
@@ -1650,10 +1650,11 @@ F test/userauth01.test e740a2697a7b40d7c5003a7d7edaee16acd349a9
 F test/utf16align.test 54cd35a27c005a9b6e7815d887718780b6a462ae
 F test/vacuum-into.test 48f4cec354fb6f27c98ef58d2fe49a11b71ff131af0cd9140efacc9858b9f670
 F test/vacuum.test ce91c39f7f91a4273bf620efad21086b5aa6ef1d
-F test/vacuum2.test aa048abee196c16c9ba308465494009057b79f9b
+F test/vacuum2.test 9fd45ce6ce29f5614c249e03938d3567c06a9e772d4f155949f8eafe2d8af520
 F test/vacuum3.test 77ecdd54592b45a0bcb133339f99f1ae0ae94d0d
 F test/vacuum4.test 7ea76b769fffeb41f925303b04cbcf5a5bbeabe55e4c60ae754ff24eeeb7c010
 F test/vacuum5.test 263b144d537e92ad8e9ca8a73cc6e1583f41cfd0dda9432b87f7806174a2f48c
+F test/vacuum6.test 7d3aea27ef3c671dafccbad5adc7848205c38ab14ee7a1dabe592e0794e132d7
 F test/vacuummem.test 7b42abb3208bd82dd23a7536588396f295a314f2
 F test/varint.test bbce22cda8fc4d135bcc2b589574be8410614e62
 F test/veryquick.test 57ab846bacf7b90cf4e9a672721ea5c5b669b661
@@ -1888,10 +1889,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 4b286129138d44e6f8e9b3450289941e01d20fdfb9d0b5d846031425e8ca6b49
+P c90e063ca9ddcdd1e9f1a2e25a3f7d6e7ee798373ad8acf65b90536b0a124c0d
-R ebe894c92d077c4bc1f846403c247ebe
+R 306b82fcb1780d3a4b54ba51609ce03f
-T *branch * vacuum-lomem
-T *sym-vacuum-lomem *
-T -sym-trunk *
 U dan
-Z e00eb83dabf2298b62b079f62bce894b
+Z fc13f09d46076f2b65a2bba6a14de43b

manifest.uuid

@@ -1 +1 @@
-c90e063ca9ddcdd1e9f1a2e25a3f7d6e7ee798373ad8acf65b90536b0a124c0d
+0d2c3776065dc94119899ae4164995193b82fca7ac31868f3141b729d0b65ab9

src/btree.c

@@ -1143,6 +1143,24 @@ static SQLITE_NOINLINE void btreeParseCellAdjustSizeForOverflow(
   pInfo->nSize = (u16)(&pInfo->pPayload[pInfo->nLocal] - pCell) + 4;
 }
 
+/*
+** Given a record with nPayload bytes of payload stored within btree
+** page pPage, return the number of bytes of payload stored locally.
+*/
+static int btreePayloadToLocal(MemPage *pPage, int nPayload){
+  int maxLocal;  /* Maximum amount of payload held locally */
+  maxLocal = pPage->maxLocal;
+  if( nPayload<=maxLocal ){
+    return nPayload;
+  }else{
+    int minLocal;  /* Minimum amount of payload held locally */
+    int surplus;   /* Overflow payload available for local storage */
+    minLocal = pPage->minLocal;
+    surplus = minLocal + (nPayload - minLocal)%(pPage->pBt->usableSize-4);
+    return ( surplus <= maxLocal ) ? surplus : minLocal;
+  }
+}
+
 /*
 ** The following routines are implementations of the MemPage.xParseCell()
 ** method.
@@ -6500,7 +6518,7 @@ static int fillInCell(
   /* Fill in the header. */
   nHeader = pPage->childPtrSize;
   if( pPage->intKey ){
-    if( pX->pData==(const void*)pPage->pBt->pTmpSpace ){
+    if( pX->nZero<0 ){
       *pnSize = pX->nData;
       return SQLITE_OK;
     }
@@ -8924,72 +8942,89 @@ int sqlite3BtreeTransfer(
 ){
   int rc = SQLITE_OK;
   BtreePayload x;
+  Pager *pSrcPager = pSrc->pBt->pPager;
+  u8 *pCell = pDest->pBt->pTmpSpace;
+  u8 *aOut;                     /* Pointer to next output buffer */
+  int nOut;                     /* Size of output buffer aOut[] */
+  const u8 *aIn;                /* Pointer to next input buffer */
+  int nIn;                      /* Size of input buffer aIn[] */
+  int nRem;                     /* Bytes of data still to copy */
+  u8 *pPgnoOut = 0;
+  Pgno ovflIn = 0;
+  DbPage *pPageIn = 0;
+  MemPage *pPageOut = 0;
 
   memset(&x, 0, sizeof(x));
   x.nKey = iKey;
   getCellInfo(pSrc);
-  if( pDest->pBt->usableSize!=pSrc->pBt->usableSize ){
+
-    void *pFree = 0;
+  x.pData = pCell;
-    x.nData = pSrc->info.nPayload;
+  x.nData = putVarint32(pCell, pSrc->info.nPayload);
-    if( pSrc->info.nLocal==pSrc->info.nPayload ){
+  x.nData += putVarint(&pCell[x.nData], iKey);
-      x.pData = pSrc->info.pPayload;
+  x.nZero = -1;
-    }else{
+
-      x.pData = pFree = sqlite3_malloc(pSrc->info.nPayload);
+  nOut = btreePayloadToLocal(pDest->pPage, pSrc->info.nPayload);
-      if( pFree==0 ){
+  aOut = &pCell[x.nData];
-        rc = SQLITE_NOMEM_BKPT;
+  nIn = pSrc->info.nLocal;
-      }else{
+  aIn = pSrc->info.pPayload;
-        rc = sqlite3BtreePayload(pSrc, 0, x.nData, pFree);
+  nRem = pSrc->info.nPayload;
+
+  x.nData += nOut;
+  if( nOut<pSrc->info.nPayload ){
+    pPgnoOut = &pCell[x.nData];
+    x.nData += 4;
+  }
+
+  if( nRem>nIn ){
+    ovflIn = get4byte(&pSrc->info.pPayload[nIn]);
+  }
+
+  do {
+    nRem -= nOut;
+    do{
+      assert( nOut>0 );
+      if( nIn>0 ){
+        int nCopy = MIN(nOut, nIn);
+        memcpy(aOut, aIn, nCopy);
+        nOut -= nCopy;
+        nIn -= nCopy;
+        aOut += nCopy;
+        aIn += nCopy;
+      }
+      if( nOut>0 ){
+        sqlite3PagerUnref(pPageIn);
+        pPageIn = 0;
+        rc = sqlite3PagerGet(pSrcPager, ovflIn, &pPageIn, PAGER_GET_READONLY);
+        if( rc==SQLITE_OK ){
+          aIn = (const u8*)sqlite3PagerGetData(pPageIn);
+          ovflIn = get4byte(aIn);
+          aIn += 4;
+          nIn = pSrc->pBt->usableSize - 4;
+        }
+      }
+    }while( rc==SQLITE_OK && nOut>0 );
+
+    if( rc==SQLITE_OK && nRem>0 ){
+      Pgno pgnoNew;
+      MemPage *pNew = 0;
+      rc = allocateBtreePage(pDest->pBt, &pNew, &pgnoNew, 0, 0);
+      put4byte(pPgnoOut, pgnoNew);
+      releasePage(pPageOut);
+      pPageOut = pNew;
+      if( pPageOut ){
+        pPgnoOut = pPageOut->aData;
+        put4byte(pPgnoOut, 0);
+        aOut = &pPgnoOut[4];
+        nOut = MIN(pDest->pBt->usableSize - 4, nRem);
       }
     }
-    if( rc==SQLITE_OK ){
+  }while( nRem>0 && rc==SQLITE_OK );
-      rc = sqlite3BtreeInsert(pDest, &x, OPFLAG_APPEND, seekResult);
+
-    }
+  releasePage(pPageOut);
-    sqlite3_free(pFree);
+  sqlite3PagerUnref(pPageIn);
-  }else{
+
-    /* Page sizes match. This means each overflow page (if any) and the 
+  if( rc==SQLITE_OK ){
-    ** cell itself can be copied verbatim. */
+    rc = sqlite3BtreeInsert(pDest, &x, OPFLAG_APPEND, seekResult);
-    u8 *pCell = pDest->pBt->pTmpSpace;
-    x.pData = pCell;
-    x.nData = putVarint32(pCell, pSrc->info.nPayload);
-    x.nData += putVarint(&pCell[x.nData], iKey);
-    memcpy(&pCell[x.nData], pSrc->info.pPayload, pSrc->info.nLocal);
-    x.nData += pSrc->info.nLocal;
-    assert( pSrc->info.nLocal<=pSrc->info.nPayload );
-    if( pSrc->info.nLocal<pSrc->info.nPayload ){
-      Pager *pSrcPager = pSrc->pBt->pPager;
-      u8 *pPgno = &pCell[x.nData];
-      Pgno ovfl; 
-      x.nData += 4;
-      ovfl = get4byte(pSrc->info.pPayload + pSrc->info.nLocal);
-      do{
-        MemPage *pNew = 0;
-        Pgno pgnoNew = 0;
-        if( rc==SQLITE_OK ){
-          rc = allocateBtreePage(pDest->pBt, &pNew, &pgnoNew, 0, 0);
-          put4byte(pPgno, pgnoNew);
-        }
-        if( rc==SQLITE_OK ){
-          pPgno = pNew->aData;
-          rc = sqlite3PagerWrite(pNew->pDbPage);
-        }
-        if( rc==SQLITE_OK ){
-          DbPage *pOrig = 0;
-          void *pOrigData;
-          rc = sqlite3PagerGet(pSrcPager, ovfl, &pOrig, PAGER_GET_READONLY);
-          if( rc==SQLITE_OK ){
-            pOrigData = sqlite3PagerGetData(pOrig);
-            memcpy(pNew->aData, pOrigData, pDest->pBt->usableSize);
-            put4byte(pNew->aData, 0);
-            ovfl = get4byte(pOrigData);
-          }
-          sqlite3PagerUnref(pOrig);
-        }
-        releasePage(pNew);
-      }while( rc==SQLITE_OK && ovfl>0 );
-    }
-    if( rc==SQLITE_OK ){
-      rc = sqlite3BtreeInsert(pDest, &x, OPFLAG_APPEND, seekResult);
-    }
   }
 
   return rc;

test/vacuum2.test

@@ -56,7 +56,7 @@ do_test vacuum2-2.1 {
   }
   hexio_get_int [hexio_read test.db 24 4]
 } [expr {[hexio_get_int [hexio_read test.db 24 4]]+3}]
-do_test vacuum2-2.1 {
+do_test vacuum2-2.2 {
   execsql {
     VACUUM
   }

test/vacuum6.test

@@ -0,0 +1,94 @@
+# 2016-08-19
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# 
+# This file implements a test for VACUUM on attached databases.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix vacuum6
+
+# If the VACUUM statement is disabled in the current build, skip all
+# the tests in this file.
+#
+ifcapable !vacuum {
+  finish_test
+  return
+}
+
+
+do_execsql_test 1.0 {
+  CREATE TABLE t1(x INTEGER PRIMARY KEY, y);
+  INSERT INTO t1 VALUES(1, 1);
+} {}
+do_execsql_test 1.1 {
+  VACUUM
+}
+
+#-------------------------------------------------------------------------
+reset_db
+foreach {tn sz} {1 400 2 4000 3 9999} {
+  reset_db
+  do_execsql_test 2.$tn.1 {
+    CREATE TABLE t1(a INTEGER PRIMARY KEY, b);
+    WITH s(i) AS (
+        SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100
+    )
+    INSERT INTO t1 SELECT i, randomblob($sz) FROM s;
+  }
+
+  do_execsql_test 2.$tn.2 {
+    vacuum;
+  }
+
+  do_execsql_test 2.$tn.3 {
+    PRAGMA integrity_check;
+  } {ok}
+}
+
+reset_db
+do_execsql_test 3.0 {
+  PRAGMA page_size = 1024;
+  CREATE TABLE t1(x INTEGER PRIMARY KEY, y);
+  INSERT INTO t1 VALUES(2, randomblob(1200));
+} {}
+do_execsql_test 3.1 {
+  PRAGMA page_size = 512;
+  VACUUM;
+}
+do_execsql_test 3.2 {
+  PRAGMA integrity_check
+} {ok}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 4.0 {
+  CREATE TABLE tx(a, b);
+  CREATE INDEX i1 ON tx(b);
+  WITH s(i) AS (
+      SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<10000
+  )
+  INSERT INTO tx SELECT i, randomblob(i) FROM s;
+}
+foreach {tn pgsz} {
+  1 2048
+  2 1024
+  3 65536
+  4 8192
+} {
+  do_execsql_test 4.1.$tn.1 "
+    PRAGMA page_size = $pgsz
+  "
+  do_execsql_test 4.1.$tn.2 VACUUM
+  integrity_check 4.1.$tn.3
+}
+
+finish_test