Create a new pager type, PAGER_SORTER, for use in the external merge sort.

Such pagers are always held in memory but do report when they are under
memory pressure by calling pagerStress.

FossilOrigin-Name: c71d73201d950355862dd8d5de142c9673888755

manifest

@@ -1,5 +1,5 @@
-C Reorder\ssome\sof\sthe\sbranches\sin\sbackup.c\sin\sorder\sto\smake\sthe\scode\neasier\sto\stest.
+C Create\sa\snew\spager\stype,\sPAGER_SORTER,\sfor\suse\sin\sthe\sexternal\smerge\ssort.\nSuch\spagers\sare\salways\sheld\sin\smemory\sbut\sdo\sreport\swhen\sthey\sare\sunder\nmemory\spressure\sby\scalling\spagerStress.
-D 2011-08-25T20:18:47.446
+D 2011-08-26T00:34:45.360
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 8c930e7b493d59099ea1304bd0f2aed152eb3315
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -124,10 +124,10 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34
 F src/backup.c 28a4fe55327ff708bfaf9d4326d02686f7a553c3
 F src/bitvec.c af50f1c8c0ff54d6bdb7a80e2fceca5a93670bef
 F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7
-F src/btree.c 97cf3ba4ff067e716753b33661035e50853aebba
+F src/btree.c ed13fdefdbe671d5777773dcfb3a162ddb4623ae
-F src/btree.h f5d775cd6cfc7ac32a2535b70e8d2af48ef5f2ce
+F src/btree.h 9ddf04226eac592d4cc3709c5a8b33b2351ff5f7
 F src/btreeInt.h 67978c014fa4f7cc874032dd3aacadd8db656bc3
-F src/build.c 20784c6b4e4514c90aeeec2bee0fb9d79a4e2189
+F src/build.c 2d5de52df616a3bf5a659cbca85211c46e2ba9bd
 F src/callback.c 0425c6320730e6d3981acfb9202c1bed9016ad1a
 F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac
 F src/ctime.c caf51429be3e0d4114056a8273b0fff812ff8ae9
@@ -167,8 +167,8 @@ F src/os_common.h 65a897143b64667d23ed329a7984b9b405accb58
 F src/os_os2.c 4a75888ba3dfc820ad5e8177025972d74d7f2440
 F src/os_unix.c 1a34ca3794ced80e4a4ebcc3ba1f4c516762e534
 F src/os_win.c 19fa09046f1f86590a188abdcf5630b8fe8279cf
-F src/pager.c 120550e7ef01dafaa2cbb4a0528c0d87c8f12b41
+F src/pager.c 817f7f7140c9fa2641f28e6330e924708ddd870d
-F src/pager.h 3f8c783de1d4706b40b1ac15b64f5f896bcc78d1
+F src/pager.h 2bab1b2ea4eac58663b5833e3522e36b5ff63447
 F src/parse.y 12b7ebd61ea54f0e1b1083ff69cc2c8ce9353d58
 F src/pcache.c 49e718c095810c6b3334e3a6d89970aceaddefce
 F src/pcache.h c683390d50f856d4cd8e24342ae62027d1bb6050
@@ -238,14 +238,14 @@ F src/update.c 74a6cfb34e9732c1e2a86278b229913b4b51eeec
 F src/utf.c c53eb7404b3eb5c1cbb5655c6a7a0e0ce6bd50f0
 F src/util.c 06302ffd2b80408d4f6c7af71f7090e0cf8d8ff7
 F src/vacuum.c 05513dca036a1e7848fe18d5ed1265ac0b32365e
-F src/vdbe.c 8f18a857e7cc1eba1a59d2ab011d4cf0d05f48ad
+F src/vdbe.c 4a7191c0f8e918b74e8c84cbdd77746d6b7e3bcf
 F src/vdbe.h 2bf6ec77d8b9980fc19da6e0b0a36d0dbf884ce4
 F src/vdbeInt.h f9250326f264ca5f100acc19e9c07096bb889096
 F src/vdbeapi.c 11dc47987abacb76ad016dcf5abc0dc422482a98
 F src/vdbeaux.c 11b0df8822ecf61e543562247207df75e2ebb617
 F src/vdbeblob.c f024f0bf420f36b070143c32b15cc7287341ffd3
 F src/vdbemem.c 0498796b6ffbe45e32960d6a1f5adfb6e419883b
-F src/vdbesort.c d2c872322c94caae7abd39fe88eef177f66240cf
+F src/vdbesort.c 8a61a6d731cbe612217edf9eece6197f37c9489e
 F src/vdbetrace.c 5d0dc3d5fd54878cc8d6d28eb41deb8d5885b114
 F src/vtab.c 901791a47318c0562cd0c676a2c6ff1bc530e582
 F src/wal.c 3154756177d6219e233d84291d5b05f4e06ff5e9
@@ -961,7 +961,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5
 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576
 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
 F tool/warnings.sh b7fdb2cc525f5ef4fa43c80e771636dd3690f9d2
-P 472c74b3452c5a07dfb006010441232b09599ad5
+P 2c443d47ecee7b43a89f0a4bf299c46c66e3f80d
-R 8c5e28ffe9a7799230a8adf4f20dce05
+R 5271767b893f002dc634ff718e14ca01
 U drh
-Z ae50a88981aafd0a9a503f248a20233b
+Z 986c77c3e73f3db4a3199ce94583c644

manifest.uuid

@@ -1 +1 @@
-2c443d47ecee7b43a89f0a4bf299c46c66e3f80d
+c71d73201d950355862dd8d5de142c9673888755

src/btree.c

@@ -1734,11 +1734,22 @@ int sqlite3BtreeOpen(
   /* A BTREE_SINGLE database is always a temporary and/or ephemeral */
   assert( (flags & BTREE_SINGLE)==0 || isTempDb );
 
+  /* The BTREE_SORTER flag is only used if SQLITE_OMIT_MERGE_SORT is undef */
+#ifdef SQLITE_OMIT_MERGE_SORT
+  assert( (flags & BTREE_SORTER)==0 );
+#endif
+
+  /* BTREE_SORTER is always on a BTREE_SINGLE, BTREE_OMIT_JOURNAL */
+  assert( (flags & BTREE_SORTER)==0 ||
+          (flags & (BTREE_SINGLE|BTREE_OMIT_JOURNAL))
+                                        ==(BTREE_SINGLE|BTREE_OMIT_JOURNAL) );
+
   if( db->flags & SQLITE_NoReadlock ){
     flags |= BTREE_NO_READLOCK;
   }
   if( isMemdb ){
     flags |= BTREE_MEMORY;
+    flags &= ~BTREE_SORTER;
   }
   if( (vfsFlags & SQLITE_OPEN_MAIN_DB)!=0 && (isMemdb || isTempDb) ){
     vfsFlags = (vfsFlags & ~SQLITE_OPEN_MAIN_DB) | SQLITE_OPEN_TEMP_DB;
@@ -8174,5 +8185,3 @@ int sqlite3BtreeSetVersion(Btree *pBtree, int iVersion){
   pBt->doNotUseWAL = 0;
   return rc;
 }
-
-

src/btree.h

@@ -61,6 +61,7 @@ int sqlite3BtreeOpen(
 #define BTREE_MEMORY        4  /* This is an in-memory DB */
 #define BTREE_SINGLE        8  /* The file contains at most 1 b-tree */
 #define BTREE_UNORDERED    16  /* Use of a hash implementation is OK */
+#define BTREE_SORTER       32  /* Used as accumulator in external merge sort */
 
 int sqlite3BtreeClose(Btree*);
 int sqlite3BtreeSetCacheSize(Btree*,int);

src/build.c

@@ -2372,6 +2372,7 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){
   if( bUseSorter ){
     iSorter = pParse->nTab++;
     sqlite3VdbeAddOp4(v, OP_OpenSorter, iSorter, 0, 0, (char*)pKey, P4_KEYINFO);
+    sqlite3VdbeChangeP5(v, BTREE_SORTER);
   }
 
   /* Open the table. Loop through all rows of the table, inserting index

src/pager.c

@@ -620,6 +620,8 @@ struct Pager {
   u8 tempFile;                /* zFilename is a temporary file */
   u8 readOnly;                /* True for a read-only database */
   u8 memDb;                   /* True to inhibit all file I/O */
+  u8 hasSeenStress;           /* pagerStress() called one or more times */
+  u8 isSorter;                /* True for a PAGER_SORTER */
 
   /**************************************************************************
   ** The following block contains those class members that change during
@@ -843,6 +845,15 @@ static int assert_pager_state(Pager *p){
     assert( pagerUseWal(p)==0 );
   }
 
+  /* A sorter is a temp file that never spills to disk and always has
+  ** the doNotSpill flag set
+  */
+  if( p->isSorter ){
+    assert( p->tempFile );
+    assert( p->doNotSpill );
+    assert( p->fd->pMethods==0 );
+  }
+
   /* If changeCountDone is set, a RESERVED lock or greater must be held
   ** on the file.
   */
@@ -3739,6 +3750,7 @@ static int pagerSyncHotJournal(Pager *pPager){
 int sqlite3PagerClose(Pager *pPager){
   u8 *pTmp = (u8 *)pPager->pTmpSpace;
 
+  assert( assert_pager_state(pPager) );
   disable_simulated_io_errors();
   sqlite3BeginBenignMalloc();
   /* pPager->errCode = 0; */
@@ -4173,6 +4185,7 @@ static int pagerStress(void *p, PgHdr *pPg){
   ** be called in the error state.  Nevertheless, we include a NEVER()
   ** test for the error state as a safeguard against future changes.
   */
+  pPager->hasSeenStress = 1;
   if( NEVER(pPager->errCode) ) return SQLITE_OK;
   if( pPager->doNotSpill ) return SQLITE_OK;
   if( pPager->doNotSyncSpill && (pPg->flags & PGHDR_NEED_SYNC)!=0 ){
@@ -4544,6 +4557,12 @@ int sqlite3PagerOpen(
   /* pPager->pBusyHandlerArg = 0; */
   pPager->xReiniter = xReinit;
   /* memset(pPager->aHash, 0, sizeof(pPager->aHash)); */
+#ifndef SQLITE_OMIT_MERGE_SORT
+  if( flags & PAGER_SORTER ){
+    pPager->doNotSpill = 1;
+    pPager->isSorter = 1;
+  }
+#endif
 
   *ppPager = pPager;
   return SQLITE_OK;
@@ -6088,6 +6107,17 @@ int sqlite3PagerIsMemdb(Pager *pPager){
   return MEMDB;
 }
 
+#ifndef SQLITE_OMIT_MERGE_SORT
+/*
+** Return true if the pager has seen a pagerStress callback.
+*/
+int sqlite3PagerUnderStress(Pager *pPager){
+  assert( pPager->isSorter );
+  assert( pPager->doNotSpill );
+  return pPager->hasSeenStress;
+}
+#endif
+
 /*
 ** Check that there are at least nSavepoint savepoints open. If there are
 ** currently less than nSavepoints open, then open one or more savepoints

src/pager.h

@@ -60,6 +60,7 @@ typedef struct PgHdr DbPage;
 #define PAGER_OMIT_JOURNAL  0x0001    /* Do not use a rollback journal */
 #define PAGER_NO_READLOCK   0x0002    /* Omit readlocks on readonly files */
 #define PAGER_MEMORY        0x0004    /* In-memory database */
+#define PAGER_SORTER        0x0020    /* Accumulator in external merge sort */
 
 /*
 ** Valid values for the second argument to sqlite3PagerLockingMode().
@@ -155,6 +156,9 @@ const char *sqlite3PagerJournalname(Pager*);
 int sqlite3PagerNosync(Pager*);
 void *sqlite3PagerTempSpace(Pager*);
 int sqlite3PagerIsMemdb(Pager*);
+#ifndef SQLITE_OMIT_MERGE_SORT
+int sqlite3PagerUnderStress(Pager*);
+#endif
 
 /* Functions used to truncate the database file. */
 void sqlite3PagerTruncateImage(Pager*,Pgno);

src/vdbe.c

@@ -3131,7 +3131,7 @@ case OP_OpenWrite: {
   break;
 }
 
-/* Opcode: OpenEphemeral P1 P2 * P4 *
+/* Opcode: OpenEphemeral P1 P2 * P4 P5
 **
 ** Open a new cursor P1 to a transient table.
 ** The cursor is always opened read/write even if 
@@ -3148,6 +3148,11 @@ case OP_OpenWrite: {
 ** to a TEMP table at the SQL level, or to a table opened by
 ** this opcode.  Then this opcode was call OpenVirtual.  But
 ** that created confusion with the whole virtual-table idea.
+**
+** The P5 parameter can be a mask of the BTREE_* flags defined
+** in btree.h.  These flags control aspects of the operation of
+** the btree.  The BTREE_OMIT_JOURNAL and BTREE_SINGLE flags are
+** added automatically.
 */
 /* Opcode: OpenAutoindex P1 P2 * P4 *
 **
@@ -3174,6 +3179,7 @@ case OP_OpenEphemeral: {
       SQLITE_OPEN_TRANSIENT_DB;
 
   assert( pOp->p1>=0 );
+  assert( (pOp->opcode==OP_OpenSorter)==((pOp->p5 & BTREE_SORTER)!=0) );
   pCx = allocateCursor(p, pOp->p1, pOp->p2, -1, 1);
   if( pCx==0 ) goto no_mem;
   pCx->nullRow = 1;

src/vdbesort.c

@@ -388,8 +388,10 @@ static int vdbeSorterBtreeToPMA(sqlite3 *db, VdbeCursor *pCsr){
   VdbeSorter *pSorter = pCsr->pSorter;
   int res = 0;
 
+  /* sqlite3BtreeFirst() cannot fail because sorter btrees are always held
+  ** in memory and so an I/O error is not possible. */
   rc = sqlite3BtreeFirst(pCsr->pCursor, &res);
-  if( rc!=SQLITE_OK || res ) return rc;
+  if( NEVER(rc!=SQLITE_OK) || res ) return rc;
   assert( pSorter->nBtree>0 );
 
   /* If the first temporary PMA file has not been opened, open it now. */
@@ -429,8 +431,9 @@ static int vdbeSorterBtreeToPMA(sqlite3 *db, VdbeCursor *pCsr){
 
       /* Write the record itself to the output file */
       if( rc==SQLITE_OK ){
+        /* sqlite3BtreeKey() cannot fail because sorter btrees held in memory */
         rc = sqlite3BtreeKey(pCsr->pCursor, 0, nKey, aMalloc);
-        if( rc==SQLITE_OK ){
+        if( ALWAYS(rc==SQLITE_OK) ){
           rc = sqlite3OsWrite(pSorter->pTemp1, aMalloc, nKey, iWriteOff);
           iWriteOff += nKey;
         }
@@ -474,6 +477,9 @@ int sqlite3VdbeSorterWrite(sqlite3 *db, VdbeCursor *pCsr, int nKey){
     Pager *pPager = sqlite3BtreePager(pCsr->pBt);
     int nPage;                    /* Current size of temporary file in pages */
 
+    /* Sorters never spill to disk */
+    assert( sqlite3PagerFile(pPager)->pMethods==0 );
+
     /* Determine how many pages the temporary b-tree has grown to */
     sqlite3PagerPagecount(pPager, &nPage);
 
@@ -483,7 +489,7 @@ int sqlite3VdbeSorterWrite(sqlite3 *db, VdbeCursor *pCsr, int nKey){
     ** also possible that sqlite3_release_memory() was called). So set the
     ** size of the working set to a little less than the current size of the 
     ** file in pages.  */
-    if( pSorter->nWorking==0 && sqlite3PagerFile(pPager)->pMethods ){
+    if( pSorter->nWorking==0 && sqlite3PagerUnderStress(pPager) ){
       pSorter->nWorking = nPage-5;
       if( pSorter->nWorking<SORTER_MIN_WORKING ){
         pSorter->nWorking = SORTER_MIN_WORKING;