Instead of a temporary b-tree, use a linked-list and merge-sort to sort records in main memory in vdbesort.c.

FossilOrigin-Name: 7769fb988d9be0f2d8129aaac19620ac88f9b4a6

manifest

@@ -1,5 +1,5 @@
-C Use\sOP_SorterOpen\sinstead\sof\sOP_OpenEphemeral\sto\simplement\sGROUP\sBY.
+C Instead\sof\sa\stemporary\sb-tree,\suse\sa\slinked-list\sand\smerge-sort\sto\ssort\srecords\sin\smain\smemory\sin\svdbesort.c.
-D 2011-09-01T16:01:27.777
+D 2011-09-02T10:31:11.173
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in d314143fa6be24828021d3f583ad37d9afdce505
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -124,16 +124,16 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34
 F src/backup.c 28a4fe55327ff708bfaf9d4326d02686f7a553c3
 F src/bitvec.c af50f1c8c0ff54d6bdb7a80e2fceca5a93670bef
 F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7
-F src/btree.c 4a2856b3bde9959986a7b9327841b3ff94023784
+F src/btree.c bc2099e7d3c22c52b2c54349b9c07c04f2a810d0
-F src/btree.h 9ddf04226eac592d4cc3709c5a8b33b2351ff5f7
+F src/btree.h f5d775cd6cfc7ac32a2535b70e8d2af48ef5f2ce
 F src/btreeInt.h 67978c014fa4f7cc874032dd3aacadd8db656bc3
-F src/build.c 2d5de52df616a3bf5a659cbca85211c46e2ba9bd
+F src/build.c dc367138cb3625e6d42b389e05d7267aece5753c
 F src/callback.c 0425c6320730e6d3981acfb9202c1bed9016ad1a
 F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac
 F src/ctime.c e3132ec65240b2e2f3d50831021eac387f27584d
 F src/date.c a3c6842bad7ae632281811de112a8ba63ff08ab3
 F src/delete.c ff68e5ef23aee08c0ff528f699a19397ed8bbed8
-F src/expr.c 4bbdfaf66bc614be9254ce0c26a17429067a3e07
+F src/expr.c cbcd8c2f1588a9862291a081699854c5e1cb28ab
 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb
 F src/fkey.c 9f00ea98f6b360d477b5a78b5b59a1fbde82431c
 F src/func.c 59bb046d7e3df1ab512ac339ccb0a6f996a17cb7
@@ -179,11 +179,11 @@ F src/printf.c 585a36b6a963df832cfb69505afa3a34ed5ef8a1
 F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50
 F src/resolve.c 36368f44569208fa074e61f4dd0b6c4fb60ca2b4
 F src/rowset.c 69afa95a97c524ba6faf3805e717b5b7ae85a697
-F src/select.c 037ee5501fe0e743fa98936902c200ed9ed69156
+F src/select.c 32d0f4e5513362706b8973e7f1b87cd0885dfbf5
 F src/shell.c bbe7818ff5bc8614105ceb81ad67b8bdc0b671dd
 F src/sqlite.h.in 0a6c9c23337fd1352c5c75a613ff9533aa7d91cb
 F src/sqlite3ext.h 1a1a4f784aa9c3b00edd287940197de52487cd93
-F src/sqliteInt.h f6debf9a9eb8463ab2ef8be4b2b740ea9af5afba
+F src/sqliteInt.h 723cda73a33c91f5a0a145f4c0ced45d94921079
 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d
 F src/status.c 7ac64842c86cec2fc1a1d0e5c16d3beb8ad332bf
 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e
@@ -238,14 +238,14 @@ F src/update.c 74a6cfb34e9732c1e2a86278b229913b4b51eeec
 F src/utf.c c53eb7404b3eb5c1cbb5655c6a7a0e0ce6bd50f0
 F src/util.c 06302ffd2b80408d4f6c7af71f7090e0cf8d8ff7
 F src/vacuum.c 05513dca036a1e7848fe18d5ed1265ac0b32365e
-F src/vdbe.c 9260e5138855399bea2611a29da336688bfa1b79
+F src/vdbe.c da9c7efc48dc79d7785f3f17a1c3df514bf18489
 F src/vdbe.h c1eeedacab6bcf1e7c2cf8203ba9763a616f9a86
-F src/vdbeInt.h 51a902e12c7d571e3b516e5407e30f996494aafe
+F src/vdbeInt.h a255da14be8c2794ce38e0d2142877bb29df9105
 F src/vdbeapi.c 11dc47987abacb76ad016dcf5abc0dc422482a98
 F src/vdbeaux.c e58acbc5ea3823922a0cd8fa21f94f39af51ee88
 F src/vdbeblob.c f024f0bf420f36b070143c32b15cc7287341ffd3
 F src/vdbemem.c 5e6effb96dd53d233361cbfaa3f0a43b9af689e9
-F src/vdbesort.c f3d043a1bab7409d4a23cd7a35287c3ac440a167
+F src/vdbesort.c 9c2e8ca23c2413a5162a4433cb72377588d896f8
 F src/vdbetrace.c 5d0dc3d5fd54878cc8d6d28eb41deb8d5885b114
 F src/vtab.c 901791a47318c0562cd0c676a2c6ff1bc530e582
 F src/wal.c 3154756177d6219e233d84291d5b05f4e06ff5e9
@@ -511,7 +511,7 @@ F test/incrvacuum_ioerr.test 22f208d01c528403240e05beecc41dc98ed01637
 F test/index.test b5429732b3b983fa810e3ac867d7ca85dae35097
 F test/index2.test ee83c6b5e3173a3d7137140d945d9a5d4fdfb9d6
 F test/index3.test 423a25c789fc8cc51aaf2a4370bbdde2d9e9eed7
-F test/index4.test c82a59c9ae2ac01804bdb100162dca057318f40f
+F test/index4.test 2983216eb8c86ee62d9ed7cb206b5cc3331c0026
 F test/indexedby.test be501e381b82b2f8ab406309ba7aac46e221f4ad
 F test/indexfault.test 31d4ab9a7d2f6e9616933eb079722362a883eb1d
 F test/init.test 15c823093fdabbf7b531fe22cf037134d09587a7
@@ -961,7 +961,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5
 F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06
 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
 F tool/warnings.sh b7fdb2cc525f5ef4fa43c80e771636dd3690f9d2
-P bab2e560f6cb989c83a96aad60f666960ede7abe
+P ebf819aaa555bd79fddfc0a6f9827a2539095d6c
-R 47ae22012e17b717f429ee6a9d305e96
+R c9d892df81c49366e067933871c3b776
-U drh
+U dan
-Z 4edda214a8d1e652ef01c1f98ad8fd39
+Z b3581bd03fb5a181801f57c568eb418b

manifest.uuid

@@ -1 +1 @@
-ebf819aaa555bd79fddfc0a6f9827a2539095d6c
+7769fb988d9be0f2d8129aaac19620ac88f9b4a6

src/btree.c

@@ -1734,22 +1734,11 @@ int sqlite3BtreeOpen(
   /* A BTREE_SINGLE database is always a temporary and/or ephemeral */
   assert( (flags & BTREE_SINGLE)==0 || isTempDb );
 
-  /* The BTREE_SORTER flag is only used if SQLITE_OMIT_MERGE_SORT is undef */
-#ifdef SQLITE_OMIT_MERGE_SORT
-  assert( (flags & BTREE_SORTER)==0 );
-#endif
-
-  /* BTREE_SORTER is always on a BTREE_SINGLE, BTREE_OMIT_JOURNAL */
-  assert( (flags & BTREE_SORTER)==0 ||
-          (flags & (BTREE_SINGLE|BTREE_OMIT_JOURNAL))
-                                        ==(BTREE_SINGLE|BTREE_OMIT_JOURNAL) );
-
   if( db->flags & SQLITE_NoReadlock ){
     flags |= BTREE_NO_READLOCK;
   }
   if( isMemdb ){
     flags |= BTREE_MEMORY;
-    flags &= ~BTREE_SORTER;
   }
   if( (vfsFlags & SQLITE_OPEN_MAIN_DB)!=0 && (isMemdb || isTempDb) ){
     vfsFlags = (vfsFlags & ~SQLITE_OPEN_MAIN_DB) | SQLITE_OPEN_TEMP_DB;

src/btree.h

@@ -61,7 +61,6 @@ int sqlite3BtreeOpen(
 #define BTREE_MEMORY        4  /* This is an in-memory DB */
 #define BTREE_SINGLE        8  /* The file contains at most 1 b-tree */
 #define BTREE_UNORDERED    16  /* Use of a hash implementation is OK */
-#define BTREE_SORTER       32  /* Used as accumulator in external merge sort */
 
 int sqlite3BtreeClose(Btree*);
 int sqlite3BtreeSetCacheSize(Btree*,int);

src/build.c

@@ -2326,6 +2326,7 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){
   int iIdx = pParse->nTab++;     /* Btree cursor used for pIndex */
   int iSorter = iTab;            /* Cursor opened by OpenSorter (if in use) */
   int addr1;                     /* Address of top of loop */
+  int addr2;                     /* Address to jump to for next iteration */
   int tnum;                      /* Root page of index */
   Vdbe *v;                       /* Generate code into this virtual machine */
   KeyInfo *pKey;                 /* KeyInfo for index */
@@ -2372,25 +2373,34 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){
   if( bUseSorter ){
     iSorter = pParse->nTab++;
     sqlite3VdbeAddOp4(v, OP_OpenSorter, iSorter, 0, 0, (char*)pKey, P4_KEYINFO);
-    sqlite3VdbeChangeP5(v, BTREE_SORTER);
   }
 
   /* Open the table. Loop through all rows of the table, inserting index
   ** records into the sorter. */
   sqlite3OpenTable(pParse, iTab, iDb, pTab, OP_OpenRead);
   addr1 = sqlite3VdbeAddOp2(v, OP_Rewind, iTab, 0);
+  addr2 = addr1 + 1;
   regRecord = sqlite3GetTempReg(pParse);
   regIdxKey = sqlite3GenerateIndexKey(pParse, pIndex, iTab, regRecord, 1);
 
   if( bUseSorter ){
-    sqlite3VdbeAddOp2(v, OP_IdxInsert, iSorter, regRecord);
+    sqlite3VdbeAddOp2(v, OP_SorterInsert, iSorter, regRecord);
     sqlite3VdbeAddOp2(v, OP_Next, iTab, addr1+1);
     sqlite3VdbeJumpHere(v, addr1);
-    addr1 = sqlite3VdbeAddOp2(v, OP_Sort, iSorter, 0);
+    addr1 = sqlite3VdbeAddOp2(v, OP_SorterSort, iSorter, 0);
-    sqlite3VdbeAddOp2(v, OP_RowKey, iSorter, regRecord);
-  }
-
     if( pIndex->onError!=OE_None ){
+      int j2 = sqlite3VdbeCurrentAddr(v) + 3;
+      sqlite3VdbeAddOp2(v, OP_Goto, 0, j2);
+      addr2 = sqlite3VdbeCurrentAddr(v);
+      sqlite3VdbeAddOp3(v, OP_SorterCompare, iSorter, j2, regRecord);
+      sqlite3HaltConstraint(
+          pParse, OE_Abort, "indexed columns are not unique", P4_STATIC
+      );
+    }else{
+      addr2 = sqlite3VdbeCurrentAddr(v);
+    }
+    sqlite3VdbeAddOp2(v, OP_SorterData, iSorter, regRecord);
+  }else if( pIndex->onError!=OE_None ){
     const int regRowid = regIdxKey + pIndex->nColumn;
     const int j2 = sqlite3VdbeCurrentAddr(v) + 2;
     void * const pRegKey = SQLITE_INT_TO_PTR(regIdxKey);
@@ -2411,7 +2421,7 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){
   sqlite3VdbeAddOp3(v, OP_IdxInsert, iIdx, regRecord, bUseSorter);
   sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT);
   sqlite3ReleaseTempReg(pParse, regRecord);
-  sqlite3VdbeAddOp2(v, OP_Next, iSorter, addr1+1);
+  sqlite3VdbeAddOp2(v, bUseSorter ? OP_SorterNext : OP_Next, iSorter, addr2);
   sqlite3VdbeJumpHere(v, addr1);
 
   sqlite3VdbeAddOp1(v, OP_Close, iTab);

src/expr.c

@@ -2287,7 +2287,7 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){
         inReg = pCol->iMem;
         break;
       }else if( pAggInfo->useSortingIdx ){
-        sqlite3VdbeAddOp3(v, OP_Column, pAggInfo->sortingIdx,
+        sqlite3VdbeAddOp3(v, OP_Column, pAggInfo->sortingIdxPTab,
                               pCol->iSorterColumn, target);
         break;
       }

src/select.c

@@ -4186,7 +4186,7 @@ int sqlite3Select(
         sqlite3ReleaseTempReg(pParse, regRecord);
         sqlite3ReleaseTempRange(pParse, regBase, nCol);
         sqlite3WhereEnd(pWInfo);
-        sortPTab = pParse->nTab++;
+        sAggInfo.sortingIdxPTab = sortPTab = pParse->nTab++;
         sortOut = sqlite3GetTempReg(pParse);
         sqlite3VdbeAddOp3(v, OP_OpenPseudo, sortPTab, sortOut, nCol);
         sqlite3VdbeAddOp2(v, OP_SorterSort, sAggInfo.sortingIdx, addrEnd);

src/sqliteInt.h

@@ -1550,6 +1550,7 @@ struct AggInfo {
   u8 useSortingIdx;       /* In direct mode, reference the sorting index rather
                           ** than the source table */
   int sortingIdx;         /* Cursor number of the sorting index */
+  int sortingIdxPTab;     /* Cursor number of pseudo-table */
   ExprList *pGroupBy;     /* The group by clause */
   int nSortingColumn;     /* Number of columns in the sorting index */
   struct AggInfo_col {    /* For each column used in source tables */

src/vdbe.c

@@ -3162,15 +3162,7 @@ case OP_OpenWrite: {
 ** by this opcode will be used for automatically created transient
 ** indices in joins.
 */
-/* Opcode: OpenSorter P1 P2 * P4 *
-**
-** This opcode works like OP_OpenEphemeral except that it opens
-** a transient index that is specifically designed to sort large
-** tables using an external merge-sort algorithm.
-*/
-case OP_OpenSorter: 
 case OP_OpenAutoindex: 
-case OP_SorterOpen:
 case OP_OpenEphemeral: {
   VdbeCursor *pCx;
   static const int vfsFlags = 
@@ -3181,7 +3173,6 @@ case OP_OpenEphemeral: {
       SQLITE_OPEN_TRANSIENT_DB;
 
   assert( pOp->p1>=0 );
-  assert( (pOp->opcode==OP_OpenSorter)==((pOp->p5 & BTREE_SORTER)!=0) );
   pCx = allocateCursor(p, pOp->p1, pOp->p2, -1, 1);
   if( pCx==0 ) goto no_mem;
   pCx->nullRow = 1;
@@ -3215,12 +3206,24 @@ case OP_OpenEphemeral: {
   }
   pCx->isOrdered = (pOp->p5!=BTREE_UNORDERED);
   pCx->isIndex = !pCx->isTable;
-  pCx->isSorter = pOp->opcode==OP_SorterOpen;
+  break;
-#ifndef SQLITE_OMIT_MERGE_SORT
+}
-  if( rc==SQLITE_OK && pOp->opcode==OP_OpenSorter ){
+
+/* Opcode: OpenSorter P1 P2 * P4 *
+**
+** This opcode works like OP_OpenEphemeral except that it opens
+** a transient index that is specifically designed to sort large
+** tables using an external merge-sort algorithm.
+*/
+case OP_SorterOpen:
+case OP_OpenSorter: {
+  VdbeCursor *pCx;
+  pCx = allocateCursor(p, pOp->p1, pOp->p2, -1, 1);
+  if( pCx==0 ) goto no_mem;
+  pCx->pKeyInfo = pOp->p4.pKeyInfo;
+  pCx->pKeyInfo->enc = ENC(p->db);
+  pCx->isSorter = 1;
   rc = sqlite3VdbeSorterInit(db, pCx);
-  }
-#endif
   break;
 }
 
@@ -4072,6 +4075,40 @@ case OP_ResetCount: {
   break;
 }
 
+/* Opcode: SorterCompare P1 P2 P3
+**
+** P1 is a sorter cursor. This instruction compares the record blob in 
+** register P3 with the entry that the sorter cursor currently points to.
+** If, excluding the rowid fields at the end, the two records are a match,
+** fall through to the next instruction. Otherwise, jump to instruction P2.
+*/
+case OP_SorterCompare: {
+  VdbeCursor *pC;
+  int res;
+
+  pC = p->apCsr[pOp->p1];
+  assert( isSorter(pC) );
+  pIn3 = &aMem[pOp->p3];
+  rc = sqlite3VdbeSorterCompare(pC, pIn3, &res);
+  if( res ){
+    pc = pOp->p2-1;
+  }
+  break;
+};
+
+/* Opcode: SorterData P1 P2 * * *
+**
+** Write into register P2 the current sorter data for sorter cursor P1.
+*/
+case OP_SorterData: {
+  VdbeCursor *pC;
+  pOut = &aMem[pOp->p2];
+  pC = p->apCsr[pOp->p1];
+  assert( pC->isSorter );
+  rc = sqlite3VdbeSorterRowkey(pC, pOut);
+  break;
+}
+
 /* Opcode: RowData P1 P2 * * *
 **
 ** Write into register P2 the complete row data for cursor P1.
@@ -4092,7 +4129,6 @@ case OP_ResetCount: {
 ** If the P1 cursor must be pointing to a valid row (not a NULL row)
 ** of a real table, not a pseudo-table.
 */
-case OP_SorterData:
 case OP_RowKey:
 case OP_RowData: {
   VdbeCursor *pC;
@@ -4106,6 +4142,7 @@ case OP_RowData: {
   /* Note that RowKey and RowData are really exactly the same instruction */
   assert( pOp->p1>=0 && pOp->p1<p->nCursor );
   pC = p->apCsr[pOp->p1];
+  assert( pC->isSorter==0 );
   assert( pC->isTable || pOp->opcode!=OP_RowData );
   assert( pC->isIndex || pOp->opcode==OP_RowData );
   assert( pC!=0 );
@@ -4113,12 +4150,6 @@ case OP_RowData: {
   assert( pC->pseudoTableReg==0 );
   assert( pC->isSorter==(pOp->opcode==OP_SorterData) );
 
-  if( isSorter(pC) ){
-    assert( pOp->opcode==OP_RowKey );
-    rc = sqlite3VdbeSorterRowkey(pC, pOut);
-    break;
-  }
-
   assert( pC->pCursor!=0 );
   pCrsr = pC->pCursor;
   assert( sqlite3BtreeCursorIsValid(pCrsr) );
@@ -4368,7 +4399,7 @@ case OP_Next: {        /* jump */
   }
   assert( pC->isSorter==(pOp->opcode==OP_SorterNext) );
   if( isSorter(pC) ){
-    assert( pOp->opcode==OP_Next );
+    assert( pOp->opcode==OP_SorterNext );
     rc = sqlite3VdbeSorterNext(db, pC, &res);
   }else{
     res = 1;
@@ -4421,18 +4452,19 @@ case OP_IdxInsert: {        /* in2 */
     assert( pC->isTable==0 );
     rc = ExpandBlob(pIn2);
     if( rc==SQLITE_OK ){
+      if( isSorter(pC) ){
+        rc = sqlite3VdbeSorterWrite(db, pC, pIn2);
+      }else{
         nKey = pIn2->n;
         zKey = pIn2->z;
-      rc = sqlite3VdbeSorterWrite(db, pC, nKey);
-      if( rc==SQLITE_OK ){
         rc = sqlite3BtreeInsert(pCrsr, zKey, nKey, "", 0, 0, pOp->p3, 
             ((pOp->p5 & OPFLAG_USESEEKRESULT) ? pC->seekResult : 0)
             );
         assert( pC->deferredMoveto==0 );
-      }
         pC->cacheStatus = CACHE_STALE;
       }
     }
+  }
   break;
 }
 

src/vdbeInt.h

@@ -403,13 +403,15 @@ void sqlite3VdbeMemStoreType(Mem *pMem);
 # define sqlite3VdbeSorterRowkey(Y,Z)    SQLITE_OK
 # define sqlite3VdbeSorterRewind(X,Y,Z)  SQLITE_OK
 # define sqlite3VdbeSorterNext(X,Y,Z)    SQLITE_OK
+# define sqlite3VdbeSorterCompare(X,Y,Z) SQLITE_OK
 #else
 int sqlite3VdbeSorterInit(sqlite3 *, VdbeCursor *);
-int sqlite3VdbeSorterWrite(sqlite3 *, VdbeCursor *, int);
 void sqlite3VdbeSorterClose(sqlite3 *, VdbeCursor *);
 int sqlite3VdbeSorterRowkey(VdbeCursor *, Mem *);
-int sqlite3VdbeSorterRewind(sqlite3 *, VdbeCursor *, int *);
 int sqlite3VdbeSorterNext(sqlite3 *, VdbeCursor *, int *);
+int sqlite3VdbeSorterRewind(sqlite3 *, VdbeCursor *, int *);
+int sqlite3VdbeSorterWrite(sqlite3 *, VdbeCursor *, Mem *);
+int sqlite3VdbeSorterCompare(VdbeCursor *, Mem *, int *);
 #endif
 
 #if !defined(SQLITE_OMIT_SHARED_CACHE) && SQLITE_THREADSAFE>0

src/vdbesort.c

@@ -21,6 +21,7 @@
 #ifndef SQLITE_OMIT_MERGE_SORT
 
 typedef struct VdbeSorterIter VdbeSorterIter;
+typedef struct SorterRecord SorterRecord;
 
 /*
 ** NOTES ON DATA STRUCTURE USED FOR N-WAY MERGES:
@@ -92,8 +93,7 @@ typedef struct VdbeSorterIter VdbeSorterIter;
 ** being merged (rounded up to the next power of 2).
 */
 struct VdbeSorter {
-  int nWorking;                   /* Start a new b-tree after this many pages */
+  int nInMemory;                  /* Current size of b-tree contents as PMA */
-  int nBtree;                     /* Current size of b-tree contents as PMA */
   int nTree;                      /* Used size of aTree/aIter (power of 2) */
   VdbeSorterIter *aIter;          /* Array of iterators to merge */
   int *aTree;                     /* Current state of incremental merge */
@@ -101,6 +101,9 @@ struct VdbeSorter {
   i64 iReadOff;                   /* Current read offset within file pTemp1 */
   sqlite3_file *pTemp1;           /* PMA file 1 */
   int nPMA;                       /* Number of PMAs stored in pTemp1 */
+  SorterRecord *pRecord;          /* Head of in-memory record list */
+  int nLimit1;                    /* Minimum PMA size, in bytes */
+  int nLimit2;                    /* Maximum PMA size, in bytes */
 };
 
 /*
@@ -117,6 +120,17 @@ struct VdbeSorterIter {
   u8 *aKey;                       /* Pointer to current key */
 };
 
+/*
+** A structure to store a single record. All in-memory records are connected
+** together into a linked list headed at VdbeSorter.pRecord using the 
+** SorterRecord.pNext pointer.
+*/
+struct SorterRecord {
+  void *pVal;
+  int nVal;
+  SorterRecord *pNext;
+};
+
 /* Minimum allowable value for the VdbeSorter.nWorking variable */
 #define SORTER_MIN_WORKING 10
 
@@ -275,6 +289,50 @@ static int vdbeSorterIterInit(
   return rc;
 }
 
+
+/*
+** Compare key1 (buffer pKey1, size nKey1 bytes) with key2 (buffer pKey2, 
+** size nKey2 bytes).  Argument pKeyInfo supplies the collation functions
+** used by the comparison. If an error occurs, return an SQLite error code.
+** Otherwise, return SQLITE_OK and set *pRes to a negative, zero or positive
+** value, depending on whether key1 is smaller, equal to or larger than key2.
+**
+** If the bOmitRowid argument is non-zero, assume both keys end in a rowid
+** field. For the purposes of the comparison, ignore it. Also, if bOmitRowid
+** is true and key1 contains even a single NULL value, it is considered to
+** be less than key2. Even if key2 also contains NULL values.
+*/
+static int vdbeSorterCompare(
+  KeyInfo *pKeyInfo,              /* Collation functions to use in comparison */
+  int bOmitRowid,                 /* Ignore rowid field at end of keys */
+  void *pKey1, int nKey1,         /* Left side of comparison */
+  void *pKey2, int nKey2,         /* Right side of comparison */
+  int *pRes                       /* OUT: Result of comparison */
+){
+  char aSpace[150];
+  UnpackedRecord *r2;
+  int i;
+
+  r2 = sqlite3VdbeRecordUnpack(pKeyInfo, nKey2, pKey2, aSpace, sizeof(aSpace));
+  if( r2==0 ) return SQLITE_NOMEM;
+  if( bOmitRowid ){
+    for(i=0; i<r2->nField-1; i++){
+      if( r2->aMem[i].flags & MEM_Null ){
+        *pRes = -1;
+        sqlite3VdbeDeleteUnpackedRecord(r2);
+        return SQLITE_OK;
+      }
+    }
+    r2->flags |= UNPACKED_PREFIX_MATCH;
+    r2->nField--;
+    assert( r2->nField>0 );
+  }
+
+  *pRes = sqlite3VdbeRecordCompare(nKey1, pKey1, r2);
+  sqlite3VdbeDeleteUnpackedRecord(r2);
+  return SQLITE_OK;
+}
+
 /*
 ** This function is called to compare two iterator keys when merging 
 ** multiple b-tree segments. Parameter iOut is the index of the aTree[] 
@@ -306,20 +364,16 @@ static int vdbeSorterDoCompare(VdbeCursor *pCsr, int iOut){
   }else if( p2->pFile==0 ){
     iRes = i1;
   }else{
-    char aSpace[150];
+    int res;
-    UnpackedRecord *r1;
+    int rc = vdbeSorterCompare(
-
+        pCsr->pKeyInfo, 0, p1->aKey, p1->nKey, p2->aKey, p2->nKey, &res
-    r1 = sqlite3VdbeRecordUnpack(
-        pCsr->pKeyInfo, p1->nKey, p1->aKey, aSpace, sizeof(aSpace)
     );
-    if( r1==0 ) return SQLITE_NOMEM;
+    if( rc!=SQLITE_OK ) return rc;
-
+    if( res<=0 ){
-    if( sqlite3VdbeRecordCompare(p2->nKey, p2->aKey, r1)>=0 ){
       iRes = i1;
     }else{
       iRes = i2;
     }
-    sqlite3VdbeDeleteUnpackedRecord(r1);
   }
 
   pSorter->aTree[iOut] = iRes;
@@ -330,9 +384,32 @@ static int vdbeSorterDoCompare(VdbeCursor *pCsr, int iOut){
 ** Initialize the temporary index cursor just opened as a sorter cursor.
 */
 int sqlite3VdbeSorterInit(sqlite3 *db, VdbeCursor *pCsr){
-  assert( pCsr->pKeyInfo && pCsr->pBt );
+  int pgsz;                       /* Page size of main database */
+
+  assert( pCsr->pKeyInfo && pCsr->pBt==0 );
   pCsr->pSorter = sqlite3DbMallocZero(db, sizeof(VdbeSorter));
-  return (pCsr->pSorter ? SQLITE_OK : SQLITE_NOMEM);
+  if( pCsr->pSorter==0 ){
+    return SQLITE_NOMEM;
+  }
+
+  pgsz = sqlite3BtreeGetPageSize(db->aDb[0].pBt);
+  pCsr->pSorter->nLimit1 = 10 * pgsz;
+  pCsr->pSorter->nLimit2 = db->aDb[0].pSchema->cache_size * pgsz;
+
+  return SQLITE_OK;
+}
+
+/*
+** Free the list of sorted records starting at pRecord.
+*/
+static void vdbeSorterRecordFree(sqlite3 *db, SorterRecord *pRecord){
+  SorterRecord *p;
+  SorterRecord *pNext;
+  for(p=pRecord; p; p=pNext){
+    pNext = p->pNext;
+    sqlite3DbFree(db, p->pVal);
+    sqlite3DbFree(db, p);
+  }
 }
 
 /*
@@ -351,6 +428,7 @@ void sqlite3VdbeSorterClose(sqlite3 *db, VdbeCursor *pCsr){
     if( pSorter->pTemp1 ){
       sqlite3OsCloseFree(pSorter->pTemp1);
     }
+    vdbeSorterRecordFree(db, pSorter->pRecord);
     sqlite3DbFree(db, pSorter);
     pCsr->pSorter = 0;
   }
@@ -370,10 +448,128 @@ static int vdbeSorterOpenTempFile(sqlite3 *db, sqlite3_file **ppFile){
   );
 }
 
+/*
+** Attemp to merge the two sorted lists p1 and p2 into a single list. If no
+** error occurs set *ppOut to the head of the new list and return SQLITE_OK.
+*/
+static int vdbeSorterMerge(
+  sqlite3 *db,                    /* Database handle */
+  KeyInfo *pKeyInfo,              /* Collation functions to use in comparison */
+  SorterRecord *p1,               /* First list to merge */
+  SorterRecord *p2,               /* Second list to merge */
+  SorterRecord **ppOut            /* OUT: Head of merged list */
+){
+  int rc = SQLITE_OK;
+  SorterRecord *pFinal = 0;
+  SorterRecord **pp = &pFinal;
+
+  while( p1 || p2 ){
+    if( p1==0 ){
+      *pp = p2;
+      p2 = 0;
+    }else if( p2==0 ){
+      *pp = p1;
+      p1 = 0;
+    }else{
+      int res;
+      rc = vdbeSorterCompare(
+          pKeyInfo, 0, p1->pVal, p1->nVal, p2->pVal, p2->nVal, &res
+      );
+      if( rc!=SQLITE_OK ){
+        vdbeSorterRecordFree(db, p1);
+        vdbeSorterRecordFree(db, p2);
+        vdbeSorterRecordFree(db, pFinal);
+        pFinal = 0;
+        break;
+      }
+      if( res<=0 ){
+        *pp = p1;
+        pp = &p1->pNext;
+        p1 = p1->pNext;
+      }else{
+        *pp = p2;
+        pp = &p2->pNext;
+        p2 = p2->pNext;
+      }
+      *pp = 0;
+    }
+  }
+
+  *ppOut = pFinal;
+  return rc;
+}
 
 /*
-** Write the current contents of the b-tree to a PMA. Return SQLITE_OK
+** Sort the linked list of records headed at pCsr->pRecord. Return SQLITE_OK
-** if successful, or an SQLite error code otherwise.
+** if successful, or an SQLite error code (i.e. SQLITE_NOMEM) if an error
+** occurs.
+*/
+static int vdbeSorterSort(sqlite3 *db, VdbeCursor *pCsr){
+  int rc = SQLITE_OK;
+  int i;
+  SorterRecord **aSlot;
+  SorterRecord *p;
+  VdbeSorter *pSorter = pCsr->pSorter;
+  KeyInfo *pKeyInfo = pCsr->pKeyInfo;
+
+  aSlot = (SorterRecord **)sqlite3MallocZero(64 * sizeof(SorterRecord *));
+  if( !aSlot ){
+    return SQLITE_NOMEM;
+  }
+
+  p = pSorter->pRecord;
+  while( p ){
+    SorterRecord *pNext = p->pNext;
+    p->pNext = 0;
+    for(i=0; rc==SQLITE_OK && aSlot[i]; i++){
+      rc = vdbeSorterMerge(db, pKeyInfo, p, aSlot[i], &p);
+      aSlot[i] = 0;
+    }
+    if( rc!=SQLITE_OK ){
+      vdbeSorterRecordFree(db, pNext);
+      break;
+    }
+    aSlot[i] = p;
+    p = pNext;
+  }
+
+  p = 0;
+  for(i=0; i<64; i++){
+    if( rc==SQLITE_OK ){
+      rc = vdbeSorterMerge(db, pKeyInfo, p, aSlot[i], &p);
+    }else{
+      vdbeSorterRecordFree(db, aSlot[i]);
+    }
+  }
+  pSorter->pRecord = p;
+
+#if 0
+  {
+    SorterRecord *pTmp1 = 0;
+    SorterRecord *pTmp2;
+    for(pTmp2=pSorter->pRecord; pTmp2 && rc==SQLITE_OK; pTmp2=pTmp2->pNext){
+      if( pTmp1 ){
+        int res;
+        rc = vdbeSorterCompare(pKeyInfo, 
+            0, pTmp1->pVal, pTmp1->nVal, pTmp2->pVal, pTmp2->nVal, &res
+        );
+        assert( rc!=SQLITE_OK || res<0 );
+      }
+      pTmp1 = pTmp2;
+    }
+  }
+#endif
+
+  if( rc!=SQLITE_OK ){
+  }
+  sqlite3_free(aSlot);
+  return rc;
+}
+
+
+/*
+** Write the current contents of the in-memory linked-list to a PMA. Return
+** SQLITE_OK if successful, or an SQLite error code otherwise.
 **
 ** The format of a PMA is:
 **
@@ -384,19 +580,19 @@ static int vdbeSorterOpenTempFile(sqlite3 *db, sqlite3_file **ppFile){
 **       Each record consists of a varint followed by a blob of data (the 
 **       key). The varint is the number of bytes in the blob of data.
 */
-static int vdbeSorterBtreeToPMA(sqlite3 *db, VdbeCursor *pCsr){
+static int vdbeSorterListToPMA(sqlite3 *db, VdbeCursor *pCsr){
   int rc = SQLITE_OK;             /* Return code */
   VdbeSorter *pSorter = pCsr->pSorter;
-  int res = 0;
 
-  /* sqlite3BtreeFirst() cannot fail because sorter btrees are always held
+  if( pSorter->nInMemory==0 ){
-  ** in memory and so an I/O error is not possible. */
+    assert( pSorter->pRecord==0 );
-  rc = sqlite3BtreeFirst(pCsr->pCursor, &res);
+    return rc;
-  if( NEVER(rc!=SQLITE_OK) || res ) return rc;
+  }
-  assert( pSorter->nBtree>0 );
+
+  rc = vdbeSorterSort(db, pCsr);
 
   /* If the first temporary PMA file has not been opened, open it now. */
-  if( pSorter->pTemp1==0 ){
+  if( rc==SQLITE_OK && pSorter->pTemp1==0 ){
     rc = vdbeSorterOpenTempFile(db, &pSorter->pTemp1);
     assert( rc!=SQLITE_OK || pSorter->pTemp1 );
     assert( pSorter->iWriteOff==0 );
@@ -404,129 +600,89 @@ static int vdbeSorterBtreeToPMA(sqlite3 *db, VdbeCursor *pCsr){
   }
 
   if( rc==SQLITE_OK ){
-    i64 iWriteOff = pSorter->iWriteOff;
+    i64 iOff = pSorter->iWriteOff;
-    void *aMalloc = 0;            /* Array used to hold a single record */
+    SorterRecord *p;
-    int nMalloc = 0;              /* Allocated size of aMalloc[] in bytes */
+    SorterRecord *pNext = 0;
 
     pSorter->nPMA++;
-    for(
+    rc = vdbeSorterWriteVarint(pSorter->pTemp1, pSorter->nInMemory, &iOff);
-      rc = vdbeSorterWriteVarint(pSorter->pTemp1, pSorter->nBtree, &iWriteOff);
+    for(p=pSorter->pRecord; rc==SQLITE_OK && p; p=pNext){
-      rc==SQLITE_OK && res==0;
+      pNext = p->pNext;
-      rc = sqlite3BtreeNext(pCsr->pCursor, &res)
+      rc = vdbeSorterWriteVarint(pSorter->pTemp1, p->nVal, &iOff);
-    ){
-      i64 nKey;                   /* Size of this key in bytes */
 
-      /* Write the size of the record in bytes to the output file */
-      (void)sqlite3BtreeKeySize(pCsr->pCursor, &nKey);
-      rc = vdbeSorterWriteVarint(pSorter->pTemp1, nKey, &iWriteOff);
-
-      /* Make sure the aMalloc[] buffer is large enough for the record */
-      if( rc==SQLITE_OK && nKey>nMalloc ){
-        aMalloc = sqlite3DbReallocOrFree(db, aMalloc, nKey);
-        if( !aMalloc ){ 
-          rc = SQLITE_NOMEM; 
-        }else{
-          nMalloc = nKey;
-        }
-      }
-
-      /* Write the record itself to the output file */
       if( rc==SQLITE_OK ){
-        /* sqlite3BtreeKey() cannot fail because sorter btrees held in memory */
+        rc = sqlite3OsWrite(pSorter->pTemp1, p->pVal, p->nVal, iOff);
-        rc = sqlite3BtreeKey(pCsr->pCursor, 0, nKey, aMalloc);
+        iOff += p->nVal;
-        if( ALWAYS(rc==SQLITE_OK) ){
-          rc = sqlite3OsWrite(pSorter->pTemp1, aMalloc, nKey, iWriteOff);
-          iWriteOff += nKey;
-        }
       }
 
-      if( rc!=SQLITE_OK ) break;
+      sqlite3DbFree(db, p->pVal);
+      sqlite3DbFree(db, p);
     }
 
     /* This assert verifies that unless an error has occurred, the size of 
     ** the PMA on disk is the same as the expected size stored in
-    ** pSorter->nBtree. */ 
+    ** pSorter->nInMemory. */ 
-    assert( rc!=SQLITE_OK || pSorter->nBtree==(
+    assert( rc!=SQLITE_OK || pSorter->nInMemory==(
-          iWriteOff-pSorter->iWriteOff-sqlite3VarintLen(pSorter->nBtree)
+          iOff-pSorter->iWriteOff-sqlite3VarintLen(pSorter->nInMemory)
     ));
 
-    pSorter->iWriteOff = iWriteOff;
+    pSorter->iWriteOff = iOff;
-    sqlite3DbFree(db, aMalloc);
+    pSorter->pRecord = p;
   }
 
-  pSorter->nBtree = 0;
   return rc;
 }
 
 /*
-** This function is called on a sorter cursor by the VDBE before each row 
+** Add a record to the sorter.
-** is inserted into VdbeCursor.pCsr. Argument nKey is the size of the key, in
-** bytes, about to be inserted.
-**
-** If it is determined that the temporary b-tree accessed via VdbeCursor.pCsr
-** is large enough, its contents are written to a sorted PMA on disk and the
-** tree emptied. This prevents the b-tree (which must be small enough to
-** fit entirely in the cache in order to support efficient inserts) from
-** growing too large.
-**
-** An SQLite error code is returned if an error occurs. Otherwise, SQLITE_OK.
 */
-int sqlite3VdbeSorterWrite(sqlite3 *db, VdbeCursor *pCsr, int nKey){
+int sqlite3VdbeSorterWrite(
-  int rc = SQLITE_OK;             /* Return code */
+  sqlite3 *db,                    /* Database handle */
+  VdbeCursor *pCsr,               /* Sorter cursor */
+  Mem *pVal                       /* Memory cell containing record */
+){
   VdbeSorter *pSorter = pCsr->pSorter;
-  if( pSorter ){
+  int rc;
-    Pager *pPager = sqlite3BtreePager(pCsr->pBt);
+  SorterRecord *pNew;
-    int nPage;                    /* Current size of temporary file in pages */
 
-    /* Sorters never spill to disk */
+  assert( pSorter );
-    assert( sqlite3PagerFile(pPager)->pMethods==0 );
+  pSorter->nInMemory += sqlite3VarintLen(pVal->n) + pVal->n;
-
-    /* Determine how many pages the temporary b-tree has grown to */
-    sqlite3PagerPagecount(pPager, &nPage);
-
-    /* If pSorter->nWorking is still zero, but the temporary file has been
-    ** created in the file-system, then the most recent insert into the
-    ** current b-tree segment probably caused the cache to overflow (it is
-    ** also possible that sqlite3_release_memory() was called). So set the
-    ** size of the working set to a little less than the current size of the 
-    ** file in pages.  */
-    if( pSorter->nWorking==0 && sqlite3PagerUnderStress(pPager) ){
-      pSorter->nWorking = nPage-5;
-      if( pSorter->nWorking<SORTER_MIN_WORKING ){
-        pSorter->nWorking = SORTER_MIN_WORKING;
-      }
-    }
-
-    /* If the number of pages used by the current b-tree segment is greater
-    ** than the size of the working set (VdbeSorter.nWorking), start a new
-    ** segment b-tree.  */
-    if( pSorter->nWorking && nPage>=pSorter->nWorking ){
-      BtCursor *p = pCsr->pCursor;/* Cursor structure to close and reopen */
-      int iRoot;                  /* Root page of new tree */
-
-      /* Copy the current contents of the b-tree into a PMA in sorted order.
-      ** Close the currently open b-tree cursor. */
-      rc = vdbeSorterBtreeToPMA(db, pCsr);
-      sqlite3BtreeCloseCursor(p);
 
+  pNew = (SorterRecord *)sqlite3DbMallocZero(db, sizeof(SorterRecord));
+  if( pNew==0 ){
+    rc = SQLITE_NOMEM;
+  }else{
+    rc = sqlite3VdbeMemMakeWriteable(pVal);
     if( rc==SQLITE_OK ){
-        rc = sqlite3BtreeDropTable(pCsr->pBt, 2, 0);
+      pNew->pVal = pVal->z;
-#ifdef SQLITE_DEBUG
+      pNew->nVal = pVal->n;
-        sqlite3PagerPagecount(pPager, &nPage);
+      pVal->zMalloc = 0;
-        assert( rc!=SQLITE_OK || nPage==1 );
+      sqlite3VdbeMemSetNull(pVal);
-#endif
+      pNew->pNext = pSorter->pRecord;
-      }
+      pSorter->pRecord = pNew;
-      if( rc==SQLITE_OK ){
+    }else{
-        rc = sqlite3BtreeCreateTable(pCsr->pBt, &iRoot, BTREE_BLOBKEY);
+      sqlite3DbFree(db, pNew);
-      }
+      rc = SQLITE_NOMEM;
-      if( rc==SQLITE_OK ){
-        assert( iRoot==2 );
-        rc = sqlite3BtreeCursor(pCsr->pBt, iRoot, 1, pCsr->pKeyInfo, p);
     }
   }
 
-    pSorter->nBtree += sqlite3VarintLen(nKey) + nKey;
+  /* See if the contents of the sorter should now be written out. They
+  ** are written out when either of the following are true:
+  **
+  **   * The total memory allocated for the in-memory list is greater 
+  **     than (page-size * cache-size), or
+  **
+  **   * The total memory allocated for the in-memory list is greater 
+  **     than (page-size * 10) and sqlite3HeapNearlyFull() returns true.
+  */
+  if( rc==SQLITE_OK && (
+        (pSorter->nInMemory>pSorter->nLimit2)
+     || (pSorter->nInMemory>pSorter->nLimit1 && sqlite3HeapNearlyFull())
+  )){
+    rc = vdbeSorterListToPMA(db, pCsr);
+    pSorter->nInMemory = 0;
   }
+
   return rc;
 }
 
@@ -577,8 +733,7 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, VdbeCursor *pCsr, int *pbEof){
   assert( pSorter );
 
   /* Write the current b-tree to a PMA. Close the b-tree cursor. */
-  rc = vdbeSorterBtreeToPMA(db, pCsr);
+  rc = vdbeSorterListToPMA(db, pCsr);
-  sqlite3BtreeCloseCursor(pCsr->pCursor);
   if( rc!=SQLITE_OK ) return rc;
   if( pSorter->nPMA==0 ){
     *pbEof = 1;
@@ -692,14 +847,7 @@ int sqlite3VdbeSorterRowkey(VdbeCursor *pCsr, Mem *pOut){
   VdbeSorterIter *pIter;
 
   pIter = &pSorter->aIter[ pSorter->aTree[1] ];
-
+  if( sqlite3VdbeMemGrow(pOut, pIter->nKey, 0) ){
-  /* Coverage testing note: As things are currently, this call will always
-  ** succeed. This is because the memory cell passed by the VDBE layer 
-  ** happens to be the same one as was used to assemble the keys before they
-  ** were passed to the sorter - meaning it is always large enough for the
-  ** largest key. But this could change very easily, so we leave the call
-  ** to sqlite3VdbeMemGrow() in. */
-  if( NEVER(sqlite3VdbeMemGrow(pOut, pIter->nKey, 0)) ){
     return SQLITE_NOMEM;
   }
   pOut->n = pIter->nKey;
@@ -709,4 +857,30 @@ int sqlite3VdbeSorterRowkey(VdbeCursor *pCsr, Mem *pOut){
   return SQLITE_OK;
 }
 
+/*
+** Compare the key in memory cell pVal with the key that the sorter cursor
+** passed as the first argument currently points to. For the purposes of
+** the comparison, ignore the rowid field at the end of each record.
+**
+** If an error occurs, return an SQLite error code (i.e. SQLITE_NOMEM).
+** Otherwise, set *pRes to a negative, zero or positive value if the
+** key in pVal is smaller than, equal to or larger than the current sorter
+** key.
+*/
+int sqlite3VdbeSorterCompare(
+  VdbeCursor *pCsr,               /* Sorter cursor */
+  Mem *pVal,                      /* Value to compare to current sorter key */
+  int *pRes                       /* OUT: Result of comparison */
+){
+  int rc;
+  VdbeSorter *pSorter = pCsr->pSorter;
+  VdbeSorterIter *pIter;
+  pIter = &pSorter->aIter[ pSorter->aTree[1] ];
+  rc = vdbeSorterCompare(pCsr->pKeyInfo, 1,
+      pVal->z, pVal->n, pIter->aKey, pIter->nKey, pRes
+  );
+  assert( rc!=SQLITE_OK || *pRes<=0 );
+  return rc;
+}
+
 #endif /* #ifndef SQLITE_OMIT_MERGE_SORT */

test/index4.test

@@ -108,5 +108,19 @@ do_execsql_test 1.8 {
   PRAGMA integrity_check
 } {ok}
 
+do_execsql_test 2.1 {
+  BEGIN;
+    CREATE TABLE t2(x);
+    INSERT INTO t2 VALUES(14);
+    INSERT INTO t2 VALUES(35);
+    INSERT INTO t2 VALUES(15);
+    INSERT INTO t2 VALUES(35);
+    INSERT INTO t2 VALUES(16);
+  COMMIT;
+}
+do_catchsql_test 2.2 {
+  CREATE UNIQUE INDEX i3 ON t2(x);
+} {1 {indexed columns are not unique}}
+
 
 finish_test