Merge latest changes from orderby-planning branch.

FossilOrigin-Name: 4c7fb5423430f3b936befaa7c309f8e1968ee7d8

manifest

@@ -1,5 +1,5 @@
-C Merge\sfrom\strunk\sthe\sfix\sfor\sthe\scrash\son\sa\scorrupt\sdatabase.
+C Merge\slatest\schanges\sfrom\sorderby-planning\sbranch.
-D 2014-03-26T19:45:01.037
+D 2014-03-28T19:18:16.969
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -164,7 +164,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34
 F src/backup.c a729e63cf5cd1829507cb7b8e89f99b95141bb53
 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb
 F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7
-F src/btree.c 8d7e432bdd27d63182865c708ea0e7606489b6d1
+F src/btree.c a59a199f21338ae1847d69f5db87c3e8ef1b1578
 F src/btree.h 232836cb51753f2e96aa8ce0f052c6df850f76ba
 F src/btreeInt.h 0be66063468a520e4d66b80c7a1dc26d04ee6ea4
 F src/build.c 0d50ef95aad63f4c4fc47f3fa2670d4557c45db0
@@ -217,12 +217,12 @@ F src/printf.c e5a0005f8b3de21f85da6a709d2fbee76775bf4b
 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece
 F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66
 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0
-F src/select.c 269c3e31a450fce642a10569221a49180348c88e
+F src/select.c 20055cf917222e660c4222fea306bd13a0623caa
 F src/shell.c f48b63f8e582e7998ecefd051d697f91fb1453df
 F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80
 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e
 F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc
-F src/sqliteInt.h fb667a3d602d405be6abf0fb21246aac7bb23e76
+F src/sqliteInt.h 3f5190a4e07ca227035334da8d66ebe227071528
 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d
 F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158
 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e
@@ -279,20 +279,20 @@ F src/update.c 5b3e74a03b3811e586b4f2b4cbd7c49f01c93115
 F src/utf.c 6dc9ec9f1b3db43ae8ba0365377f11df1ee4c01c
 F src/util.c c46c90459ef9bdc0c6c73803cf4c55425b4771cf
 F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179
-F src/vdbe.c 74c7386e83eee56f921a17bb4a0396c9551f5bc7
+F src/vdbe.c 02f2de0b2f3b198438e3e64a2ceba9407bb8348b
-F src/vdbe.h fb2c48c198300a7c632f09fc940011d2ad2fc2ae
+F src/vdbe.h 394464909ed682334aa3d5831aae0c2fe2abef94
-F src/vdbeInt.h 2b9a6849166d0014c843ae3fd83a062be4efa325
+F src/vdbeInt.h e6d83e5bfd62fc6685ba1ed6153f7099f82de9f7
 F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4
-F src/vdbeaux.c f81ef920dcf76aceaa1ce77081e9fc5d7a0993dd
+F src/vdbeaux.c 1153175fb57a8454e1c8cf79b59b7bf92b26779d
 F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa
 F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447
-F src/vdbesort.c c3e427de848b78e9e9feaa25f68fb64686bab6cd
+F src/vdbesort.c 01068b89364fa2bffeba9b929367ed04661e97f7
 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767
 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd
 F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8
 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4
 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45
-F src/where.c da8ec216f14af617505799b0b4e52c73dda7a5ca
+F src/where.c 7d539cedb1c6a6d6b5d2075b8fea3a48db4838eb
 F src/whereInt.h 2564055b440e44ebec8b47f237bbccae6719b7af
 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2
 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2
@@ -404,9 +404,9 @@ F test/corruptC.test 02405cf7ed0c1e989060e1aab6d02ffbc3906fbb
 F test/corruptD.test b3c205fac7952b1de645ce44bb02335cd9e3e040
 F test/corruptE.test 193b4ca4e927e77c1d5f4f56203ddc998432a7ee
 F test/corruptF.test be9fde98e4c93648f1ba52b74e5318edc8f59fe4
-F test/corruptG.test 58ec333a01997fe655e34e5bea52b7a2a6b9704d
+F test/corruptG.test 1ab3bf97ee7bdba70e0ff3ba2320657df55d1804
 F test/corruptH.test 88ed71a086e13591c917aac6de32750e7c7281cb
-F test/corruptI.test 1b796461e5b635e0a74e3c4ecb1121c82d319dff
+F test/corruptI.test b3e4203d420490fc3d3062711597bc1dea06a789
 F test/count.test 42a251178e32f617eda33f76236a7f79825a50b5
 F test/coveridxscan.test cdb47d01acc4a634a34fd25abe85189e0d0f1e62
 F test/crash.test fb9dc4a02dcba30d4aa5c2c226f98b220b2b959f
@@ -817,7 +817,7 @@ F test/skipscan1.test bed8cbe9d554c8c27afb6c88500f704c86a9196f
 F test/skipscan2.test 5a4db0799c338ddbacb154aaa5589c0254b36a8d
 F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f
 F test/softheap1.test 40562fe6cac6d9827b7b42b86d45aedf12c15e24
-F test/sort.test 0e4456e729e5a92a625907c63dcdedfbe72c5dc5
+F test/sort.test 79dc647c4e9b123a64e57b7080b7f9a2df43f87a
 F test/speed1.test f2974a91d79f58507ada01864c0e323093065452
 F test/speed1p.explain d841e650a04728b39e6740296b852dccdca9b2cb
 F test/speed1p.test b180e98609c7677382cf618c0ec9b69f789033a8
@@ -846,7 +846,7 @@ F test/tclsqlite.test 37a61c2da7e3bfe3b8c1a2867199f6b860df5d43
 F test/tempdb.test 19d0f66e2e3eeffd68661a11c83ba5e6ace9128c
 F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30
 F test/temptrigger.test 8ec228b0db5d7ebc4ee9b458fc28cb9e7873f5e1
-F test/tester.tcl f31bea1483ea1d39620f982130026e76f872d744
+F test/tester.tcl bc0889a2f86d9c17307992ca1e70391794780265
 F test/thread001.test 9f22fd3525a307ff42a326b6bc7b0465be1745a5
 F test/thread002.test e630504f8a06c00bf8bbe68528774dd96aeb2e58
 F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7
@@ -1057,7 +1057,7 @@ F test/wal3.test b22eb662bcbc148c5f6d956eaf94b047f7afe9c0
 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c
 F test/wal5.test 8f888b50f66b78821e61ed0e233ded5de378224b
 F test/wal6.test 527581f5527bf9c24394991e2be83000aace5f9e
-F test/wal64k.test 63828c2161ad76ddd4109dee0a096b6ef6895698
+F test/wal64k.test 163655ecd2cb8afef4737cac2a40fdd2eeaf20b8
 F test/wal7.test 2ae8f427d240099cc4b2dfef63cff44e2a68a1bd
 F test/wal8.test 75c42e1bc4545c277fed212f8fc9b7723cd02216
 F test/wal9.test 378e76a9ad09cd9bee06c172ad3547b0129a6750
@@ -1122,7 +1122,7 @@ F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5
 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce
 F tool/lemon.c 07aba6270d5a5016ba8107b09e431eea4ecdc123
 F tool/lempar.c 01ca97f87610d1dac6d8cd96ab109ab1130e76dc
-F tool/logest.c 7ad625cac3d54012b27d468b7af6612f78b9ba75
+F tool/logest.c 388c318c7ac8b52b7c08ca1e2de0f4ca9a8f7e81
 F tool/mkautoconfamal.sh f8d8dbf7d62f409ebed5134998bf5b51d7266383
 F tool/mkkeywordhash.c c9e05e4a7bcab8fab9f583d5b321fb72f565ad97
 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e
@@ -1160,7 +1160,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01
 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff
-P 1cab83577c814feb35b4fb91af0d52a9751d99bc f585f5d7a0f9bf8c590388654a3638231eba8892
+P 8cb2b02baa7ef9aa96319e977f0315328f944237 3047a25f1c41e83f0b4772f7c36fbfec0f12dc7e
-R 60773da8b0081a052cb0d48785e94e94
+R 132477cf9a18d083088d044c20454723
-U drh
+U dan
-Z 140bbab50f5dc3d30ae8cb405819380a
+Z b189ca5a1e8d41a36f80415af9011f68

manifest.uuid

@@ -1 +1 @@
-8cb2b02baa7ef9aa96319e977f0315328f944237
+4c7fb5423430f3b936befaa7c309f8e1968ee7d8

src/btree.c

@@ -4588,6 +4588,7 @@ int sqlite3BtreeMovetoUnpacked(
 
   if( pIdxKey ){
     xRecordCompare = sqlite3VdbeFindCompare(pIdxKey);
+    pIdxKey->isCorrupt = 0;
     assert( pIdxKey->default_rc==1 
          || pIdxKey->default_rc==0 
          || pIdxKey->default_rc==-1
@@ -4711,6 +4712,7 @@ int sqlite3BtreeMovetoUnpacked(
           c = xRecordCompare(nCell, pCellKey, pIdxKey, 0);
           sqlite3_free(pCellKey);
         }
+        assert( pIdxKey->isCorrupt==0 || c==0 );
         if( c<0 ){
           lwr = idx+1;
         }else if( c>0 ){
@@ -4720,6 +4722,7 @@ int sqlite3BtreeMovetoUnpacked(
           *pRes = 0;
           rc = SQLITE_OK;
           pCur->aiIdx[pCur->iPage] = (u16)idx;
+          if( pIdxKey->isCorrupt ) rc = SQLITE_CORRUPT;
           goto moveto_finish;
         }
         if( lwr>upr ) break;

src/select.c

@@ -455,26 +455,42 @@ static KeyInfo *keyInfoFromExprList(
 );
 
 /*
-** Insert code into "v" that will push the record in register regData
+** Generate code that will push the record in registers regData
-** into the sorter.
+** through regData+nData-1 onto the sorter.
 */
 static void pushOntoSorter(
   Parse *pParse,         /* Parser context */
   SortCtx *pSort,        /* Information about the ORDER BY clause */
   Select *pSelect,       /* The whole SELECT statement */
-  int regData            /* Register holding data to be sorted */
+  int regData,           /* First register holding data to be sorted */
+  int nData,             /* Number of elements in the data array */
+  int nPrefixReg         /* No. of reg prior to regData available for use */
 ){
-  Vdbe *v = pParse->pVdbe;
+  Vdbe *v = pParse->pVdbe;                         /* Stmt under construction */
-  int nExpr = pSort->pOrderBy->nExpr;
+  int bSeq = ((pSort->sortFlags & SORTFLAG_UseSorter)==0);
-  int regBase = sqlite3GetTempRange(pParse, nExpr+2);
+  int nExpr = pSort->pOrderBy->nExpr;              /* No. of ORDER BY terms */
-  int regRecord = sqlite3GetTempReg(pParse);
+  int nBase = nExpr + bSeq + nData;                /* Fields in sorter record */
-  int nOBSat = pSort->nOBSat;
+  int regBase;                                     /* Regs for sorter record */
-  int op;
+  int regRecord = sqlite3GetTempReg(pParse);       /* Assembled sorter record */
-  sqlite3ExprCacheClear(pParse);
+  int nOBSat = pSort->nOBSat;                      /* ORDER BY terms to skip */
-  sqlite3ExprCodeExprList(pParse, pSort->pOrderBy, regBase, 0);
+  int op;                            /* Opcode to add sorter record to sorter */
-  sqlite3VdbeAddOp2(v, OP_Sequence, pSort->iECursor, regBase+nExpr);
+
-  sqlite3ExprCodeMove(pParse, regData, regBase+nExpr+1, 1);
+  assert( bSeq==0 || bSeq==1 );
-  sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase+nOBSat, nExpr+2-nOBSat, regRecord);
+  if( nPrefixReg ){
+    assert( nPrefixReg==nExpr+bSeq );
+    regBase = regData - nExpr - bSeq;
+  }else{
+    regBase = sqlite3GetTempRange(pParse, nBase);
+  }
+  sqlite3ExprCodeExprList(pParse, pSort->pOrderBy, regBase, SQLITE_ECEL_DUP);
+  if( bSeq ){
+    sqlite3VdbeAddOp2(v, OP_Sequence, pSort->iECursor, regBase+nExpr);
+  }
+  if( nPrefixReg==0 ){
+    sqlite3VdbeAddOp3(v, OP_Move, regData, regBase+nExpr+bSeq, nData);
+  }
+
+  sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase+nOBSat, nBase-nOBSat, regRecord);
   if( nOBSat>0 ){
     int regPrevKey;   /* The first nOBSat columns of the previous row */
     int addrFirst;    /* Address of the OP_IfNot opcode */
@@ -485,8 +501,13 @@ static void pushOntoSorter(
 
     regPrevKey = pParse->nMem+1;
     pParse->nMem += pSort->nOBSat;
-    nKey = nExpr - pSort->nOBSat + 1;
+    nKey = nExpr - pSort->nOBSat + bSeq;
-    addrFirst = sqlite3VdbeAddOp1(v, OP_IfNot, regBase+nExpr); VdbeCoverage(v);
+    if( bSeq ){
+      addrFirst = sqlite3VdbeAddOp1(v, OP_IfNot, regBase+nExpr); 
+    }else{
+      addrFirst = sqlite3VdbeAddOp1(v, OP_SequenceTest, pSort->iECursor);
+    }
+    VdbeCoverage(v);
     sqlite3VdbeAddOp3(v, OP_Compare, regPrevKey, regBase, pSort->nOBSat);
     pOp = sqlite3VdbeGetOp(v, pSort->addrSortIndex);
     if( pParse->db->mallocFailed ) return;
@@ -513,7 +534,9 @@ static void pushOntoSorter(
   sqlite3VdbeAddOp2(v, op, pSort->iECursor, regRecord);
   if( nOBSat==0 ){
     sqlite3ReleaseTempReg(pParse, regRecord);
-    sqlite3ReleaseTempRange(pParse, regBase, nExpr+2);
+    if( nPrefixReg==0 ){
+      sqlite3ReleaseTempRange(pParse, regBase, nBase);
+    }
   }
   if( pSelect->iLimit ){
     int addr1, addr2;
@@ -629,6 +652,7 @@ static void selectInnerLoop(
   int eDest = pDest->eDest;   /* How to dispose of results */
   int iParm = pDest->iSDParm; /* First argument to disposal method */
   int nResultCol;             /* Number of result columns */
+  int nPrefixReg = 0;         /* Number of extra registers before regResult */
 
   assert( v );
   assert( pEList!=0 );
@@ -644,6 +668,11 @@ static void selectInnerLoop(
   nResultCol = pEList->nExpr;
 
   if( pDest->iSdst==0 ){
+    if( pSort ){
+      nPrefixReg = pSort->pOrderBy->nExpr;
+      if( !(pSort->sortFlags & SORTFLAG_UseSorter) ) nPrefixReg++;
+      pParse->nMem += nPrefixReg;
+    }
     pDest->iSdst = pParse->nMem+1;
     pParse->nMem += nResultCol;
   }else if( pDest->iSdst+nResultCol > pParse->nMem ){
@@ -760,10 +789,10 @@ static void selectInnerLoop(
     case SRT_DistFifo:
     case SRT_Table:
     case SRT_EphemTab: {
-      int r1 = sqlite3GetTempReg(pParse);
+      int r1 = sqlite3GetTempRange(pParse, nPrefixReg+1);
       testcase( eDest==SRT_Table );
       testcase( eDest==SRT_EphemTab );
-      sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r1);
+      sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r1+nPrefixReg);
 #ifndef SQLITE_OMIT_CTE
       if( eDest==SRT_DistFifo ){
         /* If the destination is DistFifo, then cursor (iParm+1) is open
@@ -778,7 +807,7 @@ static void selectInnerLoop(
       }
 #endif
       if( pSort ){
-        pushOntoSorter(pParse, pSort, p, r1);
+        pushOntoSorter(pParse, pSort, p, r1+nPrefixReg, 1, nPrefixReg);
       }else{
         int r2 = sqlite3GetTempReg(pParse);
         sqlite3VdbeAddOp2(v, OP_NewRowid, iParm, r2);
@@ -786,7 +815,7 @@ static void selectInnerLoop(
         sqlite3VdbeChangeP5(v, OPFLAG_APPEND);
         sqlite3ReleaseTempReg(pParse, r2);
       }
-      sqlite3ReleaseTempReg(pParse, r1);
+      sqlite3ReleaseTempRange(pParse, r1, nPrefixReg+1);
       break;
     }
 
@@ -804,7 +833,7 @@ static void selectInnerLoop(
         ** ORDER BY in this case since the order of entries in the set
         ** does not matter.  But there might be a LIMIT clause, in which
         ** case the order does matter */
-        pushOntoSorter(pParse, pSort, p, regResult);
+        pushOntoSorter(pParse, pSort, p, regResult, 1, nPrefixReg);
       }else{
         int r1 = sqlite3GetTempReg(pParse);
         sqlite3VdbeAddOp4(v, OP_MakeRecord, regResult,1,r1, &pDest->affSdst, 1);
@@ -830,7 +859,7 @@ static void selectInnerLoop(
     case SRT_Mem: {
       assert( nResultCol==1 );
       if( pSort ){
-        pushOntoSorter(pParse, pSort, p, regResult);
+        pushOntoSorter(pParse, pSort, p, regResult, 1, nPrefixReg);
       }else{
         sqlite3ExprCodeMove(pParse, regResult, iParm, 1);
         /* The LIMIT clause will jump out of the loop for us */
@@ -844,10 +873,7 @@ static void selectInnerLoop(
       testcase( eDest==SRT_Coroutine );
       testcase( eDest==SRT_Output );
       if( pSort ){
-        int r1 = sqlite3GetTempReg(pParse);
+        pushOntoSorter(pParse, pSort, p, regResult, nResultCol, nPrefixReg);
-        sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r1);
-        pushOntoSorter(pParse, pSort, p, r1);
-        sqlite3ReleaseTempReg(pParse, r1);
       }else if( eDest==SRT_Coroutine ){
         sqlite3VdbeAddOp1(v, OP_Yield, pDest->iSDParm);
       }else{
@@ -1127,46 +1153,62 @@ static void generateSortTail(
   int addr;
   int addrOnce = 0;
   int iTab;
-  int pseudoTab = 0;
   ExprList *pOrderBy = pSort->pOrderBy;
   int eDest = pDest->eDest;
   int iParm = pDest->iSDParm;
   int regRow;
   int regRowid;
   int nKey;
+  int iSortTab;                   /* Sorter cursor to read from */
+  int nSortData;                  /* Trailing values to read from sorter */
+  u8 p5;                          /* p5 parameter for 1st OP_Column */
+  int i;
+  int bSeq;                       /* True if sorter record includes seq. no. */
+#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS
+  struct ExprList_item *aOutEx = p->pEList->a;
+#endif
 
   if( pSort->labelBkOut ){
     sqlite3VdbeAddOp2(v, OP_Gosub, pSort->regReturn, pSort->labelBkOut);
     sqlite3VdbeAddOp2(v, OP_Goto, 0, addrBreak);
     sqlite3VdbeResolveLabel(v, pSort->labelBkOut);
-    addrOnce = sqlite3CodeOnce(pParse); VdbeCoverage(v);
   }
   iTab = pSort->iECursor;
-  regRow = sqlite3GetTempReg(pParse);
   if( eDest==SRT_Output || eDest==SRT_Coroutine ){
-    pseudoTab = pParse->nTab++;
-    sqlite3VdbeAddOp3(v, OP_OpenPseudo, pseudoTab, regRow, nColumn);
     regRowid = 0;
+    regRow = pDest->iSdst;
+    nSortData = nColumn;
   }else{
     regRowid = sqlite3GetTempReg(pParse);
+    regRow = sqlite3GetTempReg(pParse);
+    nSortData = 1;
   }
   nKey = pOrderBy->nExpr - pSort->nOBSat;
   if( pSort->sortFlags & SORTFLAG_UseSorter ){
     int regSortOut = ++pParse->nMem;
-    int ptab2 = pParse->nTab++;
+    iSortTab = pParse->nTab++;
-    sqlite3VdbeAddOp3(v, OP_OpenPseudo, ptab2, regSortOut, nKey+2);
+    if( pSort->labelBkOut ){
+      addrOnce = sqlite3CodeOnce(pParse); VdbeCoverage(v);
+    }
+    sqlite3VdbeAddOp3(v, OP_OpenPseudo, iSortTab, regSortOut, nKey+1+nSortData);
     if( addrOnce ) sqlite3VdbeJumpHere(v, addrOnce);
     addr = 1 + sqlite3VdbeAddOp2(v, OP_SorterSort, iTab, addrBreak);
     VdbeCoverage(v);
     codeOffset(v, p->iOffset, addrContinue);
     sqlite3VdbeAddOp2(v, OP_SorterData, iTab, regSortOut);
-    sqlite3VdbeAddOp3(v, OP_Column, ptab2, nKey+1, regRow);
+    p5 = OPFLAG_CLEARCACHE;
-    sqlite3VdbeChangeP5(v, OPFLAG_CLEARCACHE);
+    bSeq = 0;
   }else{
-    if( addrOnce ) sqlite3VdbeJumpHere(v, addrOnce);
     addr = 1 + sqlite3VdbeAddOp2(v, OP_Sort, iTab, addrBreak); VdbeCoverage(v);
     codeOffset(v, p->iOffset, addrContinue);
-    sqlite3VdbeAddOp3(v, OP_Column, iTab, nKey+1, regRow);
+    iSortTab = iTab;
+    p5 = 0;
+    bSeq = 1;
+  }
+  for(i=0; i<nSortData; i++){
+    sqlite3VdbeAddOp3(v, OP_Column, iSortTab, nKey+bSeq+i, regRow+i);
+    if( i==0 ) sqlite3VdbeChangeP5(v, p5);
+    VdbeComment((v, "%s", aOutEx[i].zName ? aOutEx[i].zName : aOutEx[i].zSpan));
   }
   switch( eDest ){
     case SRT_Table:
@@ -1195,17 +1237,9 @@ static void generateSortTail(
     }
 #endif
     default: {
-      int i;
       assert( eDest==SRT_Output || eDest==SRT_Coroutine ); 
       testcase( eDest==SRT_Output );
       testcase( eDest==SRT_Coroutine );
-      for(i=0; i<nColumn; i++){
-        assert( regRow!=pDest->iSdst+i );
-        sqlite3VdbeAddOp3(v, OP_Column, pseudoTab, i, pDest->iSdst+i);
-        if( i==0 ){
-          sqlite3VdbeChangeP5(v, OPFLAG_CLEARCACHE);
-        }
-      }
       if( eDest==SRT_Output ){
         sqlite3VdbeAddOp2(v, OP_ResultRow, pDest->iSdst, nColumn);
         sqlite3ExprCacheAffinityChange(pParse, pDest->iSdst, nColumn);
@@ -1215,9 +1249,10 @@ static void generateSortTail(
       break;
     }
   }
-  sqlite3ReleaseTempReg(pParse, regRow);
+  if( regRowid ){
-  sqlite3ReleaseTempReg(pParse, regRowid);
+    sqlite3ReleaseTempReg(pParse, regRow);
-
+    sqlite3ReleaseTempReg(pParse, regRowid);
+  }
   /* The bottom of the loop
   */
   sqlite3VdbeResolveLabel(v, addrContinue);
@@ -1228,9 +1263,6 @@ static void generateSortTail(
   }
   if( pSort->regReturn ) sqlite3VdbeAddOp1(v, OP_Return, pSort->regReturn);
   sqlite3VdbeResolveLabel(v, addrBreak);
-  if( eDest==SRT_Output || eDest==SRT_Coroutine ){
-    sqlite3VdbeAddOp2(v, OP_Close, pseudoTab, 0);
-  }
 }
 
 /*
@@ -4772,8 +4804,9 @@ int sqlite3Select(
     sSort.iECursor = pParse->nTab++;
     sSort.addrSortIndex =
       sqlite3VdbeAddOp4(v, OP_OpenEphemeral,
-                           sSort.iECursor, sSort.pOrderBy->nExpr+2, 0,
+          sSort.iECursor, sSort.pOrderBy->nExpr+1+pEList->nExpr, 0,
-                           (char*)pKeyInfo, P4_KEYINFO);
+          (char*)pKeyInfo, P4_KEYINFO
+      );
   }else{
     sSort.addrSortIndex = -1;
   }
@@ -4891,7 +4924,7 @@ int sqlite3Select(
     sNC.pSrcList = pTabList;
     sNC.pAggInfo = &sAggInfo;
     sAggInfo.mnReg = pParse->nMem+1;
-    sAggInfo.nSortingColumn = pGroupBy ? pGroupBy->nExpr+1 : 0;
+    sAggInfo.nSortingColumn = pGroupBy ? pGroupBy->nExpr : 0;
     sAggInfo.pGroupBy = pGroupBy;
     sqlite3ExprAnalyzeAggList(&sNC, pEList);
     sqlite3ExprAnalyzeAggList(&sNC, sSort.pOrderBy);
@@ -4983,8 +5016,8 @@ int sqlite3Select(
 
         groupBySort = 1;
         nGroupBy = pGroupBy->nExpr;
-        nCol = nGroupBy + 1;
+        nCol = nGroupBy;
-        j = nGroupBy+1;
+        j = nGroupBy;
         for(i=0; i<sAggInfo.nColumn; i++){
           if( sAggInfo.aCol[i].iSorterColumn>=j ){
             nCol++;
@@ -4994,8 +5027,7 @@ int sqlite3Select(
         regBase = sqlite3GetTempRange(pParse, nCol);
         sqlite3ExprCacheClear(pParse);
         sqlite3ExprCodeExprList(pParse, pGroupBy, regBase, 0);
-        sqlite3VdbeAddOp2(v, OP_Sequence, sAggInfo.sortingIdx,regBase+nGroupBy);
+        j = nGroupBy;
-        j = nGroupBy+1;
         for(i=0; i<sAggInfo.nColumn; i++){
           struct AggInfo_col *pCol = &sAggInfo.aCol[i];
           if( pCol->iSorterColumn>=j ){

src/sqliteInt.h

@@ -1628,6 +1628,7 @@ struct UnpackedRecord {
   KeyInfo *pKeyInfo;  /* Collation and sort-order information */
   u16 nField;         /* Number of entries in apMem[] */
   i8 default_rc;      /* Comparison result if keys are equal */
+  u8 isCorrupt;       /* Corruption detected by xRecordCompare() */
   Mem *aMem;          /* Values */
   int r1;             /* Value to return if (lhs > rhs) */
   int r2;             /* Value to return if (rhs < lhs) */

src/vdbe.c

@@ -3410,6 +3410,24 @@ case OP_SorterOpen: {
   break;
 }
 
+/* Opcode: SequenceTest P1 P2 * * *
+** Synopsis: if( cursor[P1].ctr++ ) pc = P2
+**
+** P1 is a sorter cursor. If the sequence counter is currently zero, jump
+** to P2. Regardless of whether or not the jump is taken, increment the
+** the sequence value.
+*/
+case OP_SequenceTest: {
+  VdbeCursor *pC;
+  assert( pOp->p1>=0 && pOp->p1<p->nCursor );
+  pC = p->apCsr[pOp->p1];
+  assert( pC->pSorter );
+  if( (pC->seqCount++)==0 ){
+    pc = pOp->p2 - 1;
+  }
+  break;
+}
+
 /* Opcode: OpenPseudo P1 P2 P3 * *
 ** Synopsis: P3 columns in r[P2]
 **

src/vdbe.h

@@ -211,10 +211,10 @@ void sqlite3VdbeSetVarmask(Vdbe*, int);
 #endif
 
 void sqlite3VdbeRecordUnpack(KeyInfo*,int,const void*,UnpackedRecord*);
-int sqlite3VdbeRecordCompare(int,const void*,const UnpackedRecord*,int);
+int sqlite3VdbeRecordCompare(int,const void*,UnpackedRecord*,int);
 UnpackedRecord *sqlite3VdbeAllocUnpackedRecord(KeyInfo *, char *, int, char **);
 
-typedef int (*RecordCompare)(int,const void*,const UnpackedRecord*,int);
+typedef int (*RecordCompare)(int,const void*,UnpackedRecord*,int);
 RecordCompare sqlite3VdbeFindCompare(UnpackedRecord*);
 
 #ifndef SQLITE_OMIT_TRIGGER

src/vdbeInt.h

@@ -392,7 +392,7 @@ u32 sqlite3VdbeSerialGet(const unsigned char*, u32, Mem*);
 void sqlite3VdbeDeleteAuxData(Vdbe*, int, int);
 
 int sqlite2BtreeKeyCompare(BtCursor *, const void *, int, int, int *);
-int sqlite3VdbeIdxKeyCompare(VdbeCursor*,const UnpackedRecord*,int*);
+int sqlite3VdbeIdxKeyCompare(VdbeCursor*,UnpackedRecord*,int*);
 int sqlite3VdbeIdxRowid(sqlite3*, BtCursor *, i64 *);
 int sqlite3MemCompare(const Mem*, const Mem*, const CollSeq*);
 int sqlite3VdbeExec(Vdbe*);

src/vdbeaux.c

@@ -3405,10 +3405,13 @@ static i64 vdbeRecordDecodeInt(u32 serial_type, const u8 *aKey){
 ** Key1 and Key2 do not have to contain the same number of fields. If all 
 ** fields that appear in both keys are equal, then pPKey2->default_rc is 
 ** returned.
+**
+** If database corruption is discovered, set pPKey2->isCorrupt to non-zero
+** and return 0.
 */
 int sqlite3VdbeRecordCompare(
   int nKey1, const void *pKey1,   /* Left key */
-  const UnpackedRecord *pPKey2,   /* Right key */
+  UnpackedRecord *pPKey2,         /* Right key */
   int bSkip                       /* If true, skip the first field */
 ){
   u32 d1;                         /* Offset into aKey[] of next data element */
@@ -3434,7 +3437,10 @@ int sqlite3VdbeRecordCompare(
   }else{
     idx1 = getVarint32(aKey1, szHdr1);
     d1 = szHdr1;
-    if( d1>(unsigned)nKey1 ) return 1;  /* Corruption */
+    if( d1>(unsigned)nKey1 ){ 
+      pPKey2->isCorrupt = (u8)SQLITE_CORRUPT_BKPT;
+      return 0;  /* Corruption */
+    }
     i = 0;
   }
 
@@ -3511,7 +3517,8 @@ int sqlite3VdbeRecordCompare(
         testcase( (d1+mem1.n)==(unsigned)nKey1 );
         testcase( (d1+mem1.n+1)==(unsigned)nKey1 );
         if( (d1+mem1.n) > (unsigned)nKey1 ){
-          rc = 1;                /* Corruption */
+          pPKey2->isCorrupt = (u8)SQLITE_CORRUPT_BKPT;
+          return 0;                /* Corruption */
         }else if( pKeyInfo->aColl[i] ){
           mem1.enc = pKeyInfo->enc;
           mem1.db = pKeyInfo->db;
@@ -3537,7 +3544,8 @@ int sqlite3VdbeRecordCompare(
         testcase( (d1+nStr)==(unsigned)nKey1 );
         testcase( (d1+nStr+1)==(unsigned)nKey1 );
         if( (d1+nStr) > (unsigned)nKey1 ){
-          rc = 1;                /* Corruption */
+          pPKey2->isCorrupt = (u8)SQLITE_CORRUPT_BKPT;
+          return 0;                /* Corruption */
         }else{
           int nCmp = MIN(nStr, pRhs->n);
           rc = memcmp(&aKey1[d1], pRhs->z, nCmp);
@@ -3596,7 +3604,7 @@ int sqlite3VdbeRecordCompare(
 */
 static int vdbeRecordCompareInt(
   int nKey1, const void *pKey1, /* Left key */
-  const UnpackedRecord *pPKey2, /* Right key */
+  UnpackedRecord *pPKey2,       /* Right key */
   int bSkip                     /* Ignored */
 ){
   const u8 *aKey = &((const u8*)pKey1)[*(const u8*)pKey1 & 0x3F];
@@ -3694,7 +3702,7 @@ static int vdbeRecordCompareInt(
 */
 static int vdbeRecordCompareString(
   int nKey1, const void *pKey1, /* Left key */
-  const UnpackedRecord *pPKey2, /* Right key */
+  UnpackedRecord *pPKey2,       /* Right key */
   int bSkip
 ){
   const u8 *aKey1 = (const u8*)pKey1;
@@ -3715,7 +3723,10 @@ static int vdbeRecordCompareString(
     int szHdr = aKey1[0];
 
     nStr = (serial_type-12) / 2;
-    if( (szHdr + nStr) > nKey1 ) return 0;    /* Corruption */
+    if( (szHdr + nStr) > nKey1 ){
+      pPKey2->isCorrupt = (u8)SQLITE_CORRUPT_BKPT;
+      return 0;    /* Corruption */
+    }
     nCmp = MIN( pPKey2->aMem[0].n, nStr );
     res = memcmp(&aKey1[szHdr], pPKey2->aMem[0].z, nCmp);
 
@@ -3880,7 +3891,7 @@ idx_rowid_corruption:
 */
 int sqlite3VdbeIdxKeyCompare(
   VdbeCursor *pC,                  /* The cursor to compare against */
-  const UnpackedRecord *pUnpacked, /* Unpacked version of key */
+  UnpackedRecord *pUnpacked,       /* Unpacked version of key */
   int *res                         /* Write the comparison result here */
 ){
   i64 nCellKey = 0;

src/vdbesort.c

@@ -90,6 +90,7 @@ struct SorterThread {
   int nConsolidate;               /* For THREAD_CONS, max final PMAs */
   SorterRecord *pList;            /* List of records for pThread to sort */
   int nInMemory;                  /* Expected size of PMA based on pList */
+  u8 *aListMemory;                /* Records memory (or NULL) */
 
   int nPMA;                       /* Number of PMAs currently in pTemp1 */
   i64 iTemp1Off;                  /* Offset to write to in pTemp1 */
@@ -183,6 +184,9 @@ struct VdbeSorter {
   int bUsePMA;                    /* True if one or more PMAs created */
   SorterRecord *pRecord;          /* Head of in-memory record list */
   SorterMerger *pMerger;          /* For final merge of PMAs (by caller) */ 
+  u8 *aMemory;                    /* Block of memory to alloc records from */
+  int iMemory;                    /* Offset of first free byte in aMemory */
+  int nMemory;                    /* Size of aMemory allocation in bytes */
   SorterThread aThread[SQLITE_MAX_SORTER_THREAD];
 };
 
@@ -200,6 +204,7 @@ struct VdbeSorterIter {
   u8 *aKey;                       /* Pointer to current key */
   u8 *aBuffer;                    /* Current read buffer */
   int nBuffer;                    /* Size of read buffer in bytes */
+  u8 *aMap;                       /* Pointer to mapping of entire file */
 };
 
 /*
@@ -220,15 +225,37 @@ struct FileWriter {
 
 /*
 ** A structure to store a single record. All in-memory records are connected
-** together into a linked list headed at VdbeSorter.pRecord using the 
+** together into a linked list headed at VdbeSorter.pRecord.
-** SorterRecord.pNext pointer.
+**
+** How the linked list is connected depends on how memory is being managed
+** by this module. If using a separate allocation for each in-memory record
+** (VdbeSorter.aMemory==0), then the list is always connected using the
+** SorterRecord.u.pNext pointers.
+**
+** Or, if using the single large allocation method (VdbeSorter.aMemory!=0),
+** then while records are being accumulated the list is linked using the
+** SorterRecord.u.iNext offset. This is because the aMemory[] array may
+** be sqlite3Realloc()ed while records are being accumulated. Once the VM
+** has finished passing records to the sorter, or when the in-memory buffer
+** is full, the list is sorted. As part of the sorting process, it is
+** converted to use the SorterRecord.u.pNext pointers. See function
+** vdbeSorterSort() for details.
 */
 struct SorterRecord {
-  void *pVal;
   int nVal;
-  SorterRecord *pNext;
+  union {
+    SorterRecord *pNext;          /* Pointer to next record in list */
+    int iNext;                    /* Offset within aMemory of next record */
+  } u;
 };
 
+/* Return a pointer to the buffer containing the record data for SorterRecord
+** object p. Should be used as if:
+**
+**   void *SRVAL(SorterRecord *p) { return (void*)&p[1]; }
+*/
+#define SRVAL(p) ((void*)((SorterRecord*)(p) + 1))
+
 /* The minimum PMA size is set to this value multiplied by the database
 ** page size in bytes.  */
 #define SORTER_MIN_WORKING 10
@@ -243,6 +270,7 @@ struct SorterRecord {
 static void vdbeSorterIterZero(VdbeSorterIter *pIter){
   sqlite3_free(pIter->aAlloc);
   sqlite3_free(pIter->aBuffer);
+  if( pIter->aMap ) sqlite3OsUnfetch(pIter->pFile, 0, pIter->aMap);
   memset(pIter, 0, sizeof(VdbeSorterIter));
 }
 
@@ -262,6 +290,13 @@ static int vdbeSorterIterRead(
 ){
   int iBuf;                       /* Offset within buffer to read from */
   int nAvail;                     /* Bytes of data available in buffer */
+
+  if( p->aMap ){
+    *ppOut = &p->aMap[p->iReadOff];
+    p->iReadOff += nByte;
+    return SQLITE_OK;
+  }
+
   assert( p->aBuffer );
 
   /* If there is no more data to be read from the buffer, read the next 
@@ -345,18 +380,22 @@ static int vdbeSorterIterRead(
 static int vdbeSorterIterVarint(VdbeSorterIter *p, u64 *pnOut){
   int iBuf;
 
-  iBuf = p->iReadOff % p->nBuffer;
+  if( p->aMap ){
-  if( iBuf && (p->nBuffer-iBuf)>=9 ){
+    p->iReadOff += sqlite3GetVarint(&p->aMap[p->iReadOff], pnOut);
-    p->iReadOff += sqlite3GetVarint(&p->aBuffer[iBuf], pnOut);
   }else{
-    u8 aVarint[16], *a;
+    iBuf = p->iReadOff % p->nBuffer;
-    int i = 0, rc;
+    if( iBuf && (p->nBuffer-iBuf)>=9 ){
-    do{
+      p->iReadOff += sqlite3GetVarint(&p->aBuffer[iBuf], pnOut);
-      rc = vdbeSorterIterRead(p, 1, &a);
+    }else{
-      if( rc ) return rc;
+      u8 aVarint[16], *a;
-      aVarint[(i++)&0xf] = a[0];
+      int i = 0, rc;
-    }while( (a[0]&0x80)!=0 );
+      do{
-    sqlite3GetVarint(aVarint, pnOut);
+        rc = vdbeSorterIterRead(p, 1, &a);
+        if( rc ) return rc;
+        aVarint[(i++)&0xf] = a[0];
+      }while( (a[0]&0x80)!=0 );
+      sqlite3GetVarint(aVarint, pnOut);
+    }
   }
 
   return SQLITE_OK;
@@ -400,6 +439,7 @@ static int vdbeSorterIterInit(
 ){
   int rc = SQLITE_OK;
   int nBuf = pThread->pgsz;
+  void *pMap = 0;                 /* Mapping of temp file */
 
   assert( pThread->iTemp1Off>iStart );
   assert( pIter->aAlloc==0 );
@@ -408,33 +448,41 @@ static int vdbeSorterIterInit(
   pIter->iReadOff = iStart;
   pIter->nAlloc = 128;
   pIter->aAlloc = (u8*)sqlite3Malloc(pIter->nAlloc);
-  pIter->nBuffer = nBuf;
-  pIter->aBuffer = (u8*)sqlite3Malloc(nBuf);
 
-  if( !pIter->aBuffer ){
+  /* Try to xFetch() a mapping of the entire temp file. If this is possible,
-    rc = SQLITE_NOMEM;
+  ** the PMA will be read via the mapping. Otherwise, use xRead().  */
-  }else{
+  rc = sqlite3OsFetch(pIter->pFile, 0, pThread->iTemp1Off, &pMap);
-    int iBuf;
 
-    iBuf = iStart % nBuf;
+  if( rc==SQLITE_OK ){
-    if( iBuf ){
+    if( pMap ){
-      int nRead = nBuf - iBuf;
+      pIter->aMap = (u8*)pMap;
-      if( (iStart + nRead) > pThread->iTemp1Off ){
+    }else{
-        nRead = (int)(pThread->iTemp1Off - iStart);
+      pIter->nBuffer = nBuf;
+      pIter->aBuffer = (u8*)sqlite3Malloc(nBuf);
+      if( !pIter->aBuffer ){
+        rc = SQLITE_NOMEM;
+      }else{
+        int iBuf = iStart % nBuf;
+        if( iBuf ){
+          int nRead = nBuf - iBuf;
+          if( (iStart + nRead) > pThread->iTemp1Off ){
+            nRead = (int)(pThread->iTemp1Off - iStart);
+          }
+          rc = sqlite3OsRead(
+              pThread->pTemp1, &pIter->aBuffer[iBuf], nRead, iStart
+              );
+          assert( rc!=SQLITE_IOERR_SHORT_READ );
+        }
       }
-      rc = sqlite3OsRead(
-          pThread->pTemp1, &pIter->aBuffer[iBuf], nRead, iStart
-      );
-      assert( rc!=SQLITE_IOERR_SHORT_READ );
     }
+  }
 
-    if( rc==SQLITE_OK ){
+  if( rc==SQLITE_OK ){
-      u64 nByte;
+    u64 nByte;                    /* Size of PMA in bytes */
-      pIter->iEof = pThread->iTemp1Off;
+    pIter->iEof = pThread->iTemp1Off;
-      rc = vdbeSorterIterVarint(pIter, &nByte);
+    rc = vdbeSorterIterVarint(pIter, &nByte);
-      pIter->iEof = pIter->iReadOff + nByte;
+    pIter->iEof = pIter->iReadOff + nByte;
-      *pnByte += nByte;
+    *pnByte += nByte;
-    }
   }
 
   if( rc==SQLITE_OK ){
@@ -549,34 +597,46 @@ int sqlite3VdbeSorterInit(sqlite3 *db, VdbeCursor *pCsr){
   VdbeSorter *pSorter;            /* The new sorter */
   KeyInfo *pKeyInfo;              /* Copy of pCsr->pKeyInfo with db==0 */
   int szKeyInfo;                  /* Size of pCsr->pKeyInfo in bytes */
+  int rc = SQLITE_OK;
 
   assert( pCsr->pKeyInfo && pCsr->pBt==0 );
   szKeyInfo = sizeof(KeyInfo) + (pCsr->pKeyInfo->nField-1)*sizeof(CollSeq*);
   pSorter = (VdbeSorter*)sqlite3DbMallocZero(db, sizeof(VdbeSorter)+szKeyInfo);
   pCsr->pSorter = pSorter;
   if( pSorter==0 ){
-    return SQLITE_NOMEM;
+    rc = SQLITE_NOMEM;
-  }
+  }else{
-  pKeyInfo = (KeyInfo*)&pSorter[1];
+    pKeyInfo = (KeyInfo*)&pSorter[1];
-  memcpy(pKeyInfo, pCsr->pKeyInfo, szKeyInfo);
+    memcpy(pKeyInfo, pCsr->pKeyInfo, szKeyInfo);
-  pKeyInfo->db = 0;
+    pKeyInfo->db = 0;
-  pgsz = sqlite3BtreeGetPageSize(db->aDb[0].pBt);
+    pgsz = sqlite3BtreeGetPageSize(db->aDb[0].pBt);
 
-  for(i=0; i<SQLITE_MAX_SORTER_THREAD; i++){
+    for(i=0; i<SQLITE_MAX_SORTER_THREAD; i++){
-    SorterThread *pThread = &pSorter->aThread[i];
+      SorterThread *pThread = &pSorter->aThread[i];
-    pThread->pKeyInfo = pKeyInfo;
+      pThread->pKeyInfo = pKeyInfo;
-    pThread->pVfs = db->pVfs;
+      pThread->pVfs = db->pVfs;
-    pThread->pgsz = pgsz;
+      pThread->pgsz = pgsz;
+    }
+
+    if( !sqlite3TempInMemory(db) ){
+      pSorter->mnPmaSize = SORTER_MIN_WORKING * pgsz;
+      mxCache = db->aDb[0].pSchema->cache_size;
+      if( mxCache<SORTER_MIN_WORKING ) mxCache = SORTER_MIN_WORKING;
+      pSorter->mxPmaSize = mxCache * pgsz;
+
+      /* If the application is using memsys3 or memsys5, use a separate 
+      ** allocation for each sort-key in memory. Otherwise, use a single big
+      ** allocation at pSorter->aMemory for all sort-keys.  */
+      if( sqlite3GlobalConfig.pHeap==0 ){
+        assert( pSorter->iMemory==0 );
+        pSorter->nMemory = pgsz;
+        pSorter->aMemory = (u8*)sqlite3Malloc(pgsz);
+        if( !pSorter->aMemory ) rc = SQLITE_NOMEM;
+      }
+    }
   }
 
-  if( !sqlite3TempInMemory(db) ){
+  return rc;
-    pSorter->mnPmaSize = SORTER_MIN_WORKING * pgsz;
-    mxCache = db->aDb[0].pSchema->cache_size;
-    if( mxCache<SORTER_MIN_WORKING ) mxCache = SORTER_MIN_WORKING;
-    pSorter->mxPmaSize = mxCache * pgsz;
-  }
-
-  return SQLITE_OK;
 }
 
 /*
@@ -586,7 +646,7 @@ static void vdbeSorterRecordFree(sqlite3 *db, SorterRecord *pRecord){
   SorterRecord *p;
   SorterRecord *pNext;
   for(p=pRecord; p; p=pNext){
-    pNext = p->pNext;
+    pNext = p->u.pNext;
     sqlite3DbFree(db, p);
   }
 }
@@ -598,7 +658,12 @@ static void vdbeSorterRecordFree(sqlite3 *db, SorterRecord *pRecord){
 static void vdbeSorterThreadCleanup(sqlite3 *db, SorterThread *pThread){
   sqlite3DbFree(db, pThread->pUnpacked);
   pThread->pUnpacked = 0;
-  vdbeSorterRecordFree(0, pThread->pList);
+  if( pThread->aListMemory==0 ){
+    vdbeSorterRecordFree(0, pThread->pList);
+  }else{
+    sqlite3_free(pThread->aListMemory);
+    pThread->aListMemory = 0;
+  }
   pThread->pList = 0;
   if( pThread->pTemp1 ){
     sqlite3OsCloseFree(pThread->pTemp1);
@@ -678,11 +743,14 @@ void sqlite3VdbeSorterReset(sqlite3 *db, VdbeSorter *pSorter){
     SorterThread *pThread = &pSorter->aThread[i];
     vdbeSorterThreadCleanup(db, pThread);
   }
-  vdbeSorterRecordFree(0, pSorter->pRecord);
+  if( pSorter->aMemory==0 ){
+    vdbeSorterRecordFree(0, pSorter->pRecord);
+  }
   vdbeSorterMergerReset(pSorter->pMerger);
   pSorter->pRecord = 0;
   pSorter->nInMemory = 0;
   pSorter->bUsePMA = 0;
+  pSorter->iMemory = 0;
 }
 
 /*
@@ -693,6 +761,7 @@ void sqlite3VdbeSorterClose(sqlite3 *db, VdbeCursor *pCsr){
   if( pSorter ){
     sqlite3VdbeSorterReset(db, pSorter);
     vdbeSorterMergerFree(pSorter->pMerger);
+    sqlite3_free(pSorter->aMemory);
     sqlite3DbFree(db, pSorter);
     pCsr->pSorter = 0;
   }
@@ -704,12 +773,17 @@ void sqlite3VdbeSorterClose(sqlite3 *db, VdbeCursor *pCsr){
 ** Otherwise, set *ppFile to 0 and return an SQLite error code.
 */
 static int vdbeSorterOpenTempFile(sqlite3_vfs *pVfs, sqlite3_file **ppFile){
-  int dummy;
+  int rc;
-  return sqlite3OsOpenMalloc(pVfs, 0, ppFile,
+  rc = sqlite3OsOpenMalloc(pVfs, 0, ppFile,
       SQLITE_OPEN_TEMP_JOURNAL |
       SQLITE_OPEN_READWRITE    | SQLITE_OPEN_CREATE |
-      SQLITE_OPEN_EXCLUSIVE    | SQLITE_OPEN_DELETEONCLOSE, &dummy
+      SQLITE_OPEN_EXCLUSIVE    | SQLITE_OPEN_DELETEONCLOSE, &rc
   );
+  if( rc==SQLITE_OK ){
+    i64 max = SQLITE_MAX_MMAP_SIZE;
+    sqlite3OsFileControlHint( *ppFile, SQLITE_FCNTL_MMAP_SIZE, (void*)&max);
+  }
+  return rc;
 }
 
 /*
@@ -724,22 +798,22 @@ static void vdbeSorterMerge(
 ){
   SorterRecord *pFinal = 0;
   SorterRecord **pp = &pFinal;
-  void *pVal2 = p2 ? p2->pVal : 0;
+  void *pVal2 = p2 ? SRVAL(p2) : 0;
 
   while( p1 && p2 ){
     int res;
-    vdbeSorterCompare(pThread, 0, p1->pVal, p1->nVal, pVal2, p2->nVal, &res);
+    vdbeSorterCompare(pThread, 0, SRVAL(p1), p1->nVal, pVal2, p2->nVal, &res);
     if( res<=0 ){
       *pp = p1;
-      pp = &p1->pNext;
+      pp = &p1->u.pNext;
-      p1 = p1->pNext;
+      p1 = p1->u.pNext;
       pVal2 = 0;
     }else{
       *pp = p2;
-       pp = &p2->pNext;
+       pp = &p2->u.pNext;
-      p2 = p2->pNext;
+      p2 = p2->u.pNext;
       if( p2==0 ) break;
-      pVal2 = p2->pVal;
+      pVal2 = SRVAL(p2);
     }
   }
   *pp = p1 ? p1 : p2;
@@ -763,8 +837,19 @@ static int vdbeSorterSort(SorterThread *pThread){
 
   p = pThread->pList;
   while( p ){
-    SorterRecord *pNext = p->pNext;
+    SorterRecord *pNext;
-    p->pNext = 0;
+    if( pThread->aListMemory ){
+      if( (u8*)p==pThread->aListMemory ){
+        pNext = 0;
+      }else{
+        assert( p->u.iNext<sqlite3MallocSize(pThread->aListMemory) );
+        pNext = (SorterRecord*)&pThread->aListMemory[p->u.iNext];
+      }
+    }else{
+      pNext = p->u.pNext;
+    }
+
+    p->u.pNext = 0;
     for(i=0; aSlot[i]; i++){
       vdbeSorterMerge(pThread, p, aSlot[i], &p);
       aSlot[i] = 0;
@@ -867,6 +952,30 @@ static void fileWriterWriteVarint(FileWriter *p, u64 iVal){
   fileWriterWrite(p, aByte, nByte);
 }
 
+#if SQLITE_MAX_MMAP_SIZE>0
+/*
+** The first argument is a file-handle open on a temporary file. The file
+** is guaranteed to be nByte bytes or smaller in size. This function
+** attempts to extend the file to nByte bytes in size and to ensure that
+** the VFS has memory mapped it.
+**
+** Whether or not the file does end up memory mapped of course depends on
+** the specific VFS implementation.
+*/
+static int vdbeSorterExtendFile(sqlite3_file *pFile, i64 nByte){
+  int rc = sqlite3OsTruncate(pFile, nByte);
+  if( rc==SQLITE_OK ){
+    void *p = 0;
+    sqlite3OsFetch(pFile, 0, nByte, &p);
+    sqlite3OsUnfetch(pFile, 0, p);
+  }
+  return rc;
+}
+#else
+# define vdbeSorterExtendFile(x,y) SQLITE_OK
+#endif
+
+
 /*
 ** Write the current contents of the in-memory linked-list to a PMA. Return
 ** SQLITE_OK if successful, or an SQLite error code otherwise.
@@ -895,6 +1004,13 @@ static int vdbeSorterListToPMA(SorterThread *pThread){
     assert( pThread->nPMA==0 );
   }
 
+  /* Try to get the file to memory map */
+  if( rc==SQLITE_OK ){
+    rc = vdbeSorterExtendFile(
+        pThread->pTemp1, pThread->iTemp1Off + pThread->nInMemory + 9
+    );
+  }
+
   if( rc==SQLITE_OK ){
     SorterRecord *p;
     SorterRecord *pNext = 0;
@@ -903,15 +1019,16 @@ static int vdbeSorterListToPMA(SorterThread *pThread){
     pThread->nPMA++;
     fileWriterWriteVarint(&writer, pThread->nInMemory);
     for(p=pThread->pList; p; p=pNext){
-      pNext = p->pNext;
+      pNext = p->u.pNext;
       fileWriterWriteVarint(&writer, p->nVal);
-      fileWriterWrite(&writer, p->pVal, p->nVal);
+      fileWriterWrite(&writer, SRVAL(p), p->nVal);
-      sqlite3_free(p);
+      if( pThread->aListMemory==0 ) sqlite3_free(p);
     }
     pThread->pList = p;
     rc = fileWriterFinish(&writer, &pThread->iTemp1Off);
   }
 
+  assert( pThread->pList==0 || rc!=SQLITE_OK );
   return rc;
 }
 
@@ -966,6 +1083,7 @@ static void *vdbeSorterThreadMain(void *pCtx){
       rc = SQLITE_NOMEM;
       goto thread_out;
     }
+    pThread->pUnpacked->nField = pThread->pKeyInfo->nField;
   }
 
   if( pThread->eWork==SORTER_THREAD_CONS ){
@@ -987,6 +1105,9 @@ static void *vdbeSorterThreadMain(void *pCtx){
 
       /* Open a second temp file to write merged data to */
       rc = vdbeSorterOpenTempFile(pThread->pVfs, &pTemp2);
+      if( rc==SQLITE_OK ){
+        rc = vdbeSorterExtendFile(pTemp2, pThread->iTemp1Off);
+      }
       if( rc!=SQLITE_OK ){
         vdbeSorterMergerFree(pMerger);
         break;
@@ -1094,6 +1215,8 @@ static int vdbeSorterFlushPMA(sqlite3 *db, const VdbeCursor *pCsr, int bFg){
   }
 
   if( rc==SQLITE_OK ){
+    int bUseFg = (bFg || i==(SQLITE_MAX_SORTER_THREAD-1));
+
     assert( pThread->pThread==0 && pThread->bDone==0 );
     pThread->eWork = SORTER_THREAD_TO_PMA;
     pThread->pList = pSorter->pRecord;
@@ -1101,12 +1224,29 @@ static int vdbeSorterFlushPMA(sqlite3 *db, const VdbeCursor *pCsr, int bFg){
     pSorter->nInMemory = 0;
     pSorter->pRecord = 0;
 
-    if( bFg || i<(SQLITE_MAX_SORTER_THREAD-1) ){
+    if( pSorter->aMemory ){
+      u8 *aMem = pThread->aListMemory;
+      pThread->aListMemory = pSorter->aMemory;
+      pSorter->aMemory = aMem;
+    }
+
+    if( bUseFg==0 ){
+      /* Launch a background thread for this operation */
       void *pCtx = (void*)pThread;
+      if( pSorter->aMemory==0 ){
+        pSorter->aMemory = sqlite3Malloc(pSorter->nMemory);
+        if( pSorter->aMemory==0 ) return SQLITE_NOMEM;
+      }else{
+        pSorter->nMemory = sqlite3MallocSize(pSorter->aMemory);
+      }
       rc = sqlite3ThreadCreate(&pThread->pThread, vdbeSorterThreadMain, pCtx);
     }else{
       /* Use the foreground thread for this operation */
+      u8 *aMem;
       rc = vdbeSorterRunThread(pThread);
+      aMem = pThread->aListMemory;
+      pThread->aListMemory = pSorter->aMemory;
+      pSorter->aMemory = aMem;
     }
   }
 
@@ -1125,22 +1265,21 @@ int sqlite3VdbeSorterWrite(
   int rc = SQLITE_OK;             /* Return Code */
   SorterRecord *pNew;             /* New list element */
 
+  int bFlush;                     /* True to flush contents of memory to PMA */
+  int nReq;                       /* Bytes of memory required */
+  int nPMA;                       /* Bytes of PMA space required */
+
   assert( pSorter );
-  pSorter->nInMemory += sqlite3VarintLen(pVal->n) + pVal->n;
 
-  pNew = (SorterRecord *)sqlite3Malloc(pVal->n + sizeof(SorterRecord));
+  /* Figure out whether or not the current contents of memory should be
-  if( pNew==0 ){
+  ** flushed to a PMA before continuing. If so, do so.
-    rc = SQLITE_NOMEM;
+  **
-  }else{
+  ** If using the single large allocation mode (pSorter->aMemory!=0), then
-    pNew->pVal = (void *)&pNew[1];
+  ** flush the contents of memory to a new PMA if (a) at least one value is
-    memcpy(pNew->pVal, pVal->z, pVal->n);
+  ** already in memory and (b) the new value will not fit in memory.
-    pNew->nVal = pVal->n;
+  ** 
-    pNew->pNext = pSorter->pRecord;
+  ** Or, if using separate allocations for each record, flush the contents
-    pSorter->pRecord = pNew;
+  ** of memory to a PMA if either of the following are true:
-  }
-
-  /* See if the contents of the sorter should now be written out. They
-  ** are written out when either of the following are true:
   **
   **   * The total memory allocated for the in-memory list is greater 
   **     than (page-size * cache-size), or
@@ -1148,12 +1287,56 @@ int sqlite3VdbeSorterWrite(
   **   * The total memory allocated for the in-memory list is greater 
   **     than (page-size * 10) and sqlite3HeapNearlyFull() returns true.
   */
-  if( rc==SQLITE_OK && pSorter->mxPmaSize>0 && (
+  nReq = pVal->n + sizeof(SorterRecord);
-        (pSorter->nInMemory>pSorter->mxPmaSize)
+  nPMA = pVal->n + sqlite3VarintLen(pVal->n);
-     || (pSorter->nInMemory>pSorter->mnPmaSize && sqlite3HeapNearlyFull())
+  if( pSorter->aMemory ){
-  )){
+    bFlush = pSorter->iMemory && (pSorter->iMemory+nReq) > pSorter->mxPmaSize;
-    rc = vdbeSorterFlushPMA(db, pCsr, 0);
+  }else{
+    bFlush = (
+        (pSorter->nInMemory > pSorter->mxPmaSize)
+     || (pSorter->nInMemory > pSorter->mnPmaSize && sqlite3HeapNearlyFull())
+    );
   }
+  if( bFlush ){
+    rc = vdbeSorterFlushPMA(db, pCsr, 0);
+    pSorter->nInMemory = 0;
+    pSorter->iMemory = 0;
+    assert( rc!=SQLITE_OK || pSorter->pRecord==0 );
+  }
+
+  pSorter->nInMemory += nPMA;
+
+  if( pSorter->aMemory ){
+    int nMin = pSorter->iMemory + nReq;
+
+    if( nMin>pSorter->nMemory ){
+      u8 *aNew;
+      int nNew = pSorter->nMemory * 2;
+      while( nNew < nMin ) nNew = nNew*2;
+      if( nNew > pSorter->mxPmaSize ) nNew = pSorter->mxPmaSize;
+      if( nNew < nMin ) nNew = nMin;
+
+      aNew = sqlite3Realloc(pSorter->aMemory, nNew);
+      if( !aNew ) return SQLITE_NOMEM;
+      pSorter->pRecord = aNew + ((u8*)pSorter->pRecord - pSorter->aMemory);
+      pSorter->aMemory = aNew;
+      pSorter->nMemory = nNew;
+    }
+
+    pNew = (SorterRecord*)&pSorter->aMemory[pSorter->iMemory];
+    pSorter->iMemory += ROUND8(nReq);
+    pNew->u.iNext = (u8*)(pSorter->pRecord) - pSorter->aMemory;
+  }else{
+    pNew = (SorterRecord *)sqlite3Malloc(pVal->n+sizeof(SorterRecord));
+    if( pNew==0 ){
+      return SQLITE_NOMEM;
+    }
+    pNew->u.pNext = pSorter->pRecord;
+  }
+
+  memcpy(SRVAL(pNew), pVal->z, pVal->n);
+  pNew->nVal = pVal->n;
+  pSorter->pRecord = pNew;
 
   return rc;
 }
@@ -1189,7 +1372,10 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){
       *pbEof = 0;
       pThread->pList = pSorter->pRecord;
       pThread->eWork = SORTER_THREAD_SORT;
+      assert( pThread->aListMemory==0 );
+      pThread->aListMemory = pSorter->aMemory;
       rc = vdbeSorterRunThread(pThread);
+      pThread->aListMemory = 0;
       pSorter->pRecord = pThread->pList;
       pThread->pList = 0;
     }else{
@@ -1282,9 +1468,9 @@ int sqlite3VdbeSorterNext(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){
     rc = vdbeSorterNext(&pSorter->aThread[0], pSorter->pMerger, pbEof);
   }else{
     SorterRecord *pFree = pSorter->pRecord;
-    pSorter->pRecord = pFree->pNext;
+    pSorter->pRecord = pFree->u.pNext;
-    pFree->pNext = 0;
+    pFree->u.pNext = 0;
-    vdbeSorterRecordFree(db, pFree);
+    if( pSorter->aMemory==0 ) vdbeSorterRecordFree(db, pFree);
     *pbEof = !pSorter->pRecord;
     rc = SQLITE_OK;
   }
@@ -1307,7 +1493,7 @@ static void *vdbeSorterRowkey(
     pKey = pIter->aKey;
   }else{
     *pnKey = pSorter->pRecord->nVal;
-    pKey = pSorter->pRecord->pVal;
+    pKey = SRVAL(pSorter->pRecord);
   }
   return pKey;
 }

src/where.c

@@ -4328,18 +4328,34 @@ static int whereLoopAddBtree(
           )
       ){
         pNew->iSortIdx = b ? iSortIdx : 0;
+        /* TUNING:  The base cost of an index scan is N + log2(N).
+        ** The log2(N) is for the initial seek to the beginning and the N
+        ** is for the scan itself. */
+        pNew->rRun = sqlite3LogEstAdd(rSize, rLogSize);
         if( m==0 ){
           /* TUNING: Cost of a covering index scan is K*(N + log2(N)).
           **  +  The extra factor K of between 1.1 and 3.0 that depends
           **     on the relative sizes of the table and the index.  K
           **     is smaller for smaller indices, thus favoring them.
+          **     The upper bound on K (3.0) matches the penalty factor
+          **     on a full table scan that tries to encourage the use of
+          **     indexed lookups over full scans.
           */
-          pNew->rRun = sqlite3LogEstAdd(rSize,rLogSize) + 1 +
+          pNew->rRun +=  1 + (15*pProbe->szIdxRow)/pTab->szTabRow;
-                        (15*pProbe->szIdxRow)/pTab->szTabRow;
         }else{
-          /* TUNING: Cost of scanning a non-covering index is (N+1)*log2(N)
+          /* TUNING: The cost of scanning a non-covering index is multiplied
-          ** which we will simplify to just N*log2(N) */
+          ** by log2(N) to account for the binary search of the main table
-          pNew->rRun = rSize + rLogSize;
+          ** that must happen for each row of the index.
+          ** TODO: Should there be a multiplier here, analogous to the 3x
+          ** multiplier for a fulltable scan or covering index scan, to
+          ** further discourage the use of an index scan?  Or is the log2(N)
+          ** term sufficient discouragement?
+          ** TODO: What if some or all of the WHERE clause terms can be
+          ** computed without reference to the original table.  Then the
+          ** penality should reduce to logK where K is the number of output
+          ** rows.
+          */
+          pNew->rRun += rLogSize;
         }
         whereLoopOutputAdjust(pWC, pNew);
         rc = whereLoopInsert(pBuilder, pNew);
@@ -4920,7 +4936,7 @@ static i8 wherePathSatisfiesOrderBy(
       }
     }
   } /* End the loop over all WhereLoops from outer-most down to inner-most */
-  if( obSat==obDone ) return nOrderBy;
+  if( obSat==obDone ) return (i8)nOrderBy;
   if( !isOrderDistinct ){
     for(i=nOrderBy-1; i>0; i--){
       Bitmask m = MASKBIT(i) - 1;
@@ -5041,11 +5057,19 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
                        pWInfo->pOrderBy, pFrom, pWInfo->wctrlFlags,
                        iLoop, pWLoop, &revMask);
           if( isOrdered>=0 && isOrdered<nOrderBy ){
-            /* TUNING: Estimated cost of sorting cost as roughly N*log(N).
+            /* TUNING: Estimated cost of sorting is N*log(N).
-            ** If some but not all of the columns are in sorted order, then
+            ** If the order-by clause has X terms but only the last Y terms
-            ** scale down the log(N) term. */
+            ** are out of order, then block-sorting will reduce the sorting
-            LogEst rScale = sqlite3LogEst((nOrderBy-isOrdered)*100/nOrderBy);
+            ** cost to N*log(N)*log(Y/X).  The log(Y/X) term is computed
-            LogEst rSortCost = nRowEst + estLog(nRowEst) + rScale - 66;
+            ** by rScale.
+            ** TODO: Should the sorting cost get a small multiplier to help
+            ** discourage the use of sorting and encourage the use of index
+            ** scans instead?
+            */
+            LogEst rScale, rSortCost;
+            assert( nOrderBy>0 );
+            rScale = sqlite3LogEst((nOrderBy-isOrdered)*100/nOrderBy) - 66;
+            rSortCost = nRowEst + estLog(nRowEst) + rScale;
             /* TUNING: The cost of implementing DISTINCT using a B-TREE is
             ** also N*log(N) but it has a larger constant of proportionality.
             ** Multiply by 3.0. */

test/corruptG.test

@@ -47,12 +47,12 @@ do_test 1.2 {
   catchsql {
     SELECT c FROM t1 WHERE a>'abc';
   }
-} {0 {}}
+} {1 {database disk image is malformed}}
 do_test 1.3 {
   catchsql {
      PRAGMA integrity_check
   }
-} {0 ok}
+} {1 {database disk image is malformed}}
 do_test 1.4 {
   catchsql {
     SELECT c FROM t1 ORDER BY a;
@@ -71,11 +71,6 @@ do_test 2.1 {
   catchsql {
     SELECT rowid FROM t1 WHERE a='abc' and b='xyz123456789XYZ';
   }
-  # The following test result is brittle.  The point above is to try to
+} {1 {database disk image is malformed}}
-  # force a buffer overread by a corrupt database file.  If we get an
-  # incorrect answer from a corrupt database file, that is OK.  If the
-  # result below changes, that just means that "undefined behavior" has
-  # changed.
-} {/0 .*/}
 
 finish_test

test/corruptI.test

@@ -51,7 +51,7 @@ do_test 1.3 {
   hexio_write test.db $off FFFF7f02
   sqlite3 db test.db
   catchsql { SELECT * FROM t1 WHERE a = 10 }
-} {0 {}}
+} {1 {database disk image is malformed}}
 
 do_test 2.0 {
   execsql {

test/sort.test

@@ -464,4 +464,26 @@ do_test sort-12.1 {
   }
 } {1 2 xxx 1 3 yyy 1 1 zzz}
 
+#-------------------------------------------------------------------------
+# Check that the sorter in vdbesort.c sorts in a stable fashion.
+#
+do_execsql_test sort-13.0 {
+  CREATE TABLE t10(a, b);
+}
+do_test sort-13.1 {
+  db transaction {
+    for {set i 0} {$i < 100000} {incr i} {
+      execsql { INSERT INTO t10 VALUES( $i/10, $i%10 ) }
+    }
+  }
+} {}
+do_execsql_test sort-13.2 {
+  SELECT a, b FROM t10 ORDER BY a;
+} [db eval {SELECT a, b FROM t10 ORDER BY a, b}]
+do_execsql_test sort-13.3 {
+  PRAGMA cache_size = 5;
+  SELECT a, b FROM t10 ORDER BY a;
+} [db eval {SELECT a, b FROM t10 ORDER BY a, b}]
+
+
 finish_test

test/tester.tcl

@@ -1076,6 +1076,7 @@ proc explain_i {sql {db db}} {
   foreach opcode {
       Seek SeekGe SeekGt SeekLe SeekLt NotFound Last Rewind
       NoConflict Next Prev VNext VPrev VFilter
+      SorterSort SorterNext
   } {
     set color($opcode) $B
   }
@@ -1098,6 +1099,7 @@ proc explain_i {sql {db db}} {
 
     if {$opcode=="Next"  || $opcode=="Prev" 
      || $opcode=="VNext" || $opcode=="VPrev"
+     || $opcode=="SorterNext"
     } {
       for {set i $p2} {$i<$addr} {incr i} {
         incr x($i) 2

test/wal64k.test

@@ -19,6 +19,11 @@ set testprefix wal64k
 
 ifcapable !wal {finish_test ; return }
 
+if {$tcl_platform(platform) != "unix"} {
+  finish_test
+  return
+}
+
 db close
 test_syscall pagesize 65536
 sqlite3 db test.db
@@ -44,4 +49,3 @@ integrity_check 1.3
 db close
 test_syscall pagesize -1
 finish_test
-

tool/logest.c

@@ -17,13 +17,7 @@
 **
 **      ./LogEst ARGS
 **
-** Arguments:
+** See the showHelp() routine for a description of valid arguments.
-**
-**    'x'    Multiple the top two elements of the stack
-**    '+'    Add the top two elements of the stack
-**    NUM    Convert NUM from integer to LogEst and push onto the stack
-**   ^NUM    Interpret NUM as a LogEst and push onto stack.
-**
 ** Examples:
 **
 ** To convert 123 from LogEst to integer:
@@ -97,12 +91,31 @@ static LogEst logEstFromDouble(double x){
   return e*10;
 }
 
+int isInteger(const char *z){
+  while( z[0]>='0' && z[0]<='9' ) z++;
+  return z[0]==0;
+}
+
 int isFloat(const char *z){
-  while( z[0] ){
+  char c;
-    if( z[0]=='.' || z[0]=='E' || z[0]=='e' ) return 1;
+  while( ((c=z[0])>='0' && c<='9') || c=='.' || c=='E' || c=='e'
-    z++;
+          || c=='+' || c=='-'  ) z++;
-  }
+  return z[0]==0;
-  return 0;
+}
+
+static void showHelp(const char *zArgv0){
+  printf("Usage: %s ARGS...\n", zArgv0);
+  printf("Arguments:\n"
+    "  NUM    Convert NUM from integer to LogEst and push onto the stack\n"
+    " ^NUM    Interpret NUM as a LogEst and push onto stack\n"
+    "  x      Multiple the top two elements of the stack\n"
+    "  +      Add the top two elements of the stack\n"
+    "  dup    Dupliate the top element on the stack\n"
+    "  inv    Take the reciprocal of the top of stack.  N = 1/N.\n"
+    "  log    Find the LogEst of the number on top of stack\n"
+    "  nlogn  Compute NlogN where N is the top of stack\n"
+  );
+  exit(1);
 }
 
 int main(int argc, char **argv){
@@ -111,30 +124,43 @@ int main(int argc, char **argv){
   LogEst a[100];
   for(i=1; i<argc; i++){
     const char *z = argv[i];
-    if( z[0]=='+' ){
+    if( strcmp(z,"+")==0 ){
       if( n>=2 ){
         a[n-2] = logEstAdd(a[n-2],a[n-1]);
         n--;
       }
-    }else if( z[0]=='x' ){
+    }else if( strcmp(z,"x")==0 ){
       if( n>=2 ){
         a[n-2] = logEstMultiply(a[n-2],a[n-1]);
         n--;
       }
+    }else if( strcmp(z,"dup")==0 ){
+      if( n>0 ){
+        a[n] = a[n-1];
+        n++;
+      }
+    }else if( strcmp(z,"log")==0 ){
+      if( n>0 ) a[n-1] = logEstFromInteger(a[n-1]) - 33;
+    }else if( strcmp(z,"nlogn")==0 ){
+      if( n>0 ) a[n-1] += logEstFromInteger(a[n-1]) - 33;
+    }else if( strcmp(z,"inv")==0 ){
+      if( n>0 ) a[n-1] = -a[n-1];
     }else if( z[0]=='^' ){
       a[n++] = atoi(z+1);
-    }else if( isFloat(z) ){
+    }else if( isInteger(z) ){
+      a[n++] = logEstFromInteger(atoi(z));
+    }else if( isFloat(z) && z[0]!='-' ){
       a[n++] = logEstFromDouble(atof(z));
     }else{
-      a[n++] = logEstFromInteger(atoi(z));
+      showHelp(argv[0]);
     }
   }
   for(i=n-1; i>=0; i--){
     if( a[i]<0 ){
-      printf("%d (%f)\n", a[i], 1.0/(double)logEstToInt(-a[i]));
+      printf("%5d (%f)\n", a[i], 1.0/(double)logEstToInt(-a[i]));
     }else{
       sqlite3_uint64 x = logEstToInt(a[i]+100)*100/1024;
-      printf("%d (%lld.%02lld)\n", a[i], x/100, x%100);
+      printf("%5d (%lld.%02lld)\n", a[i], x/100, x%100);
     }
   }
   return 0;