Slightly faster INSERTs from a SELECT by avoiding an intermediate table.

But it didn't make nearly as much difference as I had hoped. (CVS 732) FossilOrigin-Name: 723362e74f79c784314d042e3a8c8a9bf07cbd5e
2025-08-07 02:42:48 +03:00 · 2002-08-28 03:00:58 +00:00
parent 66105a8ea0
commit 142e30df99
8 changed files with 184 additions and 46 deletions
--- a/22
+++ b/22
@@ -1,5 +1,5 @@
-C Change\sthe\stokenizer\sto\signore\sC-style\scomments\s/*...*/\sin\saccordance\swith\nSQL99.\s(CVS\s731)
-D 2002-08-27T14:28:30
+C Slightly\sfaster\sINSERTs\sfrom\sa\sSELECT\sby\savoiding\san\sintermediate\stable.\nBut\sit\sdidn't\smake\snearly\sas\smuch\sdifference\sas\sI\shad\shoped.\s(CVS\s732)
+D 2002-08-28T03:00:58
 F Makefile.in bcb81f40d9a17bd94f59e67157b1e1c54c046c2b
 F Makefile.linux-gcc b86a99c493a5bfb402d1d9178dcdc4bd4b32f906
 F README f1de682fbbd94899d50aca13d387d1b3fd3be2dd
@@ -27,7 +27,7 @@ F src/expr.c ee027b908a1e157fc21644121811fa6ec1eec798
 F src/func.c e45cd908b9b723d9b91473d09e12c23f786b3fc2
 F src/hash.c 6a6236b89c8c060c65dabd300a1c8ce7c10edb72
 F src/hash.h cd0433998bc1a3759d244e1637fe5a3c13b53bf8
-F src/insert.c 8aefc998c86a3bd53082e2f8fdd049345fcf3463
+F src/insert.c a2f5455009904476b43ec5304a181b505235f72f
 F src/main.c 9f2633cb20cb9cc740353f57178450319c12b743
 F src/md5.c 0ae1f3e2cac92d06fc6246d1b4b8f61a2fe66d3b
 F src/os.c 00d10655e1dc9a52b4aabca58c8d8e45048057b0
@@ -37,11 +37,11 @@ F src/pager.h 6991c9c2dc5e4c7f2df4d4ba47d1c6458f763a32
 F src/parse.y 1b180e14b6346e323bd4279469748716f412cc1c
 F src/printf.c 5c50fc1da75c8f5bf432b1ad17d91d6653acd167
 F src/random.c 19e8e00fe0df32a742f115773f57651be327cabe
-F src/select.c e79db94a78368e981af31be4f186bb78ca4081ce
+F src/select.c 6cd3673edbb36a8f8027341093085e01c04dd3d4
 F src/shell.c 9e9a6eb6bca07f01e6472a603f908a0127ea50ff
 F src/shell.tcl 27ecbd63dd88396ad16d81ab44f73e6c0ea9d20e
 F src/sqlite.h.in d3999a9c6374675779058d6cfe5431131618e92b
-F src/sqliteInt.h 5bb95f64a2f86b2d14a66e35edba4a2564a6ecd7
+F src/sqliteInt.h 4d42c8685693ecf9d99edf52c9a404da2b2df7fd
 F src/table.c eed2098c9b577aa17f8abe89313a9c4413f57d63
 F src/tclsqlite.c c502819c209011659e1bbb428cbac5670cce7f79
 F src/test1.c 456cb080db85056be723e770435d9509afc3a83a
@@ -52,9 +52,9 @@ F src/tokenize.c 62c98842447effe92eba9622bb2f9a2a8a4b97ad
 F src/trigger.c c90a292a4bef25e478fd5deda6d300319be6a023
 F src/update.c f07e6ed2c517c92871e54d3f5886d1cf56121b11
 F src/util.c c70d5da5357e01b58392faebae3c3620c1d71f14
-F src/vdbe.c ea41a3ac96511399e95ae18d16d32ec41ca91ec0
-F src/vdbe.h 52ec880c63c6ca74bb6377432149260b1b237873
-F src/where.c ce42cce65d7bf42341627f3fb0a17f69fea6a4f4
+F src/vdbe.c d4969d78fdd2408706a329c7d4554b9596d1be94
+F src/vdbe.h 7cfeb3aab6a901336532d93494cdedbddf30b7ec
+F src/where.c 53959c9d94adaf93b409271815e26eafa6ddd515
 F test/all.test efd958d048c70a3247997c482f0b33561f7759f0
 F test/bigrow.test 8ab252dba108f12ad64e337b0f2ff31a807ac578
 F test/btree.test bf326f546a666617367a7033fa2c07451bd4f8e1
@@ -147,7 +147,7 @@ F www/speed.tcl a20a792738475b68756ea7a19321600f23d1d803
 F www/sqlite.tcl ae3dcfb077e53833b59d4fcc94d8a12c50a44098
 F www/tclsqlite.tcl 1db15abeb446aad0caf0b95b8b9579720e4ea331
 F www/vdbe.tcl 2013852c27a02a091d39a766bc87cff329f21218
-P 45847390d007718a4b7a4e9fa445136d013113f8
-R 37afa768fcedc8d2fc0a05d007792ef5
+P f1534489484afdb835ad8e6f97909fbe76dbe414
+R e0d3b63d78fe10e969ff04668037a39d
 U drh
-Z 451eca83ef39ddfc50a8575792ef9f91
+Z 1807e27a4648862e140898b176210cca
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-f1534489484afdb835ad8e6f97909fbe76dbe414
+723362e74f79c784314d042e3a8c8a9bf07cbd5e
--- a/src/insert.c
+++ b/src/insert.c
@@ -12,7 +12,7 @@
 ** This file contains C code routines that are called by the parser
 ** to handle INSERT statements in SQLite.
 **
-** $Id: insert.c,v 1.65 2002/07/31 00:32:50 drh Exp $
+** $Id: insert.c,v 1.66 2002/08/28 03:00:58 drh Exp $
 */
 #include "sqliteInt.h"

@@ -30,6 +30,58 @@
 ** statement above, and pSelect is NULL.  For the second form, pList is
 ** NULL and pSelect is a pointer to the select statement used to generate
 ** data for the insert.
+**
+** The code generated follows one of three templates.  For a simple
+** select with data coming from a VALUES clause, the code executes
+** once straight down through.  The template looks like this:
+**
+**         open write cursor to <table> and its indices
+**         puts VALUES clause expressions onto the stack
+**         write the resulting record into <table>
+**         cleanup
+**
+** If the statement is of the form
+**
+**   INSERT INTO <table> SELECT ...
+**
+** And the SELECT clause does not read from <table> at any time, then
+** the generated code follows this template:
+**
+**         goto B
+**      A: setup for the SELECT
+**         loop over the tables in the SELECT
+**           gosub C
+**         end loop
+**         cleanup after the SELECT
+**         goto D
+**      B: open write cursor to <table> and its indices
+**         goto A
+**      C: insert the select result into <table>
+**         return
+**      D: cleanup
+**
+** The third template is used if the insert statement takes its
+** values from a SELECT but the data is being inserted into a table
+** that is also read as part of the SELECT.  In the third form,
+** we have to use a intermediate table to store the results of
+** the select.  The template is like this:
+**
+**         goto B
+**      A: setup for the SELECT
+**         loop over the tables in the SELECT
+**           gosub C
+**         end loop
+**         cleanup after the SELECT
+**         goto D
+**      C: insert the select result into the intermediate table
+**         return
+**      B: open a cursor to an intermediate table
+**         goto A
+**      D: open write cursor to <table> and its indices
+**         loop over the intermediate table
+**           transfer values form intermediate table into <table>
+**         end the loop
+**         cleanup
 */
 void sqliteInsert(
  Parse *pParse,        /* Parser context */
@@ -44,7 +96,6 @@ void sqliteInsert(
  int i, j, idx;        /* Loop counters */
  Vdbe *v;              /* Generate code into this virtual machine */
  Index *pIdx;          /* For looping over indices of the table */
-  int srcTab;           /* Date comes from this temporary cursor if >=0 */
  int nColumn;          /* Number of columns in the data */
  int base;             /* First available cursor */
  int iCont, iBreak;    /* Beginning and end of the loop over srcTab */
@@ -52,6 +103,12 @@ void sqliteInsert(
  int openOp;           /* Opcode used to open cursors */
  int keyColumn = -1;   /* Column that is the INTEGER PRIMARY KEY */
  int endOfLoop;        /* Label for the end of the insertion loop */
+  int useTempTable;     /* Store SELECT results in intermediate table */
+  int srcTab;           /* Data comes from this temporary cursor if >=0 */
+  int iSelectLoop;      /* Address of code that implements the SELECT */
+  int iCleanup;         /* Address of the cleanup code */
+  int iInsertBlock;     /* Address of the subroutine used to insert data */
+  int iCntMem;          /* Memory cell used for the row counter */

  int row_triggers_exist = 0; /* True if there are FOR EACH ROW triggers */
  int newIdx = -1;
@@ -111,23 +168,66 @@ void sqliteInsert(
  }

  /* Figure out how many columns of data are supplied.  If the data
-  ** is coming from a SELECT statement, then this step has to generate
-  ** all the code to implement the SELECT statement and leave the data
-  ** in a temporary table.  If data is coming from an expression list,
-  ** then we just have to count the number of expressions.
+  ** is coming from a SELECT statement, then this step also generates
+  ** all the code to implement the SELECT statement and invoke a subroutine
+  ** to process each row of the result. (Template 2.) If the SELECT
+  ** statement uses the the table that is being inserted into, then the
+  ** subroutine is also coded here.  That subroutine stores the SELECT
+  ** results in a temporary table. (Template 3.)
  */
  if( pSelect ){
-    int rc;
-    srcTab = pParse->nTab++;
-    sqliteVdbeAddOp(v, OP_OpenTemp, srcTab, 0);
-    rc = sqliteSelect(pParse, pSelect, SRT_Table, srcTab, 0,0,0);
+    /* Data is coming from a SELECT.  Generate code to implement that SELECT
+    */
+    int rc, iInitCode;
+    int opCode;
+    iInitCode = sqliteVdbeAddOp(v, OP_Goto, 0, 0);
+    iSelectLoop = sqliteVdbeCurrentAddr(v);
+    iInsertBlock = sqliteVdbeMakeLabel(v);
+    rc = sqliteSelect(pParse, pSelect, SRT_Subroutine, iInsertBlock, 0,0,0);
    if( rc || pParse->nErr || sqlite_malloc_failed ) goto insert_cleanup;
+    iCleanup = sqliteVdbeMakeLabel(v);
+    sqliteVdbeAddOp(v, OP_Goto, 0, iCleanup);
    assert( pSelect->pEList );
    nColumn = pSelect->pEList->nExpr;
+
+    /* Set useTempTable to TRUE if the result of the SELECT statement
+    ** should be written into a temporary table.  Set to FALSE if each
+    ** row of the SELECT can be written directly into the result table.
+    */
+    opCode = pTab->isTemp ? OP_OpenTemp : OP_Open;
+    useTempTable = row_triggers_exist || sqliteVdbeFindOp(v,opCode,pTab->tnum);
+
+    if( useTempTable ){
+      /* Generate the subroutine that SELECT calls to process each row of
+      ** the result.  Store the result in a temporary table
+      */
+      srcTab = pParse->nTab++;
+      sqliteVdbeResolveLabel(v, iInsertBlock);
+      sqliteVdbeAddOp(v, OP_MakeRecord, nColumn, 0);
+      sqliteVdbeAddOp(v, OP_NewRecno, srcTab, 0);
+      sqliteVdbeAddOp(v, OP_Pull, 1, 0);
+      sqliteVdbeAddOp(v, OP_PutIntKey, srcTab, 0);
+      sqliteVdbeAddOp(v, OP_Return, 0, 0);
+
+      /* The following code runs first because the GOTO at the very top
+      ** of the program jumps to it.  Create the temporary table, then jump
+      ** back up and execute the SELECT code above.
+      */
+      sqliteVdbeChangeP2(v, iInitCode, sqliteVdbeCurrentAddr(v));
+      sqliteVdbeAddOp(v, OP_OpenTemp, srcTab, 0);
+      sqliteVdbeAddOp(v, OP_Goto, 0, iSelectLoop);
+      sqliteVdbeResolveLabel(v, iCleanup);
+    }else{
+      sqliteVdbeChangeP2(v, iInitCode, sqliteVdbeCurrentAddr(v));
+    }
  }else{
+    /* This is the case if the data for the INSERT is coming from a VALUES
+    ** clause
+    */
    SrcList dummy;
    assert( pList!=0 );
    srcTab = -1;
+    useTempTable = 0;
    assert( pList );
    nColumn = pList->nExpr;
    dummy.nSrc = 0;
@@ -208,15 +308,18 @@ void sqliteInsert(
    keyColumn = pTab->iPKey;
  }

-  /* Open the temp table for FOR EACH ROW triggers */
+  /* Open the temp table for FOR EACH ROW triggers
+  */
  if( row_triggers_exist ){
    sqliteVdbeAddOp(v, OP_OpenTemp, newIdx, 0);
  }
    
  /* Initialize the count of rows to be inserted
  */
-  if( db->flags & SQLITE_CountRows && !pParse->trigStack ){
-    sqliteVdbeAddOp(v, OP_Integer, 0, 0);  /* Initialize the row count */
+  if( db->flags & SQLITE_CountRows ){
+    iCntMem = pParse->nMem++;
+    sqliteVdbeAddOp(v, OP_Integer, 0, 0);
+    sqliteVdbeAddOp(v, OP_MemStore, iCntMem, 1);
  }

  /* Open tables and indices if there are no row triggers */
@@ -232,15 +335,18 @@ void sqliteInsert(
    pParse->nTab += idx;
  }

-  /* If the data source is a SELECT statement, then we have to create
+  /* If the data source is a temporary table, then we have to create
  ** a loop because there might be multiple rows of data.  If the data
-  ** source is an expression list, then exactly one row will be inserted
-  ** and the loop is not used.
+  ** source is a subroutine call from the SELECT statement, then we need
+  ** to launch the SELECT statement processing.
  */
-  if( srcTab>=0 ){
+  if( useTempTable ){
    iBreak = sqliteVdbeMakeLabel(v);
    sqliteVdbeAddOp(v, OP_Rewind, srcTab, iBreak);
    iCont = sqliteVdbeCurrentAddr(v);
+  }else if( pSelect ){
+    sqliteVdbeAddOp(v, OP_Goto, 0, iSelectLoop);
+    sqliteVdbeResolveLabel(v, iInsertBlock);
  }

  endOfLoop = sqliteVdbeMakeLabel(v);
@@ -259,8 +365,10 @@ void sqliteInsert(
      if( pColumn && j>=pColumn->nId ){
        sqliteVdbeAddOp(v, OP_String, 0, 0);
        sqliteVdbeChangeP3(v, -1, pTab->aCol[i].zDflt, P3_STATIC);
-      }else if( srcTab>=0 ){
+      }else if( useTempTable ){
        sqliteVdbeAddOp(v, OP_Column, srcTab, j); 
+      }else if( pSelect ){
+        sqliteVdbeAddOp(v, OP_Dup, nColumn-j-1, 1);
      }else{
        sqliteExprCode(pParse, pList->a[j].pExpr);
      }
@@ -296,8 +404,10 @@ void sqliteInsert(
  */
  if( !pTab->pSelect ){
    if( keyColumn>=0 ){
-      if( srcTab>=0 ){
+      if( useTempTable ){
        sqliteVdbeAddOp(v, OP_Column, srcTab, keyColumn);
+      }else if( pSelect ){
+        sqliteVdbeAddOp(v, OP_Dup, nColumn - keyColumn - 1, 1);
      }else{
        sqliteExprCode(pParse, pList->a[keyColumn].pExpr);
      }
@@ -334,8 +444,10 @@ void sqliteInsert(
      if( pColumn && j>=pColumn->nId ){
        sqliteVdbeAddOp(v, OP_String, 0, 0);
        sqliteVdbeChangeP3(v, -1, pTab->aCol[i].zDflt, P3_STATIC);
-      }else if( srcTab>=0 ){
+      }else if( useTempTable ){
        sqliteVdbeAddOp(v, OP_Column, srcTab, j); 
+      }else if( pSelect ){
+        sqliteVdbeAddOp(v, OP_Dup, i+nColumn-j, 1);
      }else{
        sqliteExprCode(pParse, pList->a[j].pExpr);
      }
@@ -349,8 +461,8 @@ void sqliteInsert(

    /* Update the count of rows that are inserted
    */
-    if( (db->flags & SQLITE_CountRows)!=0 && !pParse->trigStack){
-      sqliteVdbeAddOp(v, OP_AddImm, 1, 0);
+    if( (db->flags & SQLITE_CountRows)!=0 ){
+      sqliteVdbeAddOp(v, OP_MemIncr, iCntMem, 0);
    }
  }

@@ -373,10 +485,14 @@ void sqliteInsert(
  /* The bottom of the loop, if the data source is a SELECT statement
  */
  sqliteVdbeResolveLabel(v, endOfLoop);
-  if( srcTab>=0 ){
+  if( useTempTable ){
    sqliteVdbeAddOp(v, OP_Next, srcTab, iCont);
    sqliteVdbeResolveLabel(v, iBreak);
    sqliteVdbeAddOp(v, OP_Close, srcTab, 0);
+  }else if( pSelect ){
+    sqliteVdbeAddOp(v, OP_Pop, nColumn, 0);
+    sqliteVdbeAddOp(v, OP_Return, 0, 0);
+    sqliteVdbeResolveLabel(v, iCleanup);
  }

  if( !row_triggers_exist ){
@@ -392,10 +508,11 @@ void sqliteInsert(
  /*
  ** Return the number of rows inserted.
  */
-  if( db->flags & SQLITE_CountRows && !pParse->trigStack ){
+  if( db->flags & SQLITE_CountRows ){
    sqliteVdbeAddOp(v, OP_ColumnCount, 1, 0);
    sqliteVdbeAddOp(v, OP_ColumnName, 0, 0);
    sqliteVdbeChangeP3(v, -1, "rows inserted", P3_STATIC);
+    sqliteVdbeAddOp(v, OP_MemLoad, iCntMem, 0);
    sqliteVdbeAddOp(v, OP_Callback, 1, 0);
  }

--- a/src/select.c
+++ b/src/select.c
@@ -12,7 +12,7 @@
 ** This file contains C code routines that are called by the parser
 ** to handle SELECT statements in SQLite.
 **
-** $Id: select.c,v 1.110 2002/08/25 19:20:40 drh Exp $
+** $Id: select.c,v 1.111 2002/08/28 03:00:59 drh Exp $
 */
 #include "sqliteInt.h"

@@ -520,6 +520,14 @@ static int selectInnerLoop(
      break;
    }

+    /* Invoke a subroutine to handle the results.  The subroutine itself
+    ** is responsible for popping the results off of the stack.
+    */
+    case SRT_Subroutine: {
+      sqliteVdbeAddOp(v, OP_Gosub, 0, iParm);
+      break;
+    }
+
    /* Discard the results.  This is used for SELECT statements inside
    ** the body of a TRIGGER.  The purpose of such selects is to call
    ** user-defined functions that have side effects.  We do not care
@@ -1075,7 +1083,6 @@ static int multiSelect(Parse *pParse, Select *p, int eDest, int iParm){
  int rc;             /* Success code from a subroutine */
  Select *pPrior;     /* Another SELECT immediately to our left */
  Vdbe *v;            /* Generate code to this VDBE */
-  int base;           /* Baseline value for pParse->nTab */

  /* Make sure there is no ORDER BY clause on prior SELECTs.  Only the 
  ** last SELECT in the series may have an ORDER BY.
@@ -1103,7 +1110,6 @@ static int multiSelect(Parse *pParse, Select *p, int eDest, int iParm){

  /* Generate code for the left and right SELECT statements.
  */
-  base = pParse->nTab;
  switch( p->op ){
    case TK_ALL: {
      if( p->pOrderBy==0 ){
@@ -1258,7 +1264,6 @@ static int multiSelect(Parse *pParse, Select *p, int eDest, int iParm){
    pParse->nErr++;
    return 1;
  }
-  pParse->nTab = base;
  return 0;
 }

@@ -2080,7 +2085,7 @@ int sqliteSelect(
  ** successful coding of the SELECT.
  */
 select_end:
-  pParse->nTab = base;
+  /* pParse->nTab = base; */
  sqliteAggregateInfoReset(pParse);
  return rc;
 }
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -11,7 +11,7 @@
 *************************************************************************
 ** Internal interface definitions for SQLite.
 **
-** @(#) $Id: sqliteInt.h,v 1.143 2002/08/24 18:24:55 drh Exp $
+** @(#) $Id: sqliteInt.h,v 1.144 2002/08/28 03:00:59 drh Exp $
 */
 #include "sqlite.h"
 #include "hash.h"
@@ -610,6 +610,7 @@ struct Select {
 #define SRT_TempTable    8  /* Store result in a trasient table */
 #define SRT_Discard      9  /* Do not save the results anywhere */
 #define SRT_Sorter      10  /* Store results in the sorter */
+#define SRT_Subroutine  11  /* Call a subroutine to handle results */

 /*
 ** When a SELECT uses aggregate functions (like "count(*)" or "avg(f1)")
--- a/src/vdbe.c
+++ b/src/vdbe.c
@@ -30,7 +30,7 @@
 ** But other routines are also provided to help in building up
 ** a program instruction by instruction.
 **
-** $Id: vdbe.c,v 1.172 2002/08/26 19:55:08 drh Exp $
+** $Id: vdbe.c,v 1.173 2002/08/28 03:01:00 drh Exp $
 */
 #include "sqliteInt.h"
 #include <ctype.h>
@@ -539,6 +539,18 @@ void sqliteVdbeCompressSpace(Vdbe *p, int addr){
  z[j] = 0;
 }

+/*
+** Search for the current program for the given opcode and P2
+** value.  Return 1 if found and 0 if not found.
+*/
+int sqliteVdbeFindOp(Vdbe *p, int op, int p2){
+  int i;
+  for(i=0; i<p->nOp; i++){
+    if( p->aOp[i].opcode==op && p->aOp[i].p2==p2 ) return 1;
+  }
+  return 0;
+}
+
 /*
 ** The following group or routines are employed by installable functions
 ** to return their results.
@@ -5231,7 +5243,7 @@ cleanup:
  }
  sqliteBtreeCommitCkpt(pBt);
  if( db->pBeTemp ) sqliteBtreeCommitCkpt(db->pBeTemp);
-  assert( p->tos<pc );
+  assert( p->tos<pc || sqlite_malloc_failed==1 );
  return rc;

  /* Jump to here if a malloc() fails.  It's hard to get a malloc()
--- a/src/vdbe.h
+++ b/src/vdbe.h
@@ -15,7 +15,7 @@
 ** or VDBE.  The VDBE implements an abstract machine that runs a
 ** simple program to access and modify the underlying database.
 **
-** $Id: vdbe.h,v 1.58 2002/08/25 19:20:42 drh Exp $
+** $Id: vdbe.h,v 1.59 2002/08/28 03:01:01 drh Exp $
 */
 #ifndef _SQLITE_VDBE_H_
 #define _SQLITE_VDBE_H_
@@ -231,6 +231,7 @@ void sqliteVdbeChangeP1(Vdbe*, int addr, int P1);
 void sqliteVdbeChangeP2(Vdbe*, int addr, int P2);
 void sqliteVdbeChangeP3(Vdbe*, int addr, const char *zP1, int N);
 void sqliteVdbeDequoteP3(Vdbe*, int addr);
+int sqliteVdbeFindOp(Vdbe*, int, int);
 int sqliteVdbeMakeLabel(Vdbe*);
 void sqliteVdbeDelete(Vdbe*);
 int sqliteVdbeOpcode(const char *zName);
--- a/src/where.c
+++ b/src/where.c
@@ -13,7 +13,7 @@
 ** the WHERE clause of SQL statements.  Also found here are subroutines
 ** to generate VDBE code to evaluate expressions.
 **
-** $Id: where.c,v 1.63 2002/08/15 13:50:50 drh Exp $
+** $Id: where.c,v 1.64 2002/08/28 03:01:01 drh Exp $
 */
 #include "sqliteInt.h"

@@ -1102,9 +1102,11 @@ void sqliteWhereEnd(WhereInfo *pWInfo){
      sqliteVdbeAddOp(v, OP_Close, pLevel->iCur, 0);
    }
  }
+#if 0  /* Never reuse a cursor */
  if( pWInfo->pParse->nTab==pWInfo->peakNTab ){
    pWInfo->pParse->nTab = pWInfo->savedNTab;
  }
+#endif
  sqliteFree(pWInfo);
  return;
 }