diff --git a/doc/lemon.html b/doc/lemon.html index 2b8c613640..773c68e6ad 100644 --- a/doc/lemon.html +++ b/doc/lemon.html @@ -5,14 +5,17 @@

The Lemon Parser Generator

-

Lemon is an LALR(1) parser generator for C or C++. -It does the same job as ``bison'' and ``yacc''. -But lemon is not another bison or yacc clone. It +

Lemon is an LALR(1) parser generator for C. +It does the same job as "bison" and "yacc". +But lemon is not a bison or yacc clone. Lemon uses a different grammar syntax which is designed to -reduce the number of coding errors. Lemon also uses a more -sophisticated parsing engine that is faster than yacc and -bison and which is both reentrant and thread-safe. -Furthermore, Lemon implements features that can be used +reduce the number of coding errors. Lemon also uses a +parsing engine that is faster than yacc and +bison and which is both reentrant and threadsafe. +(Update: Since the previous sentence was written, bison +has also been updated so that it too can generate a +reentrant and threadsafe parser.) +Lemon also implements features that can be used to eliminate resource leaks, making is suitable for use in long-running programs such as graphical user interfaces or embedded controllers.

@@ -44,18 +47,18 @@ one and three files of outputs. automaton. By default, all three of these output files are generated. -The header file is suppressed if the ``-m'' command-line option is -used and the report file is omitted when ``-q'' is selected.

+The header file is suppressed if the "-m" command-line option is +used and the report file is omitted when "-q" is selected.

-

The grammar specification file uses a ``.y'' suffix, by convention. +

The grammar specification file uses a ".y" suffix, by convention. In the examples used in this document, we'll assume the name of the -grammar file is ``gram.y''. A typical use of Lemon would be the +grammar file is "gram.y". A typical use of Lemon would be the following command: 

    lemon gram.y
-This command will generate three output files named ``gram.c'', -``gram.h'' and ``gram.out''. +This command will generate three output files named "gram.c", +"gram.h" and "gram.out". The first is C code to implement the parser. The second is the header file that defines numerical values for all terminal symbols, and the last is the report that explains @@ -71,39 +74,35 @@ with a brief explanation of what each does by typing As of this writing, the following command-line options are supported: -The ``-b'' option reduces the amount of text in the report file by -printing only the basis of each parser state, rather than the full -configuration. -The ``-c'' option suppresses action table compression. Using -c -will make the parser a little larger and slower but it will detect -syntax errors sooner. -The ``-g'' option causes no output files to be generated at all. -Instead, the input grammar file is printed on standard output but -with all comments, actions and other extraneous text deleted. This -is a useful way to get a quick summary of a grammar. -The ``-m'' option causes the output C source file to be compatible -with the ``makeheaders'' program. -Makeheaders is a program that automatically generates header files -from C source code. When the ``-m'' option is used, the header -file is not output since the makeheaders program will take care -of generated all header files automatically. -The ``-q'' option suppresses the report file. -Using ``-s'' causes a brief summary of parser statistics to be -printed. Like this: -
-   Parser statistics: 74 terminals, 70 nonterminals, 179 rules
-                      340 states, 2026 parser table entries, 0 conflicts
-
-Finally, the ``-x'' option causes Lemon to print its version number -and then stops without attempting to read the grammar or generate a parser.

The Parser Interface

@@ -121,12 +120,12 @@ A new parser is created as follows: The ParseAlloc() routine allocates and initializes a new parser and returns a pointer to it. -The actual data structure used to represent a parser is opaque -- +The actual data structure used to represent a parser is opaque — its internal structure is not visible or usable by the calling routine. For this reason, the ParseAlloc() routine returns a pointer to void rather than a pointer to some particular structure. The sole argument to the ParseAlloc() routine is a pointer to the -subroutine used to allocate memory. Typically this means ``malloc()''.

+subroutine used to allocate memory. Typically this means malloc().

After a program is finished using a parser, it can reclaim all memory allocated by that parser by calling @@ -151,13 +150,13 @@ type of the next token in the data stream. There is one token type for each terminal symbol in the grammar. The gram.h file generated by Lemon contains #define statements that map symbolic terminal symbol names into appropriate integer values. -(A value of 0 for the second argument is a special flag to the -parser to indicate that the end of input has been reached.) +A value of 0 for the second argument is a special flag to the +parser to indicate that the end of input has been reached. The third argument is the value of the given token. By default, the type of the third argument is integer, but the grammar will usually redefine this type to be some kind of structure. Typically the second argument will be a broad category of tokens -such as ``identifier'' or ``number'' and the third argument will +such as "identifier" or "number" and the third argument will be the name of the identifier or the value of the number.

The Parse() function may have either three or four arguments, @@ -193,7 +192,7 @@ following: This example shows a user-written routine that parses a file of text and returns a pointer to the parse tree. -(We've omitted all error-handling from this example to keep it +(All error-handling code is omitted from this example to keep it simple.) We assume the existence of some kind of tokenizer which is created using TokenizerCreate() on line 8 and deleted by TokenizerFree() @@ -287,7 +286,7 @@ tokens) and it honors the same commenting conventions as C and C++.

Terminals and Nonterminals

A terminal symbol (token) is any string of alphanumeric -and underscore characters +and/or underscore characters that begins with an upper case letter. A terminal can contain lowercase letters after the first character, but the usual convention is to make terminals all upper case. @@ -314,7 +313,7 @@ must have alphanumeric names.

The main component of a Lemon grammar file is a sequence of grammar rules. Each grammar rule consists of a nonterminal symbol followed by -the special symbol ``::='' and then a list of terminals and/or nonterminals. +the special symbol "::=" and then a list of terminals and/or nonterminals. The rule is terminated by a period. The list of terminals and nonterminals on the right-hand side of the rule can be empty. @@ -330,9 +329,9 @@ A typical sequence of grammar rules might look something like this:

-

There is one non-terminal in this example, ``expr'', and five -terminal symbols or tokens: ``PLUS'', ``TIMES'', ``LPAREN'', -``RPAREN'' and ``VALUE''.

+

There is one non-terminal in this example, "expr", and five +terminal symbols or tokens: "PLUS", "TIMES", "LPAREN", +"RPAREN" and "VALUE".

Like yacc and bison, Lemon allows the grammar to specify a block of C code that will be executed whenever a grammar rule is reduced @@ -348,15 +347,15 @@ For example:

In order to be useful, grammar actions must normally be linked to their associated grammar rules. -In yacc and bison, this is accomplished by embedding a ``$$'' in the +In yacc and bison, this is accomplished by embedding a "$$" in the action to stand for the value of the left-hand side of the rule and -symbols ``$1'', ``$2'', and so forth to stand for the value of +symbols "$1", "$2", and so forth to stand for the value of the terminal or nonterminal at position 1, 2 and so forth on the right-hand side of the rule. This idea is very powerful, but it is also very error-prone. The single most common source of errors in a yacc or bison grammar is to miscount the number of symbols on the right-hand side of a grammar -rule and say ``$7'' when you really mean ``$8''.

+rule and say "$7" when you really mean "$8".

Lemon avoids the need to count grammar symbols by assigning symbolic names to each symbol in a grammar rule and then using those symbolic @@ -386,7 +385,7 @@ For example, the rule 

   expr(A) ::= expr(B) PLUS expr(C).  { A = B; }
-will generate an error because the linking symbol ``C'' is used +will generate an error because the linking symbol "C" is used in the grammar rule but not in the reduce action.

The Lemon notation for linking grammar rules to reduce actions @@ -394,6 +393,7 @@ also facilitates the use of destructors for reclaiming memory allocated by the values of terminals and nonterminals on the right-hand side of a rule.

+

Precedence Rules

Lemon resolves parsing ambiguities in exactly the same way as @@ -405,7 +405,10 @@ whichever rule comes first in the grammar file.

yacc and bison, Lemon allows a measure of control over the resolution of paring conflicts using precedence rules. A precedence value can be assigned to any terminal symbol -using the %left, %right or %nonassoc directives. Terminal symbols +using the +%left, +%right or +%nonassoc directives. Terminal symbols mentioned in earlier directives have a lower precedence that terminal symbols mentioned in later directives. For example:

@@ -525,7 +528,11 @@ other than that, the order of directives in Lemon is arbitrary.

  • %default_destructor
  • %default_type
  • %destructor +
  • %endif
  • %extra_argument +
  • %fallback +
  • %ifdef +
  • %ifndef
  • %include
  • %left
  • %name @@ -537,49 +544,57 @@ other than that, the order of directives in Lemon is arbitrary.

  • %stack_size
  • %start_symbol
  • %syntax_error +
  • %token_class
  • %token_destructor
  • %token_prefix
  • %token_type
  • %type +
  • %wildcard Each of these directives will be described separately in the following sections:

    +

    The %code directive

    -

    The %code directive is used to specify addition C/C++ code that +

    The %code directive is used to specify addition C code that is added to the end of the main output file. This is similar to -the %include directive except that %include is inserted at the -beginning of the main output file.

    +the %include directive except that %include +is inserted at the beginning of the main output file.

    %code is typically used to include some action routines or perhaps -a tokenizer as part of the output file.

    +a tokenizer or even the "main()" function +as part of the output file.

    +

    The %default_destructor directive

    The %default_destructor directive specifies a destructor to use for non-terminals that do not have their own destructor specified by a separate %destructor directive. See the documentation -on the %destructor directive below for additional information.

    +on the %destructor directive below for +additional information.

    In some grammers, many different non-terminal symbols have the same datatype and hence the same destructor. This directive is a convenience way to specify the same destructor for all those non-terminals using a single statement.

    +

    The %default_type directive

    The %default_type directive specifies the datatype of non-terminal symbols that do no have their own datatype defined using a separate -%type directive. See the documentation on %type below for addition -information.

    +%type directive. +

    +

    The %destructor directive

    The %destructor directive is used to specify a destructor for a non-terminal symbol. -(See also the %token_destructor directive which is used to -specify a destructor for terminal symbols.)

    +(See also the %token_destructor +directive which is used to specify a destructor for terminal symbols.)

    A non-terminal's destructor is called to dispose of the non-terminal's value whenever the non-terminal is popped from @@ -602,26 +617,25 @@ or other resources held by that non-terminal.

    This example is a bit contrived but it serves to illustrate how destructors work. The example shows a non-terminal named -``nt'' that holds values of type ``void*''. When the rule for -an ``nt'' reduces, it sets the value of the non-terminal to +"nt" that holds values of type "void*". When the rule for +an "nt" reduces, it sets the value of the non-terminal to space obtained from malloc(). Later, when the nt non-terminal is popped from the stack, the destructor will fire and call free() on this malloced space, thus avoiding a memory leak. -(Note that the symbol ``$$'' in the destructor code is replaced +(Note that the symbol "$$" in the destructor code is replaced by the value of the non-terminal.)

    It is important to note that the value of a non-terminal is passed to the destructor whenever the non-terminal is removed from the stack, unless the non-terminal is used in a C-code action. If the non-terminal is used by C-code, then it is assumed that the -C-code will take care of destroying it if it should really -be destroyed. More commonly, the value is used to build some +C-code will take care of destroying it. +More commonly, the value is used to build some larger structure and we don't want to destroy it, which is why the destructor is not called in this circumstance.

    -

    By appropriate use of destructors, it is possible to -build a parser using Lemon that can be used within a long-running -program, such as a GUI, that will not leak memory or other resources. +

    Destructors help avoid memory leaks by automatically freeing +allocated objects when they go out of scope. To do the same using yacc or bison is much more difficult.

    @@ -638,17 +652,66 @@ and so forth. For example, if the grammar file contains:

    Then the Parse() function generated will have an 4th parameter -of type ``MyStruct*'' and all action routines will have access to -a variable named ``pAbc'' that is the value of the 4th parameter +of type "MyStruct*" and all action routines will have access to +a variable named "pAbc" that is the value of the 4th parameter in the most recent call to Parse().

    + +

    The %fallback directive

    + +

    The %fallback directive specifies an alternative meaning for one +or more tokens. The alternative meaning is tried if the original token +would have generated a syntax error. + +

    The %fallback directive was added to support robust parsing of SQL +syntax in SQLite. +The SQL language contains a large assortment of keywords, each of which +appears as a different token to the language parser. SQL contains so +many keywords, that it can be difficult for programmers to keep up with +them all. Programmers will, therefore, sometimes mistakenly use an +obscure language keyword for an identifier. The %fallback directive +provides a mechanism to tell the parser: "If you are unable to parse +this keyword, try treating it as an identifier instead." + +

    The syntax of %fallback is as follows: + +

    +%fallback ID TOKEN... . +
    + +

    In words, the %fallback directive is followed by a list of token names +terminated by a period. The first token name is the fallback token - the +token to which all the other tokens fall back to. The second and subsequent +arguments are tokens which fall back to the token identified by the first +argument. + + +

    The %ifdef, %ifndef, and %endif directives.

    + +

    The %ifdef, %ifndef, and %endif directives are similar to +#ifdef, #ifndef, and #endif in the C-preprocessor, just not as general. +Each of these directives must begin at the left margin. No whitespace +is allowed between the "%" and the directive name. + +

    Grammar text in between "%ifdef MACRO" and the next nested "%endif" is +ignored unless the "-DMACRO" command-line option is used. Grammar text +betwen "%ifndef MACRO" and the next nested "%endif" is included except when +the "-DMACRO" command-line option is used. + +

    Note that the argument to %ifdef and %ifndef must be a single +preprocessor symbol name, not a general expression. There is no "%else" +directive. + + +

    The %include directive

    The %include directive specifies C code that is included at the top of the generated parser. You can include any text you want -- the Lemon parser generator copies it blindly. If you have multiple -%include directives in your grammar file the value of the last -%include directive overwrites all the others.

    The %include directive is very handy for getting some extra #include preprocessor statements at the beginning of the generated parser. @@ -661,12 +724,13 @@ For example:

    This might be needed, for example, if some of the C actions in the grammar call functions that are prototyed in unistd.h.

    +

    The %left directive

    -The %left directive is used (along with the %right and -%nonassoc directives) to declare precedences of terminal -symbols. Every terminal symbol whose name appears after -a %left directive but before the next period (``.'') is +The %left directive is used (along with the %right and +%nonassoc directives) to declare precedences of +terminal symbols. Every terminal symbol whose name appears after +a %left directive but before the next period (".") is given the same left-associative precedence value. Subsequent %left directives have higher precedence. For example:

    @@ -687,10 +751,11 @@ a large amount of stack space if you make heavy use or right-associative operators. For this reason, it is recommended that you use %left rather than %right whenever possible.

    +

    The %name directive

    By default, the functions generated by Lemon all begin with the -five-character string ``Parse''. You can change this string to something +five-character string "Parse". You can change this string to something different using the %name directive. For instance:

    @@ -709,16 +774,19 @@ The %name directive allows you to generator two or more different
 parsers and link them all into the same executable.
 
    
 
+
 
    The %nonassoc directive
    
 
 
    This directive is used to assign non-associative precedence to
-one or more terminal symbols.  See the section on precedence rules
-or on the %left directive for additional information.
    
+one or more terminal symbols.  See the section on 
+precedence rules
+or on the %left directive for additional information.
    
 
+
 
    The %parse_accept directive
    
 
 
    The %parse_accept directive specifies a block of C code that is
-executed whenever the parser accepts its input string.  To ``accept''
+executed whenever the parser accepts its input string.  To "accept"
 an input string means that the parser was able to process all tokens
 without error.
    
 
@@ -730,7 +798,7 @@ without error.
    
    }

    - +

    The %parse_failure directive

    The %parse_failure directive specifies a block of C code that @@ -745,12 +813,15 @@ only invoked when parsing is unable to continue.

    }

    +

    The %right directive

    This directive is used to assign right-associative precedence to -one or more terminal symbols. See the section on precedence rules -or on the %left directive for additional information.

    +one or more terminal symbols. See the section on +precedence rules +or on the %left directive for additional information.

    +

    The %stack_overflow directive

    The %stack_overflow directive specifies a block of C code that @@ -779,6 +850,7 @@ Not like this: list ::= . +

    The %stack_size directive

    If stack overflow is a problem and you can't resolve the trouble @@ -791,6 +863,7 @@ with a stack of the requested size. The default value is 100.

    %stack_size 2000

    +

    The %start_symbol directive

    By default, the start-symbol for the grammar that Lemon generates @@ -801,6 +874,7 @@ can choose a different start-symbol using the %start_symbol directive.

    %start_symbol prog

    +

    The %token_destructor directive

    The %destructor directive assigns a destructor to a non-terminal @@ -813,6 +887,7 @@ the %token_type directive) and so they use a common destructor. Other than that, the token destructor works just like the non-terminal destructors.

    +

    The %token_prefix directive

    Lemon generates #defines that assign small integer constants @@ -838,6 +913,7 @@ to cause Lemon to produce these symbols instead: #define TOKEN_PLUS 4 +

    The %token_type and %type directives

    These directives are used to specify the data types for values @@ -853,7 +929,7 @@ token structure. Like this:

    If the data type of terminals is not specified, the default value -is ``int''.

    +is "int".

    Non-terminal symbols can each have their own data types. Typically the data type of a non-terminal is a pointer to the root of a parse-tree @@ -874,6 +950,17 @@ non-terminal whose data type requires 1K of storage, then your 100 entry parser stack will require 100K of heap space. If you are willing and able to pay that price, fine. You just need to know.

    + +

    The %wildcard directive

    + +

    The %wildcard directive is followed by a single token name and a +period. This directive specifies that the identified token should +match any input token. + +

    When the generated parser has the choice of matching an input against +the wildcard token and some other token, the other token is always used. +The wildcard token is only matched if there are no other alternatives. +

    Error Processing

    After extensive experimentation over several years, it has been @@ -885,7 +972,7 @@ first invokes the code specified by the %syntax_error directive, if any. It then enters its error recovery strategy. The error recovery strategy is to begin popping the parsers stack until it enters a state where it is permitted to shift a special non-terminal symbol -named ``error''. It then shifts this non-terminal and continues +named "error". It then shifts this non-terminal and continues parsing. But the %syntax_error routine will not be called again until at least three new tokens have been successfully shifted.

    @@ -894,7 +981,7 @@ is unable to shift the error symbol, then the %parse_failed routine is invoked and the parser resets itself to its start state, ready to begin parsing a new file. This is what will happen at the very first syntax error, of course, if there are no instances of the -``error'' non-terminal in your grammar.

    +"error" non-terminal in your grammar.

    diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index 1e9be8117d..0bc61414b0 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -2500,7 +2500,7 @@ static int fts5ExprPopulatePoslistsCb( Fts5ExprTerm *pTerm; if( p->aPopulator[i].bOk==0 ) continue; for(pTerm=&pExpr->apExprPhrase[i]->aTerm[0]; pTerm; pTerm=pTerm->pSynonym){ - int nTerm = strlen(pTerm->zTerm); + int nTerm = (int)strlen(pTerm->zTerm); if( (nTerm==nToken || (nTermbPrefix)) && memcmp(pTerm->zTerm, pToken, nTerm)==0 ){ diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 323e6cefdc..44fba94c84 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -3453,18 +3453,35 @@ static int fts5AllocateSegid(Fts5Index *p, Fts5Structure *pStruct){ if( pStruct->nSegment>=FTS5_MAX_SEGMENT ){ p->rc = SQLITE_FULL; }else{ - while( iSegid==0 ){ - int iLvl, iSeg; - sqlite3_randomness(sizeof(u32), (void*)&iSegid); - iSegid = iSegid & ((1 << FTS5_DATA_ID_B)-1); - for(iLvl=0; iLvlnLevel; iLvl++){ - for(iSeg=0; iSegaLevel[iLvl].nSeg; iSeg++){ - if( iSegid==pStruct->aLevel[iLvl].aSeg[iSeg].iSegid ){ - iSegid = 0; - } + /* FTS5_MAX_SEGMENT is currently defined as 2000. So the following + ** array is 63 elements, or 252 bytes, in size. */ + u32 aUsed[(FTS5_MAX_SEGMENT+31) / 32]; + int iLvl, iSeg; + int i; + u32 mask; + memset(aUsed, 0, sizeof(aUsed)); + for(iLvl=0; iLvlnLevel; iLvl++){ + for(iSeg=0; iSegaLevel[iLvl].nSeg; iSeg++){ + int iId = pStruct->aLevel[iLvl].aSeg[iSeg].iSegid; + if( iId<=FTS5_MAX_SEGMENT ){ + aUsed[(iId-1) / 32] |= 1 << ((iId-1) % 32); } } } + + for(i=0; aUsed[i]==0xFFFFFFFF; i++); + mask = aUsed[i]; + for(iSegid=0; mask & (1 << iSegid); iSegid++); + iSegid += 1 + i*32; + +#ifdef SQLITE_DEBUG + for(iLvl=0; iLvlnLevel; iLvl++){ + for(iSeg=0; iSegaLevel[iLvl].nSeg; iSeg++){ + assert( iSegid!=pStruct->aLevel[iLvl].aSeg[iSeg].iSegid ); + } + } + assert( iSegid>0 && iSegid<=FTS5_MAX_SEGMENT ); +#endif } } @@ -3909,7 +3926,9 @@ static void fts5WriteFinish( fts5WriteFlushLeaf(p, pWriter); } *pnLeaf = pLeaf->pgno-1; - fts5WriteFlushBtree(p, pWriter); + if( pLeaf->pgno>1 ){ + fts5WriteFlushBtree(p, pWriter); + } } fts5BufferFree(&pLeaf->term); fts5BufferFree(&pLeaf->buf); diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c index e4f5dd8f51..e052d03cd2 100644 --- a/ext/fts5/fts5_storage.c +++ b/ext/fts5/fts5_storage.c @@ -145,6 +145,7 @@ static int fts5StorageGetStmt( } *ppStmt = p->aStmt[eStmt]; + sqlite3_reset(*ppStmt); return rc; } @@ -1121,5 +1122,3 @@ int sqlite3Fts5StorageConfigValue( } return rc; } - - diff --git a/ext/fts5/tool/fts5txt2db.tcl b/ext/fts5/tool/fts5txt2db.tcl index d5df971d4d..4766b00b06 100644 --- a/ext/fts5/tool/fts5txt2db.tcl +++ b/ext/fts5/tool/fts5txt2db.tcl @@ -17,6 +17,7 @@ proc process_cmdline {} { {detail "full" "Fts5 detail mode to use"} {repeat 1 "Load each file this many times"} {prefix "" "Fts prefix= option"} + {trans 1 "True to use a transaction"} database file... } { @@ -214,7 +215,7 @@ foreach c [lrange $cols 1 end] { } append sql ")" -db eval BEGIN +if {$A(trans)} { db eval BEGIN } while {$i < $N} { foreach c $cols s $A(colsize) { set R($c) [lrange $tokens $i [expr $i+$s-1]] @@ -222,7 +223,7 @@ db eval BEGIN } db eval $sql } -db eval COMMIT +if {$A(trans)} { db eval COMMIT } diff --git a/ext/rbu/rbu1.test b/ext/rbu/rbu1.test index 19b3d27a9f..51d2ce8171 100644 --- a/ext/rbu/rbu1.test +++ b/ext/rbu/rbu1.test @@ -611,7 +611,7 @@ foreach {tn3 create_vfs destroy_vfs} { 9 { CREATE TABLE t1(a, b PRIMARY KEY) WITHOUT ROWID; CREATE TABLE rbu.data_t1(a, b, rbu_control); - INSERT INTO rbu.data_t1 VALUES(1, 2, 2); + INSERT INTO rbu.data_t1 VALUES(1, 2, 3); } {SQLITE_ERROR - invalid rbu_control value} 10 { diff --git a/ext/rbu/rbudiff.test b/ext/rbu/rbudiff.test index 10be9e0d53..0863f6e807 100644 --- a/ext/rbu/rbudiff.test +++ b/ext/rbu/rbudiff.test @@ -18,16 +18,7 @@ if {![info exists testdir]} { source $testdir/tester.tcl set testprefix rbudiff -if {$tcl_platform(platform)=="windows"} { - set PROG "sqldiff.exe" -} else { - set PROG "./sqldiff" -} -if {![file exe $PROG]} { - puts "rbudiff.test cannot run because $PROG is not available" - finish_test - return -} +set PROG [test_find_sqldiff] db close proc get_rbudiff_sql {db1 db2} { @@ -45,6 +36,7 @@ proc step_rbu {target rbu} { } proc apply_rbudiff {sql target} { + test_rbucount $sql forcedelete rbu.db sqlite3 rbudb rbu.db rbudb eval $sql @@ -52,6 +44,27 @@ proc apply_rbudiff {sql target} { step_rbu $target rbu.db } +# The only argument is the output of an [sqldiff -rbu] run. This command +# tests that the contents of the rbu_count table is correct. An exception +# is thrown if it is not. +# +proc test_rbucount {sql} { + sqlite3 tmpdb "" + tmpdb eval $sql + tmpdb eval { + SELECT name FROM sqlite_master WHERE name LIKE 'data%' AND type='table' + } { + set a [tmpdb eval "SELECT count(*) FROM $name"] + set b [tmpdb eval {SELECT cnt FROM rbu_count WHERE tbl = $name}] + if {$a != $b} { + tmpdb close + error "rbu_count error - tbl = $name" + } + } + tmpdb close + return "" +} + proc rbudiff_cksum {db1} { set txt "" diff --git a/ext/rbu/rbuprogress.test b/ext/rbu/rbuprogress.test new file mode 100644 index 0000000000..6afbffe8ed --- /dev/null +++ b/ext/rbu/rbuprogress.test @@ -0,0 +1,415 @@ +# 2016 March 18 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +source [file join [file dirname [info script]] rbu_common.tcl] +set ::testprefix rbuprogress + + +proc create_db_file {filename sql} { + forcedelete $filename + sqlite3 tmpdb $filename + tmpdb eval $sql + tmpdb close +} + +# Create a simple RBU database. That expects to write to a table: +# +# CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); +# +proc create_rbu1 {filename} { + create_db_file $filename { + CREATE TABLE data_t1(a, b, c, rbu_control); + INSERT INTO data_t1 VALUES(1, 2, 3, 0); + INSERT INTO data_t1 VALUES(2, 'two', 'three', 0); + INSERT INTO data_t1 VALUES(3, NULL, 8.2, 0); + + CREATE TABLE rbu_count(tbl, cnt); + INSERT INTO rbu_count VALUES('data_t1', 3); + } + return $filename +} + + +do_execsql_test 1.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); +} + +do_test 1.1 { + create_rbu1 rbu.db + sqlite3rbu rbu test.db rbu.db + rbu bp_progress +} {0 0} +do_test 1.2 { rbu step ; rbu bp_progress } {3333 0} +do_test 1.3 { rbu step ; rbu bp_progress } {6666 0} +do_test 1.4 { rbu step ; rbu bp_progress } {10000 0} +do_test 1.5 { rbu step ; rbu bp_progress } {10000 0} +do_test 1.6 { rbu step ; rbu bp_progress } {10000 0} +do_test 1.7 { rbu step ; rbu bp_progress } {10000 5000} +do_test 1.8 { rbu step ; rbu bp_progress } {10000 10000} +do_test 1.9 { rbu step ; rbu bp_progress } {10000 10000} + +do_test 1.10 { + rbu close +} {SQLITE_DONE} + +#------------------------------------------------------------------------- +# +proc do_sp_test {tn bReopen target rbu reslist} { + uplevel [list do_test $tn [subst -nocommands { + if {$bReopen==0} { sqlite3rbu rbu $target $rbu } + set res [list] + while 1 { + if {$bReopen} { sqlite3rbu rbu $target $rbu } + set rc [rbu step] + if {[set rc] != "SQLITE_OK"} { rbu close ; error "error 1" } + lappend res [lindex [rbu bp_progress] 0] + if {[lindex [set res] end]==10000} break + if {$bReopen} { rbu close } + } + if {[set res] != [list $reslist]} { + rbu close + error "1. reslist incorrect (expect=$reslist got=[set res])" + } + + # One step to clean up the temporary tables used to update the only + # target table in the rbu database. And one more to move the *-oal + # file to *-wal. After each of these steps, the progress remains + # at "10000 0". + # + if {[lindex [list $reslist] 0]!=-1} { + rbu step + set res [rbu bp_progress] + if {[set res] != [list 10000 0]} { + rbu close + error "2. reslist incorrect (expect=10000 0 got=[set res])" + } + } + + rbu step + set res [rbu bp_progress] + if {[set res] != [list 10000 0]} { + rbu close + error "3. reslist incorrect (expect=10000 0 got=[set res])" + } + + # Do the checkpoint. + while {[rbu step]=="SQLITE_OK"} { + foreach {a b} [rbu bp_progress] {} + if {[set a]!=10000 || [set b]<=0 || [set b]>10000} { + rbu close + error "4. reslist incorrect (expect=10000 1..10000 got=[set a] [set b])" + } + } + + set res [rbu bp_progress] + if {[set res] != [list 10000 10000]} { + rbu close + error "5. reslist is incorrect (expect=10000 10000 got=[set res])" + } + + rbu close + }] {SQLITE_DONE}] +} + +foreach {bReopen} { 0 1 } { + reset_db + do_test 2.$bReopen.1.0 { + execsql { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + } + create_db_file rbu.db { + CREATE TABLE data_t1(a, b, c, rbu_control); + INSERT INTO data_t1 VALUES(4, 4, 4, 0); + INSERT INTO data_t1 VALUES(5, 5, 5, 0); + + CREATE TABLE rbu_count(tbl, cnt); + INSERT INTO rbu_count VALUES('data_t1', 2); + } + } {} + do_sp_test 2.$bReopen.1.1 $bReopen test.db rbu.db {5000 10000} + + reset_db + do_test 2.$bReopen.2.0 { + execsql { CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c) } + create_rbu1 rbu.db + } {rbu.db} + do_sp_test 2.$bReopen.2.1 $bReopen test.db rbu.db {3333 6666 10000} + + reset_db + do_test 2.$bReopen.3.0 { + execsql { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b); + INSERT INTO t1 VALUES(1, 1, 1); + INSERT INTO t1 VALUES(2, 2, 2); + INSERT INTO t1 VALUES(3, 3, 3); + } + create_db_file rbu.db { + CREATE TABLE data_t1(a, b, c, rbu_control); + INSERT INTO data_t1 VALUES(4, 4, 4, 0); + INSERT INTO data_t1 VALUES(2, NULL, NULL, 1); + INSERT INTO data_t1 VALUES(5, NULL, NULL, 1); + + CREATE TABLE rbu_count(tbl, cnt); + INSERT INTO rbu_count VALUES('data_t1', 3); + } + } {} + do_sp_test 2.$bReopen.3.1 $bReopen test.db rbu.db {1666 3333 6000 8000 10000} + + reset_db + do_test 2.$bReopen.4.0 { + execsql { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b); + INSERT INTO t1 VALUES(1, 1, 1); + INSERT INTO t1 VALUES(2, 2, 2); + INSERT INTO t1 VALUES(3, 3, 3); + } + create_db_file rbu.db { + CREATE TABLE data_t1(a, b, c, rbu_control); + INSERT INTO data_t1 VALUES(2, 4, 4, '.xx'); + + CREATE TABLE rbu_count(tbl, cnt); + INSERT INTO rbu_count VALUES('data_t1', 1); + } + } {} + do_sp_test 2.$bReopen.4.1 $bReopen test.db rbu.db {3333 6666 10000} + + reset_db + do_test 2.$bReopen.5.0 { + execsql { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b); + INSERT INTO t1 VALUES(1, 1, 1); + INSERT INTO t1 VALUES(2, 2, 2); + INSERT INTO t1 VALUES(3, 3, 3); + } + create_db_file rbu.db { + CREATE TABLE data_t1(a, b, c, rbu_control); + INSERT INTO data_t1 VALUES(4, NULL, 4, '.xx'); + + CREATE TABLE rbu_count(tbl, cnt); + INSERT INTO rbu_count VALUES('data_t1', 1); + } + } {} + do_sp_test 2.$bReopen.5.1 $bReopen test.db rbu.db {10000} + + reset_db + do_test 2.$bReopen.6.0 { + execsql { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b); + INSERT INTO t1 VALUES(1, 1, 1); + INSERT INTO t1 VALUES(2, 2, 2); + INSERT INTO t1 VALUES(3, 3, 3); + } + create_db_file rbu.db { + CREATE TABLE data_t1(a, b, c, rbu_control); + INSERT INTO data_t1 VALUES(4, 4, 4, 0); + INSERT INTO data_t1 VALUES(2, NULL, NULL, 1); + INSERT INTO data_t1 VALUES(5, NULL, NULL, 1); + } + } {} + do_sp_test 2.$bReopen.6.1 $bReopen test.db rbu.db {-1 -1 -1 -1 -1 10000} +} + +#------------------------------------------------------------------------- +# The following tests verify that the API works when resuming an update +# during the incremental checkpoint stage. +# +proc do_phase2_test {tn bReopen target rbu nStep} { + uplevel [list do_test $tn [subst -nocommands { + + # Build the OAL/WAL file: + sqlite3rbu rbu $target $rbu + while {[lindex [rbu bp_progress] 0]<10000} { + set rc [rbu step] + if {"SQLITE_OK" != [set rc]} { rbu close } + } + + # Clean up the temp tables and move the *-oal file to *-wal. + rbu step + rbu step + + for {set i 0} {[set i] < $nStep} {incr i} { + if {$bReopen} { + rbu close + sqlite3rbu rbu $target $rbu + } + rbu step + set res [rbu bp_progress] + set expect [expr (1 + [set i]) * 10000 / $nStep] + if {[lindex [set res] 1] != [set expect]} { + error "Have [set res], expected 10000 [set expect]" + } + } + + set rc [rbu step] + if {[set rc] != "SQLITE_DONE"} { + error "Have [set rc], expected SQLITE_DONE" + } + + rbu close + }] {SQLITE_DONE}] +} + +foreach bReopen {0 1} { + do_test 3.$bReopen.1.0 { + reset_db + execsql { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + CREATE TABLE t2(a INTEGER PRIMARY KEY, b); + CREATE TABLE t3(a INTEGER PRIMARY KEY, b); + CREATE TABLE t4(a INTEGER PRIMARY KEY, b); + } + create_db_file rbu.db { + CREATE TABLE data_t1(a, b, rbu_control); + CREATE TABLE data_t2(a, b, rbu_control); + CREATE TABLE data_t3(a, b, rbu_control); + CREATE TABLE data_t4(a, b, rbu_control); + INSERT INTO data_t1 VALUES(1, 2, 0); + INSERT INTO data_t2 VALUES(1, 2, 0); + INSERT INTO data_t3 VALUES(1, 2, 0); + INSERT INTO data_t4 VALUES(1, 2, 0); + + CREATE TABLE rbu_count(tbl, cnt); + INSERT INTO rbu_count VALUES('data_t1', 1); + INSERT INTO rbu_count VALUES('data_t2', 1); + INSERT INTO rbu_count VALUES('data_t3', 1); + INSERT INTO rbu_count VALUES('data_t4', 1); + } + } {} + do_phase2_test 3.$bReopen.1.1 $bReopen test.db rbu.db 5 +} + + +foreach {bReopen} { 0 1 } { + foreach {tn tbl} { + ipk { CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c) } + wr { CREATE TABLE t1(a INT PRIMARY KEY, b, c) WITHOUT ROWID } + pk { CREATE TABLE t1(a INT PRIMARY KEY, b, c) } + } { + + foreach {tn2 rbusql r1 r3} { + 1 { + CREATE TABLE data0_t1(a, b, c, rbu_control); + INSERT INTO data0_t1 VALUES(15, 15, 15, 0); + INSERT INTO data0_t1 VALUES(20, 20, 20, 0); + CREATE TABLE rbu_count(tbl, cnt); + INSERT INTO rbu_count VALUES('data0_t1', 2); + } + {2500 5000 7500 10000} + {1666 3333 5000 6666 8333 10000} + + 2 { + CREATE TABLE data0_t1(a, b, c, rbu_control); + INSERT INTO data0_t1 VALUES(10, 10, 10, 2); + CREATE TABLE rbu_count(tbl, cnt); + INSERT INTO rbu_count VALUES('data0_t1', 1); + } + {3333 6666 10000} + {2000 4000 6000 8000 10000} + + 3 { + CREATE TABLE data0_t1(a, b, c, rbu_control); + INSERT INTO data0_t1 VALUES(7, 7, 7, 2); + INSERT INTO data0_t1 VALUES(10, 10, 10, 2); + CREATE TABLE rbu_count(tbl, cnt); + INSERT INTO rbu_count VALUES('data0_t1', 2); + } + {2500 4000 6000 8000 10000} + {1666 2500 3750 5000 6250 7500 8750 10000} + + } { + + reset_db ; execsql $tbl + do_test 4.$tn.$bReopen.$tn2.0 { + execsql { + CREATE INDEX t1c ON t1(c); + INSERT INTO t1 VALUES(1, 1, 1); + INSERT INTO t1 VALUES(5, 5, 5); + INSERT INTO t1 VALUES(10, 10, 10); + } + create_db_file rbu.db $rbusql + } {} + + set R(ipk) $r1 + set R(wr) $r1 + set R(pk) $r3 + do_sp_test 4.$tn.$bReopen.$tn2.1 $bReopen test.db rbu.db $R($tn) + } + } +} + +foreach {bReopen} { 0 1 } { + foreach {tn tbl} { + nopk { + CREATE TABLE t1(a, b, c); + CREATE INDEX t1c ON t1(c); + } + vtab { + CREATE VIRTUAL TABLE t1 USING fts5(a, b, c); + } + } { + + foreach {tn2 rbusql r1 r2} { + 1 { + CREATE TABLE data0_t1(a, b, c, rbu_rowid, rbu_control); + INSERT INTO data0_t1 VALUES(15, 15, 15, 4, 0); + INSERT INTO data0_t1 VALUES(20, 20, 20, 5, 0); + CREATE TABLE rbu_count(tbl, cnt); + INSERT INTO rbu_count VALUES('data0_t1', 2); + } + {2500 5000 7500 10000} + {5000 10000} + + 2 { + CREATE TABLE data0_t1(rbu_rowid, a, b, c, rbu_control); + INSERT INTO data0_t1 VALUES(0, 7, 7, 7, 2); + INSERT INTO data0_t1 VALUES(2, 10, 10, 10, 2); + CREATE TABLE rbu_count(tbl, cnt); + INSERT INTO rbu_count VALUES('data0_t1', 2); + } + {2500 4000 6000 8000 10000} + {5000 10000} + + 3 { + CREATE TABLE data0_t1(rbu_rowid, a, b, c, rbu_control); + INSERT INTO data0_t1 VALUES(1, NULL, NULL, NULL, 1); + INSERT INTO data0_t1 VALUES(2, NULL, NULL, 7, '..x'); + CREATE TABLE rbu_count(tbl, cnt); + INSERT INTO rbu_count VALUES('data0_t1', 2); + } + {2500 4000 6000 8000 10000} + {5000 10000} + } { + + reset_db ; execsql $tbl + do_test 5.$tn.$bReopen.$tn2.0 { + execsql { + INSERT INTO t1 VALUES(1, 1, 1); + INSERT INTO t1 VALUES(5, 5, 5); + INSERT INTO t1 VALUES(10, 10, 10); + } + create_db_file rbu.db $rbusql + } {} + + set R(nopk) $r1 + set R(vtab) $r2 + do_sp_test 5.$tn.$bReopen.$tn2.1 $bReopen test.db rbu.db $R($tn) + } + } +} + + +finish_test + diff --git a/ext/rbu/sqlite3rbu.c b/ext/rbu/sqlite3rbu.c index 474e39fe8d..93b756aa6a 100644 --- a/ext/rbu/sqlite3rbu.c +++ b/ext/rbu/sqlite3rbu.c @@ -147,14 +147,15 @@ ** RBU_STATE_OALSZ: ** Valid if STAGE==1. The size in bytes of the *-oal file. */ -#define RBU_STATE_STAGE 1 -#define RBU_STATE_TBL 2 -#define RBU_STATE_IDX 3 -#define RBU_STATE_ROW 4 -#define RBU_STATE_PROGRESS 5 -#define RBU_STATE_CKPT 6 -#define RBU_STATE_COOKIE 7 -#define RBU_STATE_OALSZ 8 +#define RBU_STATE_STAGE 1 +#define RBU_STATE_TBL 2 +#define RBU_STATE_IDX 3 +#define RBU_STATE_ROW 4 +#define RBU_STATE_PROGRESS 5 +#define RBU_STATE_CKPT 6 +#define RBU_STATE_COOKIE 7 +#define RBU_STATE_OALSZ 8 +#define RBU_STATE_PHASEONESTEP 9 #define RBU_STAGE_OAL 1 #define RBU_STAGE_MOVE 2 @@ -200,6 +201,7 @@ struct RbuState { i64 nProgress; u32 iCookie; i64 iOalSz; + i64 nPhaseOneStep; }; struct RbuUpdateStmt { @@ -244,6 +246,7 @@ struct RbuObjIter { int iTnum; /* Root page of current object */ int iPkTnum; /* If eType==EXTERNAL, root of PK index */ int bUnique; /* Current index is unique */ + int nIndex; /* Number of aux. indexes on table zTbl */ /* Statements created by rbuObjIterPrepareAll() */ int nCol; /* Number of columns in current object */ @@ -297,6 +300,43 @@ struct RbuFrame { /* ** RBU handle. +** +** nPhaseOneStep: +** If the RBU database contains an rbu_count table, this value is set to +** a running estimate of the number of b-tree operations required to +** finish populating the *-oal file. This allows the sqlite3_bp_progress() +** API to calculate the permyriadage progress of populating the *-oal file +** using the formula: +** +** permyriadage = (10000 * nProgress) / nPhaseOneStep +** +** nPhaseOneStep is initialized to the sum of: +** +** nRow * (nIndex + 1) +** +** for all source tables in the RBU database, where nRow is the number +** of rows in the source table and nIndex the number of indexes on the +** corresponding target database table. +** +** This estimate is accurate if the RBU update consists entirely of +** INSERT operations. However, it is inaccurate if: +** +** * the RBU update contains any UPDATE operations. If the PK specified +** for an UPDATE operation does not exist in the target table, then +** no b-tree operations are required on index b-trees. Or if the +** specified PK does exist, then (nIndex*2) such operations are +** required (one delete and one insert on each index b-tree). +** +** * the RBU update contains any DELETE operations for which the specified +** PK does not exist. In this case no operations are required on index +** b-trees. +** +** * the RBU update contains REPLACE operations. These are similar to +** UPDATE operations. +** +** nPhaseOneStep is updated to account for the conditions above during the +** first pass of each source table. The updated nPhaseOneStep value is +** stored in the rbu_state table if the RBU update is suspended. */ struct sqlite3rbu { int eStage; /* Value of RBU_STATE_STAGE field */ @@ -314,6 +354,7 @@ struct sqlite3rbu { const char *zVfsName; /* Name of automatically created rbu vfs */ rbu_file *pTargetFd; /* File handle open on target db */ i64 iOalSz; + i64 nPhaseOneStep; /* The following state variables are used as part of the incremental ** checkpoint stage (eStage==RBU_STAGE_CKPT). See comments surrounding @@ -1144,6 +1185,7 @@ static void rbuObjIterCacheIndexedCols(sqlite3rbu *p, RbuObjIter *pIter){ ); } + pIter->nIndex = 0; while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pList) ){ const char *zIdx = (const char*)sqlite3_column_text(pList, 1); sqlite3_stmt *pXInfo = 0; @@ -1157,6 +1199,12 @@ static void rbuObjIterCacheIndexedCols(sqlite3rbu *p, RbuObjIter *pIter){ } rbuFinalize(p, pXInfo); bIndex = 1; + pIter->nIndex++; + } + + if( pIter->eType==RBU_PK_WITHOUT_ROWID ){ + /* "PRAGMA index_list" includes the main PK b-tree */ + pIter->nIndex--; } rbuFinalize(p, pList); @@ -1270,6 +1318,7 @@ static int rbuObjIterCacheTableInfo(sqlite3rbu *p, RbuObjIter *pIter){ rbuFinalize(p, pStmt); rbuObjIterCacheIndexedCols(p, pIter); assert( pIter->eType!=RBU_PK_VTAB || pIter->abIndexed==0 ); + assert( pIter->eType!=RBU_PK_VTAB || pIter->nIndex==0 ); } return p->rc; @@ -1823,6 +1872,14 @@ static void rbuTmpInsertFunc( int rc = SQLITE_OK; int i; + assert( sqlite3_value_int(apVal[0])!=0 + || p->objiter.eType==RBU_PK_EXTERNAL + || p->objiter.eType==RBU_PK_NONE + ); + if( sqlite3_value_int(apVal[0])!=0 ){ + p->nPhaseOneStep += p->objiter.nIndex; + } + for(i=0; rc==SQLITE_OK && iobjiter.pTmpInsert, i+1, apVal[i]); } @@ -2567,6 +2624,17 @@ static void rbuStepOneOp(sqlite3rbu *p, int eType){ assert( p->rc==SQLITE_OK ); assert( eType!=RBU_DELETE || pIter->zIdx==0 ); + assert( eType==RBU_DELETE || eType==RBU_IDX_DELETE + || eType==RBU_INSERT || eType==RBU_IDX_INSERT + ); + + /* If this is a delete, decrement nPhaseOneStep by nIndex. If the DELETE + ** statement below does actually delete a row, nPhaseOneStep will be + ** incremented by the same amount when SQL function rbu_tmp_insert() + ** is invoked by the trigger. */ + if( eType==RBU_DELETE ){ + p->nPhaseOneStep -= p->objiter.nIndex; + } if( eType==RBU_IDX_DELETE || eType==RBU_DELETE ){ pWriter = pIter->pDelete; @@ -2638,11 +2706,14 @@ static int rbuStep(sqlite3rbu *p){ ); assert( eType!=RBU_UPDATE || pIter->zIdx==0 ); - if( pIter->zIdx==0 && eType==RBU_IDX_DELETE ){ + if( pIter->zIdx==0 && (eType==RBU_IDX_DELETE || eType==RBU_IDX_INSERT) ){ rbuBadControlError(p); } else if( eType==RBU_REPLACE ){ - if( pIter->zIdx==0 ) rbuStepOneOp(p, RBU_DELETE); + if( pIter->zIdx==0 ){ + p->nPhaseOneStep += p->objiter.nIndex; + rbuStepOneOp(p, RBU_DELETE); + } if( p->rc==SQLITE_OK ) rbuStepOneOp(p, RBU_INSERT); } else if( eType!=RBU_UPDATE ){ @@ -2652,6 +2723,7 @@ static int rbuStep(sqlite3rbu *p){ sqlite3_value *pVal; sqlite3_stmt *pUpdate = 0; assert( eType==RBU_UPDATE ); + p->nPhaseOneStep -= p->objiter.nIndex; rbuGetUpdateStmt(p, pIter, zMask, &pUpdate); if( pUpdate ){ int i; @@ -2729,6 +2801,7 @@ static void rbuSaveState(sqlite3rbu *p, int eStage){ "(%d, %d), " "(%d, %lld), " "(%d, %lld), " + "(%d, %lld), " "(%d, %lld) ", p->zStateDb, RBU_STATE_STAGE, eStage, @@ -2738,7 +2811,8 @@ static void rbuSaveState(sqlite3rbu *p, int eStage){ RBU_STATE_PROGRESS, p->nProgress, RBU_STATE_CKPT, p->iWalCksum, RBU_STATE_COOKIE, (i64)p->pTargetFd->iCookie, - RBU_STATE_OALSZ, p->iOalSz + RBU_STATE_OALSZ, p->iOalSz, + RBU_STATE_PHASEONESTEP, p->nPhaseOneStep ) ); assert( pInsert==0 || rc==SQLITE_OK ); @@ -2925,6 +2999,10 @@ static RbuState *rbuLoadState(sqlite3rbu *p){ pRet->iOalSz = (u32)sqlite3_column_int64(pStmt, 1); break; + case RBU_STATE_PHASEONESTEP: + pRet->nPhaseOneStep = sqlite3_column_int64(pStmt, 1); + break; + default: rc = SQLITE_CORRUPT; break; @@ -3032,6 +3110,100 @@ static void rbuDeleteVfs(sqlite3rbu *p){ } } +/* +** This user-defined SQL function is invoked with a single argument - the +** name of a table expected to appear in the target database. It returns +** the number of auxilliary indexes on the table. +*/ +static void rbuIndexCntFunc( + sqlite3_context *pCtx, + int nVal, + sqlite3_value **apVal +){ + sqlite3rbu *p = (sqlite3rbu*)sqlite3_user_data(pCtx); + sqlite3_stmt *pStmt = 0; + char *zErrmsg = 0; + int rc; + + assert( nVal==1 ); + + rc = prepareFreeAndCollectError(p->dbMain, &pStmt, &zErrmsg, + sqlite3_mprintf("SELECT count(*) FROM sqlite_master " + "WHERE type='index' AND tbl_name = %Q", sqlite3_value_text(apVal[0])) + ); + if( rc!=SQLITE_OK ){ + sqlite3_result_error(pCtx, zErrmsg, -1); + }else{ + int nIndex = 0; + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + nIndex = sqlite3_column_int(pStmt, 0); + } + rc = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ){ + sqlite3_result_int(pCtx, nIndex); + }else{ + sqlite3_result_error(pCtx, sqlite3_errmsg(p->dbMain), -1); + } + } + + sqlite3_free(zErrmsg); +} + +/* +** If the RBU database contains the rbu_count table, use it to initialize +** the sqlite3rbu.nPhaseOneStep variable. The schema of the rbu_count table +** is assumed to contain the same columns as: +** +** CREATE TABLE rbu_count(tbl TEXT PRIMARY KEY, cnt INTEGER) WITHOUT ROWID; +** +** There should be one row in the table for each data_xxx table in the +** database. The 'tbl' column should contain the name of a data_xxx table, +** and the cnt column the number of rows it contains. +** +** sqlite3rbu.nPhaseOneStep is initialized to the sum of (1 + nIndex) * cnt +** for all rows in the rbu_count table, where nIndex is the number of +** indexes on the corresponding target database table. +*/ +static void rbuInitPhaseOneSteps(sqlite3rbu *p){ + if( p->rc==SQLITE_OK ){ + sqlite3_stmt *pStmt = 0; + int bExists = 0; /* True if rbu_count exists */ + + p->nPhaseOneStep = -1; + + p->rc = sqlite3_create_function(p->dbRbu, + "rbu_index_cnt", 1, SQLITE_UTF8, (void*)p, rbuIndexCntFunc, 0, 0 + ); + + /* Check for the rbu_count table. If it does not exist, or if an error + ** occurs, nPhaseOneStep will be left set to -1. */ + if( p->rc==SQLITE_OK ){ + p->rc = prepareAndCollectError(p->dbRbu, &pStmt, &p->zErrmsg, + "SELECT 1 FROM sqlite_master WHERE tbl_name = 'rbu_count'" + ); + } + if( p->rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + bExists = 1; + } + p->rc = sqlite3_finalize(pStmt); + } + + if( p->rc==SQLITE_OK && bExists ){ + p->rc = prepareAndCollectError(p->dbRbu, &pStmt, &p->zErrmsg, + "SELECT sum(cnt * (1 + rbu_index_cnt(rbu_target_name(tbl))))" + "FROM rbu_count" + ); + if( p->rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + p->nPhaseOneStep = sqlite3_column_int64(pStmt, 0); + } + p->rc = sqlite3_finalize(pStmt); + } + } + } +} + /* ** Open and return a new RBU handle. */ @@ -3077,9 +3249,11 @@ sqlite3rbu *sqlite3rbu_open( if( pState->eStage==0 ){ rbuDeleteOalFile(p); + rbuInitPhaseOneSteps(p); p->eStage = RBU_STAGE_OAL; }else{ p->eStage = pState->eStage; + p->nPhaseOneStep = pState->nPhaseOneStep; } p->nProgress = pState->nProgress; p->iOalSz = pState->iOalSz; @@ -3243,6 +3417,42 @@ sqlite3_int64 sqlite3rbu_progress(sqlite3rbu *pRbu){ return pRbu->nProgress; } +/* +** Return permyriadage progress indications for the two main stages of +** an RBU update. +*/ +void sqlite3rbu_bp_progress(sqlite3rbu *p, int *pnOne, int *pnTwo){ + const int MAX_PROGRESS = 10000; + switch( p->eStage ){ + case RBU_STAGE_OAL: + if( p->nPhaseOneStep>0 ){ + *pnOne = (int)(MAX_PROGRESS * (i64)p->nProgress/(i64)p->nPhaseOneStep); + }else{ + *pnOne = -1; + } + *pnTwo = 0; + break; + + case RBU_STAGE_MOVE: + *pnOne = MAX_PROGRESS; + *pnTwo = 0; + break; + + case RBU_STAGE_CKPT: + *pnOne = MAX_PROGRESS; + *pnTwo = (int)(MAX_PROGRESS * (i64)p->nStep / (i64)p->nFrame); + break; + + case RBU_STAGE_DONE: + *pnOne = MAX_PROGRESS; + *pnTwo = MAX_PROGRESS; + break; + + default: + assert( 0 ); + } +} + int sqlite3rbu_savestate(sqlite3rbu *p){ int rc = p->rc; diff --git a/ext/rbu/sqlite3rbu.h b/ext/rbu/sqlite3rbu.h index f1a0f3cd84..f379bb5b41 100644 --- a/ext/rbu/sqlite3rbu.h +++ b/ext/rbu/sqlite3rbu.h @@ -400,6 +400,48 @@ int sqlite3rbu_close(sqlite3rbu *pRbu, char **pzErrmsg); */ sqlite3_int64 sqlite3rbu_progress(sqlite3rbu *pRbu); +/* +** Obtain permyriadage (permyriadage is to 10000 as percentage is to 100) +** progress indications for the two stages of an RBU update. This API may +** be useful for driving GUI progress indicators and similar. +** +** An RBU update is divided into two stages: +** +** * Stage 1, in which changes are accumulated in an oal/wal file, and +** * Stage 2, in which the contents of the wal file are copied into the +** main database. +** +** The update is visible to non-RBU clients during stage 2. During stage 1 +** non-RBU reader clients may see the original database. +** +** If this API is called during stage 2 of the update, output variable +** (*pnOne) is set to 10000 to indicate that stage 1 has finished and (*pnTwo) +** to a value between 0 and 10000 to indicate the permyriadage progress of +** stage 2. A value of 5000 indicates that stage 2 is half finished, +** 9000 indicates that it is 90% finished, and so on. +** +** If this API is called during stage 1 of the update, output variable +** (*pnTwo) is set to 0 to indicate that stage 2 has not yet started. The +** value to which (*pnOne) is set depends on whether or not the RBU +** database contains an "rbu_count" table. The rbu_count table, if it +** exists, must contain the same columns as the following: +** +** CREATE TABLE rbu_count(tbl TEXT PRIMARY KEY, cnt INTEGER) WITHOUT ROWID; +** +** There must be one row in the table for each source (data_xxx) table within +** the RBU database. The 'tbl' column should contain the name of the source +** table. The 'cnt' column should contain the number of rows within the +** source table. +** +** If the rbu_count table is present and populated correctly and this +** API is called during stage 1, the *pnOne output variable is set to the +** permyriadage progress of the same stage. If the rbu_count table does +** not exist, then (*pnOne) is set to -1 during stage 1. If the rbu_count +** table exists but is not correctly populated, the value of the *pnOne +** output variable during stage 1 is undefined. +*/ +void sqlite3rbu_bp_progress(sqlite3rbu *pRbu, int *pnOne, int *pnTwo); + /* ** Create an RBU VFS named zName that accesses the underlying file-system ** via existing VFS zParent. Or, if the zParent parameter is passed NULL, diff --git a/ext/rbu/test_rbu.c b/ext/rbu/test_rbu.c index 3fa85b7569..5e8640a9f8 100644 --- a/ext/rbu/test_rbu.c +++ b/ext/rbu/test_rbu.c @@ -66,6 +66,7 @@ static int test_sqlite3rbu_cmd( {"create_rbu_delta", 2, ""}, /* 2 */ {"savestate", 2, ""}, /* 3 */ {"dbMain_eval", 3, "SQL"}, /* 4 */ + {"bp_progress", 2, ""}, /* 5 */ {0,0,0} }; int iCmd; @@ -136,6 +137,18 @@ static int test_sqlite3rbu_cmd( break; } + case 5: /* bp_progress */ { + int one, two; + Tcl_Obj *pObj; + sqlite3rbu_bp_progress(pRbu, &one, &two); + + pObj = Tcl_NewObj(); + Tcl_ListObjAppendElement(interp, pObj, Tcl_NewIntObj(one)); + Tcl_ListObjAppendElement(interp, pObj, Tcl_NewIntObj(two)); + Tcl_SetObjResult(interp, pObj); + break; + } + default: /* seems unlikely */ assert( !"cannot happen" ); break; diff --git a/manifest b/manifest index c10bdd869b..561e39edc2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sall\srecent\senhancements\sfrom\strunk. -D 2016-03-16T01:16:30.929 +C Merge\s3.12.0\sbeta\schanges\sfrom\strunk. +D 2016-03-21T15:32:19.793 F Makefile.in e812bb732d7af01baa09f1278bd4f4a2e3a09449 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc e6ee58b849c116d5554024f524cbf61f064f6f01 @@ -33,7 +33,7 @@ F config.sub 9ebe4c3b3dab6431ece34f16828b594fb420da55 F configure c01a159fdf7ea0171ad01ce2937283f2c7972bde x F configure.ac 89e4e02a83d8a1528011f8395621b8c3186b4089 F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad -F doc/lemon.html c30255bea0fd87a81f082d17a72c9dffbc3f6dd9 +F doc/lemon.html e2118945e5f07ed146b45c9cd2b2dd6eabb8ebf2 F doc/pager-invariants.txt 27fed9a70ddad2088750c4a2b493b63853da2710 F doc/vfs-shm.txt e101f27ea02a8387ce46a05be2b1a902a021d37a F ext/README.txt 913a7bd3f4837ab14d7e063304181787658b14e1 @@ -102,11 +102,11 @@ F ext/fts5/fts5Int.h 4060504b7979601d99e1385c2b5713036854979a F ext/fts5/fts5_aux.c daa57fb45216491814520bbb587e97bf81ced458 F ext/fts5/fts5_buffer.c 4c1502d4c956cd092c89ce4480867f9d8bf325cd F ext/fts5/fts5_config.c 5af9c360e99669d29f06492c370892394aba0857 -F ext/fts5/fts5_expr.c be309fb227003c931107bfcc12d5be4f2fd2bb8c +F ext/fts5/fts5_expr.c 35e9d92c89e7c7ea0759b73d24da1ecb7630a24b F ext/fts5/fts5_hash.c f3a7217c86eb8f272871be5f6aa1b6798960a337 -F ext/fts5/fts5_index.c d4f0c12e4f04bbc3a06b6da052039f2ce3e45438 +F ext/fts5/fts5_index.c d3759c2f7d878e9e0a392b027a1c6e05c356007d F ext/fts5/fts5_main.c b8501e1a6a11591c53b18ce7aea7e5386cfb0421 -F ext/fts5/fts5_storage.c f8343db90d8c95a4d4b52f6676e354b4649ffd6e +F ext/fts5/fts5_storage.c 2a38c6fa5db193a6a00588865134450ef5812daa F ext/fts5/fts5_tcl.c f8731e0508299bd43f1a2eff7dbeaac870768966 F ext/fts5/fts5_test_mi.c 783b86697ebf773c18fc109992426c0173a055bc F ext/fts5/fts5_test_tok.c db08af63673c3a7d39f053b36fd6e065017706be @@ -194,7 +194,7 @@ F ext/fts5/test/fts5update.test 57c7012a7919889048947addae10e0613df45529 F ext/fts5/test/fts5version.test 978f59541d8cef7e8591f8be2115ec5ccb863e2e F ext/fts5/test/fts5vocab.test 480d780aa6b699816c5066225fbd86f3a0239477 F ext/fts5/tool/fts5speed.tcl b0056f91a55b2d1a3684ec05729de92b042e2f85 -F ext/fts5/tool/fts5txt2db.tcl 1343745b89ca2a1e975c23f836d0cee410052975 +F ext/fts5/tool/fts5txt2db.tcl 526a9979c963f1c54fd50976a05a502e533a4c59 F ext/fts5/tool/loadfts5.tcl 95b03429ee6b138645703c6ca192c3ac96eaf093 F ext/fts5/tool/mkfts5c.tcl d1c2a9ab8e0ec690a52316f33dd9b1d379942f45 F ext/fts5/tool/showfts5.tcl d54da0e067306663e2d5d523965ca487698e722c @@ -221,7 +221,7 @@ F ext/misc/vfslog.c fe40fab5c077a40477f7e5eba994309ecac6cc95 F ext/misc/vtshim.c babb0dc2bf116029e3e7c9a618b8a1377045303e F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 F ext/rbu/rbu.c ba3983dceffa0938532e79142f391737513de023 -F ext/rbu/rbu1.test 57601977588603e82700a43c279bd55282ffa482 +F ext/rbu/rbu1.test 42bd835e019eff789ec241017965277baeb658b1 F ext/rbu/rbu10.test 046b0980041d30700464a800bbf6733ed2df515d F ext/rbu/rbu11.test 9bc68c2d3dbeb1720153626e3bd0466dcc017702 F ext/rbu/rbu12.test bde22ed0004dd5d1888c72a84ae407e574aeae16 @@ -238,14 +238,15 @@ F ext/rbu/rbuB.test c25bc325b8072a766e56bb76c001866b405925c2 F ext/rbu/rbuC.test efe47db508a0269b683cb2a1913a425ffd39a831 F ext/rbu/rbu_common.tcl 0398545fed614f807d5f0ba55a85a51f08ba8f1a F ext/rbu/rbucrash.test 8d2ed5d4b05fef6c00c2a6b5f7ead71fa172a695 -F ext/rbu/rbudiff.test 6cc806dc36389292f2a8f5842d0103721df4a07d +F ext/rbu/rbudiff.test 2df0a8a7d998ecf81764c21eeda3cde5611c5091 F ext/rbu/rbufault.test cc0be8d5d392d98b0c2d6a51be377ea989250a89 F ext/rbu/rbufault2.test 9a7f19edd6ea35c4c9f807d8a3db0a03a5670c06 F ext/rbu/rbufts.test 828cd689da825f0a7b7c53ffc1f6f7fdb6fa5bda +F ext/rbu/rbuprogress.test 2023a7df2c523e3df1cb532eff811cda385a789a F ext/rbu/rbusave.test 0f43b6686084f426ddd040b878426452fd2c2f48 -F ext/rbu/sqlite3rbu.c 5956f8bee63b5ab2b04e65c1801ea0f5920dac92 -F ext/rbu/sqlite3rbu.h 0bdeb3be211aaba7d85445fa36f4701a25a3dbde -F ext/rbu/test_rbu.c 4a4cdcef4ef9379fc2a21f008805c80b27bcf573 +F ext/rbu/sqlite3rbu.c 007fc4db8c0b95c7ef10162b5864921ef5cc8106 +F ext/rbu/sqlite3rbu.h d7cc99350c10134f358fe1a8997d9225b3f712b2 +F ext/rbu/test_rbu.c 3505641a78b723589b8780d5f9b2faeeb73e037d F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 0b870ccb7b58b734a2a8e1e2755a7c0ded070920 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e @@ -324,7 +325,7 @@ F src/ctime.c 60e135af364d777a9ab41c97e5e89cd224da6198 F src/date.c 0b73e681c11fca867fec554750c07fe0d4e417c1 F src/dbstat.c c845548d4346e606e2f2b7d2e714ace2b8a7dd1b F src/delete.c eeac28b3d3d88e3541bdf93e91ea7492a7b67842 -F src/expr.c c329d581e5d631153456369684d7d4bcd94c907d +F src/expr.c 289ffac5240b60fee0a824d3d5ab2d7bd2630c94 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 5da47ff524e2f687997a74737ce598f043e1342a F src/func.c 552d300265aed09eea21f68ac742a440550c0062 @@ -334,8 +335,8 @@ F src/hash.h c8f3c31722cf3277d03713909761e152a5b81094 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 F src/insert.c 8f4e9fcbd8e95e85f15647ba8b413b18d556ec2b F src/legacy.c 75d3023be8f0d2b99d60f905090341a03358c58e -F src/loadext.c 9e2a41adcaff16ebc1ebff1f336cbf33de55396f -F src/main.c 0a3dc6c0bac5d4abdc74449f895e1599542acfca +F src/loadext.c e70f8f9e97624a232870ea5486e682c813ac3002 +F src/main.c 915aeae74e2526b8fa33f9b99b1ed13eab968ac2 F src/malloc.c 1443d1ad95d67c21d77af7ae3f44678252f0efec F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c 6919bcf12f221868ea066eec27e579fed95ce98b @@ -350,20 +351,20 @@ F src/mutex_noop.c 9d4309c075ba9cc7249e19412d3d62f7f94839c4 F src/mutex_unix.c 27bb6cc49485ee46711a6580ab7b3f1402211d23 F src/mutex_w32.c 5e6fe1c298fb5a8a15aaed4161d5759311431c17 F src/notify.c 9711a7575036f0d3040ba61bc6e217f13a9888e7 -F src/os.c f89e3ca1c2e3d5015b847aec60371c474acbac82 -F src/os.h 3e57a24e2794a94d3cf2342c6d9a884888cd96bf +F src/os.c ca10edb445ad2c5fdc7285b49d72bcdf261fa23e +F src/os.h 91ff889115ecd01f436d3611f7f5ea4dc12d92f1 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa -F src/os_unix.c f5bac8e74aaefc4ea520e43b4540793c3b8a9e8f -F src/os_win.c cbf8c442a0d818d05bcf40b093cb3ebad435b9be +F src/os_unix.c 50103f69121bca969761b821e2b0e393b55fe869 +F src/os_win.c 551d973ada67127430e41d9e514e53f6beb6c5a7 F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca F src/pager.c 38718a019ca762ba4f6795425d5a54db70d1790d F src/pager.h e1d38a2f14849e219df0f91f8323504d134c8a56 F src/parse.y 5ea8c81c5c41b27887f41b4a7e1c58470d7d3821 F src/pcache.c 647bb53a86b7bbcf55ad88089b3ea5a9170b90df F src/pcache.h 4d0ccaad264d360981ec5e6a2b596d6e85242545 -F src/pcache1.c 72f644dc9e1468c72922eff5904048427b817051 -F src/pragma.c 04baa9343771f913f1c86b2720f768be8a3ad52a +F src/pcache1.c c40cdb93586e21b5dd826b5e671240bd91c26b05 +F src/pragma.c e7e8f380efec6075a722822306435afc1eeca88a F src/pragma.h 64c78a648751b9f4f297276c4eb7507b14b4628c F src/prepare.c 22df6171aec1d86904ed2ad30c2348a5748aa04e F src/printf.c 63e6fb12bbe702dd664dc3703776c090383a5a26 @@ -372,15 +373,15 @@ F src/resolve.c b8f7174e5f8c33c44ded3a25a973d0bb89228c20 F src/rowset.c 9fe4b3ad7cc00944386bb600233d8f523de07a6e F src/select.c 6dd2097bb158efe3b8d68683dcc3b4a49e907a34 F src/shell.c cd3f82fdc5c895b817a375b7ab8319cb41f447ce -F src/sqlite.h.in 86884a006a451c22d342da18d8e373aa70e65ec0 +F src/sqlite.h.in a1d7a367997f80c7075303aaf73e5fc46261c11c F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 -F src/sqlite3ext.h dfbe62ffd95b99afe2140d8c35b180d11924072d -F src/sqliteInt.h a8838eed1eb20f794ce929597864d070944f15b7 +F src/sqlite3ext.h 98f72cbfe00169c39089115427d06ea05fe4b4a2 +F src/sqliteInt.h f5cebbea5909189efbffd7a659195436f503ebb7 F src/sqliteLimit.h 7b28cf72cbd52f178bfc97ea266445e351f2cd24 F src/status.c 70912d7be68e9e2dbc4010c93d344af61d4c59ba F src/table.c 5226df15ab9179b9ed558d89575ea0ce37b03fc9 F src/tclsqlite.c 56569acc73d36e836b64aefecbbb709a92ba0077 -F src/test1.c 52965bd684ddcd7f22328ebd7d50fd0b6e51f0d4 +F src/test1.c 7187b7e924bfc97780e6fd2a40dad94a32bddca0 F src/test2.c 5586f43fcd9a1be0830793cf9d354082c261b25b F src/test3.c a8887dabbbee3059af338f20d290084a63ed1b0f F src/test4.c d168f83cc78d02e8d35567bb5630e40dcd85ac1e @@ -395,7 +396,7 @@ F src/test_backup.c 2e6e6a081870150f20c526a2e9d0d29cda47d803 F src/test_bestindex.c 29af3cc3b963ffe5760c85d142b9b3e5302c1e3d F src/test_blob.c b2551a9b5573232db5f66f292307c37067937239 F src/test_btree.c 2e9978eca99a9a4bfa8cae949efb00886860a64f -F src/test_config.c a0e8eeb86a7d9393f3bbf7315932c1cccd0e8010 +F src/test_config.c 7003f6f35134de6f19c6588f44783e43390ea277 F src/test_demovfs.c 0de72c2c89551629f58486fde5734b7d90758852 F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc F src/test_fs.c f10f840ca4f8c72e4837908bd8347ac4bcab074b @@ -434,15 +435,15 @@ F src/treeview.c e4b41a37530a191579d3c53142cc44ee2eb99373 F src/trigger.c e14840ee0c3e549e758ec9bf3e4146e166002280 F src/update.c 14c6916d0194a7f25ad429292f4831b8c31e93d9 F src/utf.c 699001c79f28e48e9bcdf8a463da029ea660540c -F src/util.c 34ef7be420f82415ec48131404995ddb6ee7502f +F src/util.c cf7dce85ab9af5280b8a45985df2591efbfefe56 F src/vacuum.c feb1eabb20987983d9350cad98299b21fa811f52 -F src/vdbe.c 8e9452b8bc241ae2031c35764b78ae8b4804effb -F src/vdbe.h 594aef1a7dcfc2944e2f266f148140c3427fd0f0 +F src/vdbe.c 399bb1d353da8ce2e52e5182aabca18da0296749 +F src/vdbe.h c16ba943d407baa1c7085eefea73a063fc631863 F src/vdbeInt.h 9b704336a69f62ba189eafb06a46171e3463d27b F src/vdbeapi.c ba85b78fe08dc4a9ce747e62c89a2b4a4547e74c -F src/vdbeaux.c 325dcf1b944e6d339a5c5210b7b8c5724a8496fb +F src/vdbeaux.c 03b2a622bf7714153bf06f9a6789e94aad9a5f89 F src/vdbeblob.c c9f2f494b911c6fa34efd9803f0a10807da80f77 -F src/vdbemem.c 0bcc4af472ebd36e57603d733a6501573a82b526 +F src/vdbemem.c 5cfef60e60e19cab6275d1b975bf4c791d575beb F src/vdbesort.c 307460bfa4de4d1c3901fcd42089159131e34062 F src/vdbetrace.c f75c5455d8cf389ef86a8bfdfd3177e0e3692484 F src/vtab.c fd69fd398e23e57ea4ea377d8a44b6998fc569c7 @@ -450,7 +451,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 10deb6b43887662691e5f53d10b3c171c401169b F src/wal.h 2f7c831cf3b071fa548bf2d5cac640846a7ff19c F src/walker.c 0f142b5bd3ed2041fc52d773880748b212e63354 -F src/where.c 5533002ddf4fbc256f450cb629668a200b06a3ce +F src/where.c fe7925faafbe9a458972035c0bb4753d672f04ed F src/whereInt.h 93297d56edd137b7ea004490690fb6e2ce028a34 F src/wherecode.c 863aedf086131743763c1960637fde904eadc442 F src/whereexpr.c fb87944b1254234e5bba671aaf6dee476241506a @@ -540,9 +541,9 @@ F test/busy.test 76b4887f8b9160ba903c1ac22e8ff406ad6ae2f0 F test/cache.test 13bc046b26210471ca6f2889aceb1ea52dc717de F test/cacheflush.test af25bb1509df04c1da10e38d8f322d66eceedf61 F test/capi2.test 011c16da245fdc0106a2785035de6b242c05e738 -F test/capi3.test bf6f0308bbbba1e770dac13aa08e5c2ac61c7324 +F test/capi3.test 860dafdc04f651a67781018cb1a0b179d22d7d15 F test/capi3b.test efb2b9cfd127efa84433cd7a2d72ce0454ae0dc4 -F test/capi3c.test 06f6261f9e9b4ef6f76afcd9900f3665408af1c8 +F test/capi3c.test b28ec47692f0fc50eb61b2d464d8d52e816b3732 F test/capi3d.test 485048dc5cd07bc68011e4917ad035ad6047ab82 F test/capi3e.test 3d49c01ef2a1a55f41d73cba2b23b5059ec460fe F test/cast.test 4c275cbdc8202d6f9c54a3596701719868ac7dc3 @@ -649,7 +650,7 @@ F test/enc4.test c8f1ce3618508fd0909945beb8b8831feef2c020 F test/eqp.test 3fe051af50921284189d1970eb653f9fcf5117d2 F test/errmsg.test f31592a594b44ee121371d25ddd5d63497bb3401 F test/eval.test a64c9105d6ff163df7cf09d6ac29cdad5922078c -F test/exclusive.test f48243eaf40e0957215501a12f510a8644d13a02 +F test/exclusive.test 9a57bd66e39144b888ca75c309914fcdefb4e3f9 F test/exclusive2.test 32798111aae78a5deec980eee383213f189df308 F test/exec.test e949714dc127eaa5ecc7d723efec1ec27118fdd7 F test/exists.test 8f7b27b61c2fbe5822f0a1f899c715d14e416e30 @@ -840,7 +841,7 @@ F test/insert5.test 394f96728d1258f406fe5f5aeb0aaf29487c39a6 F test/instr.test 737bbf80685232033f3abedc6ae92f75860b5dd2 F test/intarray.test 066b7d7ac38d25bf96f87f1b017bfc687551cdd4 F test/interrupt.test dfe9a67a94b0b2d8f70545ba1a6cca10780d71cc -F test/intpkey.test 7506090fc08e028712a8bf47e5f54111947e3844 +F test/intpkey.test 70aab09756b9bd3bcb6b0358f971d7bbace46522 F test/io.test f95bca1783b01ea7761671560d023360d2dfa4cc F test/ioerr.test 2a24bd6ed5a8b062e64bfe1f6cf94fb25e92210d F test/ioerr2.test 2593563599e2cc6b6b4fcf5878b177bdd5d8df26 @@ -909,7 +910,7 @@ F test/manydb.test 28385ae2087967aa05c38624cec7d96ec74feb3e F test/mem5.test c6460fba403c5703141348cd90de1c294188c68f F test/memdb.test c1f2a343ad14398d5d6debda6ea33e80d0dafcc7 F test/memleak.test 10b9c6c57e19fc68c32941495e9ba1c50123f6e2 -F test/memsubsys1.test 0311bfb4edd2615e3aa56c7a9cf44574e4229077 +F test/memsubsys1.test 69924593856040e266fdd9aa1ecb4d5a0888eb12 F test/memsubsys2.test 3e4a8d0c05fd3e5fa92017c64666730a520c7e08 F test/minmax.test 42fbad0e81afaa6e0de41c960329f2b2c3526efd F test/minmax2.test b44bae787fc7b227597b01b0ca5575c7cb54d3bc @@ -933,7 +934,7 @@ F test/multiplex.test efd015ca0b5b4a57dc9535b8feb1273eebeadb60 F test/multiplex2.test 580ca5817c7edbe4cc68fa150609c9473393003a F test/multiplex3.test d228f59eac91839a977eac19f21d053f03e4d101 F test/multiplex4.test e8ae4c4bd70606a5727743241f13b5701990abe4 -F test/mutex1.test e0a44072d98189003deae4b091106f085d94bea8 +F test/mutex1.test ea2cc74d97f077b9e74c84cbd024f14d79a8126f F test/mutex2.test bfeaeac2e73095b2ac32285d2756e3a65e681660 F test/nan.test dacc57f80859c06a433d30839336fe227d2038b3 F test/nolock.test 96e922d2d3db71c2dd6557c98e8027a28277b415 @@ -991,7 +992,7 @@ F test/rdonly.test 64e2696c322e3538df0b1ed624e21f9a23ed9ff8 F test/regexp1.test 497ea812f264d12b6198d6e50a76be4a1973a9d8 F test/regexp2.test aa7ffcc21350007a78361b82bcf3b74d12227144 F test/reindex.test 44edd3966b474468b823d481eafef0c305022254 -F test/releasetest.tcl a246ecb14ed594bf44bf77bd21df873971d779bf +F test/releasetest.tcl f2519836bebefcc7db36b4946e8028d4885baa59 F test/resolver01.test f4022acafda7f4d40eca94dbf16bc5fc4ac30ceb F test/rollback.test 458fe73eb3ffdfdf9f6ba3e9b7350a6220414dea F test/rollback2.test 8435d6ff0f13f51d2a4181c232e706005fa90fc5 @@ -1082,7 +1083,7 @@ F test/speedtest1.c 1478cb3fb64ad30f291ddca87ca9dbd72ff552aa F test/spellfix.test f9c1f431e2c096c8775fec032952320c0e4700db F test/spellfix2.test dfc8f519a3fc204cb2dfa8b4f29821ae90f6f8c3 F test/spellfix3.test 0f9efaaa502a0e0a09848028518a6fb096c8ad33 -F test/sqldiff1.test 8f6bc7c6a5b3585d350d779c6078869ba402f8f5 +F test/sqldiff1.test 28cd737cf1b0078b1ec1bbf425e674c47785835e F test/sqllimits1.test a74ee2a3740b9f9c2437c246d8fb77354862a142 F test/sqllog.test 6af6cb0b09f4e44e1917e06ce85be7670302517a F test/stat.test b65bad7120c52583b8f0054d99eff80718119a77 @@ -1106,7 +1107,7 @@ F test/tclsqlite.test e1306001a0ca92250b691ea6d3cecaca5b6342aa F test/tempdb.test bd92eba8f20e16a9136e434e20b280794de3cdb6 F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30 F test/temptrigger.test 8ec228b0db5d7ebc4ee9b458fc28cb9e7873f5e1 -F test/tester.tcl ac5f9558e2fbef44facc1639e3738394dc110852 +F test/tester.tcl 7b740ee852c55e1e72b6ebe5044acee7aa4e5553 F test/thread001.test 9f22fd3525a307ff42a326b6bc7b0465be1745a5 F test/thread002.test e630504f8a06c00bf8bbe68528774dd96aeb2e58 F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7 @@ -1325,8 +1326,8 @@ F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 5253bff2355a9a3f014c15337da7e177ab0ef8ad F test/wal.test 613efec03e517e1775d86b993a54877d2e29a477 -F test/wal2.test 25ae059e900dbb584e0775627e45415ba5940df1 -F test/wal3.test 5dd734147f1f8f958c5261a1f2775d346d7013ce +F test/wal2.test 56b0bc95b8693a0be294f8d210c49025dd094bd7 +F test/wal3.test 2a93004bc0fb2b5c29888964024695bade278ab2 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c F test/wal5.test 9c11da7aeccd83a46d79a556ad11a18d3cb15aa9 F test/wal6.test a9d6aa635b9d63607dabdc11406f5f96ca986635 @@ -1407,8 +1408,8 @@ F tool/fuzzershell.c 94019b185caceffc9f7c7b678a6489e42bc2aefa F tool/genfkey.README cf68fddd4643bbe3ff8e31b8b6d8b0a1b85e20f4 F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce -F tool/lemon.c 251f5c3f21b553240cbdd42dd187a51bb2372cd3 -F tool/lempar.c d5114c7d13aa3af1e27ff3d02e4dea6eadec7ddf +F tool/lemon.c cfbfe061a4b2766512f6b484882eee2c86a14506 +F tool/lempar.c 404ea3dc27dbeed343f0e61b1d36e97b9f5f0fb6 F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862 F tool/logest.c eef612f8adf4d0993dafed0416064cf50d5d33c6 F tool/mkautoconfamal.sh e855df211ecbcc7131dee817110ff386cfb112f7 @@ -1444,7 +1445,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 7ce07da76b5e745783e703a834417d725b7d45fd F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c -F tool/sqldiff.c 5a26205111e6fa856d9b1535b1637744dcdb930b +F tool/sqldiff.c ca315aca4e2d24233e8f2000edea5880c53d1875 F tool/srcck1.c 4f65e1a6748e42f24c0ea629dddc934d821c729a F tool/stack_usage.tcl f8e71b92cdb099a147dad572375595eae55eca43 F tool/symbols-mingw.sh 4dbcea7e74768305384c9fd2ed2b41bbf9f0414d @@ -1477,7 +1478,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 17fd8f3cf0ec565e08403dc8e10a1cffc2bbe165 45f7f0c80bd91a0c7ff859c27fd9e82e551bd83e -R 7b1ff29ad9cf7b5c8ae4f4645405d6a4 +P 6a7ee04b0ddac36a87d5ed2ac89a53e537f4d5a3 4bd12b57ea177cfb1f44d54bfa7aedfb8a8c0c64 +R 23117f3fb3eb670baa9fc58a11fb65fe U drh -Z 05e24502adf6b1cda5add372ab8596b8 +Z 887267d056670df55047d1f847ebaae7 diff --git a/manifest.uuid b/manifest.uuid index d4116a6eca..d75bc5442c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6a7ee04b0ddac36a87d5ed2ac89a53e537f4d5a3 \ No newline at end of file +3296a0ceedef43c2790f0b36471f91138a575243 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 3672d02df1..998c94f231 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1283,7 +1283,8 @@ u32 sqlite3ExprListFlags(const ExprList *pList){ if( pList ){ for(i=0; inExpr; i++){ Expr *pExpr = pList->a[i].pExpr; - if( ALWAYS(pExpr) ) m |= pExpr->flags; + assert( pExpr!=0 ); + m |= pExpr->flags; } } return m; @@ -3548,6 +3549,13 @@ void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ sqlite3ExprIfFalse(pParse, pExpr->pLeft, dest, jumpIfNull); break; } + case TK_IS: + case TK_ISNOT: + testcase( op==TK_IS ); + testcase( op==TK_ISNOT ); + op = (op==TK_IS) ? TK_EQ : TK_NE; + jumpIfNull = SQLITE_NULLEQ; + /* Fall thru */ case TK_LT: case TK_LE: case TK_GT: @@ -3563,23 +3571,12 @@ void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ assert(TK_LE==OP_Le); testcase(op==OP_Le); VdbeCoverageIf(v,op==OP_Le); assert(TK_GT==OP_Gt); testcase(op==OP_Gt); VdbeCoverageIf(v,op==OP_Gt); assert(TK_GE==OP_Ge); testcase(op==OP_Ge); VdbeCoverageIf(v,op==OP_Ge); - assert(TK_EQ==OP_Eq); testcase(op==OP_Eq); VdbeCoverageIf(v,op==OP_Eq); - assert(TK_NE==OP_Ne); testcase(op==OP_Ne); VdbeCoverageIf(v,op==OP_Ne); - testcase( regFree1==0 ); - testcase( regFree2==0 ); - break; - } - case TK_IS: - case TK_ISNOT: { - testcase( op==TK_IS ); - testcase( op==TK_ISNOT ); - r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); - r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); - op = (op==TK_IS) ? TK_EQ : TK_NE; - codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, - r1, r2, dest, SQLITE_NULLEQ); - VdbeCoverageIf(v, op==TK_EQ); - VdbeCoverageIf(v, op==TK_NE); + assert(TK_EQ==OP_Eq); testcase(op==OP_Eq); + VdbeCoverageIf(v, op==OP_Eq && jumpIfNull==SQLITE_NULLEQ); + VdbeCoverageIf(v, op==OP_Eq && jumpIfNull!=SQLITE_NULLEQ); + assert(TK_NE==OP_Ne); testcase(op==OP_Ne); + VdbeCoverageIf(v, op==OP_Ne && jumpIfNull==SQLITE_NULLEQ); + VdbeCoverageIf(v, op==OP_Ne && jumpIfNull!=SQLITE_NULLEQ); testcase( regFree1==0 ); testcase( regFree2==0 ); break; @@ -3704,6 +3701,13 @@ void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ sqlite3ExprIfTrue(pParse, pExpr->pLeft, dest, jumpIfNull); break; } + case TK_IS: + case TK_ISNOT: + testcase( pExpr->op==TK_IS ); + testcase( pExpr->op==TK_ISNOT ); + op = (pExpr->op==TK_IS) ? TK_NE : TK_EQ; + jumpIfNull = SQLITE_NULLEQ; + /* Fall thru */ case TK_LT: case TK_LE: case TK_GT: @@ -3719,23 +3723,12 @@ void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ assert(TK_LE==OP_Le); testcase(op==OP_Le); VdbeCoverageIf(v,op==OP_Le); assert(TK_GT==OP_Gt); testcase(op==OP_Gt); VdbeCoverageIf(v,op==OP_Gt); assert(TK_GE==OP_Ge); testcase(op==OP_Ge); VdbeCoverageIf(v,op==OP_Ge); - assert(TK_EQ==OP_Eq); testcase(op==OP_Eq); VdbeCoverageIf(v,op==OP_Eq); - assert(TK_NE==OP_Ne); testcase(op==OP_Ne); VdbeCoverageIf(v,op==OP_Ne); - testcase( regFree1==0 ); - testcase( regFree2==0 ); - break; - } - case TK_IS: - case TK_ISNOT: { - testcase( pExpr->op==TK_IS ); - testcase( pExpr->op==TK_ISNOT ); - r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); - r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); - op = (pExpr->op==TK_IS) ? TK_NE : TK_EQ; - codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, - r1, r2, dest, SQLITE_NULLEQ); - VdbeCoverageIf(v, op==TK_EQ); - VdbeCoverageIf(v, op==TK_NE); + assert(TK_EQ==OP_Eq); testcase(op==OP_Eq); + VdbeCoverageIf(v, op==OP_Eq && jumpIfNull!=SQLITE_NULLEQ); + VdbeCoverageIf(v, op==OP_Eq && jumpIfNull==SQLITE_NULLEQ); + assert(TK_NE==OP_Ne); testcase(op==OP_Ne); + VdbeCoverageIf(v, op==OP_Ne && jumpIfNull!=SQLITE_NULLEQ); + VdbeCoverageIf(v, op==OP_Ne && jumpIfNull==SQLITE_NULLEQ); testcase( regFree1==0 ); testcase( regFree2==0 ); break; @@ -4258,3 +4251,26 @@ void sqlite3ClearTempRegCache(Parse *pParse){ pParse->nTempReg = 0; pParse->nRangeReg = 0; } + +/* +** Validate that no temporary register falls within the range of +** iFirst..iLast, inclusive. This routine is only call from within assert() +** statements. +*/ +#ifdef SQLITE_DEBUG +int sqlite3NoTempsInRange(Parse *pParse, int iFirst, int iLast){ + int i; + if( pParse->nRangeReg>0 + && pParse->iRangeReg+pParse->nRangeRegiRangeReg>=iFirst + ){ + return 0; + } + for(i=0; inTempReg; i++){ + if( pParse->aTempReg[i]>=iFirst && pParse->aTempReg[i]<=iLast ){ + return 0; + } + } + return 1; +} +#endif /* SQLITE_DEBUG */ diff --git a/src/loadext.c b/src/loadext.c index 3469fbb73d..495001e55a 100644 --- a/src/loadext.c +++ b/src/loadext.c @@ -414,7 +414,9 @@ static const sqlite3_api_routines sqlite3Apis = { /* Version 3.10.0 and later */ sqlite3_status64, sqlite3_strlike, - sqlite3_db_cacheflush + sqlite3_db_cacheflush, + /* Version 3.12.0 and later */ + sqlite3_system_errno }; /* diff --git a/src/main.c b/src/main.c index 40a192e163..d3114b588b 100644 --- a/src/main.c +++ b/src/main.c @@ -2261,6 +2261,9 @@ int sqlite3_extended_errcode(sqlite3 *db){ } return db->errCode; } +int sqlite3_system_errno(sqlite3 *db){ + return db ? db->iSysErrno : 0; +} /* ** Return a string that describes the kind of error specified in the diff --git a/src/os.c b/src/os.c index eed7828538..bfcc9cdd05 100644 --- a/src/os.c +++ b/src/os.c @@ -262,6 +262,9 @@ int sqlite3OsRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){ int sqlite3OsSleep(sqlite3_vfs *pVfs, int nMicro){ return pVfs->xSleep(pVfs, nMicro); } +int sqlite3OsGetLastError(sqlite3_vfs *pVfs){ + return pVfs->xGetLastError ? pVfs->xGetLastError(pVfs, 0, 0) : 0; +} int sqlite3OsCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *pTimeOut){ int rc; /* IMPLEMENTATION-OF: R-49045-42493 SQLite will use the xCurrentTimeInt64() diff --git a/src/os.h b/src/os.h index 2c1b86f913..f813541561 100644 --- a/src/os.h +++ b/src/os.h @@ -197,6 +197,7 @@ void sqlite3OsDlClose(sqlite3_vfs *, void *); #endif /* SQLITE_OMIT_LOAD_EXTENSION */ int sqlite3OsRandomness(sqlite3_vfs *, int, char *); int sqlite3OsSleep(sqlite3_vfs *, int); +int sqlite3OsGetLastError(sqlite3_vfs*); int sqlite3OsCurrentTimeInt64(sqlite3_vfs *, sqlite3_int64*); /* diff --git a/src/os_unix.c b/src/os_unix.c index d6fbd80588..d593b952f5 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -1357,6 +1357,10 @@ static int fileHasMoved(unixFile *pFile){ static void verifyDbFile(unixFile *pFile){ struct stat buf; int rc; + + /* These verifications occurs for the main database only */ + if( pFile->ctrlFlags & UNIXFILE_NOLOCK ) return; + rc = osFstat(pFile->h, &buf); if( rc!=0 ){ sqlite3_log(SQLITE_WARNING, "cannot fstat db file %s", pFile->zPath); @@ -5803,9 +5807,6 @@ static int unixOpen( p->openFlags = openFlags; } #endif - - noLock = eType!=SQLITE_OPEN_MAIN_DB; - #if defined(__APPLE__) || SQLITE_ENABLE_LOCKING_STYLE if( fstatfs(fd, &fsInfo) == -1 ){ @@ -5824,6 +5825,7 @@ static int unixOpen( /* Set up appropriate ctrlFlags */ if( isDelete ) ctrlFlags |= UNIXFILE_DELETE; if( isReadonly ) ctrlFlags |= UNIXFILE_RDONLY; + noLock = eType!=SQLITE_OPEN_MAIN_DB; if( noLock ) ctrlFlags |= UNIXFILE_NOLOCK; if( syncDir ) ctrlFlags |= UNIXFILE_DIRSYNC; if( flags & SQLITE_OPEN_URI ) ctrlFlags |= UNIXFILE_URI; @@ -6262,23 +6264,18 @@ static int unixCurrentTime(sqlite3_vfs *NotUsed, double *prNow){ # define unixCurrentTime 0 #endif -#ifndef SQLITE_OMIT_DEPRECATED /* -** We added the xGetLastError() method with the intention of providing -** better low-level error messages when operating-system problems come up -** during SQLite operation. But so far, none of that has been implemented -** in the core. So this routine is never called. For now, it is merely -** a place-holder. +** The xGetLastError() method is designed to return a better +** low-level error message when operating-system problems come up +** during SQLite operation. Only the integer return code is currently +** used. */ static int unixGetLastError(sqlite3_vfs *NotUsed, int NotUsed2, char *NotUsed3){ UNUSED_PARAMETER(NotUsed); UNUSED_PARAMETER(NotUsed2); UNUSED_PARAMETER(NotUsed3); - return 0; + return errno; } -#else -# define unixGetLastError 0 -#endif /* diff --git a/src/os_win.c b/src/os_win.c index 6ff50c554b..fd95c00e3a 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -5584,8 +5584,10 @@ static int winCurrentTime(sqlite3_vfs *pVfs, double *prNow){ ** sqlite3_errmsg(), possibly making IO errors easier to debug. */ static int winGetLastError(sqlite3_vfs *pVfs, int nBuf, char *zBuf){ + DWORD e = osGetLastError(); UNUSED_PARAMETER(pVfs); - return winGetLastErrorMsg(osGetLastError(), nBuf, zBuf); + if( nBuf>0 ) winGetLastErrorMsg(e, nBuf, zBuf); + return e; } /* diff --git a/src/pcache1.c b/src/pcache1.c index 7147f6a7a8..d168e7fbcb 100644 --- a/src/pcache1.c +++ b/src/pcache1.c @@ -348,7 +348,6 @@ static void *pcache1Alloc(int nByte){ ** Free an allocated buffer obtained from pcache1Alloc(). */ static void pcache1Free(void *p){ - int nFreed = 0; if( p==0 ) return; if( SQLITE_WITHIN(p, pcache1.pStart, pcache1.pEnd) ){ PgFreeslot *pSlot; @@ -365,10 +364,13 @@ static void pcache1Free(void *p){ assert( sqlite3MemdebugHasType(p, MEMTYPE_PCACHE) ); sqlite3MemdebugSetType(p, MEMTYPE_HEAP); #ifndef SQLITE_DISABLE_PAGECACHE_OVERFLOW_STATS - nFreed = sqlite3MallocSize(p); - sqlite3_mutex_enter(pcache1.mutex); - sqlite3StatusDown(SQLITE_STATUS_PAGECACHE_OVERFLOW, nFreed); - sqlite3_mutex_leave(pcache1.mutex); + { + int nFreed = 0; + nFreed = sqlite3MallocSize(p); + sqlite3_mutex_enter(pcache1.mutex); + sqlite3StatusDown(SQLITE_STATUS_PAGECACHE_OVERFLOW, nFreed); + sqlite3_mutex_leave(pcache1.mutex); + } #endif sqlite3_free(p); } diff --git a/src/pragma.c b/src/pragma.c index 70c2950c8e..65c43ad1e5 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -1441,6 +1441,8 @@ void sqlite3Pragma( Hash *pTbls; int *aRoot; int cnt = 0; + int mxIdx = 0; + int nIdx; if( OMIT_TEMPDB && i==1 ) continue; if( iDb>=0 && i!=iDb ) continue; @@ -1462,7 +1464,8 @@ void sqlite3Pragma( Table *pTab = sqliteHashData(x); Index *pIdx; if( HasRowid(pTab) ) cnt++; - for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ cnt++; } + for(nIdx=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, nIdx++){ cnt++; } + if( nIdx>mxIdx ) mxIdx = nIdx; } aRoot = sqlite3DbMallocRawNN(db, sizeof(int)*(cnt+1)); if( aRoot==0 ) break; @@ -1477,7 +1480,7 @@ void sqlite3Pragma( aRoot[cnt] = 0; /* Make sure sufficient number of registers have been allocated */ - pParse->nMem = MAX( pParse->nMem, 14 ); + pParse->nMem = MAX( pParse->nMem, 8+mxIdx ); /* Do the b-tree integrity checks */ sqlite3VdbeAddOp4(v, OP_IntegrityCk, 2, cnt, 1, (char*)aRoot,P4_INTARRAY); @@ -1514,7 +1517,8 @@ void sqlite3Pragma( for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ sqlite3VdbeAddOp2(v, OP_Integer, 0, 8+j); /* index entries counter */ } - pParse->nMem = MAX(pParse->nMem, 8+j); + assert( pParse->nMem>=8+j ); + assert( sqlite3NoTempsInRange(pParse,1,7+j) ); sqlite3VdbeAddOp2(v, OP_Rewind, iDataCur, 0); VdbeCoverage(v); loopTop = sqlite3VdbeAddOp2(v, OP_AddImm, 7, 1); /* Verify that all NOT NULL columns really are NOT NULL */ @@ -1706,7 +1710,9 @@ void sqlite3Pragma( ** PRAGMA [schema.]user_version ** PRAGMA [schema.]user_version = ** - ** PRAGMA [schema.]freelist_count = + ** PRAGMA [schema.]freelist_count + ** + ** PRAGMA [schema.]data_version ** ** PRAGMA [schema.]application_id ** PRAGMA [schema.]application_id = @@ -1762,6 +1768,7 @@ void sqlite3Pragma( aOp[1].p3 = iCookie; sqlite3VdbeSetNumCols(v, 1); sqlite3VdbeSetColName(v, 0, COLNAME_NAME, zLeft, SQLITE_TRANSIENT); + sqlite3VdbeReusable(v); } } break; @@ -1783,6 +1790,7 @@ void sqlite3Pragma( sqlite3VdbeLoadString(v, 1, zOpt); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); } + sqlite3VdbeReusable(v); } break; #endif /* SQLITE_OMIT_COMPILEOPTION_DIAGS */ diff --git a/src/sqlite.h.in b/src/sqlite.h.in index cce23aa225..c9b80d6567 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -8010,6 +8010,18 @@ SQLITE_EXPERIMENTAL int sqlite3_preupdate_count(sqlite3 *); SQLITE_EXPERIMENTAL int sqlite3_preupdate_depth(sqlite3 *); SQLITE_EXPERIMENTAL int sqlite3_preupdate_new(sqlite3 *, int, sqlite3_value **); +/* +** CAPI3REF: Low-level system error code +** +** ^Attempt to return the underlying operating system error code or error +** number that caused the most reason I/O error or failure to open a file. +** The return value is OS-dependent. For example, on unix systems, after +** [sqlite3_open_v2()] returns [SQLITE_CANTOPEN], this interface could be +** called to get back the underlying "errno" that caused the problem, such +** as ENOSPC, EAUTH, EISDIR, and so forth. +*/ +int sqlite3_system_errno(sqlite3*); + /* ** CAPI3REF: Database Snapshot ** KEYWORDS: {snapshot} diff --git a/src/sqlite3ext.h b/src/sqlite3ext.h index 2e1c764a52..20a2fcdf02 100644 --- a/src/sqlite3ext.h +++ b/src/sqlite3ext.h @@ -279,6 +279,8 @@ struct sqlite3_api_routines { int (*status64)(int,sqlite3_int64*,sqlite3_int64*,int); int (*strlike)(const char*,const char*,unsigned int); int (*db_cacheflush)(sqlite3*); + /* Version 3.12.0 and later */ + int (*system_errno)(sqlite3*); }; /* @@ -522,6 +524,8 @@ struct sqlite3_api_routines { #define sqlite3_status64 sqlite3_api->status64 #define sqlite3_strlike sqlite3_api->strlike #define sqlite3_db_cacheflush sqlite3_api->db_cacheflush +/* Version 3.12.0 and later */ +#define sqlite3_system_errno sqlite3_api->system_errno #endif /* !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) */ #if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) diff --git a/src/sqliteInt.h b/src/sqliteInt.h index de357537e2..da051a4149 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1221,6 +1221,7 @@ struct sqlite3 { unsigned int openFlags; /* Flags passed to sqlite3_vfs.xOpen() */ int errCode; /* Most recent error code (SQLITE_*) */ int errMask; /* & result codes with this before returning */ + int iSysErrno; /* Errno value from last system error */ u16 dbOptFlags; /* Flags to enable/disable optimizations */ u8 enc; /* Text encoding */ u8 autoCommit; /* The auto-commit flag. */ @@ -3431,6 +3432,9 @@ void sqlite3ReleaseTempReg(Parse*,int); int sqlite3GetTempRange(Parse*,int); void sqlite3ReleaseTempRange(Parse*,int,int); void sqlite3ClearTempRegCache(Parse*); +#ifdef SQLITE_DEBUG +int sqlite3NoTempsInRange(Parse*,int,int); +#endif Expr *sqlite3ExprAlloc(sqlite3*,int,const Token*,int); Expr *sqlite3Expr(sqlite3*,int,const char*); void sqlite3ExprAttachSubtrees(sqlite3*,Expr*,Expr*,Expr*); @@ -3775,6 +3779,7 @@ int sqlite3Atoi64(const char*, i64*, int, u8); int sqlite3DecOrHexToI64(const char*, i64*); void sqlite3ErrorWithMsg(sqlite3*, int, const char*,...); void sqlite3Error(sqlite3*,int); +void sqlite3SystemError(sqlite3*,int); void *sqlite3HexToBlob(sqlite3*, const char *z, int n); u8 sqlite3HexToInt(int h); int sqlite3TwoPartName(Parse *, Token *, Token *, Token **); diff --git a/src/test1.c b/src/test1.c index 8ad653ca6c..ead9bd47ba 100644 --- a/src/test1.c +++ b/src/test1.c @@ -1931,6 +1931,8 @@ static int test_load_extension( #ifdef SQLITE_OMIT_LOAD_EXTENSION rc = SQLITE_ERROR; zErr = sqlite3_mprintf("this build omits sqlite3_load_extension()"); + (void)zProc; + (void)zFile; #else rc = sqlite3_load_extension(db, zFile, zProc, &zErr); #endif @@ -4847,6 +4849,29 @@ static int test_db_cacheflush( return TCL_OK; } +/* +** Usage: sqlite3_system_errno DB +** +** Return the low-level system errno value. +*/ +static int test_system_errno( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + sqlite3 *db; + int iErrno; + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB"); + return TCL_ERROR; + } + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; + iErrno = sqlite3_system_errno(db); + Tcl_SetObjResult(interp, Tcl_NewIntObj(iErrno)); + return TCL_OK; +} + /* ** Usage: sqlite3_db_filename DB DBNAME ** @@ -7084,6 +7109,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ { "sqlite3_release_memory", test_release_memory, 0}, { "sqlite3_db_release_memory", test_db_release_memory, 0}, { "sqlite3_db_cacheflush", test_db_cacheflush, 0}, + { "sqlite3_system_errno", test_system_errno, 0}, { "sqlite3_db_filename", test_db_filename, 0}, { "sqlite3_db_readonly", test_db_readonly, 0}, { "sqlite3_soft_heap_limit", test_soft_heap_limit, 0}, diff --git a/src/test_config.c b/src/test_config.c index f712ebac63..9ce9bfcaca 100644 --- a/src/test_config.c +++ b/src/test_config.c @@ -79,6 +79,13 @@ static void set_options(Tcl_Interp *interp){ Tcl_SetVar2(interp, "sqlite_options", "debug", "0", TCL_GLOBAL_ONLY); #endif +#ifdef SQLITE_DEFAULT_CKPTFULLFSYNC + Tcl_SetVar2(interp, "sqlite_options", "default_ckptfullfsync", + SQLITE_DEFAULT_CKPTFULLFSYNC ? "1" : "0", TCL_GLOBAL_ONLY); +#else + Tcl_SetVar2(interp, "sqlite_options", "default_ckptfullfsync", "0", TCL_GLOBAL_ONLY); +#endif + #ifdef SQLITE_DIRECT_OVERFLOW_READ Tcl_SetVar2(interp, "sqlite_options", "direct_read", "1", TCL_GLOBAL_ONLY); #else @@ -97,6 +104,12 @@ static void set_options(Tcl_Interp *interp){ Tcl_SetVar2(interp, "sqlite_options", "lfs", "1", TCL_GLOBAL_ONLY); #endif +#ifdef SQLITE_DISABLE_PAGECACHE_OVERFLOW_STATS + Tcl_SetVar2(interp, "sqlite_options", "pagecache_overflow_stats","0",TCL_GLOBAL_ONLY); +#else + Tcl_SetVar2(interp, "sqlite_options", "pagecache_overflow_stats","1",TCL_GLOBAL_ONLY); +#endif + #if SQLITE_MAX_MMAP_SIZE>0 Tcl_SetVar2(interp, "sqlite_options", "mmap", "1", TCL_GLOBAL_ONLY); #else @@ -588,7 +601,11 @@ Tcl_SetVar2(interp, "sqlite_options", "mergesort", "1", TCL_GLOBAL_ONLY); #endif Tcl_SetVar2(interp, "sqlite_options", "threadsafe", - STRINGVALUE(SQLITE_THREADSAFE), TCL_GLOBAL_ONLY); + SQLITE_THREADSAFE ? "1" : "0", TCL_GLOBAL_ONLY); + Tcl_SetVar2(interp, "sqlite_options", "threadsafe1", + SQLITE_THREADSAFE==1 ? "1" : "0", TCL_GLOBAL_ONLY); + Tcl_SetVar2(interp, "sqlite_options", "threadsafe2", + SQLITE_THREADSAFE==2 ? "1" : "0", TCL_GLOBAL_ONLY); assert( sqlite3_threadsafe()==SQLITE_THREADSAFE ); #ifdef SQLITE_OMIT_TEMPDB diff --git a/src/util.c b/src/util.c index d6a6f6b954..428dfd046c 100644 --- a/src/util.c +++ b/src/util.c @@ -117,13 +117,37 @@ const char *sqlite3StrNext(const char *z){ return z + strlen(z) + 1; } +/* +** Helper function for sqlite3Error() - called rarely. Broken out into +** a separate routine to avoid unnecessary register saves on entry to +** sqlite3Error(). +*/ +static SQLITE_NOINLINE void sqlite3ErrorFinish(sqlite3 *db, int err_code){ + if( db->pErr ) sqlite3ValueSetNull(db->pErr); + sqlite3SystemError(db, err_code); +} + /* ** Set the current error code to err_code and clear any prior error message. +** Also set iSysErrno (by calling sqlite3System) if the err_code indicates +** that would be appropriate. */ void sqlite3Error(sqlite3 *db, int err_code){ assert( db!=0 ); db->errCode = err_code; - if( db->pErr ) sqlite3ValueSetNull(db->pErr); + if( err_code || db->pErr ) sqlite3ErrorFinish(db, err_code); +} + +/* +** Load the sqlite3.iSysErrno field if that is an appropriate thing +** to do based on the SQLite error code in rc. +*/ +void sqlite3SystemError(sqlite3 *db, int rc){ + if( rc==SQLITE_IOERR_NOMEM ) return; + rc &= 0xff; + if( rc==SQLITE_CANTOPEN || rc==SQLITE_IOERR ){ + db->iSysErrno = sqlite3OsGetLastError(db->pVfs); + } } /* @@ -150,6 +174,7 @@ void sqlite3Error(sqlite3 *db, int err_code){ void sqlite3ErrorWithMsg(sqlite3 *db, int err_code, const char *zFormat, ...){ assert( db!=0 ); db->errCode = err_code; + sqlite3SystemError(db, err_code); if( zFormat==0 ){ sqlite3Error(db, err_code); }else if( db->pErr || (db->pErr = sqlite3ValueNew(db))!=0 ){ diff --git a/src/vdbe.c b/src/vdbe.c index 2757a624ea..829eddf43d 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -202,11 +202,11 @@ static VdbeCursor *allocateCursor( ** be freed lazily via the sqlite3_release_memory() API. This ** minimizes the number of malloc calls made by the system. ** - ** Memory cells for cursors are allocated at the top of the address - ** space. Memory cell (p->nMem) corresponds to cursor 0. Space for - ** cursor 1 is managed by memory cell (p->nMem-1), etc. + ** The memory cell for cursor 0 is aMem[0]. The rest are allocated from + ** the top of the register space. Cursor 1 is at Mem[p->nMem-1]. + ** Cursor 2 is at Mem[p->nMem-2]. And so forth. */ - Mem *pMem = &p->aMem[p->nMem-iCur]; + Mem *pMem = iCur>0 ? &p->aMem[p->nMem-iCur] : p->aMem; int nByte; VdbeCursor *pCx = 0; @@ -214,7 +214,7 @@ static VdbeCursor *allocateCursor( ROUND8(sizeof(VdbeCursor)) + 2*sizeof(u32)*nField + (eCurType==CURTYPE_BTREE?sqlite3BtreeCursorSize():0); - assert( iCurnCursor ); + assert( iCur>=0 && iCurnCursor ); if( p->apCsr[iCur] ){ sqlite3VdbeFreeCursor(p, p->apCsr[iCur]); p->apCsr[iCur] = 0; @@ -539,7 +539,7 @@ static SQLITE_NOINLINE Mem *out2PrereleaseWithClear(Mem *pOut){ static Mem *out2Prerelease(Vdbe *p, VdbeOp *pOp){ Mem *pOut; assert( pOp->p2>0 ); - assert( pOp->p2<=(p->nMem-p->nCursor) ); + assert( pOp->p2<=(p->nMem+1 - p->nCursor) ); pOut = &p->aMem[pOp->p2]; memAboutToChange(p, pOut); if( VdbeMemDynamic(pOut) ){ @@ -677,33 +677,33 @@ int sqlite3VdbeExec( assert( pOp->opflags==sqlite3OpcodeProperty[pOp->opcode] ); if( (pOp->opflags & OPFLG_IN1)!=0 ){ assert( pOp->p1>0 ); - assert( pOp->p1<=(p->nMem-p->nCursor) ); + assert( pOp->p1<=(p->nMem+1 - p->nCursor) ); assert( memIsValid(&aMem[pOp->p1]) ); assert( sqlite3VdbeCheckMemInvariants(&aMem[pOp->p1]) ); REGISTER_TRACE(pOp->p1, &aMem[pOp->p1]); } if( (pOp->opflags & OPFLG_IN2)!=0 ){ assert( pOp->p2>0 ); - assert( pOp->p2<=(p->nMem-p->nCursor) ); + assert( pOp->p2<=(p->nMem+1 - p->nCursor) ); assert( memIsValid(&aMem[pOp->p2]) ); assert( sqlite3VdbeCheckMemInvariants(&aMem[pOp->p2]) ); REGISTER_TRACE(pOp->p2, &aMem[pOp->p2]); } if( (pOp->opflags & OPFLG_IN3)!=0 ){ assert( pOp->p3>0 ); - assert( pOp->p3<=(p->nMem-p->nCursor) ); + assert( pOp->p3<=(p->nMem+1 - p->nCursor) ); assert( memIsValid(&aMem[pOp->p3]) ); assert( sqlite3VdbeCheckMemInvariants(&aMem[pOp->p3]) ); REGISTER_TRACE(pOp->p3, &aMem[pOp->p3]); } if( (pOp->opflags & OPFLG_OUT2)!=0 ){ assert( pOp->p2>0 ); - assert( pOp->p2<=(p->nMem-p->nCursor) ); + assert( pOp->p2<=(p->nMem+1 - p->nCursor) ); memAboutToChange(p, &aMem[pOp->p2]); } if( (pOp->opflags & OPFLG_OUT3)!=0 ){ assert( pOp->p3>0 ); - assert( pOp->p3<=(p->nMem-p->nCursor) ); + assert( pOp->p3<=(p->nMem+1 - p->nCursor) ); memAboutToChange(p, &aMem[pOp->p3]); } #endif @@ -802,7 +802,7 @@ check_for_interrupt: ** and then jump to address P2. */ case OP_Gosub: { /* jump */ - assert( pOp->p1>0 && pOp->p1<=(p->nMem-p->nCursor) ); + assert( pOp->p1>0 && pOp->p1<=(p->nMem+1 - p->nCursor) ); pIn1 = &aMem[pOp->p1]; assert( VdbeMemDynamic(pIn1)==0 ); memAboutToChange(p, pIn1); @@ -842,7 +842,7 @@ case OP_Return: { /* in1 */ ** See also: EndCoroutine */ case OP_InitCoroutine: { /* jump */ - assert( pOp->p1>0 && pOp->p1<=(p->nMem-p->nCursor) ); + assert( pOp->p1>0 && pOp->p1<=(p->nMem+1 - p->nCursor) ); assert( pOp->p2>=0 && pOp->p2nOp ); assert( pOp->p3>=0 && pOp->p3nOp ); pOut = &aMem[pOp->p1]; @@ -1111,7 +1111,7 @@ case OP_String: { /* out2 */ #ifndef SQLITE_LIKE_DOESNT_MATCH_BLOBS if( pOp->p5 ){ assert( pOp->p3>0 ); - assert( pOp->p3<=(p->nMem-p->nCursor) ); + assert( pOp->p3<=(p->nMem+1 - p->nCursor) ); pIn3 = &aMem[pOp->p3]; assert( pIn3->flags & MEM_Int ); if( pIn3->u.i ) pOut->flags = MEM_Blob|MEM_Static|MEM_Term; @@ -1137,7 +1137,7 @@ case OP_Null: { /* out2 */ u16 nullFlag; pOut = out2Prerelease(p, pOp); cnt = pOp->p3-pOp->p2; - assert( pOp->p3<=(p->nMem-p->nCursor) ); + assert( pOp->p3<=(p->nMem+1 - p->nCursor) ); pOut->flags = nullFlag = pOp->p1 ? (MEM_Null|MEM_Cleared) : MEM_Null; while( cnt>0 ){ pOut++; @@ -1158,7 +1158,7 @@ case OP_Null: { /* out2 */ ** previously copied using OP_SCopy, the copies will continue to be valid. */ case OP_SoftNull: { - assert( pOp->p1>0 && pOp->p1<=(p->nMem-p->nCursor) ); + assert( pOp->p1>0 && pOp->p1<=(p->nMem+1 - p->nCursor) ); pOut = &aMem[pOp->p1]; pOut->flags = (pOut->flags|MEM_Null)&~MEM_Undefined; break; @@ -1225,8 +1225,8 @@ case OP_Move: { pIn1 = &aMem[p1]; pOut = &aMem[p2]; do{ - assert( pOut<=&aMem[(p->nMem-p->nCursor)] ); - assert( pIn1<=&aMem[(p->nMem-p->nCursor)] ); + assert( pOut<=&aMem[(p->nMem+1 - p->nCursor)] ); + assert( pIn1<=&aMem[(p->nMem+1 - p->nCursor)] ); assert( memIsValid(pIn1) ); memAboutToChange(p, pOut); sqlite3VdbeMemMove(pOut, pIn1); @@ -1326,7 +1326,7 @@ case OP_ResultRow: { int i; assert( p->nResColumn==pOp->p2 ); assert( pOp->p1>0 ); - assert( pOp->p1+pOp->p2<=(p->nMem-p->nCursor)+1 ); + assert( pOp->p1+pOp->p2<=(p->nMem+1 - p->nCursor)+1 ); #ifndef SQLITE_OMIT_PROGRESS_CALLBACK /* Run the progress counter just before returning. @@ -1638,8 +1638,8 @@ case OP_Function0: { assert( pOp->p4type==P4_FUNCDEF ); n = pOp->p5; - assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) ); - assert( n==0 || (pOp->p2>0 && pOp->p2+n<=(p->nMem-p->nCursor)+1) ); + assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) ); + assert( n==0 || (pOp->p2>0 && pOp->p2+n<=(p->nMem+1 - p->nCursor)+1) ); assert( pOp->p3p2 || pOp->p3>=pOp->p2+n ); pCtx = sqlite3DbMallocRawNN(db, sizeof(*pCtx) + (n-1)*sizeof(sqlite3_value*)); if( pCtx==0 ) goto no_mem; @@ -2139,11 +2139,11 @@ case OP_Compare: { if( aPermute ){ int k, mx = 0; for(k=0; kmx ) mx = aPermute[k]; - assert( p1>0 && p1+mx<=(p->nMem-p->nCursor)+1 ); - assert( p2>0 && p2+mx<=(p->nMem-p->nCursor)+1 ); + assert( p1>0 && p1+mx<=(p->nMem+1 - p->nCursor)+1 ); + assert( p2>0 && p2+mx<=(p->nMem+1 - p->nCursor)+1 ); }else{ - assert( p1>0 && p1+n<=(p->nMem-p->nCursor)+1 ); - assert( p2>0 && p2+n<=(p->nMem-p->nCursor)+1 ); + assert( p1>0 && p1+n<=(p->nMem+1 - p->nCursor)+1 ); + assert( p2>0 && p2+n<=(p->nMem+1 - p->nCursor)+1 ); } #endif /* SQLITE_DEBUG */ for(i=0; ip3>0 && pOp->p3<=(p->nMem-p->nCursor) ); + assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) ); pDest = &aMem[pOp->p3]; memAboutToChange(p, pDest); assert( pOp->p1>=0 && pOp->p1nCursor ); @@ -2648,7 +2648,7 @@ case OP_Affinity: { assert( zAffinity[pOp->p2]==0 ); pIn1 = &aMem[pOp->p1]; while( (cAff = *(zAffinity++))!=0 ){ - assert( pIn1 <= &p->aMem[(p->nMem-p->nCursor)] ); + assert( pIn1 <= &p->aMem[(p->nMem+1 - p->nCursor)] ); assert( memIsValid(pIn1) ); applyAffinity(pIn1, cAff, encoding); pIn1++; @@ -2710,7 +2710,7 @@ case OP_MakeRecord: { nZero = 0; /* Number of zero bytes at the end of the record */ nField = pOp->p1; zAffinity = pOp->p4.z; - assert( nField>0 && pOp->p2>0 && pOp->p2+nField<=(p->nMem-p->nCursor)+1 ); + assert( nField>0 && pOp->p2>0 && pOp->p2+nField<=(p->nMem+1 - p->nCursor)+1 ); pData0 = &aMem[nField]; nField = pOp->p2; pLast = &pData0[nField-1]; @@ -2800,7 +2800,7 @@ case OP_MakeRecord: { assert( i==nHdr ); assert( j==nByte ); - assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) ); + assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) ); pOut->n = (int)nByte; pOut->flags = MEM_Blob; if( nZero ){ @@ -3386,7 +3386,7 @@ case OP_OpenWrite: } if( pOp->p5 & OPFLAG_P2ISREG ){ assert( p2>0 ); - assert( p2<=(p->nMem-p->nCursor) ); + assert( p2<=(p->nMem+1 - p->nCursor) ); pIn2 = &aMem[p2]; assert( memIsValid(pIn2) ); assert( (pIn2->flags & MEM_Int)!=0 ); @@ -4181,7 +4181,7 @@ case OP_NewRowid: { /* out2 */ pMem = &pFrame->aMem[pOp->p3]; }else{ /* Assert that P3 is a valid memory cell. */ - assert( pOp->p3<=(p->nMem-p->nCursor) ); + assert( pOp->p3<=(p->nMem+1 - p->nCursor) ); pMem = &aMem[pOp->p3]; memAboutToChange(p, pMem); } @@ -5007,7 +5007,7 @@ case OP_IdxDelete: { UnpackedRecord r; assert( pOp->p3>0 ); - assert( pOp->p2>0 && pOp->p2+pOp->p3<=(p->nMem-p->nCursor)+1 ); + assert( pOp->p2>0 && pOp->p2+pOp->p3<=(p->nMem+1 - p->nCursor)+1 ); assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); @@ -5513,7 +5513,7 @@ case OP_IntegrityCk: { aRoot = pOp->p4.ai; assert( nRoot>0 ); assert( aRoot[nRoot]==0 ); - assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) ); + assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) ); pnErr = &aMem[pOp->p3]; assert( (pnErr->flags & MEM_Int)!=0 ); assert( (pnErr->flags & (MEM_Str|MEM_Blob))==0 ); @@ -5703,6 +5703,8 @@ case OP_Program: { /* jump */ ** variable nMem (and later, VdbeFrame.nChildMem) to this value. */ nMem = pProgram->nMem + pProgram->nCsr; + assert( nMem>0 ); + if( pProgram->nCsr==0 ) nMem++; nByte = ROUND8(sizeof(VdbeFrame)) + nMem * sizeof(Mem) + pProgram->nCsr * sizeof(VdbeCursor *) @@ -5739,7 +5741,8 @@ case OP_Program: { /* jump */ } }else{ pFrame = pRt->u.pFrame; - assert( pProgram->nMem+pProgram->nCsr==pFrame->nChildMem ); + assert( pProgram->nMem+pProgram->nCsr==pFrame->nChildMem + || (pProgram->nCsr==0 && pProgram->nMem+1==pFrame->nChildMem) ); assert( pProgram->nCsr==pFrame->nChildCsr ); assert( (int)(pOp - aOp)==pFrame->pc ); } @@ -5754,10 +5757,10 @@ case OP_Program: { /* jump */ p->pAuxData = 0; p->nChange = 0; p->pFrame = pFrame; - p->aMem = aMem = &VdbeFrameMem(pFrame)[-1]; + p->aMem = aMem = VdbeFrameMem(pFrame); p->nMem = pFrame->nChildMem; p->nCursor = (u16)pFrame->nChildCsr; - p->apCsr = (VdbeCursor **)&aMem[p->nMem+1]; + p->apCsr = (VdbeCursor **)&aMem[p->nMem]; p->aOp = aOp = pProgram->aOp; p->nOp = pProgram->nOp; p->aOnceFlag = (u8 *)&p->apCsr[p->nCursor]; @@ -6003,8 +6006,8 @@ case OP_AggStep0: { assert( pOp->p4type==P4_FUNCDEF ); n = pOp->p5; - assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) ); - assert( n==0 || (pOp->p2>0 && pOp->p2+n<=(p->nMem-p->nCursor)+1) ); + assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) ); + assert( n==0 || (pOp->p2>0 && pOp->p2+n<=(p->nMem+1 - p->nCursor)+1) ); assert( pOp->p3p2 || pOp->p3>=pOp->p2+n ); pCtx = sqlite3DbMallocRawNN(db, sizeof(*pCtx) + (n-1)*sizeof(sqlite3_value*)); if( pCtx==0 ) goto no_mem; @@ -6083,7 +6086,7 @@ case OP_AggStep: { */ case OP_AggFinal: { Mem *pMem; - assert( pOp->p1>0 && pOp->p1<=(p->nMem-p->nCursor) ); + assert( pOp->p1>0 && pOp->p1<=(p->nMem+1 - p->nCursor) ); pMem = &aMem[pOp->p1]; assert( (pMem->flags & ~(MEM_Null|MEM_Agg))==0 ); rc = sqlite3VdbeMemFinalize(pMem, pOp->p4.pFunc); @@ -6525,7 +6528,7 @@ case OP_VColumn: { VdbeCursor *pCur = p->apCsr[pOp->p1]; assert( pCur->eCurType==CURTYPE_VTAB ); - assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) ); + assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) ); pDest = &aMem[pOp->p3]; memAboutToChange(p, pDest); if( pCur->nullRow ){ @@ -6883,6 +6886,7 @@ abort_due_to_error: sqlite3VdbeError(p, "%s", sqlite3ErrStr(rc)); } p->rc = rc; + sqlite3SystemError(db, rc); testcase( sqlite3GlobalConfig.xLog!=0 ); sqlite3_log(rc, "statement aborts at %d: [%s] %s", (int)(pOp - aOp), p->zSql, p->zErrMsg); diff --git a/src/vdbe.h b/src/vdbe.h index f6acb7d51f..ab0de8f3eb 100644 --- a/src/vdbe.h +++ b/src/vdbe.h @@ -204,6 +204,7 @@ void sqlite3VdbeUsesBtree(Vdbe*, int); VdbeOp *sqlite3VdbeGetOp(Vdbe*, int); int sqlite3VdbeMakeLabel(Vdbe*); void sqlite3VdbeRunOnlyOnce(Vdbe*); +void sqlite3VdbeReusable(Vdbe*); void sqlite3VdbeDelete(Vdbe*); void sqlite3VdbeClearObject(sqlite3*,Vdbe*); void sqlite3VdbeMakeReady(Vdbe*,Parse*); diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 50f5e99c50..932821f383 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -392,6 +392,13 @@ void sqlite3VdbeRunOnlyOnce(Vdbe *p){ p->runOnlyOnce = 1; } +/* +** Mark the VDBE as one that can only be run multiple times. +*/ +void sqlite3VdbeReusable(Vdbe *p){ + p->runOnlyOnce = 0; +} + #ifdef SQLITE_DEBUG /* sqlite3AssertMayAbort() logic */ /* @@ -1795,7 +1802,7 @@ void sqlite3VdbeRewind(Vdbe *p){ p->magic = VDBE_MAGIC_RUN; #ifdef SQLITE_DEBUG - for(i=1; inMem; i++){ + for(i=0; inMem; i++){ assert( p->aMem[i].db==p->db ); } #endif @@ -1860,16 +1867,13 @@ void sqlite3VdbeMakeReady( nOnce = pParse->nOnce; if( nOnce==0 ) nOnce = 1; /* Ensure at least one byte in p->aOnceFlag[] */ - /* For each cursor required, also allocate a memory cell. Memory - ** cells (nMem+1-nCursor)..nMem, inclusive, will never be used by - ** the vdbe program. Instead they are used to allocate memory for - ** VdbeCursor/BtCursor structures. The blob of memory associated with - ** cursor 0 is stored in memory cell nMem. Memory cell (nMem-1) - ** stores the blob of memory associated with cursor 1, etc. - ** + /* Each cursor uses a memory cell. The first cursor (cursor 0) can + ** use aMem[0] which is not otherwise used by the VDBE program. Allocate + ** space at the end of aMem[] for cursors 1 and greater. ** See also: allocateCursor(). */ nMem += nCursor; + if( nCursor==0 && nMem>0 ) nMem++; /* Space for aMem[0] even if not used */ /* Figure out how much reusable memory is available at the end of the ** opcode array. This extra memory will be reallocated for other elements @@ -1931,9 +1935,8 @@ void sqlite3VdbeMakeReady( pParse->nzVar = 0; pParse->azVar = 0; if( p->aMem ){ - p->aMem--; /* aMem[] goes from 1..nMem */ - p->nMem = nMem; /* not from 0..nMem-1 */ - for(n=1; n<=nMem; n++){ + p->nMem = nMem; + for(n=0; naMem[n].flags = MEM_Undefined; p->aMem[n].db = db; } @@ -2043,7 +2046,7 @@ static void closeAllCursors(Vdbe *p){ assert( p->nFrame==0 ); closeCursorsInFrame(p); if( p->aMem ){ - releaseMemArray(&p->aMem[1], p->nMem); + releaseMemArray(p->aMem, p->nMem); } while( p->pDelFrame ){ VdbeFrame *pDel = p->pDelFrame; @@ -2068,7 +2071,7 @@ static void Cleanup(Vdbe *p){ int i; if( p->apCsr ) for(i=0; inCursor; i++) assert( p->apCsr[i]==0 ); if( p->aMem ){ - for(i=1; i<=p->nMem; i++) assert( p->aMem[i].flags==MEM_Undefined ); + for(i=0; inMem; i++) assert( p->aMem[i].flags==MEM_Undefined ); } #endif diff --git a/src/vdbemem.c b/src/vdbemem.c index abe2dd2516..f9396e570c 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -761,7 +761,7 @@ int sqlite3VdbeMemTooBig(Mem *p){ void sqlite3VdbeMemAboutToChange(Vdbe *pVdbe, Mem *pMem){ int i; Mem *pX; - for(i=1, pX=&pVdbe->aMem[1]; i<=pVdbe->nMem; i++, pX++){ + for(i=0, pX=pVdbe->aMem; inMem; i++, pX++){ if( pX->pScopyFrom==pMem ){ pX->flags |= MEM_Undefined; pX->pScopyFrom = 0; diff --git a/src/where.c b/src/where.c index 83d72ee397..cf23aa62a4 100644 --- a/src/where.c +++ b/src/where.c @@ -289,6 +289,7 @@ static WhereTerm *whereScanInit( j = iColumn; iColumn = pIdx->aiColumn[j]; if( iColumn==XN_EXPR ) pScan->pIdxExpr = pIdx->aColExpr->a[j].pExpr; + if( iColumn==pIdx->pTable->iPKey ) iColumn = XN_ROWID; } if( pIdx && iColumn>=0 ){ pScan->idxaff = pIdx->pTable->aCol[iColumn].affinity; @@ -3929,7 +3930,7 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){ int j; Table *pTab; Index *pIdx; - + pWInfo = pBuilder->pWInfo; if( pWInfo->wctrlFlags & WHERE_FORCE_TABLE ) return 0; assert( pWInfo->pTabList->nSrc>=1 ); diff --git a/test/capi3.test b/test/capi3.test index 163bb19ada..8096d29bec 100644 --- a/test/capi3.test +++ b/test/capi3.test @@ -172,14 +172,15 @@ do_test capi3-3.3 { catch { set db2 [sqlite3_open /bogus/path/test.db {}] } - sqlite3_extended_errcode $db2 -} {SQLITE_CANTOPEN} + set ::capi3_errno [sqlite3_system_errno $db2] + list [sqlite3_extended_errcode $db2] [expr {$::capi3_errno!=0}] +} {SQLITE_CANTOPEN 1} do_test capi3-3.4 { sqlite3_errmsg $db2 } {unable to open database file} do_test capi3-3.5 { - sqlite3_close $db2 -} {SQLITE_OK} + list [sqlite3_system_errno $db2] [sqlite3_close $db2] +} [list $::capi3_errno SQLITE_OK] if {[clang_sanitize_address]==0} { do_test capi3-3.6.1-misuse { sqlite3_close $db2 @@ -925,19 +926,20 @@ do_test capi3-11.9.3 { do_test capi3-11.10 { sqlite3_step $STMT } {SQLITE_ROW} -ifcapable !autoreset { - # If SQLITE_OMIT_AUTORESET is defined, then the statement must be - # reset() before it can be passed to step() again. - do_test capi3-11.11a { sqlite3_step $STMT } {SQLITE_MISUSE} - do_test capi3-11.11b { sqlite3_reset $STMT } {SQLITE_ABORT} -} do_test capi3-11.11 { sqlite3_step $STMT } {SQLITE_DONE} -do_test capi3-11.12 { - sqlite3_step $STMT - sqlite3_step $STMT -} {SQLITE_ROW} +ifcapable api_armor { + do_test capi3-11.12armor { + sqlite3_step $STMT + sqlite3_step $STMT + } {SQLITE_MISUSE} +} else { + do_test capi3-11.12 { + sqlite3_step $STMT + sqlite3_step $STMT + } {SQLITE_ROW} +} do_test capi3-11.13 { sqlite3_finalize $STMT } {SQLITE_OK} diff --git a/test/capi3c.test b/test/capi3c.test index 15307a7f7a..91c02561b1 100644 --- a/test/capi3c.test +++ b/test/capi3c.test @@ -865,19 +865,20 @@ do_test capi3c-11.9.3 { do_test capi3c-11.10 { sqlite3_step $STMT } {SQLITE_ROW} -ifcapable !autoreset { - # If SQLITE_OMIT_AUTORESET is defined, then the statement must be - # reset() before it can be passed to step() again. - do_test capi3-11.11a { sqlite3_step $STMT } {SQLITE_MISUSE} - do_test capi3-11.11b { sqlite3_reset $STMT } {SQLITE_ABORT} -} do_test capi3c-11.11 { sqlite3_step $STMT } {SQLITE_DONE} -do_test capi3c-11.12 { - sqlite3_step $STMT - sqlite3_step $STMT -} {SQLITE_ROW} +ifcapable api_armor { + do_test capi3c-11.12armor { + sqlite3_step $STMT + sqlite3_step $STMT + } {SQLITE_MISUSE} +} else { + do_test capi3c-11.12 { + sqlite3_step $STMT + sqlite3_step $STMT + } {SQLITE_ROW} +} do_test capi3c-11.13 { sqlite3_finalize $STMT } {SQLITE_OK} diff --git a/test/exclusive.test b/test/exclusive.test index c7b88cfbca..45f9318205 100644 --- a/test/exclusive.test +++ b/test/exclusive.test @@ -423,7 +423,7 @@ do_test exclusive-5.1 { # (2016-03-04) The statement-journal is now opened lazily set sqlite_open_file_count expr $sqlite_open_file_count-$extrafds -} [expr 2 - ($TEMP_STORE>=2)] +} {2} do_test exclusive-5.2 { execsql { COMMIT; @@ -450,7 +450,7 @@ do_test exclusive-5.4 { # 2016-03-04: The statement-journal open is deferred set sqlite_open_file_count expr $sqlite_open_file_count-$extrafds -} [expr 2 - ($TEMP_STORE>=2)] +} {2} do_test exclusive-5.5 { execsql { COMMIT; @@ -459,7 +459,7 @@ do_test exclusive-5.5 { # 2016-03-04: The statement-journal open is deferred set sqlite_open_file_count expr $sqlite_open_file_count-$extrafds -} [expr 2 - ($TEMP_STORE>=2)] +} {2} do_test exclusive-5.6 { execsql { PRAGMA locking_mode = normal; diff --git a/test/intpkey.test b/test/intpkey.test index 41858e5d46..a149f43fb2 100644 --- a/test/intpkey.test +++ b/test/intpkey.test @@ -296,7 +296,7 @@ do_test intpkey-3.8 { count { SELECT * FROM t1 WHERE c=='world' AND a>7; } -} {11 hello world 4} +} {11 hello world 3} do_test intpkey-3.9 { count { SELECT * FROM t1 WHERE 71 || $MEMORY_MANAGEMENT==0)*1024] +ifcapable pagecache_overflow_stats { + ifcapable !malloc_usable_size { + do_test memsubsys1-2.3 { + set pg_ovfl [lindex [sqlite3_status SQLITE_STATUS_PAGECACHE_OVERFLOW 0] 2] + } [expr ($TEMP_STORE>1 || $MEMORY_MANAGEMENT==0)*1024] + } } do_test memsubsys1-2.4 { set pg_used [lindex [sqlite3_status SQLITE_STATUS_PAGECACHE_USED 0] 2] diff --git a/test/mutex1.test b/test/mutex1.test index 340e271175..f567a0d930 100644 --- a/test/mutex1.test +++ b/test/mutex1.test @@ -97,7 +97,7 @@ do_test mutex1-1.9 { # * Multi-threaded mode, # * Single-threaded mode. # -ifcapable threadsafe&&shared_cache { +ifcapable threadsafe1&&shared_cache { set enable_shared_cache [sqlite3_enable_shared_cache 1] foreach {mode mutexes} { singlethread {} diff --git a/test/releasetest.tcl b/test/releasetest.tcl index 2bcf1aa511..15f64e3218 100644 --- a/test/releasetest.tcl +++ b/test/releasetest.tcl @@ -174,24 +174,47 @@ array set ::Configs [strip_comments { -O2 -DSQLITE_ENABLE_LOCKING_STYLE=1 } - "OS-X" { + "Apple" { -O1 # Avoid a compiler bug in gcc 4.2.1 build 5658 - -DSQLITE_OMIT_LOAD_EXTENSION=1 - -DSQLITE_DEFAULT_MEMSTATUS=0 - -DSQLITE_THREADSAFE=2 - -DSQLITE_OS_UNIX=1 - -DSQLITE_ENABLE_JSON1=1 - -DSQLITE_ENABLE_LOCKING_STYLE=1 - -DUSE_PREAD=1 - -DSQLITE_ENABLE_RTREE=1 + -DHAVE_GMTIME_R=1 + -DHAVE_ISNAN=1 + -DHAVE_LOCALTIME_R=1 + -DHAVE_PREAD=1 + -DHAVE_PWRITE=1 + -DHAVE_USLEEP=1 + -DHAVE_USLEEP=1 + -DHAVE_UTIME=1 + -DSQLITE_DEFAULT_CACHE_SIZE=1000 + -DSQLITE_DEFAULT_CKPTFULLFSYNC=1 + -DSQLITE_DEFAULT_MEMSTATUS=1 + -DSQLITE_DEFAULT_PAGE_SIZE=1024 + -DSQLITE_DISABLE_PAGECACHE_OVERFLOW_STATS=1 + -DSQLITE_ENABLE_API_ARMOR=1 + -DSQLITE_ENABLE_AUTO_PROFILE=1 + -DSQLITE_ENABLE_FLOCKTIMEOUT=1 -DSQLITE_ENABLE_FTS3=1 -DSQLITE_ENABLE_FTS3_PARENTHESIS=1 - -DSQLITE_DEFAULT_CACHE_SIZE=1000 + -DSQLITE_ENABLE_FTS3_TOKENIZER=1 + if:os=="Darwin" -DSQLITE_ENABLE_LOCKING_STYLE=1 + -DSQLITE_ENABLE_PERSIST_WAL=1 + -DSQLITE_ENABLE_PURGEABLE_PCACHE=1 + -DSQLITE_ENABLE_RTREE=1 + -DSQLITE_ENABLE_SNAPSHOT=1 + # -DSQLITE_ENABLE_SQLLOG=1 + -DSQLITE_ENABLE_UPDATE_DELETE_LIMIT=1 -DSQLITE_MAX_LENGTH=2147483645 -DSQLITE_MAX_VARIABLE_NUMBER=500000 - -DSQLITE_DEBUG=1 + # -DSQLITE_MEMDEBUG=1 + -DSQLITE_NO_SYNC=1 + -DSQLITE_OMIT_AUTORESET=1 + -DSQLITE_OMIT_LOAD_EXTENSION=1 -DSQLITE_PREFER_PROXY_LOCKING=1 - -DSQLITE_ENABLE_API_ARMOR=1 + -DSQLITE_SERIES_CONSTRAINT_VERIFY=1 + -DSQLITE_THREADSAFE=2 + -DSQLITE_USE_URI=1 + -DSQLITE_WRITE_WALFRAME_PREBUFFERED=1 + -DUSE_GUARDED_FD=1 + -DUSE_PREAD=1 --enable-json1 --enable-fts5 } "Extra-Robustness" { @@ -248,6 +271,7 @@ array set ::Platforms [strip_comments { "Device-Two" test "No-lookaside" test "Devkit" test + "Apple" test "Sanitize" {QUICKTEST_OMIT=func4.test,nan.test test} "Device-One" fulltest "Default" "threadtest fulltest" @@ -264,12 +288,12 @@ array set ::Platforms [strip_comments { Darwin-i386 { "Locking-Style" "mptest test" "Have-Not" test - "OS-X" "threadtest fulltest" + "Apple" "threadtest fulltest" } Darwin-x86_64 { "Locking-Style" "mptest test" "Have-Not" test - "OS-X" "threadtest fulltest" + "Apple" "threadtest fulltest" } "Windows NT-intel" { "Have-Not" test @@ -588,13 +612,23 @@ proc add_test_suite {listvar name testtarget config} { set opts "" set title ${name}($testtarget) set configOpts $::WITHTCL + set skip 0 regsub -all {#[^\n]*\n} $config \n config foreach arg $config { + if {$skip} { + set skip 0 + continue + } if {[regexp {^-[UD]} $arg]} { lappend opts $arg } elseif {[regexp {^[A-Z]+=} $arg]} { lappend testtarget $arg + } elseif {[regexp {^if:([a-z]+)(.*)} $arg all key tail]} { + # Arguments of the form 'if:os=="Linux"' will cause the subsequent + # argument to be skipped if the $tcl_platform(os) is not "Linux", for + # example... + set skip [expr !(\$::tcl_platform($key)$tail)] } elseif {[regexp {^--(enable|disable)-} $arg]} { if {$::MSVC} { if {$arg eq "--disable-amalgamation"} { diff --git a/test/sqldiff1.test b/test/sqldiff1.test index 3201fb3654..ea4e1f9993 100644 --- a/test/sqldiff1.test +++ b/test/sqldiff1.test @@ -14,16 +14,8 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl -if {$tcl_platform(platform)=="windows"} { - set PROG "sqldiff.exe" -} else { - set PROG "./sqldiff" -} -if {![file exe $PROG]} { - puts "sqldiff cannot run because $PROG is not available" - finish_test - return -} +set PROG [test_find_sqldiff] + db close forcedelete test.db test2.db sqlite3 db test.db diff --git a/test/tester.tcl b/test/tester.tcl index dc0c73eab0..2fcae7b5a2 100644 --- a/test/tester.tcl +++ b/test/tester.tcl @@ -2142,24 +2142,41 @@ proc test_restore_config_pagecache {} { sqlite3 db test.db } +proc test_find_binary {nm} { + if {$::tcl_platform(platform)=="windows"} { + set ret "$nm.exe" + } else { + set ret $nm + } + set ret [file normalize [file join $::cmdlinearg(TESTFIXTURE_HOME) $ret]] + if {![file executable $ret]} { + finish_test + return "" + } + return $ret +} + # Find the name of the 'shell' executable (e.g. "sqlite3.exe") to use for # the tests in shell[1-5].test. If no such executable can be found, invoke # [finish_test ; return] in the callers context. # proc test_find_cli {} { - if {$::tcl_platform(platform)=="windows"} { - set ret "sqlite3.exe" - } else { - set ret "sqlite3" - } - set ret [file normalize [file join $::cmdlinearg(TESTFIXTURE_HOME) $ret]] - if {![file executable $ret]} { - finish_test - return -code return - } - return $ret + set prog [test_find_binary sqlite3] + if {$prog==""} { return -code return } + return $prog } +# Find the name of the 'sqldiff' executable (e.g. "sqlite3.exe") to use for +# the tests in sqldiff tests. If no such executable can be found, invoke +# [finish_test ; return] in the callers context. +# +proc test_find_sqldiff {} { + set prog [test_find_binary sqldiff] + if {$prog==""} { return -code return } + return $prog +} + + # If the library is compiled with the SQLITE_DEFAULT_AUTOVACUUM macro set # to non-zero, then set the global variable $AUTOVACUUM to 1. set AUTOVACUUM $sqlite_options(default_autovacuum) diff --git a/test/wal2.test b/test/wal2.test index 4b9bbf315f..0b15b15461 100644 --- a/test/wal2.test +++ b/test/wal2.test @@ -1194,6 +1194,9 @@ foreach {tn sql reslist} { 2 { PRAGMA checkpoint_fullfsync = 1 } {10 4 4 2 6 2} 3 { PRAGMA checkpoint_fullfsync = 0 } {10 0 4 0 6 0} } { + ifcapable default_ckptfullfsync { + if {[string trim $sql]==""} continue + } faultsim_delete_and_reopen execsql {PRAGMA auto_vacuum = 0; PRAGMA synchronous = FULL;} diff --git a/test/wal3.test b/test/wal3.test index da3d318773..56f40ab539 100644 --- a/test/wal3.test +++ b/test/wal3.test @@ -220,6 +220,7 @@ foreach {tn syncmode synccount} { sqlite3 db test.db -vfs T execsql "PRAGMA synchronous = $syncmode" + execsql "PRAGMA checkpoint_fullfsync = 0" execsql { PRAGMA journal_mode = WAL } execsql { CREATE TABLE filler(a,b,c); } diff --git a/tool/lemon.c b/tool/lemon.c index cefdf80174..903fe97cfb 100644 --- a/tool/lemon.c +++ b/tool/lemon.c @@ -290,6 +290,7 @@ struct rule { const char *codeSuffix; /* Breakdown code after code[] above */ struct symbol *precsym; /* Precedence symbol for this rule */ int index; /* An index number for this rule */ + int iRule; /* Rule number as used in the generated tables */ Boolean canReduce; /* True if this rule is ever reduced */ struct rule *nextlhs; /* Next rule with the same LHS */ struct rule *next; /* Next rule in the global list */ @@ -372,6 +373,7 @@ struct plink { struct lemon { struct state **sorted; /* Table of states sorted by state number */ struct rule *rule; /* List of all rules */ + struct rule *startRule; /* First rule */ int nstate; /* Number of states */ int nxstate; /* nstate with tail degenerate states removed */ int nrule; /* Number of rules */ @@ -858,12 +860,12 @@ void FindStates(struct lemon *lemp) ErrorMsg(lemp->filename,0, "The specified start symbol \"%s\" is not \ in a nonterminal of the grammar. \"%s\" will be used as the start \ -symbol instead.",lemp->start,lemp->rule->lhs->name); +symbol instead.",lemp->start,lemp->startRule->lhs->name); lemp->errorcnt++; - sp = lemp->rule->lhs; + sp = lemp->startRule->lhs; } }else{ - sp = lemp->rule->lhs; + sp = lemp->startRule->lhs; } /* Make sure the start symbol doesn't occur on the right-hand side of @@ -1117,9 +1119,9 @@ void FindActions(struct lemon *lemp) /* Add the accepting token */ if( lemp->start ){ sp = Symbol_find(lemp->start); - if( sp==0 ) sp = lemp->rule->lhs; + if( sp==0 ) sp = lemp->startRule->lhs; }else{ - sp = lemp->rule->lhs; + sp = lemp->startRule->lhs; } /* Add to the first state (which is always the starting state of the ** finite state machine) an action to ACCEPT if the lookahead is the @@ -1497,6 +1499,54 @@ static void handle_T_option(char *z){ lemon_strcpy(user_templatename, z); } +/* Merge together to lists of rules order by rule.iRule */ +static struct rule *Rule_merge(struct rule *pA, struct rule *pB){ + struct rule *pFirst = 0; + struct rule **ppPrev = &pFirst; + while( pA && pB ){ + if( pA->iRuleiRule ){ + *ppPrev = pA; + ppPrev = &pA->next; + pA = pA->next; + }else{ + *ppPrev = pB; + ppPrev = &pB->next; + pB = pB->next; + } + } + if( pA ){ + *ppPrev = pA; + }else{ + *ppPrev = pB; + } + return pFirst; +} + +/* +** Sort a list of rules in order of increasing iRule value +*/ +static struct rule *Rule_sort(struct rule *rp){ + int i; + struct rule *pNext; + struct rule *x[32]; + memset(x, 0, sizeof(x)); + while( rp ){ + pNext = rp->next; + rp->next = 0; + for(i=0; iname[0]); i++); lem.nterminal = i; + /* Assign sequential rule numbers */ + for(i=0, rp=lem.rule; rp; rp=rp->next){ + rp->iRule = rp->code ? i++ : -1; + } + for(rp=lem.rule; rp; rp=rp->next){ + if( rp->iRule<0 ) rp->iRule = i++; + } + lem.startRule = lem.rule; + lem.rule = Rule_sort(lem.rule); + /* Generate a reprint of the grammar, if requested on the command line */ if( rpflag ){ Reprint(&lem); @@ -3054,13 +3115,13 @@ int PrintAction( } case REDUCE: { struct rule *rp = ap->x.rp; - fprintf(fp,"%*s reduce %-7d",indent,ap->sp->name,rp->index); + fprintf(fp,"%*s reduce %-7d",indent,ap->sp->name,rp->iRule); RulePrint(fp, rp, -1); break; } case SHIFTREDUCE: { struct rule *rp = ap->x.rp; - fprintf(fp,"%*s shift-reduce %-7d",indent,ap->sp->name,rp->index); + fprintf(fp,"%*s shift-reduce %-7d",indent,ap->sp->name,rp->iRule); RulePrint(fp, rp, -1); break; } @@ -3073,7 +3134,7 @@ int PrintAction( case SRCONFLICT: case RRCONFLICT: fprintf(fp,"%*s reduce %-7d ** Parsing conflict **", - indent,ap->sp->name,ap->x.rp->index); + indent,ap->sp->name,ap->x.rp->iRule); break; case SSCONFLICT: fprintf(fp,"%*s shift %-7d ** Parsing conflict **", @@ -3090,7 +3151,7 @@ int PrintAction( case RD_RESOLVED: if( showPrecedenceConflict ){ fprintf(fp,"%*s reduce %-7d -- dropped by precedence", - indent,ap->sp->name,ap->x.rp->index); + indent,ap->sp->name,ap->x.rp->iRule); }else{ result = 0; } @@ -3121,7 +3182,7 @@ void ReportOutput(struct lemon *lemp) while( cfp ){ char buf[20]; if( cfp->dot==cfp->rp->nrhs ){ - lemon_sprintf(buf,"(%d)",cfp->rp->index); + lemon_sprintf(buf,"(%d)",cfp->rp->iRule); fprintf(fp," %5s ",buf); }else{ fprintf(fp," "); @@ -3222,8 +3283,8 @@ PRIVATE int compute_action(struct lemon *lemp, struct action *ap) int act; switch( ap->type ){ case SHIFT: act = ap->x.stp->statenum; break; - case SHIFTREDUCE: act = ap->x.rp->index + lemp->nstate; break; - case REDUCE: act = ap->x.rp->index + lemp->nstate+lemp->nrule; break; + case SHIFTREDUCE: act = ap->x.rp->iRule + lemp->nstate; break; + case REDUCE: act = ap->x.rp->iRule + lemp->nstate+lemp->nrule; break; case ERROR: act = lemp->nstate + lemp->nrule*2; break; case ACCEPT: act = lemp->nstate + lemp->nrule*2 + 1; break; default: act = -1; break; @@ -4241,7 +4302,7 @@ void ReportTable( ** when tracing REDUCE actions. */ for(i=0, rp=lemp->rule; rp; rp=rp->next, i++){ - assert( rp->index==i ); + assert( rp->iRule==i ); fprintf(out," /* %3d */ \"", i); writeRuleText(out, rp); fprintf(out,"\",\n"); lineno++; @@ -4337,14 +4398,14 @@ void ReportTable( struct rule *rp2; /* Other rules with the same action */ if( rp->code==0 ) continue; if( rp->code[0]=='\n' && rp->code[1]==0 ) continue; /* Will be default: */ - fprintf(out," case %d: /* ", rp->index); + fprintf(out," case %d: /* ", rp->iRule); writeRuleText(out, rp); fprintf(out, " */\n"); lineno++; for(rp2=rp->next; rp2; rp2=rp2->next){ if( rp2->code==rp->code ){ - fprintf(out," case %d: /* ", rp2->index); + fprintf(out," case %d: /* ", rp2->iRule); writeRuleText(out, rp2); - fprintf(out," */ yytestcase(yyruleno==%d);\n", rp2->index); lineno++; + fprintf(out," */ yytestcase(yyruleno==%d);\n", rp2->iRule); lineno++; rp2->code = 0; } } @@ -4358,9 +4419,9 @@ void ReportTable( for(rp=lemp->rule; rp; rp=rp->next){ if( rp->code==0 ) continue; assert( rp->code[0]=='\n' && rp->code[1]==0 ); - fprintf(out," /* (%d) ", rp->index); + fprintf(out," /* (%d) ", rp->iRule); writeRuleText(out, rp); - fprintf(out, " */ yytestcase(yyruleno==%d);\n", rp->index); lineno++; + fprintf(out, " */ yytestcase(yyruleno==%d);\n", rp->iRule); lineno++; } fprintf(out," break;\n"); lineno++; tplt_xfer(lemp->name,in,out,&lineno); diff --git a/tool/lempar.c b/tool/lempar.c index db7f2dbf1b..e313904082 100644 --- a/tool/lempar.c +++ b/tool/lempar.c @@ -418,7 +418,7 @@ int ParseStackPeak(void *p){ ** Find the appropriate action for a parser given the terminal ** look-ahead token iLookAhead. */ -static int yy_find_shift_action( +static unsigned int yy_find_shift_action( yyParser *pParser, /* The parser */ YYCODETYPE iLookAhead /* The look-ahead token */ ){ @@ -606,7 +606,7 @@ static void yy_accept(yyParser*); /* Forward Declaration */ */ static void yy_reduce( yyParser *yypParser, /* The parser */ - int yyruleno /* Number of the rule by which to reduce */ + unsigned int yyruleno /* Number of the rule by which to reduce */ ){ int yygoto; /* The next state */ int yyact; /* The next action */ @@ -615,8 +615,7 @@ static void yy_reduce( ParseARG_FETCH; yymsp = &yypParser->yystack[yypParser->yyidx]; #ifndef NDEBUG - if( yyTraceFILE && yyruleno>=0 - && yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ){ + if( yyTraceFILE && yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ){ yysize = yyRuleInfo[yyruleno].nrhs; fprintf(yyTraceFILE, "%sReduce [%s], go to state %d.\n", yyTracePrompt, yyRuleName[yyruleno], yymsp[-yysize].stateno); @@ -661,7 +660,7 @@ static void yy_reduce( %% /********** End reduce actions ************************************************/ }; - assert( yyruleno>=0 && yyruleno0 ){ + Str cnt = {0, 0, 0}; + strPrintf(&cnt, "INSERT INTO rbu_count VALUES('data_%q', %d);", zTab, nRow); + fprintf(out, "%s\n", cnt.z); + strFree(&cnt); + } strFree(&ct); strFree(&sql); @@ -1757,8 +1765,10 @@ int main(int argc, char **argv){ char *zTab = 0; FILE *out = stdout; void (*xDiff)(const char*,FILE*) = diff_one_table; +#ifndef SQLITE_OMIT_LOAD_EXTENSION int nExt = 0; char **azExt = 0; +#endif int useTransaction = 0; int neverUseTransaction = 0; @@ -1841,8 +1851,8 @@ int main(int argc, char **argv){ cmdlineError("error loading %s: %s", azExt[i], zErrMsg); } } -#endif free(azExt); +#endif zSql = sqlite3_mprintf("ATTACH %Q as aux;", zDb2); rc = sqlite3_exec(g.db, zSql, 0, 0, &zErrMsg); if( rc || zErrMsg ){ @@ -1854,7 +1864,13 @@ int main(int argc, char **argv){ } if( neverUseTransaction ) useTransaction = 0; - if( useTransaction ) printf("BEGIN TRANSACTION;\n"); + if( useTransaction ) fprintf(out, "BEGIN TRANSACTION;\n"); + if( xDiff==rbudiff_one_table ){ + fprintf(out, "CREATE TABLE IF NOT EXISTS rbu_count" + "(tbl TEXT PRIMARY KEY COLLATE NOCASE, cnt INTEGER) " + "WITHOUT ROWID;\n" + ); + } if( zTab ){ xDiff(zTab, out); }else{