From 99a94a124c6ff0b0428fa6ff7e880443edef8dcb Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 17 Feb 2024 20:55:01 +0000 Subject: [PATCH 01/16] Add start of extension for incremental integrity-checks to ext/intck/. FossilOrigin-Name: 444e3c9210026da7eae1ed98850722e002433aa2cc77dbc6b6f80327a6b7a390 --- Makefile.in | 2 + Makefile.msc | 2 + ext/intck/intck1.test | 192 +++++++++++ ext/intck/intck2.test | 70 ++++ ext/intck/intck_common.tcl | 49 +++ ext/intck/sqlite3intck.c | 647 +++++++++++++++++++++++++++++++++++++ ext/intck/sqlite3intck.h | 55 ++++ ext/intck/test_intck.c | 185 +++++++++++ main.mk | 4 +- manifest | 29 +- manifest.uuid | 2 +- src/test_tclsh.c | 2 + 12 files changed, 1227 insertions(+), 12 deletions(-) create mode 100644 ext/intck/intck1.test create mode 100644 ext/intck/intck2.test create mode 100644 ext/intck/intck_common.tcl create mode 100644 ext/intck/sqlite3intck.c create mode 100644 ext/intck/sqlite3intck.h create mode 100644 ext/intck/test_intck.c diff --git a/Makefile.in b/Makefile.in index cb894666d9..509ad4884f 100644 --- a/Makefile.in +++ b/Makefile.in @@ -418,6 +418,8 @@ TESTSRC = \ $(TOP)/ext/recover/sqlite3recover.c \ $(TOP)/ext/recover/dbdata.c \ $(TOP)/ext/recover/test_recover.c \ + $(TOP)/ext/intck/test_intck.c \ + $(TOP)/ext/intck/sqlite3intck.c \ $(TOP)/ext/rbu/test_rbu.c # Statically linked extensions diff --git a/Makefile.msc b/Makefile.msc index 19bfe2f386..7d9dbd2c2b 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1595,6 +1595,8 @@ TESTEXT = \ $(TOP)\ext\rtree\test_rtreedoc.c \ $(TOP)\ext\recover\sqlite3recover.c \ $(TOP)\ext\recover\test_recover.c \ + $(TOP)\ext\intck\test_intck.c \ + $(TOP)\ext\intck\sqlite3intck.c \ $(TOP)\ext\recover\dbdata.c # If use of zlib is enabled, add the "zipfile.c" source file. diff --git a/ext/intck/intck1.test b/ext/intck/intck1.test new file mode 100644 index 0000000000..c46a7d6b2d --- /dev/null +++ b/ext/intck/intck1.test @@ -0,0 +1,192 @@ +# 2008 Feb 19 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# The focus of this file is testing the r-tree extension. +# + +source [file join [file dirname [info script]] intck_common.tcl] +set testprefix intck1 + +foreach {tn sql} { + 1 "CREATE TABLE t1(a PRIMARY KEY, b)" + 2 "CREATE TABLE t2(a PRIMARY KEY, b) WITHOUT ROWID " + 3 "CREATE TABLE t3(a PRIMARY KEY, b) WITHOUT rowID;" + 4 "CREATE TABLE t4(a PRIMARY KEY, ROWID)" + 5 {CREATE TABLE t5(a PRIMARY KEY, ROWID) WITHOUT ROWID + } +} { + do_test 1.1.$tn { + db eval $sql + set {} {} + } {} +} + +set space " \n\v\t\r\f" + +do_execsql_test 1.2 { + SELECT name, (rtrim(sql, $space) LIKE '%rowid') + FROM sqlite_schema WHERE type='table' + ORDER BY 1 +} { + t1 0 + t2 1 + t3 1 + t4 0 + t5 1 +} + +do_execsql_test 1.3 { + CREATE TABLE x1(a COLLATE nocase, b INTEGER, c BLOB); + INSERT INTO x1 VALUES('lEtTeRs', 1234, 1234); +} +do_execsql_test 1.3.1 { + WITH wrapper(c1, c2, c3) AS ( + SELECT a, b, c FROM x1 + ) + SELECT * FROM wrapper WHERE c1='letters'; +} {lEtTeRs 1234 1234} +do_execsql_test 1.3.2 { + WITH wrapper(c1, c2, c3) AS ( + SELECT a, b, c FROM x1 + ) + SELECT * FROM wrapper WHERE c2='1234'; +} {lEtTeRs 1234 1234} +do_execsql_test 1.3.2 { + WITH wrapper(c1, c2, c3) AS ( + SELECT a, b, c FROM x1 + ) + SELECT * FROM wrapper WHERE c3='1234'; +} {} + +do_execsql_test 1.4 { + CREATE TABLE z1(a, b); + CREATE INDEX z1ab ON z1(a+b COLLATE nocase); +} +do_execsql_test 1.4.1 { + SELECT * FROM z1 INDEXED BY z1ab +} + +do_catchsql_test 1.5.1 { + CREATE INDEX z1b ON z1(b ASC NULLS LAST); +} {1 {unsupported use of NULLS LAST}} +do_catchsql_test 1.5.2 { + CREATE INDEX z1b ON z1(b DESC NULLS LAST); +} {1 {unsupported use of NULLS LAST}} +do_catchsql_test 1.5.3 { + CREATE INDEX z1b ON z1(b ASC NULLS FIRST); +} {1 {unsupported use of NULLS FIRST}} +do_catchsql_test 1.5.4 { + CREATE INDEX z1b ON z1(b DESC NULLS FIRST); +} {1 {unsupported use of NULLS FIRST}} + + +#------------------------------------------------------------------------- +reset_db + +do_test 2.0 { + set ic [sqlite3_intck db main ""] + $ic close +} {} + +do_execsql_test 2.1 { + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES(3, 4); + + CREATE INDEX i1 ON t1(a COLLATE nocase); + CREATE INDEX i2 ON t1(b, a); + CREATE INDEX i3 ON t1(b + a COLLATE nocase) WHERE a!=1; +} + +do_intck_test 2.2 { +} + +# Delete a row from each of the i1 and i2 indexes using the imposter +# table interface. +# +do_test 2.3 { + db eval {SELECT name, rootpage FROM sqlite_schema} { + set R($name) $rootpage + } + sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 1 $R(i1) + db eval { CREATE TABLE imp1(a PRIMARY KEY, rowid) WITHOUT ROWID; } + sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 1 $R(i2) + db eval { CREATE TABLE imp2(b, a, rowid, PRIMARY KEY(b, a)) WITHOUT ROWID; } + sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 0 0 + + db eval { + DELETE FROM imp1 WHERE rowid=1; + DELETE FROM imp2 WHERE rowid=2; + } + + db close + sqlite3 db test.db +} {} + +do_intck_test 2.4 { + {entry (1,1) missing from index i1} + {entry (4,3,2) missing from index i2} +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 3.0 { + CREATE TABLE x1(a, b, c, PRIMARY KEY(c, b)) WITHOUT ROWID; + CREATE INDEX x1a ON x1(a COLLATE nocase); + + INSERT INTO x1 VALUES(1, 2, 'three'); + INSERT INTO x1 VALUES(4, 5, 'six'); + INSERT INTO x1 VALUES(7, 8, 'nine'); +} + +do_intck_test 3.1 { } + +do_test 3.2 { + db eval {SELECT name, rootpage FROM sqlite_schema} { + set R($name) $rootpage + } + sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 1 $R(x1a) + db eval { CREATE TABLE imp1(c, b, a, PRIMARY KEY(c, b)) WITHOUT ROWID } + sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 0 0 + + db eval { + DELETE FROM imp1 WHERE a=5; + } + execsql_pp { + } + + db close + sqlite3 db test.db +} {} + +puts "[intck_sql db x1a]" +execsql_pp "EXPLAIN QUERY PLAN [intck_sql db x1a]" +do_intck_test 3.3 { + {entry (4,'six',5) missing from index x1a} +} + +#explain_i [intck_sql db x1] +#puts [intck_sql db x1] +#puts [intck_sql db x1a] + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 4.0 { + CREATE TABLE www(x, y, z); + CREATE INDEX w1 ON www( (x+1), z ); + INSERT INTO www VALUES(1, 1, 1), (2, 2, 2); +} + +do_intck_test 4.1 { } + +finish_test + + diff --git a/ext/intck/intck2.test b/ext/intck/intck2.test new file mode 100644 index 0000000000..bc9aebeea8 --- /dev/null +++ b/ext/intck/intck2.test @@ -0,0 +1,70 @@ +# 2008 Feb 19 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# The focus of this file is testing the r-tree extension. +# + +source [file join [file dirname [info script]] intck_common.tcl] +set testprefix intck2 + + +do_execsql_test 1.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b TEXT); + INSERT INTO t1 VALUES(1, 'one'); + INSERT INTO t1 VALUES(2, 'two'); + INSERT INTO t1 VALUES(3, 'three'); + CREATE INDEX i1 ON t1(b); +} + +proc imposter_edit {obj create sql} { + sqlite3 xdb test.db + set pgno [xdb one {SELECT rootpage FROM sqlite_schema WHERE name=$obj}] + + sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER xdb main 1 $pgno + xdb eval $create + sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER xdb main 0 0 + xdb eval $sql + xdb close +} + +imposter_edit i1 { + CREATE TABLE imp(b, a, PRIMARY KEY(b)) WITHOUT ROWID; +} { + DELETE FROM imp WHERE b='two'; + INSERT INTO imp(b, a) VALUES('four', 4); +} + +do_intck_test 1.1 { + {entry ('two',2) missing from index i1} + {surplus entry ('four',4) in index i1} +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 2.0 { + CREATE TABLE x1(a, b, "c d"); + CREATE INDEX x1a ON x1(a COLLATE nocase DESC , b ASC); + CREATE INDEX x1b ON x1( a || b || ' "''" ' COLLATE binary ASC ); + CREATE INDEX x1c ON x1( format('%s', a)ASC, format('%d', "c d" ) ); + INSERT INTO x1 VALUES('one', 2, 3); + INSERT INTO x1 VALUES('One', 4, 5); + INSERT INTO x1 VALUES('ONE', 6, 7); + INSERT INTO x1 VALUES(NULL, NULL, NULL); +} + +do_intck_test 2.1 {} +puts [intck_sql db x1] + + + +finish_test + + diff --git a/ext/intck/intck_common.tcl b/ext/intck/intck_common.tcl new file mode 100644 index 0000000000..0763d13266 --- /dev/null +++ b/ext/intck/intck_common.tcl @@ -0,0 +1,49 @@ +# 2024 Feb 18 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source $testdir/tester.tcl + +proc do_intck {db} { + set ic [sqlite3_intck $db main ""] + + set ret [list] + while {"SQLITE_OK"==[$ic step]} { + set msg [$ic message] + if {$msg!=""} { + lappend ret $msg + } + } + + set err [$ic error] + if {[lindex $err 0]!="SQLITE_OK"} { + error $err + } + $ic close + + return $ret +} + +proc intck_sql {db tbl} { + set ic [sqlite3_intck $db main ""] + set sql [$ic test_sql $tbl] + $ic close + return $sql +} + +proc do_intck_test {tn expect} { + uplevel [list do_test $tn [list do_intck db] [list {*}$expect]] +} + + diff --git a/ext/intck/sqlite3intck.c b/ext/intck/sqlite3intck.c new file mode 100644 index 0000000000..7610d44a9b --- /dev/null +++ b/ext/intck/sqlite3intck.c @@ -0,0 +1,647 @@ +/* +** 2024-02-08 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +*/ + +#include "sqlite3intck.h" +#include +#include +#include + +struct sqlite3_intck { + sqlite3 *db; + const char *zDb; /* Copy of zDb parameter to _open() */ + + sqlite3_stmt *pListTables; + + sqlite3_stmt *pCheck; + int nCheck; + + int rc; /* SQLite error code */ + char *zErr; /* Error message */ + + char *zTestSql; /* Returned by sqlite3_intck_test_sql() */ +}; + + +/* +** Some error has occurred while using database p->db. Save the error message +** and error code currently held by the database handle in p->rc and p->zErr. +*/ +static void intckSaveErrmsg(sqlite3_intck *p){ + const char *zDberr = sqlite3_errmsg(p->db); + p->rc = sqlite3_errcode(p->db); + if( zDberr ){ + sqlite3_free(p->zErr); + p->zErr = sqlite3_mprintf("%s", zDberr); + } +} + +static sqlite3_stmt *intckPrepare(sqlite3_intck *p, const char *zFmt, ...){ + sqlite3_stmt *pRet = 0; + va_list ap; + char *zSql = 0; + va_start(ap, zFmt); + zSql = sqlite3_vmprintf(zFmt, ap); + if( p->rc==SQLITE_OK ){ + if( zSql==0 ){ + p->rc = SQLITE_NOMEM; + }else{ + p->rc = sqlite3_prepare_v2(p->db, zSql, -1, &pRet, 0); + fflush(stdout); + if( p->rc!=SQLITE_OK ){ + printf("ERROR: %s\n", zSql); + printf("MSG: %s\n", sqlite3_errmsg(p->db)); + if( sqlite3_error_offset(p->db)>=0 ){ + int iOff = sqlite3_error_offset(p->db); + printf("AT: %.40s\n", &zSql[iOff]); + } + fflush(stdout); + intckSaveErrmsg(p); + assert( pRet==0 ); + } + } + } + sqlite3_free(zSql); + va_end(ap); + return pRet; +} + +static void intckFinalize(sqlite3_intck *p, sqlite3_stmt *pStmt){ + int rc = sqlite3_finalize(pStmt); + if( p->rc==SQLITE_OK && rc!=SQLITE_OK ){ + intckSaveErrmsg(p); + } +} + +/* +** Return an SQL statement that will itself return a single row for each +** table in the target schema. The row contains two columns: +** +** 0: table_name - name of table +** 1: without_rowid - true for WITHOUT ROWID tables, false otherwise. +** +*/ +static sqlite3_stmt *intckListTables(sqlite3_intck *p){ + return intckPrepare(p, + "WITH tables(table_name) AS (" + " SELECT name" + " FROM %Q.sqlite_schema WHERE type='table' OR type='index'" + " UNION ALL " + " SELECT 'sqlite_schema'" + ")" + "SELECT * FROM tables" + , p->zDb + ); +} + +static char *intckStrdup(sqlite3_intck *p, const char *zIn){ + char *zOut = 0; + if( p->rc==SQLITE_OK ){ + int nIn = strlen(zIn); + zOut = sqlite3_malloc(nIn+1); + if( zOut==0 ){ + p->rc = SQLITE_NOMEM; + }else{ + memcpy(zOut, zIn, nIn+1); + } + } + return zOut; +} + +/* +** Return the size in bytes of the first token in nul-terminated buffer z. +** For the purposes of this call, a token is either: +** +** * a quoted SQL string, +* * a contiguous series of ascii alphabet characters, or +* * any other single byte. +*/ +static int intckGetToken(const char *z){ + char c = z[0]; + int iRet = 1; + if( c=='\'' || c=='"' || c=='`' ){ + while( 1 ){ + if( z[iRet]==c ){ + iRet++; + if( z[iRet+1]!=c ) break; + } + iRet++; + } + } + else if( c=='[' ){ + while( z[iRet++]!=']' && z[iRet] ); + } + else if( (c>='A' && c<='Z') || (c>='a' && c<='z') ){ + while( (z[iRet]>='A' && z[iRet]<='Z') || (z[iRet]>='a' && z[iRet]<='z') ){ + iRet++; + } + } + + return iRet; +} + +static int intckIsSpace(char c){ + return (c==' ' || c=='\t' || c=='\n' || c=='\r'); +} + +static int intckTokenMatch( + const char *zToken, + int nToken, + const char *z1, + const char *z2 +){ + return ( + (strlen(z1)==nToken && 0==sqlite3_strnicmp(zToken, z1, nToken)) + || (z2 && strlen(z2)==nToken && 0==sqlite3_strnicmp(zToken, z2, nToken)) + ); +} + +/* +** Argument z points to the text of a CREATE INDEX statement. This function +** identifies the part of the text that contains either the index WHERE +** clause (if iCol<0) or the iCol'th column of the index. +** +** If (iCol<0), the identified fragment does not include the "WHERE" keyword, +** only the expression that follows it. If (iCol>=0) then the identified +** fragment does not include any trailing sort-order keywords - "ASC" or +** "DESC". +** +** If the CREATE INDEX statement does not contain the requested field or +** clause, NULL is returned and (*pnByte) is set to 0. Otherwise, a pointer to +** the identified fragment is returned and output parameter (*pnByte) set +** to its size in bytes. +*/ +static const char *intckParseCreateIndex(const char *z, int iCol, int *pnByte){ + int iOff = 0; + int iThisCol = 0; + int iStart = 0; + int nOpen = 0; + + const char *zRet = 0; + int nRet = 0; + + int iEndOfCol = 0; + + /* Skip forward until the first "(" token */ + while( z[iOff]!='(' ){ + iOff += intckGetToken(&z[iOff]); + if( z[iOff]=='\0' ) return 0; + } + assert( z[iOff]=='(' ); + + nOpen = 1; + iOff++; + iStart = iOff; + while( z[iOff] ){ + const char *zToken = &z[iOff]; + int nToken = 0; + + /* Check if this is the end of the current column - either a "," or ")" + ** when nOpen==1. */ + if( nOpen==1 ){ + if( z[iOff]==',' || z[iOff]==')' ){ + if( iCol==iThisCol ){ + int iEnd = iEndOfCol ? iEndOfCol : iOff; + nRet = (iEnd - iStart); + zRet = &z[iStart]; + break; + } + iStart = iOff+1; + while( intckIsSpace(z[iStart]) ) iStart++; + iThisCol++; + } + if( z[iOff]==')' ) break; + } + if( z[iOff]=='(' ) nOpen++; + if( z[iOff]==')' ) nOpen--; + nToken = intckGetToken(zToken); + + if( intckTokenMatch(zToken, nToken, "ASC", "DESC") ){ + iEndOfCol = iOff; + }else if( 0==intckIsSpace(zToken[0]) ){ + iEndOfCol = 0; + } + + iOff += nToken; + } + + /* iStart is now the byte offset of 1 byte passed the final ')' in the + ** CREATE INDEX statement. Try to find a WHERE clause to return. */ + while( zRet==0 && z[iOff] ){ + int n = intckGetToken(&z[iOff]); + if( n==5 && 0==sqlite3_strnicmp(&z[iOff], "where", 5) ){ + zRet = &z[iOff+5]; + nRet = strlen(zRet); + } + iOff += n; + } + + /* Trim any whitespace from the start and end of the returned string. */ + if( zRet ){ + while( intckIsSpace(zRet[0]) ){ + nRet--; + zRet++; + } + while( nRet>0 && intckIsSpace(zRet[nRet-1]) ) nRet--; + } + + *pnByte = nRet; + return zRet; +} + +static void parseCreateIndexFunc( + sqlite3_context *pCtx, + int nVal, + sqlite3_value **apVal +){ + const char *zSql = (const char*)sqlite3_value_text(apVal[0]); + int idx = sqlite3_value_int(apVal[1]); + const char *zRes = 0; + int nRes = 0; + + assert( nVal==2 ); + if( zSql ){ + zRes = intckParseCreateIndex(zSql, idx, &nRes); + } + sqlite3_result_text(pCtx, zRes, nRes, SQLITE_TRANSIENT); +} + +/* +** Return true if sqlite3_intck.db has automatic indexes enabled, false +** otherwise. +*/ +static int intckGetAutoIndex(sqlite3_intck *p){ + int bRet = 0; + sqlite3_stmt *pStmt = 0; + pStmt = intckPrepare(p, "PRAGMA automatic_index"); + if( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + bRet = sqlite3_column_int(pStmt, 0); + } + intckFinalize(p, pStmt); + return bRet; +} + +/* +** Return true if zObj is an index, or false otherwise. +*/ +static int intckIsIndex(sqlite3_intck *p, const char *zObj){ + int bRet = 0; + sqlite3_stmt *pStmt = 0; + pStmt = intckPrepare(p, + "SELECT 1 FROM %Q.sqlite_schema WHERE name=%Q AND type='index'", + p->zDb, zObj + ); + if( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + bRet = 1; + } + intckFinalize(p, pStmt); + return bRet; +} + +static void intckExec(sqlite3_intck *p, const char *zSql){ + sqlite3_stmt *pStmt = 0; + pStmt = intckPrepare(p, "%s", zSql); + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ); + intckFinalize(p, pStmt); +} + +static char *intckCheckObjectSql(sqlite3_intck *p, const char *zObj){ + char *zRet = 0; + sqlite3_stmt *pStmt = 0; + int bAutoIndex = 0; + int bIsIndex = 0; + + const char *zCommon = + /* Relation without_rowid also contains just one row. Column "b" is + ** set to true if the table being examined is a WITHOUT ROWID table, + ** or false otherwise. */ + ", without_rowid(b) AS (" + " SELECT EXISTS (" + " SELECT 1 FROM tabname, pragma_index_list(tab, db) AS l" + " WHERE origin='pk' " + " AND NOT EXISTS (SELECT 1 FROM sqlite_schema WHERE name=l.name)" + " )" + ")" + "" + /* Table idx_cols contains 1 row for each column in each index on the + ** table being checked. Columns are: + ** + ** idx_name: Name of the index. + ** idx_ispk: True if this index is the PK of a WITHOUT ROWID table. + ** col_name: Name of indexed column, or NULL for index on expression. + ** col_expr: Indexed expression, including COLLATE clause. + ** col_alias: Alias used for column in 'intck_wrapper' table. + */ + ", idx_cols(idx_name, idx_ispk, col_name, col_expr, col_alias) AS (" + " SELECT l.name, (l.origin=='pk' AND w.b), i.name, COALESCE((" + " SELECT parse_create_index(sql, i.seqno) FROM " + " sqlite_schema WHERE name = l.name" + " ), format('\"%w\"', i.name) || ' COLLATE ' || quote(i.coll))," + " 'c' || row_number() OVER ()" + " FROM " + " tabname t," + " without_rowid w," + " pragma_index_list(t.tab, t.db) l," + " pragma_index_xinfo(l.name) i" + " WHERE i.key" + " UNION ALL" + " SELECT '', 1, '_rowid_', '_rowid_', 'r1' FROM without_rowid WHERE b=0" + ")" + "" + "" + ", tabpk(db, tab, idx, o_pk, i_pk, q_pk, eq_pk, ps_pk, pk_pk) AS (" + " WITH pkfields(f, a) AS (" + " SELECT i.col_name, i.col_alias FROM idx_cols i WHERE i.idx_ispk" + " )" + " SELECT t.db, t.tab, t.idx, " + " group_concat('o.'||a, ', '), " + " group_concat('i.'||quote(f), ', '), " + " group_concat('quote(o.'||a||')', ' || '','' || '), " + " format('(%s)==(%s)'," + " group_concat('o.'||a, ', '), " + " group_concat(format('\"%w\"', f), ', ')" + " )," + " group_concat('%s', ',')," + " group_concat('quote('||a||')', ', ') " + " FROM tabname t, pkfields" + ")" + "" + ", idx(name, match_expr, partial, partial_alias, idx_ps, idx_idx) AS (" + " SELECT idx_name," + " format('(%s) IS (%s)', " + " group_concat(i.col_expr, ', ')," + " group_concat('o.'||i.col_alias, ', ')" + " ), " + " parse_create_index(" + " (SELECT sql FROM sqlite_schema WHERE name=idx_name), -1" + " )," + " 'cond' || row_number() OVER ()" + " , group_concat('%s', ',')" + " , group_concat('quote('||i.col_alias||')', ', ')" + " FROM tabpk t, " + " without_rowid w," + " idx_cols i" + " WHERE i.idx_ispk==0 " + " GROUP BY idx_name" + ")" + "" + ", wrapper_with(s) AS (" + " SELECT 'intck_wrapper AS (\n SELECT\n ' || (" + " WITH f(a, b) AS (" + " SELECT col_expr, col_alias FROM idx_cols" + " UNION ALL " + " SELECT partial, partial_alias FROM idx WHERE partial IS NOT NULL" + " )" + " SELECT group_concat(format('%s AS %s', a, b), ',\n ') FROM f" + " )" + " || format('\n FROM %Q.%Q ', t.db, t.tab)" + /* If the object being checked is a table, append "NOT INDEXED". + ** Otherwise, append "INDEXED BY ", and then, if the index + ** is a partial index " WHERE ". */ + " || CASE WHEN t.idx IS NULL THEN " + " 'NOT INDEXED'" + " ELSE" + " format('INDEXED BY %Q%s', t.idx, ' WHERE '||i.partial)" + " END" + " || '\n)'" + " FROM tabname t LEFT JOIN idx i ON (i.name=t.idx)" + ")" + "" + ; + + bAutoIndex = intckGetAutoIndex(p); + if( bAutoIndex ) intckExec(p, "PRAGMA automatic_index = 0"); + + bIsIndex = intckIsIndex(p, zObj); + if( bIsIndex ){ + pStmt = intckPrepare(p, + /* Table idxname contains a single row. The first column, "db", contains + ** the name of the db containing the table (e.g. "main") and the second, + ** "tab", the name of the table itself. */ + "WITH tabname(db, tab, idx) AS (" + " SELECT %Q, (SELECT tbl_name FROM %Q.sqlite_schema WHERE name=%Q), %Q " + ")" + "%s" /* zCommon */ + "" + ", case_statement(c) AS (" + " SELECT " + " 'CASE WHEN (' || group_concat(col_alias, ', ') || ') IS (\n ' " + " || 'SELECT ' || group_concat(col_expr, ', ') || ' FROM '" + " || format('%%Q.%%Q NOT INDEXED WHERE %%s\n)', t.db, t.tab, p.eq_pk)" + " || '\nTHEN NULL\n'" + " || 'ELSE format(''surplus entry ('" + " || group_concat('%%s', ',') || ',' || p.ps_pk" + " || ') in index ' || t.idx || ''', ' " + " || group_concat('quote('||i.col_alias||')', ', ') || ', ' || p.pk_pk" + " || ')'" + " || '\nEND'" + " FROM tabname t, tabpk p, idx_cols i WHERE i.idx_name=t.idx" + "" + ")" + "" + ", main_select(m) AS (" + " SELECT format(" + " 'WITH %%s\nSELECT %%s\nFROM intck_wrapper AS o'," + " ww.s, c" + " )" + " FROM case_statement, wrapper_with ww" + ")" + + "SELECT m FROM main_select" + , p->zDb, p->zDb, zObj, zObj + , zCommon + ); + }else{ + pStmt = intckPrepare(p, + /* Table tabname contains a single row. The first column, "db", contains + ** the name of the db containing the table (e.g. "main") and the second, + ** "tab", the name of the table itself. */ + "WITH tabname(db, tab, idx) AS (SELECT %Q, %Q, NULL)" + "" + "%s" /* zCommon */ + + /* expr(e) contains one row for each index on table zObj. Value e + ** is set to an expression that evaluates to NULL if the required + ** entry is present in the index, or an error message otherwise. */ + ", expr(e, p) AS (" + " SELECT format('CASE WHEN (%%s) IN\n" + " (SELECT %%s FROM %%Q.%%Q AS i INDEXED BY %%Q WHERE %%s%%s)\n" + " THEN NULL\n" + " ELSE format(''entry (%%s,%%s) missing from index %%s'', %%s, %%s)\n" + " END\n'" + " , t.o_pk, t.i_pk, t.db, t.tab, i.name, i.match_expr, " + " ' AND (' || partial || ')'," + " i.idx_ps, t.ps_pk, i.name, i.idx_idx, t.pk_pk)," + " CASE WHEN partial IS NULL THEN NULL ELSE i.partial_alias END" + " FROM tabpk t, idx i" + ")" + + ", numbered(ii, cond, e) AS (" + " SELECT 0, 'n.ii=0', 'NULL'" + " UNION ALL " + " SELECT row_number() OVER ()," + " '(n.ii='||row_number() OVER ()||COALESCE(' AND '||p||')', ')'), e" + " FROM expr" + ")" + + ", counter_with(w) AS (" + " SELECT 'WITH intck_counter(ii) AS (\n ' || " + " group_concat('SELECT '||ii, ' UNION ALL\n ') " + " || '\n)' FROM numbered" + ")" + "" + ", case_statement(c) AS (" + " SELECT 'CASE ' || " + " group_concat(format('\n WHEN %%s THEN (%%s)', cond, e), '') ||" + " '\nEND AS error_message'" + " FROM numbered" + ")" + + ", main_select(m) AS (" + " SELECT format(" + " '%%s, %%s\nSELECT %%s\nFROM intck_wrapper AS o" + ", intck_counter AS n ORDER BY %%s', " + " w, ww.s, c, t.o_pk" + " )" + " FROM case_statement, tabpk t, counter_with, wrapper_with ww" + ")" + + "SELECT m FROM main_select", + p->zDb, zObj, zCommon + ); + } + + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ +#if 0 + int nField = sqlite3_column_count(pStmt); + int ii; + for(ii=0; iipListTables, 0); + char *zSql = intckCheckObjectSql(p, zTab); + p->pCheck = intckPrepare(p, "%s", zSql); + sqlite3_free(zSql); +} + +int sqlite3_intck_open( + sqlite3 *db, /* Database handle to operate on */ + const char *zDbArg, /* "main", "temp" etc. */ + const char *zFile, /* Path to save-state db on disk (or NULL) */ + sqlite3_intck **ppOut /* OUT: New integrity-check handle */ +){ + sqlite3_intck *pNew = 0; + int rc = SQLITE_OK; + const char *zDb = zDbArg ? zDbArg : "main"; + int nDb = strlen(zDb); + + pNew = (sqlite3_intck*)sqlite3_malloc(sizeof(*pNew) + nDb + 1); + if( pNew==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(pNew, 0, sizeof(*pNew)); + pNew->db = db; + pNew->zDb = (const char*)&pNew[1]; + memcpy(&pNew[1], zDb, nDb+1); + sqlite3_create_function(db, "parse_create_index", + 2, SQLITE_UTF8, 0, parseCreateIndexFunc, 0, 0 + ); + } + + *ppOut = pNew; + return rc; +} + +void sqlite3_intck_close(sqlite3_intck *p){ + if( p && p->db ){ + sqlite3_create_function( + p->db, "parse_create_index", 1, SQLITE_UTF8, 0, 0, 0, 0 + ); + } + sqlite3_free(p->zTestSql); + sqlite3_free(p->zErr); + sqlite3_free(p); +} + +int sqlite3_intck_step(sqlite3_intck *p){ + if( p->rc==SQLITE_OK ){ + if( p->pListTables==0 ){ + p->pListTables = intckListTables(p); + } + assert( p->pListTables || p->rc!=SQLITE_OK ); + + if( p->rc==SQLITE_OK && p->pCheck==0 ){ + if( sqlite3_step(p->pListTables)==SQLITE_ROW ){ + intckCheckObject(p); + }else{ + int rc = sqlite3_finalize(p->pListTables); + if( rc==SQLITE_OK ){ + p->rc = SQLITE_DONE; + }else{ + intckSaveErrmsg(p); + } + p->pListTables = 0; + } + } + + if( p->rc==SQLITE_OK ){ + if( sqlite3_step(p->pCheck)==SQLITE_ROW ){ + /* Fine, whatever... */ + }else{ + if( sqlite3_finalize(p->pCheck)!=SQLITE_OK ){ + intckSaveErrmsg(p); + } + p->pCheck = 0; + } + } + } + + return p->rc; +} + +const char *sqlite3_intck_message(sqlite3_intck *p){ + if( p->pCheck ){ + return (const char*)sqlite3_column_text(p->pCheck, 0); + } + return 0; +} + +int sqlite3_intck_error(sqlite3_intck *p, const char **pzErr){ + *pzErr = p->zErr; + return (p->rc==SQLITE_DONE ? SQLITE_OK : p->rc); +} + +int sqlite3_intck_suspend(sqlite3_intck *pCk){ + return SQLITE_OK; +} + +const char *sqlite3_intck_test_sql(sqlite3_intck *p, const char *zObj){ + sqlite3_free(p->zTestSql); + p->zTestSql = intckCheckObjectSql(p, zObj); + return p->zTestSql; +} + diff --git a/ext/intck/sqlite3intck.h b/ext/intck/sqlite3intck.h new file mode 100644 index 0000000000..8846812e75 --- /dev/null +++ b/ext/intck/sqlite3intck.h @@ -0,0 +1,55 @@ +/* +** 2024-02-08 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +*/ + +#ifndef _SQLITE_INTCK_H +#define _SQLITE_INTCK_H + +#include "sqlite3.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct sqlite3_intck sqlite3_intck; + +int sqlite3_intck_open( + sqlite3 *db, + const char *zDb, + const char *zFile, + sqlite3_intck **ppOut +); + +void sqlite3_intck_close(sqlite3_intck*); + +int sqlite3_intck_step(sqlite3_intck *pCk); + +const char *sqlite3_intck_message(sqlite3_intck *pCk); + +int sqlite3_intck_error(sqlite3_intck *pCk, const char **pzErr); + +int sqlite3_intck_suspend(sqlite3_intck *pCk); + +/* +** This API is used for testing only. It returns the full-text of an SQL +** statement used to test object zObj, which may be a table or index. +** The returned buffer is valid until the next call to either this function +** or sqlite3_intck_close() on the same sqlite3_intck handle. +*/ +const char *sqlite3_intck_test_sql(sqlite3_intck *pCk, const char *zObj); + + +#ifdef __cplusplus +} /* end of the 'extern "C"' block */ +#endif + +#endif /* ifndef _SQLITE_INTCK_H */ diff --git a/ext/intck/test_intck.c b/ext/intck/test_intck.c new file mode 100644 index 0000000000..33129ec275 --- /dev/null +++ b/ext/intck/test_intck.c @@ -0,0 +1,185 @@ +/* +** 2010 August 28 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** Code for testing all sorts of SQLite interfaces. This code +** is not included in the SQLite library. +*/ + +#include "sqlite3.h" +#include "sqlite3intck.h" + +#if defined(INCLUDE_SQLITE_TCL_H) +# include "sqlite_tcl.h" +#else +# include "tcl.h" +#endif + +#include +#include + +/* In test1.c */ +int getDbPointer(Tcl_Interp *interp, const char *zA, sqlite3 **ppDb); +const char *sqlite3ErrName(int); + +typedef struct TestIntck TestIntck; +struct TestIntck { + sqlite3_intck *intck; +}; + +static int testIntckCmd( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + struct Subcmd { + const char *zName; + int nArg; + const char *zExpect; + } aCmd[] = { + {"close", 0, ""}, /* 0 */ + {"step", 0, ""}, /* 1 */ + {"message", 0, ""}, /* 2 */ + {"error", 0, ""}, /* 3 */ + {"suspend", 0, ""}, /* 4 */ + {"test_sql", 1, ""}, /* 5 */ + {0 , 0} + }; + int rc = TCL_OK; + int iIdx = -1; + TestIntck *p = (TestIntck*)clientData; + + if( objc<2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "SUB-COMMAND ..."); + return TCL_ERROR; + } + + rc = Tcl_GetIndexFromObjStruct( + interp, objv[1], aCmd, sizeof(aCmd[0]), "SUB-COMMAND", 0, &iIdx + ); + if( rc ) return rc; + + if( objc!=2+aCmd[iIdx].nArg ){ + Tcl_WrongNumArgs(interp, 2, objv, aCmd[iIdx].zExpect); + return TCL_ERROR; + } + + switch( iIdx ){ + case 0: assert( 0==strcmp("close", aCmd[iIdx].zName) ); { + Tcl_DeleteCommand(interp, Tcl_GetStringFromObj(objv[0], 0)); + break; + } + + case 1: assert( 0==strcmp("step", aCmd[iIdx].zName) ); { + int rc = sqlite3_intck_step(p->intck); + Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1)); + break; + } + + case 2: assert( 0==strcmp("message", aCmd[iIdx].zName) ); { + const char *z = sqlite3_intck_message(p->intck); + Tcl_SetObjResult(interp, Tcl_NewStringObj(z ? z : "", -1)); + break; + } + + case 3: assert( 0==strcmp("error", aCmd[iIdx].zName) ); { + const char *zErr = 0; + int rc = sqlite3_intck_error(p->intck, &zErr); + Tcl_Obj *pRes = Tcl_NewObj(); + + Tcl_ListObjAppendElement( + interp, pRes, Tcl_NewStringObj(sqlite3ErrName(rc), -1) + ); + Tcl_ListObjAppendElement( + interp, pRes, Tcl_NewStringObj(zErr ? zErr : 0, -1) + ); + Tcl_SetObjResult(interp, pRes); + break; + } + + case 4: assert( 0==strcmp("suspend", aCmd[iIdx].zName) ); { + int rc = sqlite3_intck_suspend(p->intck); + Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1)); + break; + } + + case 5: assert( 0==strcmp("test_sql", aCmd[iIdx].zName) ); { + const char *zObj = Tcl_GetString(objv[2]); + const char *zSql = sqlite3_intck_test_sql(p->intck, zObj); + Tcl_SetObjResult(interp, Tcl_NewStringObj(zSql, -1)); + break; + } + } + + return TCL_OK; +} + +/* +** Destructor for commands created by test_sqlite3_intck(). +*/ +static void testIntckFree(void *clientData){ + TestIntck *p = (TestIntck*)clientData; + sqlite3_intck_close(p->intck); + ckfree(p); +} + +/* +** tclcmd: sqlite3_intck DB DBNAME PATH +*/ +static int test_sqlite3_intck( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + char zName[64]; + int iName = 0; + Tcl_CmdInfo info; + TestIntck *p = 0; + sqlite3 *db = 0; + const char *zDb = 0; + const char *zFile = 0; + int rc = SQLITE_OK; + + if( objc!=4 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB DBNAME PATH"); + return TCL_ERROR; + } + + p = (TestIntck*)ckalloc(sizeof(TestIntck)); + memset(p, 0, sizeof(TestIntck)); + + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ){ + return TCL_ERROR; + } + zDb = Tcl_GetString(objv[2]); + zFile = Tcl_GetString(objv[3]); + + rc = sqlite3_intck_open(db, zDb, zFile, &p->intck); + if( rc!=SQLITE_OK ){ + ckfree(p); + Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3_errstr(rc), -1)); + return TCL_ERROR; + } + + do { + sprintf(zName, "intck%d", iName); + }while( Tcl_GetCommandInfo(interp, zName, &info)!=0 ); + Tcl_CreateObjCommand(interp, zName, testIntckCmd, (void*)p, testIntckFree); + Tcl_SetObjResult(interp, Tcl_NewStringObj(zName, -1)); + + return TCL_OK; +} + +int Sqlitetestintck_Init(Tcl_Interp *interp){ + Tcl_CreateObjCommand(interp, "sqlite3_intck", test_sqlite3_intck, 0, 0); + return TCL_OK; +} diff --git a/main.mk b/main.mk index 081e0cd3b5..0a0af725d0 100644 --- a/main.mk +++ b/main.mk @@ -375,7 +375,9 @@ TESTSRC += \ $(TOP)/ext/rtree/test_rtreedoc.c \ $(TOP)/ext/recover/sqlite3recover.c \ $(TOP)/ext/recover/dbdata.c \ - $(TOP)/ext/recover/test_recover.c + $(TOP)/ext/recover/test_recover.c \ + $(TOP)/ext/intck/test_intck.c \ + $(TOP)/ext/intck/sqlite3intck.c #TESTSRC += $(TOP)/ext/fts3/fts3_tokenizer.c diff --git a/manifest b/manifest index ab0f22bde5..a378ab14eb 100644 --- a/manifest +++ b/manifest @@ -1,11 +1,11 @@ -C Fix\srounding\sin\szero-precision\s%f\sand\s%g\sprintf\sconversions.\n[forum:/info/393708f4a8|Forum\spost\s393708f4a8].\s\sThis\sbug\swas\nintroduced\sby\scheck-in\s[32befb224b254639]\sand\sfirst\sappeared\sin\sversion\s3.43.0. -D 2024-02-17T03:32:31.878 +C Add\sstart\sof\sextension\sfor\sincremental\sintegrity-checks\sto\sext/intck/. +D 2024-02-17T20:55:01.343 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 -F Makefile.in 24be65ae641c5727bbc247d60286a15ecc24fb80f14f8fb2d32533bf0ec96e79 +F Makefile.in 216eea0cc5a9613d9f4f21402a4b759c2fce2a0cb9567513933562b65e30670b F Makefile.linux-gcc f3842a0b1efbfbb74ac0ef60e56b301836d05b4d867d014f714fa750048f1ab6 -F Makefile.msc df56c06ef05add5ebcdcc9d4823fb2ec66ac16b06acb6971a7420859025886fa +F Makefile.msc a496ca640052c1e102daaa6e2d2216ae482f22995498c7c9492fd7f841481400 F README.md 6358805260a03ebead84e168bbf3740ddf3f683b477e478567186aa7afb490d3 F VERSION c84541c6a9e8426462176fbb1f9ecb5cfd7d1bb56228053ff7eeba8841673eb6 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -248,6 +248,12 @@ F ext/fts5/tool/showfts5.tcl d54da0e067306663e2d5d523965ca487698e722c F ext/icu/README.txt 7ab7ced8ae78e3a645b57e78570ff589d4c672b71370f5aa9e1cd7024f400fc9 F ext/icu/icu.c c074519b46baa484bb5396c7e01e051034da8884bad1a1cb7f09bbe6be3f0282 F ext/icu/sqliteicu.h fa373836ed5a1ee7478bdf8a1650689294e41d0c89c1daab26e9ae78a32075a8 +F ext/intck/intck1.test 0fcf3696b59aff6c344553647d612921dd529600796ff7172c02679955cecdcf +F ext/intck/intck2.test dd06719eca145b317ae380c81f04cd8a096a7cfdb71074cc6b6e7f195058b0d0 +F ext/intck/intck_common.tcl 1f2599d50033d21d5df89f5ed54cc29af472d86e3927e116db50c5ba94d903b9 +F ext/intck/sqlite3intck.c 14300998e91cd8788f483d97e53be9406f2c0be8af1867f399b80fef5e3721fb +F ext/intck/sqlite3intck.h 342ee2e2c7636b4daf29fa195d0a3a658272b76b283d586fba50f6bc80fc143d +F ext/intck/test_intck.c 3f9a950978842340df7492f0a4190022979f23ff904e90873a5e262adf30b78c F ext/jni/GNUmakefile 59eb05f2a363bdfac8d15d66bed624bfe1ff289229184f3861b95f98a19cf4b2 F ext/jni/README.md d899789a9082a07b99bf30b1bbb6204ae57c060efcaa634536fa669323918f42 F ext/jni/jar-dist.make 030aaa4ae71dd86e4ec5e7c1e6cd86f9dfa47c4592c070d2e35157e42498e1fa @@ -659,7 +665,7 @@ F ext/wasm/wasmfs.make 8a4955882aaa0783b3f60a9484a1f0f3d8b6f775c0fcd17c082f31966 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 5ade0bc977aa135e79e3faaea894d5671b26107cc91e70783aa7dc83f22f3ba0 -F main.mk ef8d6b0e76b27d2b32d7b71ea30a2b2626b668f998a4f32f6171c9623a310a53 +F main.mk 678f023e03c5dca755570ed964a8355e44a6435f679e3763a6f9fe3d309f9986 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 F mptest/crash01.test 61e61469e257df0850df4293d7d4d6c2af301421 @@ -790,7 +796,7 @@ F src/test_schema.c cbfd7a9a9b6b40d4377d0c76a6c5b2a58387385977f26edab4e77eb5f90a F src/test_sqllog.c 540feaea7280cd5f926168aee9deb1065ae136d0bbbe7361e2ef3541783e187a F src/test_superlock.c 4839644b9201da822f181c5bc406c0b2385f672e F src/test_syscall.c 9fdb13b1df05e639808d44fcb8f6064aaded32b6565c00b215cfd05a060d1aca -F src/test_tclsh.c 3ff5d188a72f00807425954ea3b493dfd3a4b890ecc6700ea83bad2fd1332ecf +F src/test_tclsh.c aaf0d1de4a518a8db5ad38e5262be3e48b4a74ad1909f2dba753cecb30979d5d F src/test_tclvar.c 3273f9d59395b336e381b53cfc68ec6ebdaada4e93106a2e976ffb0550504e1c F src/test_thread.c 7ddcf0c8b79fa3c1d172f82f322302c963d923cdb503c6171f3c8081586d0b01 F src/test_vdbecov.c f60c6f135ec42c0de013a1d5136777aa328a776d33277f92abac648930453d43 @@ -2162,8 +2168,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1c33c5db2e05019d1a375109f79ad8588a3c17f81e4f4b8d66c880c3c860e87e -R e3e534a124d08ab0760f858683268942 -U drh -Z 25ef9b1be0189ee473aa53bd8732a56c +P 7fca1bc482fc2456d75392eb42f768fda72631c9070de46b8123b1126e78306f +R f8dccfe308ed2507020014beca90ae47 +T *branch * incr-integrity-check +T *sym-incr-integrity-check * +T -sym-trunk * +U dan +Z 8b8b6eda2ec7249d41ba58db982258fb # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 1e28ec46dd..7ea8d90d42 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7fca1bc482fc2456d75392eb42f768fda72631c9070de46b8123b1126e78306f \ No newline at end of file +444e3c9210026da7eae1ed98850722e002433aa2cc77dbc6b6f80327a6b7a390 \ No newline at end of file diff --git a/src/test_tclsh.c b/src/test_tclsh.c index 32aee42675..4697c3b856 100644 --- a/src/test_tclsh.c +++ b/src/test_tclsh.c @@ -108,6 +108,7 @@ const char *sqlite3TestInit(Tcl_Interp *interp){ extern int Sqlitetest_window_Init(Tcl_Interp *); extern int Sqlitetestvdbecov_Init(Tcl_Interp *); extern int TestRecover_Init(Tcl_Interp*); + extern int Sqlitetestintck_Init(Tcl_Interp*); Tcl_CmdInfo cmdInfo; @@ -175,6 +176,7 @@ const char *sqlite3TestInit(Tcl_Interp *interp){ Sqlitetest_window_Init(interp); Sqlitetestvdbecov_Init(interp); TestRecover_Init(interp); + Sqlitetestintck_Init(interp); Tcl_CreateObjCommand( interp, "load_testfixture_extensions", load_testfixture_extensions,0,0 From cfc62ceefdfb19ed162b725cdca30fdea6a055e0 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 19 Feb 2024 18:03:53 +0000 Subject: [PATCH 02/16] Add implementation of sqlite3_intck_suspend(). FossilOrigin-Name: c36ada868da74e030ff5002de1f3b31b639b0c43714b91c2e5ca0eda16bb6bc2 --- ext/intck/intck1.test | 9 +- ext/intck/intck2.test | 6 +- ext/intck/intck_common.tcl | 6 +- ext/intck/sqlite3intck.c | 191 +++++++++++++++++++++++++------------ ext/intck/test_intck.c | 2 +- manifest | 23 ++--- manifest.uuid | 2 +- 7 files changed, 157 insertions(+), 82 deletions(-) diff --git a/ext/intck/intck1.test b/ext/intck/intck1.test index c46a7d6b2d..e5a25dda8e 100644 --- a/ext/intck/intck1.test +++ b/ext/intck/intck1.test @@ -167,8 +167,8 @@ do_test 3.2 { sqlite3 db test.db } {} -puts "[intck_sql db x1a]" -execsql_pp "EXPLAIN QUERY PLAN [intck_sql db x1a]" +#puts "[intck_sql db x1a]" +#execsql_pp "EXPLAIN QUERY PLAN [intck_sql db x1a]" do_intck_test 3.3 { {entry (4,'six',5) missing from index x1a} } @@ -185,6 +185,11 @@ do_execsql_test 4.0 { INSERT INTO www VALUES(1, 1, 1), (2, 2, 2); } +#puts [intck_sql db w1] +#execsql_pp [intck_sql db www] +#execsql_pp [intck_sql db w1] +#puts [intck_sql db w1] + do_intck_test 4.1 { } finish_test diff --git a/ext/intck/intck2.test b/ext/intck/intck2.test index bc9aebeea8..90f263f880 100644 --- a/ext/intck/intck2.test +++ b/ext/intck/intck2.test @@ -43,8 +43,8 @@ imposter_edit i1 { } do_intck_test 1.1 { - {entry ('two',2) missing from index i1} {surplus entry ('four',4) in index i1} + {entry ('two',2) missing from index i1} } #------------------------------------------------------------------------- @@ -61,9 +61,7 @@ do_execsql_test 2.0 { } do_intck_test 2.1 {} -puts [intck_sql db x1] - - +#puts [intck_sql db x1] finish_test diff --git a/ext/intck/intck_common.tcl b/ext/intck/intck_common.tcl index 0763d13266..757ca82b3f 100644 --- a/ext/intck/intck_common.tcl +++ b/ext/intck/intck_common.tcl @@ -15,7 +15,7 @@ if {![info exists testdir]} { } source $testdir/tester.tcl -proc do_intck {db} { +proc do_intck {db {bSuspend 0}} { set ic [sqlite3_intck $db main ""] set ret [list] @@ -24,6 +24,7 @@ proc do_intck {db} { if {$msg!=""} { lappend ret $msg } + if {$bSuspend} { $ic suspend } } set err [$ic error] @@ -43,7 +44,8 @@ proc intck_sql {db tbl} { } proc do_intck_test {tn expect} { - uplevel [list do_test $tn [list do_intck db] [list {*}$expect]] + uplevel [list do_test $tn.a [list do_intck db] [list {*}$expect]] + uplevel [list do_test $tn.b [list do_intck db 1] [list {*}$expect]] } diff --git a/ext/intck/sqlite3intck.c b/ext/intck/sqlite3intck.c index 7610d44a9b..b08b499a43 100644 --- a/ext/intck/sqlite3intck.c +++ b/ext/intck/sqlite3intck.c @@ -20,10 +20,10 @@ struct sqlite3_intck { sqlite3 *db; const char *zDb; /* Copy of zDb parameter to _open() */ - sqlite3_stmt *pListTables; + char *zObj; /* Current object. Or NULL. */ + char *zKey; /* Key saved by _suspect() call. */ sqlite3_stmt *pCheck; - int nCheck; int rc; /* SQLite error code */ char *zErr; /* Error message */ @@ -58,11 +58,15 @@ static sqlite3_stmt *intckPrepare(sqlite3_intck *p, const char *zFmt, ...){ p->rc = sqlite3_prepare_v2(p->db, zSql, -1, &pRet, 0); fflush(stdout); if( p->rc!=SQLITE_OK ){ +#if 1 printf("ERROR: %s\n", zSql); printf("MSG: %s\n", sqlite3_errmsg(p->db)); +#endif if( sqlite3_error_offset(p->db)>=0 ){ +#if 1 int iOff = sqlite3_error_offset(p->db); printf("AT: %.40s\n", &zSql[iOff]); +#endif } fflush(stdout); intckSaveErrmsg(p); @@ -82,27 +86,6 @@ static void intckFinalize(sqlite3_intck *p, sqlite3_stmt *pStmt){ } } -/* -** Return an SQL statement that will itself return a single row for each -** table in the target schema. The row contains two columns: -** -** 0: table_name - name of table -** 1: without_rowid - true for WITHOUT ROWID tables, false otherwise. -** -*/ -static sqlite3_stmt *intckListTables(sqlite3_intck *p){ - return intckPrepare(p, - "WITH tables(table_name) AS (" - " SELECT name" - " FROM %Q.sqlite_schema WHERE type='table' OR type='index'" - " UNION ALL " - " SELECT 'sqlite_schema'" - ")" - "SELECT * FROM tables" - , p->zDb - ); -} - static char *intckStrdup(sqlite3_intck *p, const char *zIn){ char *zOut = 0; if( p->rc==SQLITE_OK ){ @@ -117,6 +100,42 @@ static char *intckStrdup(sqlite3_intck *p, const char *zIn){ return zOut; } +static void intckFindObject(sqlite3_intck *p){ + sqlite3_stmt *pStmt = 0; + char *zPrev = p->zObj; + p->zObj = 0; + + assert( p->rc==SQLITE_OK ); + pStmt = intckPrepare(p, + "WITH tables(table_name) AS (" + " SELECT name" + " FROM %Q.sqlite_schema WHERE type='table' OR type='index'" + " UNION ALL " + " SELECT 'sqlite_schema'" + ")" + "SELECT table_name FROM tables " + "WHERE ?1 IS NULL OR table_name%s?1 " + "ORDER BY 1" + , p->zDb, (p->zKey ? ">=" : ">") + ); + + if( p->rc==SQLITE_OK ){ + sqlite3_bind_text(pStmt, 1, zPrev, -1, SQLITE_TRANSIENT); + if( sqlite3_step(pStmt)==SQLITE_ROW ){ + p->zObj = intckStrdup(p, (const char*)sqlite3_column_text(pStmt, 0)); + } + } + intckFinalize(p, pStmt); + + /* If this is a new object, ensure the previous key value is cleared. */ + if( sqlite3_stricmp(p->zObj, zPrev) ){ + sqlite3_free(p->zKey); + p->zKey = 0; + } + + sqlite3_free(zPrev); +} + /* ** Return the size in bytes of the first token in nul-terminated buffer z. ** For the purposes of this call, a token is either: @@ -314,7 +333,11 @@ static void intckExec(sqlite3_intck *p, const char *zSql){ intckFinalize(p, pStmt); } -static char *intckCheckObjectSql(sqlite3_intck *p, const char *zObj){ +static char *intckCheckObjectSql( + sqlite3_intck *p, + const char *zObj, + const char *zPrev +){ char *zRet = 0; sqlite3_stmt *pStmt = 0; int bAutoIndex = 0; @@ -427,45 +450,61 @@ static char *intckCheckObjectSql(sqlite3_intck *p, const char *zObj){ /* Table idxname contains a single row. The first column, "db", contains ** the name of the db containing the table (e.g. "main") and the second, ** "tab", the name of the table itself. */ - "WITH tabname(db, tab, idx) AS (" - " SELECT %Q, (SELECT tbl_name FROM %Q.sqlite_schema WHERE name=%Q), %Q " + "WITH tabname(db, tab, idx, prev) AS (" + " SELECT %Q, (SELECT tbl_name FROM %Q.sqlite_schema WHERE name=%Q), " + " %Q, %Q " ")" "%s" /* zCommon */ "" ", case_statement(c) AS (" " SELECT " - " 'CASE WHEN (' || group_concat(col_alias, ', ') || ') IS (\n ' " + " 'CASE WHEN (' || group_concat(col_alias, ', ') || ') IS (\n ' " " || 'SELECT ' || group_concat(col_expr, ', ') || ' FROM '" - " || format('%%Q.%%Q NOT INDEXED WHERE %%s\n)', t.db, t.tab, p.eq_pk)" - " || '\nTHEN NULL\n'" + " || format('%%Q.%%Q NOT INDEXED WHERE %%s\n', t.db, t.tab, p.eq_pk)" + " || ' )\n THEN NULL\n '" " || 'ELSE format(''surplus entry ('" " || group_concat('%%s', ',') || ',' || p.ps_pk" " || ') in index ' || t.idx || ''', ' " " || group_concat('quote('||i.col_alias||')', ', ') || ', ' || p.pk_pk" " || ')'" - " || '\nEND'" + " || '\nEND AS error_message'" " FROM tabname t, tabpk p, idx_cols i WHERE i.idx_name=t.idx" + ")" "" + ", thiskey(k) AS (" + " SELECT format('format(''(%%s,%%s)'', %%s, %%s) AS thiskey', " + " group_concat('%%s', ','), p.ps_pk, " + " group_concat('quote('||i.col_alias||')',', '), p.pk_pk" + " ) FROM tabpk p, idx_cols i WHERE i.idx_name=p.idx" + ")" + "" + ", whereclause(w_c) AS (" + " SELECT CASE WHEN prev!='' THEN " + " '\nWHERE (' || group_concat(i.col_alias, ',') || ',' " + " || o_pk || ') > ' || prev" + " ELSE ''" + " END" + " FROM tabpk, tabname, idx_cols i WHERE i.idx_name=tabpk.idx" ")" "" ", main_select(m) AS (" " SELECT format(" - " 'WITH %%s\nSELECT %%s\nFROM intck_wrapper AS o'," - " ww.s, c" + " 'WITH %%s\nSELECT %%s,\n%%s\nFROM intck_wrapper AS o%%s'," + " ww.s, c, t.k, whereclause.w_c" " )" - " FROM case_statement, wrapper_with ww" + " FROM case_statement, wrapper_with ww, thiskey t, whereclause" ")" "SELECT m FROM main_select" , p->zDb, p->zDb, zObj, zObj - , zCommon + , zPrev, zCommon ); }else{ pStmt = intckPrepare(p, /* Table tabname contains a single row. The first column, "db", contains ** the name of the db containing the table (e.g. "main") and the second, ** "tab", the name of the table itself. */ - "WITH tabname(db, tab, idx) AS (SELECT %Q, %Q, NULL)" + "WITH tabname(db, tab, idx, prev) AS (SELECT %Q, %Q, NULL, %Q)" "" "%s" /* zCommon */ @@ -505,18 +544,41 @@ static char *intckCheckObjectSql(sqlite3_intck *p, const char *zObj){ " '\nEND AS error_message'" " FROM numbered" ")" + "" + /* This table contains a single row consisting of a single value - + ** the text of an SQL expression that may be used by the main SQL + ** statement to output an SQL literal that can be used to resume + ** the scan if it is suspended. e.g. for a rowid table, an expression + ** like: + ** + ** format('(%d,%d)', _rowid_, n.ii) + */ + ", thiskey(k) AS (" + " SELECT 'format(''(' || ps_pk || ',%%d)'', ' || pk_pk || ', n.ii)'" + " FROM tabpk" + ")" + "" + ", whereclause(w_c) AS (" + " SELECT CASE WHEN prev!='' THEN " + " '\nWHERE (' || o_pk ||', n.ii) > ' || prev" + " ELSE ''" + " END" + " FROM tabpk, tabname" + ")" + "" ", main_select(m) AS (" " SELECT format(" - " '%%s, %%s\nSELECT %%s\nFROM intck_wrapper AS o" - ", intck_counter AS n ORDER BY %%s', " - " w, ww.s, c, t.o_pk" + " '%%s, %%s\nSELECT %%s,\n%%s AS thiskey\nFROM intck_wrapper AS o" + ", intck_counter AS n%%s\nORDER BY %%s', " + " w, ww.s, c, thiskey.k, whereclause.w_c, t.o_pk" " )" - " FROM case_statement, tabpk t, counter_with, wrapper_with ww" + " FROM case_statement, tabpk t, counter_with, " + " wrapper_with ww, thiskey, whereclause" ")" "SELECT m FROM main_select", - p->zDb, zObj, zCommon + p->zDb, zObj, zPrev, zCommon ); } @@ -542,10 +604,11 @@ static char *intckCheckObjectSql(sqlite3_intck *p, const char *zObj){ } static void intckCheckObject(sqlite3_intck *p){ - const char *zTab = (const char*)sqlite3_column_text(p->pListTables, 0); - char *zSql = intckCheckObjectSql(p, zTab); + char *zSql = intckCheckObjectSql(p, p->zObj, p->zKey); p->pCheck = intckPrepare(p, "%s", zSql); sqlite3_free(zSql); + sqlite3_free(p->zKey); + p->zKey = 0; } int sqlite3_intck_open( @@ -582,6 +645,8 @@ void sqlite3_intck_close(sqlite3_intck *p){ p->db, "parse_create_index", 1, SQLITE_UTF8, 0, 0, 0, 0 ); } + sqlite3_free(p->zObj); + sqlite3_free(p->zKey); sqlite3_free(p->zTestSql); sqlite3_free(p->zErr); sqlite3_free(p); @@ -589,26 +654,19 @@ void sqlite3_intck_close(sqlite3_intck *p){ int sqlite3_intck_step(sqlite3_intck *p){ if( p->rc==SQLITE_OK ){ - if( p->pListTables==0 ){ - p->pListTables = intckListTables(p); - } - assert( p->pListTables || p->rc!=SQLITE_OK ); - - if( p->rc==SQLITE_OK && p->pCheck==0 ){ - if( sqlite3_step(p->pListTables)==SQLITE_ROW ){ - intckCheckObject(p); - }else{ - int rc = sqlite3_finalize(p->pListTables); - if( rc==SQLITE_OK ){ - p->rc = SQLITE_DONE; + if( p->pCheck==0 ){ + intckFindObject(p); + if( p->rc==SQLITE_OK ){ + if( p->zObj ){ + intckCheckObject(p); }else{ - intckSaveErrmsg(p); + p->rc = SQLITE_DONE; } - p->pListTables = 0; } } if( p->rc==SQLITE_OK ){ + assert( p->pCheck ); if( sqlite3_step(p->pCheck)==SQLITE_ROW ){ /* Fine, whatever... */ }else{ @@ -635,13 +693,28 @@ int sqlite3_intck_error(sqlite3_intck *p, const char **pzErr){ return (p->rc==SQLITE_DONE ? SQLITE_OK : p->rc); } -int sqlite3_intck_suspend(sqlite3_intck *pCk){ - return SQLITE_OK; +int sqlite3_intck_suspend(sqlite3_intck *p){ + if( p->pCheck && p->rc==SQLITE_OK ){ + assert( p->zKey==0 ); + p->zKey = intckStrdup(p, (const char*)sqlite3_column_text(p->pCheck, 1)); + intckFinalize(p, p->pCheck); + p->pCheck = 0; + } + return p->rc; } const char *sqlite3_intck_test_sql(sqlite3_intck *p, const char *zObj){ sqlite3_free(p->zTestSql); - p->zTestSql = intckCheckObjectSql(p, zObj); + if( zObj ){ + p->zTestSql = intckCheckObjectSql(p, zObj, 0); + }else{ + if( p->zObj ){ + p->zTestSql = intckCheckObjectSql(p, p->zObj, p->zKey); + }else{ + sqlite3_free(p->zTestSql); + p->zTestSql = 0; + } + } return p->zTestSql; } diff --git a/ext/intck/test_intck.c b/ext/intck/test_intck.c index 33129ec275..d14fc92a6c 100644 --- a/ext/intck/test_intck.c +++ b/ext/intck/test_intck.c @@ -113,7 +113,7 @@ static int testIntckCmd( case 5: assert( 0==strcmp("test_sql", aCmd[iIdx].zName) ); { const char *zObj = Tcl_GetString(objv[2]); - const char *zSql = sqlite3_intck_test_sql(p->intck, zObj); + const char *zSql = sqlite3_intck_test_sql(p->intck, zObj[0] ? zObj : 0); Tcl_SetObjResult(interp, Tcl_NewStringObj(zSql, -1)); break; } diff --git a/manifest b/manifest index a378ab14eb..bb8e46b123 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sstart\sof\sextension\sfor\sincremental\sintegrity-checks\sto\sext/intck/. -D 2024-02-17T20:55:01.343 +C Add\simplementation\sof\ssqlite3_intck_suspend(). +D 2024-02-19T18:03:53.963 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -248,12 +248,12 @@ F ext/fts5/tool/showfts5.tcl d54da0e067306663e2d5d523965ca487698e722c F ext/icu/README.txt 7ab7ced8ae78e3a645b57e78570ff589d4c672b71370f5aa9e1cd7024f400fc9 F ext/icu/icu.c c074519b46baa484bb5396c7e01e051034da8884bad1a1cb7f09bbe6be3f0282 F ext/icu/sqliteicu.h fa373836ed5a1ee7478bdf8a1650689294e41d0c89c1daab26e9ae78a32075a8 -F ext/intck/intck1.test 0fcf3696b59aff6c344553647d612921dd529600796ff7172c02679955cecdcf -F ext/intck/intck2.test dd06719eca145b317ae380c81f04cd8a096a7cfdb71074cc6b6e7f195058b0d0 -F ext/intck/intck_common.tcl 1f2599d50033d21d5df89f5ed54cc29af472d86e3927e116db50c5ba94d903b9 -F ext/intck/sqlite3intck.c 14300998e91cd8788f483d97e53be9406f2c0be8af1867f399b80fef5e3721fb +F ext/intck/intck1.test c831bc6ff67da3c5b6c9568640f87ad0442d4fc98ef97bc837133b94bcc645b3 +F ext/intck/intck2.test b65d7f627342f767e1d2c447d25619666cec36f516786dd56568bd741e5d7e67 +F ext/intck/intck_common.tcl 2895854e7aaf5e199a15f6f82538a00999fd8fc55553bc1f04619af7aa86c0d0 +F ext/intck/sqlite3intck.c 703ff16bc936192cff20d06b015d66279f4594e88371c00b18d17fec4f01ff5c F ext/intck/sqlite3intck.h 342ee2e2c7636b4daf29fa195d0a3a658272b76b283d586fba50f6bc80fc143d -F ext/intck/test_intck.c 3f9a950978842340df7492f0a4190022979f23ff904e90873a5e262adf30b78c +F ext/intck/test_intck.c eb596269c4a690a9b8ee689b1e52ff6e3306013ec706e319d5b97af05a08f0b9 F ext/jni/GNUmakefile 59eb05f2a363bdfac8d15d66bed624bfe1ff289229184f3861b95f98a19cf4b2 F ext/jni/README.md d899789a9082a07b99bf30b1bbb6204ae57c060efcaa634536fa669323918f42 F ext/jni/jar-dist.make 030aaa4ae71dd86e4ec5e7c1e6cd86f9dfa47c4592c070d2e35157e42498e1fa @@ -2168,11 +2168,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7fca1bc482fc2456d75392eb42f768fda72631c9070de46b8123b1126e78306f -R f8dccfe308ed2507020014beca90ae47 -T *branch * incr-integrity-check -T *sym-incr-integrity-check * -T -sym-trunk * +P 444e3c9210026da7eae1ed98850722e002433aa2cc77dbc6b6f80327a6b7a390 +R 6b69910b5d388b58f98078e173b37781 U dan -Z 8b8b6eda2ec7249d41ba58db982258fb +Z 3045521be4122627d743cbd6c4ab9b9c # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 7ea8d90d42..fa3faa50cd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -444e3c9210026da7eae1ed98850722e002433aa2cc77dbc6b6f80327a6b7a390 \ No newline at end of file +c36ada868da74e030ff5002de1f3b31b639b0c43714b91c2e5ca0eda16bb6bc2 \ No newline at end of file From ebea8458e2a00e7564986d05a0ce91c8613fefd7 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 19 Feb 2024 20:15:44 +0000 Subject: [PATCH 03/16] Use more efficient SQL to verify that indexes contain entries that match their tables. FossilOrigin-Name: c01e008c28895e50b14531b2a1f3f1110aab3b54df41ebdbd416fbac7b1bba94 --- ext/intck/sqlite3intck.c | 56 ++++++++++++++++++++-------------------- ext/intck/sqlite3intck.h | 2 +- ext/intck/test_intck.c | 53 +++++++++++++++++++++++++++++++++++++ manifest | 16 ++++++------ manifest.uuid | 2 +- 5 files changed, 91 insertions(+), 38 deletions(-) diff --git a/ext/intck/sqlite3intck.c b/ext/intck/sqlite3intck.c index b08b499a43..05eaa81a93 100644 --- a/ext/intck/sqlite3intck.c +++ b/ext/intck/sqlite3intck.c @@ -19,15 +19,11 @@ struct sqlite3_intck { sqlite3 *db; const char *zDb; /* Copy of zDb parameter to _open() */ - char *zObj; /* Current object. Or NULL. */ - char *zKey; /* Key saved by _suspect() call. */ - - sqlite3_stmt *pCheck; - - int rc; /* SQLite error code */ + char *zKey; /* Key saved by _intck_suspend() call. */ + sqlite3_stmt *pCheck; /* Current check statement */ + int rc; /* Error code */ char *zErr; /* Error message */ - char *zTestSql; /* Returned by sqlite3_intck_test_sql() */ }; @@ -109,7 +105,7 @@ static void intckFindObject(sqlite3_intck *p){ pStmt = intckPrepare(p, "WITH tables(table_name) AS (" " SELECT name" - " FROM %Q.sqlite_schema WHERE type='table' OR type='index'" + " FROM %Q.sqlite_schema WHERE (type='table' OR type='index') AND rootpage" " UNION ALL " " SELECT 'sqlite_schema'" ")" @@ -400,9 +396,9 @@ static char *intckCheckObjectSql( "" ", idx(name, match_expr, partial, partial_alias, idx_ps, idx_idx) AS (" " SELECT idx_name," - " format('(%s) IS (%s)', " - " group_concat(i.col_expr, ', ')," - " group_concat('o.'||i.col_alias, ', ')" + " format('(%s,%s) IS (%s,%s)', " + " group_concat(i.col_expr, ', '), i_pk," + " group_concat('o.'||i.col_alias, ', '), o_pk" " ), " " parse_create_index(" " (SELECT sql FROM sqlite_schema WHERE name=idx_name), -1" @@ -512,13 +508,12 @@ static char *intckCheckObjectSql( ** is set to an expression that evaluates to NULL if the required ** entry is present in the index, or an error message otherwise. */ ", expr(e, p) AS (" - " SELECT format('CASE WHEN (%%s) IN\n" - " (SELECT %%s FROM %%Q.%%Q AS i INDEXED BY %%Q WHERE %%s%%s)\n" + " SELECT format('CASE WHEN EXISTS \n" + " (SELECT 1 FROM %%Q.%%Q AS i INDEXED BY %%Q WHERE %%s%%s)\n" " THEN NULL\n" " ELSE format(''entry (%%s,%%s) missing from index %%s'', %%s, %%s)\n" " END\n'" - " , t.o_pk, t.i_pk, t.db, t.tab, i.name, i.match_expr, " - " ' AND (' || partial || ')'," + " , t.db, t.tab, i.name, i.match_expr, ' AND (' || partial || ')'," " i.idx_ps, t.ps_pk, i.name, i.idx_idx, t.pk_pk)," " CASE WHEN partial IS NULL THEN NULL ELSE i.partial_alias END" " FROM tabpk t, idx i" @@ -626,30 +621,35 @@ int sqlite3_intck_open( if( pNew==0 ){ rc = SQLITE_NOMEM; }else{ + sqlite3_create_function(db, "parse_create_index", + 2, SQLITE_UTF8, 0, parseCreateIndexFunc, 0, 0 + ); memset(pNew, 0, sizeof(*pNew)); pNew->db = db; pNew->zDb = (const char*)&pNew[1]; memcpy(&pNew[1], zDb, nDb+1); - sqlite3_create_function(db, "parse_create_index", - 2, SQLITE_UTF8, 0, parseCreateIndexFunc, 0, 0 - ); } *ppOut = pNew; return rc; } -void sqlite3_intck_close(sqlite3_intck *p){ - if( p && p->db ){ - sqlite3_create_function( - p->db, "parse_create_index", 1, SQLITE_UTF8, 0, 0, 0, 0 - ); +int sqlite3_intck_close(sqlite3_intck *p){ + int rc = SQLITE_OK; + if( p ){ + rc = (p->rc==SQLITE_DONE ? SQLITE_OK : p->rc); + if( p->db ){ + sqlite3_create_function( + p->db, "parse_create_index", 1, SQLITE_UTF8, 0, 0, 0, 0 + ); + } + sqlite3_free(p->zObj); + sqlite3_free(p->zKey); + sqlite3_free(p->zTestSql); + sqlite3_free(p->zErr); + sqlite3_free(p); } - sqlite3_free(p->zObj); - sqlite3_free(p->zKey); - sqlite3_free(p->zTestSql); - sqlite3_free(p->zErr); - sqlite3_free(p); + return rc; } int sqlite3_intck_step(sqlite3_intck *p){ diff --git a/ext/intck/sqlite3intck.h b/ext/intck/sqlite3intck.h index 8846812e75..c7c24e42c9 100644 --- a/ext/intck/sqlite3intck.h +++ b/ext/intck/sqlite3intck.h @@ -29,7 +29,7 @@ int sqlite3_intck_open( sqlite3_intck **ppOut ); -void sqlite3_intck_close(sqlite3_intck*); +int sqlite3_intck_close(sqlite3_intck*); int sqlite3_intck_step(sqlite3_intck *pCk); diff --git a/ext/intck/test_intck.c b/ext/intck/test_intck.c index d14fc92a6c..75bcfa298a 100644 --- a/ext/intck/test_intck.c +++ b/ext/intck/test_intck.c @@ -179,7 +179,60 @@ static int test_sqlite3_intck( return TCL_OK; } +/* +** tclcmd: test_do_intck DB DBNAME +*/ +static int test_do_intck( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + sqlite3 *db = 0; + const char *zDb = 0; + int rc = SQLITE_OK; + sqlite3_intck *pCk = 0; + Tcl_Obj *pRet = 0; + const char *zErr = 0; + + if( objc!=3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB DBNAME"); + return TCL_ERROR; + } + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ){ + return TCL_ERROR; + } + zDb = Tcl_GetString(objv[2]); + + pRet = Tcl_NewObj(); + Tcl_IncrRefCount(pRet); + + rc = sqlite3_intck_open(db, zDb, 0, &pCk); + if( rc==SQLITE_OK ){ + while( sqlite3_intck_step(pCk)==SQLITE_OK ){ + const char *zMsg = sqlite3_intck_message(pCk); + if( zMsg ){ + Tcl_ListObjAppendElement(interp, pRet, Tcl_NewStringObj(zMsg, -1)); + } + } + rc = sqlite3_intck_error(pCk, &zErr); + } + if( rc!=SQLITE_OK ){ + if( zErr ){ + Tcl_SetObjResult(interp, Tcl_NewStringObj(zErr, -1)); + }else{ + Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1)); + } + }else{ + Tcl_SetObjResult(interp, pRet); + } + Tcl_DecrRefCount(pRet); + sqlite3_intck_close(pCk); + return rc ? TCL_ERROR : TCL_OK; +} + int Sqlitetestintck_Init(Tcl_Interp *interp){ Tcl_CreateObjCommand(interp, "sqlite3_intck", test_sqlite3_intck, 0, 0); + Tcl_CreateObjCommand(interp, "test_do_intck", test_do_intck, 0, 0); return TCL_OK; } diff --git a/manifest b/manifest index bb8e46b123..cf37766470 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\simplementation\sof\ssqlite3_intck_suspend(). -D 2024-02-19T18:03:53.963 +C Use\smore\sefficient\sSQL\sto\sverify\sthat\sindexes\scontain\sentries\sthat\smatch\stheir\stables. +D 2024-02-19T20:15:44.802 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -251,9 +251,9 @@ F ext/icu/sqliteicu.h fa373836ed5a1ee7478bdf8a1650689294e41d0c89c1daab26e9ae78a3 F ext/intck/intck1.test c831bc6ff67da3c5b6c9568640f87ad0442d4fc98ef97bc837133b94bcc645b3 F ext/intck/intck2.test b65d7f627342f767e1d2c447d25619666cec36f516786dd56568bd741e5d7e67 F ext/intck/intck_common.tcl 2895854e7aaf5e199a15f6f82538a00999fd8fc55553bc1f04619af7aa86c0d0 -F ext/intck/sqlite3intck.c 703ff16bc936192cff20d06b015d66279f4594e88371c00b18d17fec4f01ff5c -F ext/intck/sqlite3intck.h 342ee2e2c7636b4daf29fa195d0a3a658272b76b283d586fba50f6bc80fc143d -F ext/intck/test_intck.c eb596269c4a690a9b8ee689b1e52ff6e3306013ec706e319d5b97af05a08f0b9 +F ext/intck/sqlite3intck.c 5f319b7c72b0c01cfa28bb5fdd19be5781eb11f5a5216af2a0870dc7e001414d +F ext/intck/sqlite3intck.h d9501ea480b7c41c0555f39f4f1b7c3e8d54fc1ea6d115de5e1211e0bc11d3e7 +F ext/intck/test_intck.c 06206b35f1428961015c060dd35201246c849625cfdff461e0eeaaf76bda545c F ext/jni/GNUmakefile 59eb05f2a363bdfac8d15d66bed624bfe1ff289229184f3861b95f98a19cf4b2 F ext/jni/README.md d899789a9082a07b99bf30b1bbb6204ae57c060efcaa634536fa669323918f42 F ext/jni/jar-dist.make 030aaa4ae71dd86e4ec5e7c1e6cd86f9dfa47c4592c070d2e35157e42498e1fa @@ -2168,8 +2168,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 444e3c9210026da7eae1ed98850722e002433aa2cc77dbc6b6f80327a6b7a390 -R 6b69910b5d388b58f98078e173b37781 +P c36ada868da74e030ff5002de1f3b31b639b0c43714b91c2e5ca0eda16bb6bc2 +R 830041ceb0564db12655be53580aeb6a U dan -Z 3045521be4122627d743cbd6c4ab9b9c +Z f8e44044aacffa331f677c80763fe8b6 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index fa3faa50cd..eb805d0954 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c36ada868da74e030ff5002de1f3b31b639b0c43714b91c2e5ca0eda16bb6bc2 \ No newline at end of file +c01e008c28895e50b14531b2a1f3f1110aab3b54df41ebdbd416fbac7b1bba94 \ No newline at end of file From 626d6192185c33a047966144392d5d3b6b3597ed Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 20 Feb 2024 16:04:27 +0000 Subject: [PATCH 04/16] Consider using "=" and IS operators with even low-quality indexes in cases where they are selected explicitly using an INDEXED BY clause. FossilOrigin-Name: 43cbbea82132db2d0ddb4f34cc2b6910b3a1243ae6d4e837b1b27bfe91b84834 --- ext/intck/intck1.test | 14 ++++++++++++++ manifest | 14 +++++++------- manifest.uuid | 2 +- src/where.c | 4 +++- 4 files changed, 25 insertions(+), 9 deletions(-) diff --git a/ext/intck/intck1.test b/ext/intck/intck1.test index e5a25dda8e..7f60f1fdc6 100644 --- a/ext/intck/intck1.test +++ b/ext/intck/intck1.test @@ -88,6 +88,20 @@ do_catchsql_test 1.5.4 { } {1 {unsupported use of NULLS FIRST}} +reset_db +do_execsql_test 1.6.1 { + CREATE TABLE t1(i INTEGER PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b); + ANALYZE; + INSERT INTO sqlite_stat1 VALUES('t1', 'i1', '10000 10000'); + ANALYZE sqlite_schema; +} {} +do_eqp_test 1.6.2 { + SELECT 1 FROM t1 INDEXED BY i1 WHERE (b, i) IS (?, ?); +} {SEARCH} + + + #------------------------------------------------------------------------- reset_db diff --git a/manifest b/manifest index cf37766470..e254a597a2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Use\smore\sefficient\sSQL\sto\sverify\sthat\sindexes\scontain\sentries\sthat\smatch\stheir\stables. -D 2024-02-19T20:15:44.802 +C Consider\susing\s"="\sand\sIS\soperators\swith\seven\slow-quality\sindexes\sin\scases\swhere\sthey\sare\sselected\sexplicitly\susing\san\sINDEXED\sBY\sclause. +D 2024-02-20T16:04:27.694 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -248,7 +248,7 @@ F ext/fts5/tool/showfts5.tcl d54da0e067306663e2d5d523965ca487698e722c F ext/icu/README.txt 7ab7ced8ae78e3a645b57e78570ff589d4c672b71370f5aa9e1cd7024f400fc9 F ext/icu/icu.c c074519b46baa484bb5396c7e01e051034da8884bad1a1cb7f09bbe6be3f0282 F ext/icu/sqliteicu.h fa373836ed5a1ee7478bdf8a1650689294e41d0c89c1daab26e9ae78a32075a8 -F ext/intck/intck1.test c831bc6ff67da3c5b6c9568640f87ad0442d4fc98ef97bc837133b94bcc645b3 +F ext/intck/intck1.test 5b3c9800e119b4dd50a381974f34cee6cfd5b7434286fb8da83b7c8ff1d6bb3c F ext/intck/intck2.test b65d7f627342f767e1d2c447d25619666cec36f516786dd56568bd741e5d7e67 F ext/intck/intck_common.tcl 2895854e7aaf5e199a15f6f82538a00999fd8fc55553bc1f04619af7aa86c0d0 F ext/intck/sqlite3intck.c 5f319b7c72b0c01cfa28bb5fdd19be5781eb11f5a5216af2a0870dc7e001414d @@ -830,7 +830,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 887fc4ca3f020ebb2e376f222069570834ac63bf50111ef0cbf3ae417048ed89 F src/wal.h ba252daaa94f889f4b2c17c027e823d9be47ce39da1d3799886bbd51f0490452 F src/walker.c 7c7ea0115345851c3da4e04e2e239a29983b61fb5b038b94eede6aba462640e2 -F src/where.c 0e7bf004a578eb70d54300503fdbcaaea857de6ba0204aae1adf572edb79c2c6 +F src/where.c 33eaaeef3aef10c2b9e82096e70a174d6636e35cb0b180321b8ddf804590e5cd F src/whereInt.h 82a13766f13d1a53b05387c2e60726289ef26404bc7b9b1f7770204d97357fb8 F src/wherecode.c 5d77db30a2a3dd532492ae882de114edba2fae672622056b1c7fd61f5917a8f1 F src/whereexpr.c dc5096eca5ed503999be3bdee8a90c51361289a678d396a220912e9cb73b3c00 @@ -2168,8 +2168,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c36ada868da74e030ff5002de1f3b31b639b0c43714b91c2e5ca0eda16bb6bc2 -R 830041ceb0564db12655be53580aeb6a +P c01e008c28895e50b14531b2a1f3f1110aab3b54df41ebdbd416fbac7b1bba94 +R 8db173421c176ccb487011bd9440e123 U dan -Z f8e44044aacffa331f677c80763fe8b6 +Z d4e70ffeb295fbc1a103b9adca306ecc # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index eb805d0954..4565b3d528 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c01e008c28895e50b14531b2a1f3f1110aab3b54df41ebdbd416fbac7b1bba94 \ No newline at end of file +43cbbea82132db2d0ddb4f34cc2b6910b3a1243ae6d4e837b1b27bfe91b84834 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 0e526b3372..9850d22cc3 100644 --- a/src/where.c +++ b/src/where.c @@ -2975,7 +2975,9 @@ static int whereLoopAddBtreeIndex( } if( pProbe->bUnordered || pProbe->bLowQual ){ if( pProbe->bUnordered ) opMask &= ~(WO_GT|WO_GE|WO_LT|WO_LE); - if( pProbe->bLowQual ) opMask &= ~(WO_EQ|WO_IN|WO_IS); + if( pProbe->bLowQual && pSrc->fg.isIndexedBy==0 ){ + opMask &= ~(WO_EQ|WO_IN|WO_IS); + } } assert( pNew->u.btree.nEqnColumn ); From cfcb3b9208236e6d40633bf574de2b7559ad4c9f Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 20 Feb 2024 18:17:06 +0000 Subject: [PATCH 05/16] Use fewer cycles to generate the "next key" value used by sqlite3_intck_suspend() function in the intck extension. FossilOrigin-Name: 95f01426f948cf435d0b400dc7ae06fa699eee32cff498fe77e74a1257a4e09b --- ext/intck/sqlite3intck.c | 220 ++++++++++++++++++++++++++++++++------- manifest | 12 +-- manifest.uuid | 2 +- 3 files changed, 190 insertions(+), 44 deletions(-) diff --git a/ext/intck/sqlite3intck.c b/ext/intck/sqlite3intck.c index 05eaa81a93..6d54526928 100644 --- a/ext/intck/sqlite3intck.c +++ b/ext/intck/sqlite3intck.c @@ -16,12 +16,25 @@ #include #include +/* +** apKeyVal: +** If sqlite3_intck_suspend() is called when there is a running pCheck +** statement, this array is allocated and populated with the key values +** required to restart the check. If the intck object has not been +** suspended, this is set to NULL. +** +** nKeyVal: +** The size of the apKeyVal[] array, if it is allocated. +*/ struct sqlite3_intck { sqlite3 *db; const char *zDb; /* Copy of zDb parameter to _open() */ char *zObj; /* Current object. Or NULL. */ - char *zKey; /* Key saved by _intck_suspend() call. */ + sqlite3_stmt *pCheck; /* Current check statement */ + int nKeyVal; + sqlite3_value **apKeyVal; + int rc; /* Error code */ char *zErr; /* Error message */ char *zTestSql; /* Returned by sqlite3_intck_test_sql() */ @@ -82,20 +95,119 @@ static void intckFinalize(sqlite3_intck *p, sqlite3_stmt *pStmt){ } } -static char *intckStrdup(sqlite3_intck *p, const char *zIn){ - char *zOut = 0; +/* +** Wrapper around sqlite3_malloc64() that uses the sqlite3_intck error +** code convention. +*/ +static void *intckMalloc(sqlite3_intck *p, sqlite3_int64 nByte){ + void *pRet = 0; + assert( nByte>0 ); if( p->rc==SQLITE_OK ){ - int nIn = strlen(zIn); - zOut = sqlite3_malloc(nIn+1); - if( zOut==0 ){ + pRet = sqlite3_malloc64(nByte); + if( pRet==0 ){ p->rc = SQLITE_NOMEM; - }else{ - memcpy(zOut, zIn, nIn+1); } } + return pRet; +} + +/* +** If p->rc is other than SQLITE_OK when this function is called, it +** immediately returns NULL. Otherwise, it attempts to create a copy of +** nul-terminated string zIn in a buffer obtained from sqlite3_malloc(). +** If successful, a pointer to this buffer is returned and it becomes +** the responsibility of the caller to release it using sqlite3_free() +** at some point in the future. +** +** Or, if an allocation fails within this function, p->rc is set to +** SQLITE_NOMEM and NULL is returned. +*/ +static char *intckStrdup(sqlite3_intck *p, const char *zIn){ + char *zOut = 0; + int nIn = strlen(zIn); + zOut = (char*)intckMalloc(p, nIn+1); + if( zOut ){ + memcpy(zOut, zIn, nIn+1); + } return zOut; } +/* +** A wrapper around sqlite3_mprintf() that: +** +** + Always returns 0 if p->rc is other than SQLITE_OK when it is called, and +** + Sets p->rc to SQLITE_NOMEM if an allocation fails. +*/ +static char *intckMprintf(sqlite3_intck *p, const char *zFmt, ...){ + va_list ap; + char *zRet = 0; + va_start(ap, zFmt); + zRet = sqlite3_vmprintf(zFmt, ap); + if( p->rc==SQLITE_OK ){ + if( zRet==0 ){ + p->rc = SQLITE_NOMEM; + } + }else{ + sqlite3_free(zRet); + zRet = 0; + } + return zRet; +} + +/* +** Free the sqlite3_intck.apKeyVal, if it is allocated and populated. +*/ +static void intckSavedKeyClear(sqlite3_intck *p){ + if( p->apKeyVal ){ + int ii; + for(ii=0; iinKeyVal; ii++){ + sqlite3_value_free(p->apKeyVal[ii]); + } + sqlite3_free(p->apKeyVal); + p->apKeyVal = 0; + } +} + +/* +** If the apKeyVal array is currently allocated and populated, return +** a pointer to a buffer containing a nul-terminated string representing +** the values as an SQL vector. e.g. +** +** "('abc', NULL, 2)" +** +** If apKeyVal is not allocated, return NULL. Or, if an error (e.g. OOM) +** occurs within this function, set sqlite3_intck.rc before returning +** and return NULL. +*/ +static char *intckSavedKeyToText(sqlite3_intck *p){ + char *zRet = 0; + if( p->apKeyVal ){ + int ii; + const char *zSep = "SELECT '(' || "; + char *zSql = 0; + sqlite3_stmt *pStmt = 0; + + for(ii=0; iinKeyVal; ii++){ + zSql = intckMprintf(p, "%z%squote(?)", zSql, zSep); + zSep = " || ', ' || "; + } + zSql = intckMprintf(p, "%z || ')'", zSql); + + pStmt = intckPrepare(p, "%s", zSql); + if( p->rc==SQLITE_OK ){ + for(ii=0; iinKeyVal; ii++){ + sqlite3_bind_value(pStmt, ii+1, p->apKeyVal[ii]); + } + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + zRet = intckStrdup(p, (const char*)sqlite3_column_text(pStmt, 0)); + } + intckFinalize(p, pStmt); + } + sqlite3_free(zSql); + } + return zRet; +} + static void intckFindObject(sqlite3_intck *p){ sqlite3_stmt *pStmt = 0; char *zPrev = p->zObj; @@ -112,7 +224,7 @@ static void intckFindObject(sqlite3_intck *p){ "SELECT table_name FROM tables " "WHERE ?1 IS NULL OR table_name%s?1 " "ORDER BY 1" - , p->zDb, (p->zKey ? ">=" : ">") + , p->zDb, (p->apKeyVal ? ">=" : ">") ); if( p->rc==SQLITE_OK ){ @@ -125,8 +237,7 @@ static void intckFindObject(sqlite3_intck *p){ /* If this is a new object, ensure the previous key value is cleared. */ if( sqlite3_stricmp(p->zObj, zPrev) ){ - sqlite3_free(p->zKey); - p->zKey = 0; + intckSavedKeyClear(p); } sqlite3_free(zPrev); @@ -332,7 +443,8 @@ static void intckExec(sqlite3_intck *p, const char *zSql){ static char *intckCheckObjectSql( sqlite3_intck *p, const char *zObj, - const char *zPrev + const char *zPrev, + int *pnKeyVal /* OUT: Number of key-values for this scan */ ){ char *zRet = 0; sqlite3_stmt *pStmt = 0; @@ -377,7 +489,16 @@ static char *intckCheckObjectSql( ")" "" "" - ", tabpk(db, tab, idx, o_pk, i_pk, q_pk, eq_pk, ps_pk, pk_pk) AS (" + /* + ** For a PK declared as "PRIMARY KEY(a, b) ... WITHOUT ROWID", where + ** the intck_wrapper aliases of "a" and "b" are "c1" and "c2": + ** + ** o_pk: "o.c1, o.c2" + ** i_pk: "i.'a', i.'b'" + ** ... + ** n_pk: 2 + */ + ", tabpk(db, tab, idx, o_pk, i_pk, q_pk, eq_pk, ps_pk, pk_pk, n_pk) AS (" " WITH pkfields(f, a) AS (" " SELECT i.col_name, i.col_alias FROM idx_cols i WHERE i.idx_ispk" " )" @@ -390,7 +511,8 @@ static char *intckCheckObjectSql( " group_concat(format('\"%w\"', f), ', ')" " )," " group_concat('%s', ',')," - " group_concat('quote('||a||')', ', ') " + " group_concat('quote('||a||')', ', '), " + " count(*)" " FROM tabname t, pkfields" ")" "" @@ -467,11 +589,10 @@ static char *intckCheckObjectSql( " FROM tabname t, tabpk p, idx_cols i WHERE i.idx_name=t.idx" ")" "" - ", thiskey(k) AS (" - " SELECT format('format(''(%%s,%%s)'', %%s, %%s) AS thiskey', " - " group_concat('%%s', ','), p.ps_pk, " - " group_concat('quote('||i.col_alias||')',', '), p.pk_pk" - " ) FROM tabpk p, idx_cols i WHERE i.idx_name=p.idx" + ", thiskey(k, n) AS (" + " SELECT group_concat(i.col_alias, ', ') || ', ' || p.o_pk, " + " count(*) + p.n_pk " + " FROM tabpk p, idx_cols i WHERE i.idx_name=p.idx" ")" "" ", whereclause(w_c) AS (" @@ -483,15 +604,15 @@ static char *intckCheckObjectSql( " FROM tabpk, tabname, idx_cols i WHERE i.idx_name=tabpk.idx" ")" "" - ", main_select(m) AS (" + ", main_select(m, n) AS (" " SELECT format(" " 'WITH %%s\nSELECT %%s,\n%%s\nFROM intck_wrapper AS o%%s'," " ww.s, c, t.k, whereclause.w_c" - " )" + " ), t.n" " FROM case_statement, wrapper_with ww, thiskey t, whereclause" ")" - "SELECT m FROM main_select" + "SELECT m, n FROM main_select" , p->zDb, p->zDb, zObj, zObj , zPrev, zCommon ); @@ -549,9 +670,8 @@ static char *intckCheckObjectSql( ** ** format('(%d,%d)', _rowid_, n.ii) */ - ", thiskey(k) AS (" - " SELECT 'format(''(' || ps_pk || ',%%d)'', ' || pk_pk || ', n.ii)'" - " FROM tabpk" + ", thiskey(k, n) AS (" + " SELECT o_pk || ', n.ii', n_pk+1 FROM tabpk" ")" "" ", whereclause(w_c) AS (" @@ -562,17 +682,17 @@ static char *intckCheckObjectSql( " FROM tabpk, tabname" ")" "" - ", main_select(m) AS (" + ", main_select(m, n) AS (" " SELECT format(" " '%%s, %%s\nSELECT %%s,\n%%s AS thiskey\nFROM intck_wrapper AS o" ", intck_counter AS n%%s\nORDER BY %%s', " " w, ww.s, c, thiskey.k, whereclause.w_c, t.o_pk" - " )" + " ), thiskey.n" " FROM case_statement, tabpk t, counter_with, " " wrapper_with ww, thiskey, whereclause" ")" - "SELECT m FROM main_select", + "SELECT m, n FROM main_select", p->zDb, zObj, zPrev, zCommon ); } @@ -590,6 +710,9 @@ static char *intckCheckObjectSql( fflush(stdout); #else zRet = intckStrdup(p, (const char*)sqlite3_column_text(pStmt, 0)); + if( pnKeyVal ){ + *pnKeyVal = sqlite3_column_int(pStmt, 1); + } #endif } intckFinalize(p, pStmt); @@ -599,11 +722,14 @@ static char *intckCheckObjectSql( } static void intckCheckObject(sqlite3_intck *p){ - char *zSql = intckCheckObjectSql(p, p->zObj, p->zKey); + char *zSql = 0; + char *zKey = 0; + zKey = intckSavedKeyToText(p); + zSql = intckCheckObjectSql(p, p->zObj, zKey, &p->nKeyVal); p->pCheck = intckPrepare(p, "%s", zSql); sqlite3_free(zSql); - sqlite3_free(p->zKey); - p->zKey = 0; + sqlite3_free(zKey); + intckSavedKeyClear(p); } int sqlite3_intck_open( @@ -644,7 +770,7 @@ int sqlite3_intck_close(sqlite3_intck *p){ ); } sqlite3_free(p->zObj); - sqlite3_free(p->zKey); + intckSavedKeyClear(p); sqlite3_free(p->zTestSql); sqlite3_free(p->zErr); sqlite3_free(p); @@ -668,12 +794,13 @@ int sqlite3_intck_step(sqlite3_intck *p){ if( p->rc==SQLITE_OK ){ assert( p->pCheck ); if( sqlite3_step(p->pCheck)==SQLITE_ROW ){ - /* Fine, whatever... */ + /* Normal case, do nothing. */ }else{ if( sqlite3_finalize(p->pCheck)!=SQLITE_OK ){ intckSaveErrmsg(p); } p->pCheck = 0; + p->nKeyVal = 0; } } } @@ -693,10 +820,27 @@ int sqlite3_intck_error(sqlite3_intck *p, const char **pzErr){ return (p->rc==SQLITE_DONE ? SQLITE_OK : p->rc); } + +static sqlite3_value *intckValueDup(sqlite3_intck *p, sqlite3_value *pIn){ + sqlite3_value *pRet = 0; + if( p->rc==SQLITE_OK ){ + pRet = sqlite3_value_dup(pIn); + if( pRet==0 ){ + p->rc = SQLITE_NOMEM; + } + } + return pRet; +} + int sqlite3_intck_suspend(sqlite3_intck *p){ if( p->pCheck && p->rc==SQLITE_OK ){ - assert( p->zKey==0 ); - p->zKey = intckStrdup(p, (const char*)sqlite3_column_text(p->pCheck, 1)); + const int nByte = sizeof(sqlite3_value*) * p->nKeyVal; + int ii; + assert( p->apKeyVal==0 && p->nKeyVal>0 ); + p->apKeyVal = (sqlite3_value**)intckMalloc(p, nByte); + for(ii=0; p->rc==SQLITE_OK && iinKeyVal; ii++){ + p->apKeyVal[ii] = intckValueDup(p, sqlite3_column_value(p->pCheck, ii+1)); + } intckFinalize(p, p->pCheck); p->pCheck = 0; } @@ -706,10 +850,12 @@ int sqlite3_intck_suspend(sqlite3_intck *p){ const char *sqlite3_intck_test_sql(sqlite3_intck *p, const char *zObj){ sqlite3_free(p->zTestSql); if( zObj ){ - p->zTestSql = intckCheckObjectSql(p, zObj, 0); + p->zTestSql = intckCheckObjectSql(p, zObj, 0, 0); }else{ if( p->zObj ){ - p->zTestSql = intckCheckObjectSql(p, p->zObj, p->zKey); + char *zKey = intckSavedKeyToText(p); + p->zTestSql = intckCheckObjectSql(p, p->zObj, zKey, 0); + sqlite3_free(zKey); }else{ sqlite3_free(p->zTestSql); p->zTestSql = 0; diff --git a/manifest b/manifest index e254a597a2..39fc17cfc5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Consider\susing\s"="\sand\sIS\soperators\swith\seven\slow-quality\sindexes\sin\scases\swhere\sthey\sare\sselected\sexplicitly\susing\san\sINDEXED\sBY\sclause. -D 2024-02-20T16:04:27.694 +C Use\sfewer\scycles\sto\sgenerate\sthe\s"next\skey"\svalue\sused\sby\ssqlite3_intck_suspend()\sfunction\sin\sthe\sintck\sextension. +D 2024-02-20T18:17:06.096 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -251,7 +251,7 @@ F ext/icu/sqliteicu.h fa373836ed5a1ee7478bdf8a1650689294e41d0c89c1daab26e9ae78a3 F ext/intck/intck1.test 5b3c9800e119b4dd50a381974f34cee6cfd5b7434286fb8da83b7c8ff1d6bb3c F ext/intck/intck2.test b65d7f627342f767e1d2c447d25619666cec36f516786dd56568bd741e5d7e67 F ext/intck/intck_common.tcl 2895854e7aaf5e199a15f6f82538a00999fd8fc55553bc1f04619af7aa86c0d0 -F ext/intck/sqlite3intck.c 5f319b7c72b0c01cfa28bb5fdd19be5781eb11f5a5216af2a0870dc7e001414d +F ext/intck/sqlite3intck.c 7a795f23424a29f656f3d4c7b83d23484746b57cdc25d3fb98ec805d017fc935 F ext/intck/sqlite3intck.h d9501ea480b7c41c0555f39f4f1b7c3e8d54fc1ea6d115de5e1211e0bc11d3e7 F ext/intck/test_intck.c 06206b35f1428961015c060dd35201246c849625cfdff461e0eeaaf76bda545c F ext/jni/GNUmakefile 59eb05f2a363bdfac8d15d66bed624bfe1ff289229184f3861b95f98a19cf4b2 @@ -2168,8 +2168,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c01e008c28895e50b14531b2a1f3f1110aab3b54df41ebdbd416fbac7b1bba94 -R 8db173421c176ccb487011bd9440e123 +P 43cbbea82132db2d0ddb4f34cc2b6910b3a1243ae6d4e837b1b27bfe91b84834 +R 9dcf98c64d0302d62dce4c3c1b529641 U dan -Z d4e70ffeb295fbc1a103b9adca306ecc +Z c2dbf2f2090bfeaba66446a197f3ad62 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 4565b3d528..69d33b6938 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -43cbbea82132db2d0ddb4f34cc2b6910b3a1243ae6d4e837b1b27bfe91b84834 \ No newline at end of file +95f01426f948cf435d0b400dc7ae06fa699eee32cff498fe77e74a1257a4e09b \ No newline at end of file From eb715f022f71d912e11b51ba7ab68f77d92e0234 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 20 Feb 2024 20:18:02 +0000 Subject: [PATCH 06/16] Have the intck extension better handle corruption at the b-tree layer. FossilOrigin-Name: ecd775d108f77d39a1303316c1e0f0b0ae3ffc5218222e1ebfe2ef6783829b85 --- ext/intck/intck1.test | 2 +- ext/intck/intck_common.tcl | 4 +- ext/intck/intckcorrupt.test | 235 ++++++++++++++++++++++++++++++++++++ ext/intck/sqlite3intck.c | 53 ++++++-- ext/intck/sqlite3intck.h | 3 +- ext/intck/test_intck.c | 11 +- manifest | 21 ++-- manifest.uuid | 2 +- 8 files changed, 297 insertions(+), 34 deletions(-) create mode 100644 ext/intck/intckcorrupt.test diff --git a/ext/intck/intck1.test b/ext/intck/intck1.test index 7f60f1fdc6..4110ece049 100644 --- a/ext/intck/intck1.test +++ b/ext/intck/intck1.test @@ -106,7 +106,7 @@ do_eqp_test 1.6.2 { reset_db do_test 2.0 { - set ic [sqlite3_intck db main ""] + set ic [sqlite3_intck db main] $ic close } {} diff --git a/ext/intck/intck_common.tcl b/ext/intck/intck_common.tcl index 757ca82b3f..f00a465b3c 100644 --- a/ext/intck/intck_common.tcl +++ b/ext/intck/intck_common.tcl @@ -16,7 +16,7 @@ if {![info exists testdir]} { source $testdir/tester.tcl proc do_intck {db {bSuspend 0}} { - set ic [sqlite3_intck $db main ""] + set ic [sqlite3_intck $db main] set ret [list] while {"SQLITE_OK"==[$ic step]} { @@ -37,7 +37,7 @@ proc do_intck {db {bSuspend 0}} { } proc intck_sql {db tbl} { - set ic [sqlite3_intck $db main ""] + set ic [sqlite3_intck $db main] set sql [$ic test_sql $tbl] $ic close return $sql diff --git a/ext/intck/intckcorrupt.test b/ext/intck/intckcorrupt.test new file mode 100644 index 0000000000..40f009f9c2 --- /dev/null +++ b/ext/intck/intckcorrupt.test @@ -0,0 +1,235 @@ +# 2024 Feb 21 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# The focus of this file is testing the intck extensions response +# to corruption at the b-tree level. +# + +source [file join [file dirname [info script]] intck_common.tcl] +set testprefix intckcorrupt + +#------------------------------------------------------------------------- +reset_db +do_test 1.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +| size 356352 pagesize 4096 filename crash-acaae0347204ae.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 d0 00 00 00 .....@ ........ +| 32: 40 00 ea 00 00 00 00 00 00 40 00 00 00 40 00 00 @........@...@.. +| 96: 00 00 00 00 0d 00 00 00 04 0e 9c 00 0f ad 0f 4f ...............O +| 112: 0e fc 0e 9c 00 00 00 00 00 00 00 00 00 00 00 00 ................ +| 3728: 00 00 00 00 00 00 00 00 00 00 00 00 5e 04 07 17 ............^... +| 3744: 1f 1f 01 81 0b 74 61 62 6c 65 74 31 5f 70 61 72 .....tablet1_par +| 3760: 65 6e 74 74 31 5f 70 61 72 65 6e 74 04 43 52 45 entt1_parent.CRE +| 3776: 41 54 45 20 54 41 42 4c 45 20 22 74 31 5f 70 61 ATE TABLE .t1_pa +| 3792: 72 65 6e 74 22 28 6e 6f 64 65 6e 6f 20 49 4e 54 rent.(nodeno INT +| 3808: 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 EGER PRIMARY KEY +| 3824: 2c 70 61 72 65 6e 74 6e 6f 64 65 29 51 03 06 17 ,parentnode)Q... +| 3840: 1b 1b 01 7b 74 61 62 6c 65 74 31 5f 6e 6f 64 65 ....tablet1_node +| 3856: 74 31 5f 6e 6f 64 65 03 43 52 45 41 54 45 20 54 t1_node.CREATE T +| 3872: 41 42 4c 45 20 22 74 31 5f 6e 6f 64 65 22 28 6e ABLE .t1_node.(n +| 3888: 6f 64 65 6e 6f 20 49 4e 54 45 47 45 52 20 50 52 odeno INTEGER PR +| 3904: 49 4d 41 52 59 20 4b 45 59 2c 64 61 74 61 29 5c IMARY KEY,data). +| 3920: 02 07 17 1d 1d 01 81 0b 74 61 62 6c 65 74 31 5f ........tablet1_ +| 3936: 72 6f 77 69 64 74 31 5f 72 6f 77 69 64 02 43 52 rowidt1_rowid.CR +| 3952: 45 41 54 45 20 54 41 42 4c 45 20 22 74 31 5f 72 EATE TABLE .t1_r +| 3968: 6f 77 69 64 22 28 72 6f 77 69 64 20 49 4e 54 45 owid.(rowid INTE +| 3984: 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c GER PRIMARY KEY, +| 4000: 6e 6f 64 65 6e 6f 2c 61 30 2c 61 31 29 51 01 07 nodeno,a0,a1)Q.. +| 4016: 17 11 11 08 81 0f 74 61 62 6c 65 74 31 74 31 43 ......tablet1t1C +| 4032: 52 45 41 54 45 20 56 49 52 54 55 41 4c 20 54 41 REATE VIRTUAL TA +| 4048: 42 4c 45 20 74 31 20 55 53 49 4e 47 20 72 74 72 BLE t1 USING rtr +| 4064: 65 65 28 69 64 2c 78 30 20 50 52 49 4d 41 52 59 ee(id,x0 PRIMARY +| 4080: 20 4b 45 59 2c 70 61 72 65 6e 74 6e 6f 64 65 29 KEY,parentnode) +| page 2 offset 4096 +| 0: 51 03 06 17 1b 1b 01 7b 74 61 62 6c 65 74 31 5f Q.......tablet1_ +| 16: 6e 6f 64 65 74 31 5f 6e 6f 64 65 03 43 52 45 41 nodet1_node.CREA +| 32: 54 45 20 54 41 42 4c 45 20 22 74 31 5f 6e 6f 64 TE TABLE .t1_nod +| 48: 65 22 28 6e 6f 64 65 6e 6f 20 49 4e 54 45 47 45 e.(nodeno INTEGE +| 64: 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 64 61 R PRIMARY KEY,da +| 80: 74 61 29 5c 02 07 17 1d 1d 01 81 0b 74 61 62 6c ta).........tabl +| 96: 65 74 31 5f 72 6f 77 69 64 74 31 5f 72 6f 77 69 et1_rowidt1_rowi +| 112: 64 02 43 52 45 41 54 45 20 54 41 42 4c 45 00 00 d.CREATE TABLE.. +| 128: 01 0a 02 00 00 00 01 0e 0d 00 00 00 00 24 0e 0d .............$.. +| 144: 0c 1a 06 85 50 46 60 27 70 08 00 00 00 00 00 00 ....PF`'p....... +| 3824: 00 00 00 00 00 00 00 0d 0e 05 00 09 1d 00 74 6f ..............to +| 3840: 79 20 68 61 6c 66 10 0d 05 00 09 23 00 62 6f 74 y half.....#.bot +| 3856: 74 6f 6d 20 68 61 6c 66 0f 0c 05 00 09 21 00 72 tom half.....!.r +| 3872: 69 67 68 74 20 68 61 6c 66 0e 0b 05 00 09 1f 00 ight half....... +| 3888: 6c 65 66 74 20 43 15 f6 e6 f6 46 50 34 35 24 54 left C....FP45$T +| 3904: 15 44 52 05 44 14 24 c4 52 02 27 43 15 f6 e6 f6 .DR.D.$.R.'C.... +| 3920: 46 52 22 8e 6f 64 65 6e 6f 20 49 4e 54 45 47 45 FR..odeno INTEGE +| 3936: 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 64 61 R PRIMARY KEY,da +| 3952: 74 61 29 5c 02 07 17 1d 1d 01 81 0b 74 61 62 6c ta).........tabl +| 3968: 65 74 31 5f 72 6f 74 74 6f 6d 20 65 64 67 65 0f et1_rottom edge. +| 3984: 07 05 00 09 21 00 72 69 67 68 74 20 65 64 67 65 ....!.right edge +| 4000: 0e 06 05 00 09 1f 00 6c 65 66 74 20 65 64 67 65 .......left edge +| 4016: 0b 05 05 00 09 19 00 63 65 6e 74 65 72 17 04 05 .......center... +| 4032: 00 09 31 00 75 70 70 65 72 2d 72 69 67 68 74 20 ..1.upper-right +| 4048: 63 6f 72 6e 65 72 17 03 05 00 09 31 00 6c 6f 77 corner.....1.low +| 4064: 65 72 2d 72 69 67 68 74 20 63 6f 72 6e 65 72 16 er-right corner. +| 4080: 02 05 00 09 2f 00 75 70 70 65 72 2d 6c 65 66 74 ..../.upper-left +| page 3 offset 8192 +| 0: 20 63 6f 72 6e 65 72 16 01 05 00 09 2f 01 8c 6f corner...../..o +| 16: 77 65 72 2d 6c 53 51 4c 69 74 65 20 66 6f 72 6d wer-lSQLite form +| 32: 61 74 20 33 00 10 00 01 01 00 40 20 20 00 00 00 at 3......@ ... +| 48: 00 00 00 00 2f 00 00 0d eb 13 00 00 00 03 00 00 ..../........... +| 64: 00 04 00 00 00 00 00 00 00 06 00 00 00 01 00 00 ................ +| 80: 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 ................ +| page 6 offset 20480 +| 128: 00 00 00 00 00 00 00 00 97 3d 04 ae 7c 01 00 00 .........=..|... +| 624: 00 00 00 00 00 00 21 97 3d 04 ae 7c 01 00 00 00 ......!.=..|.... +| 1120: 00 00 00 00 00 20 97 3d 04 ae 7c 01 00 00 00 00 ..... .=..|..... +| 1616: 00 00 00 00 1f 97 3d 04 ae 7c 01 00 00 00 00 00 ......=..|...... +| 2112: 00 00 00 1e 97 3d 04 ae 7c 01 00 00 00 00 00 00 .....=..|....... +| 2608: 00 00 1d 97 d3 d0 4a e7 c0 00 00 00 00 00 00 00 ......J......... +| 3088: 00 00 00 00 00 00 00 00 00 00 00 00 01 f3 00 00 ................ +| 3600: 23 97 3d 04 ae 7c 01 00 00 00 00 00 00 00 00 00 #.=..|.......... +| 4080: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 26 ...............& +| page 8 offset 28672 +| 0: 0d 00 00 00 01 04 30 00 04 30 00 00 00 00 00 00 ......0..0...... +| 1072: 97 4d 1e 14 00 ae 7c 00 00 00 00 00 00 00 00 00 .M....|......... +| 1088: 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 ................ +| 4080: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 ................ +| page 10 offset 36864 +| 0: 0d 00 00 00 01 04 30 00 04 30 00 00 00 00 00 00 ......0..0...... +| 1072: 9a ee c1 80 fd 78 1f ce 1b ae eb b4 00 00 00 00 .....x.......... +| 1088: 13 20 ff 20 00 70 00 00 00 60 50 00 00 00 11 e0 . . .p...`P..... +| 1104: 00 00 00 70 00 00 00 60 50 05 35 14 c6 97 46 52 ...p...`P.5...FR +| 1120: 06 66 f7 26 d6 17 42 03 30 01 00 00 10 10 04 02 .f.&..B.0....... +| 1136: 02 00 00 00 00 00 00 00 00 40 00 00 00 00 00 00 .........@...... +| 1152: 00 00 00 00 00 40 00 00 00 40 00 00 00 00 00 00 .....@...@...... +| 4080: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 05 ................ +| page 12 offset 45056 +| 0: 0d 00 00 00 01 04 30 00 04 30 e1 b4 30 97 4d 46 ......0..0..0.MF +| 16: 14 00 ae 7c 00 00 00 00 00 00 00 03 00 00 43 00 ...|..........C. +| page 47 offset 188416 +| 2512: 00 00 00 00 00 00 00 00 be 00 00 00 00 00 00 00 ................ +| page 87 offset 352256 +| 2512: 00 00 00 00 00 00 00 00 aa 00 00 00 00 00 00 00 ................ +| end crash-acaae0347204ae.db +}]} {} + +do_intck_test 1.1 { + {corruption found while reading database schema} +} + +#------------------------------------------------------------------------- +reset_db +do_test 2.0 { + sqlite3 db {} + db deserialize [decode_hexdb { +| size 28672 pagesize 4096 filename crash-3afa1ca9e9c1bd.db +| page 1 offset 0 +| 0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00 SQLite format 3. +| 16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 07 .....@ ........ +| 32: 00 00 00 00 00 00 00 00 00 00 00 06 00 00 00 04 ................ +| 48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 ................ +| 96: 00 00 00 00 0d 00 00 00 06 0e 88 00 0f b8 0f 6d ...............m +| 112: 0f 3a 0f 0b 0e d5 0e 88 01 00 00 00 00 00 00 00 .:.............. +| 3712: 00 00 00 00 00 00 00 00 4b 06 06 17 25 25 01 5b ........K...%%.[ +| 3728: 74 61 62 6c 65 73 71 6c 69 74 65 5f 73 74 61 74 tablesqlite_stat +| 3744: 31 73 71 6c 69 74 65 5f 73 74 61 74 31 07 43 52 1sqlite_stat1.CR +| 3760: 45 41 54 45 20 54 41 42 4c 45 20 73 71 6c 69 74 EATE TABLE sqlit +| 3776: 65 5f 73 74 61 74 31 28 74 62 6c 2c 69 64 78 2c e_stat1(tbl,idx, +| 3792: 73 74 61 74 29 34 05 06 17 13 11 01 53 69 6e 64 stat)4......Sind +| 3808: 65 78 63 31 63 63 31 06 43 52 45 41 54 45 20 55 exc1cc1.CREATE U +| 3824: 4e 49 51 55 45 20 49 4e 44 45 58 20 63 31 63 20 NIQUE INDEX c1c +| 3840: 4f 4e 20 63 31 28 63 2c 20 62 29 2d 04 06 17 13 ON c1(c, b)-.... +| 3856: 11 01 45 69 6e 64 65 78 63 31 64 63 31 05 43 52 ..Eindexc1dc1.CR +| 3872: 45 41 54 45 20 49 4e 44 45 58 20 63 31 64 20 4f EATE INDEX c1d O +| 3888: 4e 20 63 31 28 64 2c 20 62 29 31 03 06 17 13 11 N c1(d, b)1..... +| 3904: 01 4d 69 6e 64 65 78 62 31 63 62 31 05 43 52 45 .Mindexb1cb1.CRE +| 3920: 41 54 45 20 55 4e 49 51 55 45 20 49 4e 44 45 58 ATE UNIQUE INDEX +| 3936: 20 62 31 63 20 4f 4e 20 62 31 28 63 29 49 02 06 b1c ON b1(c)I.. +| 3952: 17 11 11 0f 7f 74 61 62 6c 65 63 31 63 31 03 43 .....tablec1c1.C +| 3968: 52 45 41 54 45 20 54 41 42 4c 45 20 63 31 28 61 REATE TABLE c1(a +| 3984: 20 49 4e 54 20 50 52 49 4d 41 52 59 20 4b 45 59 INT PRIMARY KEY +| 4000: 2c 20 62 2c 20 63 2c 20 64 29 20 57 49 54 48 4f , b, c, d) WITHO +| 4016: 55 54 20 52 4f 57 49 44 46 01 06 17 11 11 01 79 UT ROWIDF......y +| 4032: 74 61 62 6c 65 62 31 62 31 02 43 52 45 41 54 45 tableb1b1.CREATE +| 4048: 20 54 41 42 4c 45 20 62 31 28 61 20 49 4e 54 20 TABLE b1(a INT +| 4064: 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 62 2c 20 PRIMARY KEY, b, +| 4080: 63 29 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 c) WITHOUT ROWID +| page 2 offset 4096 +| 0: 0a 00 00 00 07 0f ca 00 0f fa 0f f2 0f ea 0f e2 ................ +| 16: 0f da 00 00 00 01 00 00 00 00 00 00 00 00 00 00 ................ +| 4032: 00 00 00 00 00 00 00 00 00 00 07 04 01 0f 01 06 ................ +| 4048: 67 07 07 04 01 0f 01 06 66 06 07 04 01 0f 01 05 g.......f....... +| 4064: 65 05 07 04 01 0f 01 04 64 04 07 04 01 0f 01 03 e.......d....... +| 4080: 63 03 07 04 01 0f 01 02 62 0f 05 04 09 0f 09 61 c.......b......a +| page 3 offset 8192 +| 0: 0a 00 00 00 07 0f bd 00 0f f9 0f ef 0f e5 0f db ................ +| 16: 0f d1 0f c7 0f bd 00 00 00 00 01 00 00 00 00 00 ................ +| 4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 09 05 01 ................ +| 4032: 0f 01 01 07 61 07 07 09 05 01 0f 01 01 06 61 06 ....a.........a. +| 4048: 06 09 05 01 0f 01 01 05 61 05 05 09 05 01 0f 01 ........a....... +| 4064: 01 04 61 04 04 09 05 01 0f 01 01 03 61 03 03 09 ..a.........a... +| 4080: 05 01 0f 01 01 02 61 0f 02 06 05 09 0f 09 09 61 ......a........a +| page 4 offset 12288 +| 0: 0a 00 00 00 07 0f d8 00 0f fc 0f f0 0f ea 0f e4 ................ +| 16: 0f de 0f d8 0f f6 00 00 00 00 00 00 00 00 00 00 ................ +| 4048: 00 00 00 00 00 00 00 00 05 03 01 01 07 07 05 03 ................ +| 4064: 01 01 06 06 05 03 01 01 05 05 05 03 01 01 04 04 ................ +| 4080: 05 03 01 01 03 03 05 03 01 01 0f 02 03 03 09 09 ................ +| page 5 offset 16384 +| 0: 0a 00 00 00 07 0f ca 00 0f fa 0f f2 0f ea 0f 00 ................ +| 4032: 00 00 00 00 00 00 00 00 00 00 07 04 01 0f 01 07 ................ +| 4048: 61 07 07 04 01 0f 01 06 61 06 07 04 01 0f 01 05 a.......a....... +| 4064: 61 05 07 04 01 1f 01 04 61 04 07 04 01 0f 01 03 a.......a....... +| 4080: 61 03 07 04 01 0f 01 02 61 02 05 04 09 0f 09 61 a.......a......a +| page 6 offset 20480 +| 0: 0a 00 00 00 07 0f ca 00 0f fa 0f ea 0f e2 00 00 ................ +| 4032: 00 00 00 00 00 00 00 00 00 00 07 04 01 0f 01 07 ................ +| 4048: 61 07 07 04 01 0f 01 06 61 06 07 04 01 0f 01 05 a.......a....... +| 4064: 61 05 07 04 01 0f 01 04 61 04 07 04 01 0f 01 03 a.......a....... +| 4080: 61 03 07 04 01 0f 01 0f 61 02 05 04 09 0f 09 61 a.......a......a +| page 7 offset 24576 +| 0: 0d 00 00 00 05 0f 1c 00 0f f0 0f e0 0f d3 0f c5 ................ +| 16: 0f b8 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ +| 4016: 00 00 00 00 00 00 00 00 0b 05 04 11 11 13 62 31 ..............b1 +| 4032: 62 31 37 20 31 0c 04 04 11 13 13 62 31 62 31 63 b17 1......b1b1c +| 4048: 37 20 31 0b 03 04 11 11 13 63 31 63 31 37 20 31 7 1......c1c17 1 +| 4064: 0e 02 04 11 13 07 63 31 63 31 64 37 20 31 20 31 ......c1c1d7 1 1 +| 4080: 0e 01 04 11 13 17 63 31 63 31 63 37 20 31 00 00 ......c1c1c7 1.. +| end crash-3afa1ca9e9c1bd.db +}]} {} + +do_intck_test 2.1 { + {corruption found while reading database schema} +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 3.0 { + PRAGMA page_size = 1024; + CREATE TABLE t1(a, b); + CREATE INDEX i1 ON t1(a); + INSERT INTO t1 VALUES(1, 1), (2, 2), (3, 3); +} + +do_test 3.1 { + set pgno [db one {SELECT rootpage FROM sqlite_schema WHERE name='t1'}] + db close + hexio_write test.db [expr ($pgno-1)*1024] 0000 +} {2} + +sqlite3 db test.db +do_intck_test 3.2 { + {corruption found while scanning database object i1} + {corruption found while scanning database object t1} +} + +finish_test + + diff --git a/ext/intck/sqlite3intck.c b/ext/intck/sqlite3intck.c index 6d54526928..f23dbbd1d7 100644 --- a/ext/intck/sqlite3intck.c +++ b/ext/intck/sqlite3intck.c @@ -35,6 +35,9 @@ struct sqlite3_intck { int nKeyVal; sqlite3_value **apKeyVal; + char *zMessage; + int bCorruptSchema; + int rc; /* Error code */ char *zErr; /* Error message */ char *zTestSql; /* Returned by sqlite3_intck_test_sql() */ @@ -67,12 +70,12 @@ static sqlite3_stmt *intckPrepare(sqlite3_intck *p, const char *zFmt, ...){ p->rc = sqlite3_prepare_v2(p->db, zSql, -1, &pRet, 0); fflush(stdout); if( p->rc!=SQLITE_OK ){ -#if 1 +#if 0 printf("ERROR: %s\n", zSql); printf("MSG: %s\n", sqlite3_errmsg(p->db)); #endif if( sqlite3_error_offset(p->db)>=0 ){ -#if 1 +#if 0 int iOff = sqlite3_error_offset(p->db); printf("AT: %.40s\n", &zSql[iOff]); #endif @@ -208,12 +211,19 @@ static char *intckSavedKeyToText(sqlite3_intck *p){ return zRet; } +/* +** Find the next database object (table or index) to check. If successful, +** set sqlite3_intck.zObj to point to a nul-terminated buffer containing +** the object's name before returning. +*/ static void intckFindObject(sqlite3_intck *p){ sqlite3_stmt *pStmt = 0; char *zPrev = p->zObj; p->zObj = 0; assert( p->rc==SQLITE_OK ); + assert( p->pCheck==0 ); + pStmt = intckPrepare(p, "WITH tables(table_name) AS (" " SELECT name" @@ -735,7 +745,6 @@ static void intckCheckObject(sqlite3_intck *p){ int sqlite3_intck_open( sqlite3 *db, /* Database handle to operate on */ const char *zDbArg, /* "main", "temp" etc. */ - const char *zFile, /* Path to save-state db on disk (or NULL) */ sqlite3_intck **ppOut /* OUT: New integrity-check handle */ ){ sqlite3_intck *pNew = 0; @@ -760,10 +769,8 @@ int sqlite3_intck_open( return rc; } -int sqlite3_intck_close(sqlite3_intck *p){ - int rc = SQLITE_OK; +void sqlite3_intck_close(sqlite3_intck *p){ if( p ){ - rc = (p->rc==SQLITE_DONE ? SQLITE_OK : p->rc); if( p->db ){ sqlite3_create_function( p->db, "parse_create_index", 1, SQLITE_UTF8, 0, 0, 0, 0 @@ -775,11 +782,19 @@ int sqlite3_intck_close(sqlite3_intck *p){ sqlite3_free(p->zErr); sqlite3_free(p); } - return rc; } int sqlite3_intck_step(sqlite3_intck *p){ if( p->rc==SQLITE_OK ){ + + if( p->zMessage ){ + sqlite3_free(p->zMessage); + p->zMessage = 0; + } + + if( p->bCorruptSchema ){ + p->rc = SQLITE_DONE; + }else if( p->pCheck==0 ){ intckFindObject(p); if( p->rc==SQLITE_OK ){ @@ -788,19 +803,29 @@ int sqlite3_intck_step(sqlite3_intck *p){ }else{ p->rc = SQLITE_DONE; } + }else if( p->rc==SQLITE_CORRUPT ){ + p->rc = SQLITE_OK; + p->zMessage = intckStrdup(p, + "corruption found while reading database schema" + ); + p->bCorruptSchema = 1; } } - if( p->rc==SQLITE_OK ){ - assert( p->pCheck ); + if( p->pCheck ){ + assert( p->rc==SQLITE_OK ); if( sqlite3_step(p->pCheck)==SQLITE_ROW ){ /* Normal case, do nothing. */ }else{ - if( sqlite3_finalize(p->pCheck)!=SQLITE_OK ){ - intckSaveErrmsg(p); - } + intckFinalize(p, p->pCheck); p->pCheck = 0; p->nKeyVal = 0; + if( p->rc==SQLITE_CORRUPT ){ + p->rc = SQLITE_OK; + p->zMessage = intckMprintf(p, + "corruption found while scanning database object %s", p->zObj + ); + } } } } @@ -809,6 +834,10 @@ int sqlite3_intck_step(sqlite3_intck *p){ } const char *sqlite3_intck_message(sqlite3_intck *p){ + assert( p->pCheck==0 || p->zMessage==0 ); + if( p->zMessage ){ + return p->zMessage; + } if( p->pCheck ){ return (const char*)sqlite3_column_text(p->pCheck, 0); } diff --git a/ext/intck/sqlite3intck.h b/ext/intck/sqlite3intck.h index c7c24e42c9..e39825fc7e 100644 --- a/ext/intck/sqlite3intck.h +++ b/ext/intck/sqlite3intck.h @@ -25,11 +25,10 @@ typedef struct sqlite3_intck sqlite3_intck; int sqlite3_intck_open( sqlite3 *db, const char *zDb, - const char *zFile, sqlite3_intck **ppOut ); -int sqlite3_intck_close(sqlite3_intck*); +void sqlite3_intck_close(sqlite3_intck*); int sqlite3_intck_step(sqlite3_intck *pCk); diff --git a/ext/intck/test_intck.c b/ext/intck/test_intck.c index 75bcfa298a..0e2aebf056 100644 --- a/ext/intck/test_intck.c +++ b/ext/intck/test_intck.c @@ -132,7 +132,7 @@ static void testIntckFree(void *clientData){ } /* -** tclcmd: sqlite3_intck DB DBNAME PATH +** tclcmd: sqlite3_intck DB DBNAME */ static int test_sqlite3_intck( void * clientData, @@ -149,8 +149,8 @@ static int test_sqlite3_intck( const char *zFile = 0; int rc = SQLITE_OK; - if( objc!=4 ){ - Tcl_WrongNumArgs(interp, 1, objv, "DB DBNAME PATH"); + if( objc!=3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB DBNAME"); return TCL_ERROR; } @@ -161,9 +161,8 @@ static int test_sqlite3_intck( return TCL_ERROR; } zDb = Tcl_GetString(objv[2]); - zFile = Tcl_GetString(objv[3]); - rc = sqlite3_intck_open(db, zDb, zFile, &p->intck); + rc = sqlite3_intck_open(db, zDb, &p->intck); if( rc!=SQLITE_OK ){ ckfree(p); Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3_errstr(rc), -1)); @@ -207,7 +206,7 @@ static int test_do_intck( pRet = Tcl_NewObj(); Tcl_IncrRefCount(pRet); - rc = sqlite3_intck_open(db, zDb, 0, &pCk); + rc = sqlite3_intck_open(db, zDb, &pCk); if( rc==SQLITE_OK ){ while( sqlite3_intck_step(pCk)==SQLITE_OK ){ const char *zMsg = sqlite3_intck_message(pCk); diff --git a/manifest b/manifest index 39fc17cfc5..6b38631176 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Use\sfewer\scycles\sto\sgenerate\sthe\s"next\skey"\svalue\sused\sby\ssqlite3_intck_suspend()\sfunction\sin\sthe\sintck\sextension. -D 2024-02-20T18:17:06.096 +C Have\sthe\sintck\sextension\sbetter\shandle\scorruption\sat\sthe\sb-tree\slayer. +D 2024-02-20T20:18:02.615 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -248,12 +248,13 @@ F ext/fts5/tool/showfts5.tcl d54da0e067306663e2d5d523965ca487698e722c F ext/icu/README.txt 7ab7ced8ae78e3a645b57e78570ff589d4c672b71370f5aa9e1cd7024f400fc9 F ext/icu/icu.c c074519b46baa484bb5396c7e01e051034da8884bad1a1cb7f09bbe6be3f0282 F ext/icu/sqliteicu.h fa373836ed5a1ee7478bdf8a1650689294e41d0c89c1daab26e9ae78a32075a8 -F ext/intck/intck1.test 5b3c9800e119b4dd50a381974f34cee6cfd5b7434286fb8da83b7c8ff1d6bb3c +F ext/intck/intck1.test 15e94ee868e939c6d3aef6884c5652d969b28d3eac940a15585511cf8aec71ad F ext/intck/intck2.test b65d7f627342f767e1d2c447d25619666cec36f516786dd56568bd741e5d7e67 -F ext/intck/intck_common.tcl 2895854e7aaf5e199a15f6f82538a00999fd8fc55553bc1f04619af7aa86c0d0 -F ext/intck/sqlite3intck.c 7a795f23424a29f656f3d4c7b83d23484746b57cdc25d3fb98ec805d017fc935 -F ext/intck/sqlite3intck.h d9501ea480b7c41c0555f39f4f1b7c3e8d54fc1ea6d115de5e1211e0bc11d3e7 -F ext/intck/test_intck.c 06206b35f1428961015c060dd35201246c849625cfdff461e0eeaaf76bda545c +F ext/intck/intck_common.tcl 6e5df126e55d6a0d7e3be9757400d5fb87bd291726d01b72a9fe77a7201967e4 +F ext/intck/intckcorrupt.test 3211ef68ac53e83951b6c8f6a8d2396506d123fe5898f97f848a25837744ec56 +F ext/intck/sqlite3intck.c 0f9674952f84084741eb4e945d7532b02996ba8ea938b060228d7874f15557f9 +F ext/intck/sqlite3intck.h dec4a37584024154ad5734cf6ee98dc484ce25c26af88652a096be402fe51833 +F ext/intck/test_intck.c eb84e825a3a0fd43ae1e4280305280a19c9ef42c2bdef3110b282b115de60463 F ext/jni/GNUmakefile 59eb05f2a363bdfac8d15d66bed624bfe1ff289229184f3861b95f98a19cf4b2 F ext/jni/README.md d899789a9082a07b99bf30b1bbb6204ae57c060efcaa634536fa669323918f42 F ext/jni/jar-dist.make 030aaa4ae71dd86e4ec5e7c1e6cd86f9dfa47c4592c070d2e35157e42498e1fa @@ -2168,8 +2169,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 43cbbea82132db2d0ddb4f34cc2b6910b3a1243ae6d4e837b1b27bfe91b84834 -R 9dcf98c64d0302d62dce4c3c1b529641 +P 95f01426f948cf435d0b400dc7ae06fa699eee32cff498fe77e74a1257a4e09b +R 2f81579b76b29691695c15522ce87fed U dan -Z c2dbf2f2090bfeaba66446a197f3ad62 +Z 5af5831e9af2fce3b2d0c2673f03eefa # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 69d33b6938..f905e93b9b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -95f01426f948cf435d0b400dc7ae06fa699eee32cff498fe77e74a1257a4e09b \ No newline at end of file +ecd775d108f77d39a1303316c1e0f0b0ae3ffc5218222e1ebfe2ef6783829b85 \ No newline at end of file From b2897c221da108280e88b1abcd9858c96dca4d99 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 21 Feb 2024 16:15:50 +0000 Subject: [PATCH 07/16] Ensure intck tests are run by testrunner.tcl. FossilOrigin-Name: 11d6816c060b6edb9cd61f29297ab95e75e2b46f29c0a796820d94fc13586f6d --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/permutations.test | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 7a0b8e41ea..43263453a2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\strunk\schanges\sinto\sthis\sbranch. -D 2024-02-21T16:12:23.138 +C Ensure\sintck\stests\sare\srun\sby\stestrunner.tcl. +D 2024-02-21T16:15:50.181 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1487,7 +1487,7 @@ F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442 F test/pendingrace.test 6aa33756b950c4529f79c4f3817a9a1e4025bd0d9961571a05c0279bd183d9c6 F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff -F test/permutations.test f7caf8dd5c7b1da74842a48df116f7f193399c656d4ffc805cd0d9658568c675 +F test/permutations.test 405542f1d659942994a6b38a9e024cf5cfd23eaa68c806aeb24a72d7c9186e80 F test/pg_common.tcl 3b27542224db1e713ae387459b5d117c836a5f6e328846922993b6d2b7640d9f F test/pragma.test cddd4b534d7fb5cf113d1308dea4231f3548e8a7f3a65d7d1cf4810c87090b5a F test/pragma2.test e5d5c176360c321344249354c0c16aec46214c9f @@ -2169,8 +2169,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ecd775d108f77d39a1303316c1e0f0b0ae3ffc5218222e1ebfe2ef6783829b85 27a2113d78b35e324e9aedda7403c96c56ad0bed8c6b139fc5a179e8800b9109 -R 73b41ee05e607fc52a1cc8e54ddf3838 +P 63e8846ac1dc1cf1f7071c4634ccbfec3c13560db6afec376cd91515b62430d3 +R f5edb7901b3b1288ef6973d26c0dbe97 U dan -Z 458475bfe24db2c8e52b55b2e9078f6a +Z 5466cb7224bcaecbaa43fd497cf85347 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index fc124a8b54..1a133ba48f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -63e8846ac1dc1cf1f7071c4634ccbfec3c13560db6afec376cd91515b62430d3 \ No newline at end of file +11d6816c060b6edb9cd61f29297ab95e75e2b46f29c0a796820d94fc13586f6d \ No newline at end of file diff --git a/test/permutations.test b/test/permutations.test index 25aa7de018..c26d6ead14 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -95,6 +95,7 @@ foreach f [glob -nocomplain \ $testdir/../ext/lsm1/test/*.test \ $testdir/../ext/recover/*.test \ $testdir/../ext/rbu/*.test \ + $testdir/../ext/intck/*.test \ ] { lappend alltests $f } From b783f56c587304d57702580ec8aff11d8d67641f Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 21 Feb 2024 19:17:45 +0000 Subject: [PATCH 08/16] Add documentation to ext/intck/sqlite3intck.h. FossilOrigin-Name: 4cc19bd74f05fe92658cc392a1d1afa173d93181a77303af6bc5684436ae832e --- ext/intck/intck_common.tcl | 2 +- ext/intck/sqlite3intck.c | 6 +- ext/intck/sqlite3intck.h | 129 +++++++++++++++++++++++++++++++++++-- ext/intck/test_intck.c | 7 +- manifest | 18 +++--- manifest.uuid | 2 +- 6 files changed, 140 insertions(+), 24 deletions(-) diff --git a/ext/intck/intck_common.tcl b/ext/intck/intck_common.tcl index f00a465b3c..cea1a247a3 100644 --- a/ext/intck/intck_common.tcl +++ b/ext/intck/intck_common.tcl @@ -24,7 +24,7 @@ proc do_intck {db {bSuspend 0}} { if {$msg!=""} { lappend ret $msg } - if {$bSuspend} { $ic suspend } + if {$bSuspend} { $ic unlock } } set err [$ic error] diff --git a/ext/intck/sqlite3intck.c b/ext/intck/sqlite3intck.c index f23dbbd1d7..d00b5ef529 100644 --- a/ext/intck/sqlite3intck.c +++ b/ext/intck/sqlite3intck.c @@ -18,7 +18,7 @@ /* ** apKeyVal: -** If sqlite3_intck_suspend() is called when there is a running pCheck +** If sqlite3_intck_unlock() is called when there is a running pCheck ** statement, this array is allocated and populated with the key values ** required to restart the check. If the intck object has not been ** suspended, this is set to NULL. @@ -845,7 +845,7 @@ const char *sqlite3_intck_message(sqlite3_intck *p){ } int sqlite3_intck_error(sqlite3_intck *p, const char **pzErr){ - *pzErr = p->zErr; + if( pzErr ) *pzErr = p->zErr; return (p->rc==SQLITE_DONE ? SQLITE_OK : p->rc); } @@ -861,7 +861,7 @@ static sqlite3_value *intckValueDup(sqlite3_intck *p, sqlite3_value *pIn){ return pRet; } -int sqlite3_intck_suspend(sqlite3_intck *p){ +int sqlite3_intck_unlock(sqlite3_intck *p){ if( p->pCheck && p->rc==SQLITE_OK ){ const int nByte = sizeof(sqlite3_value*) * p->nKeyVal; int ii; diff --git a/ext/intck/sqlite3intck.h b/ext/intck/sqlite3intck.h index e39825fc7e..e08a86f289 100644 --- a/ext/intck/sqlite3intck.h +++ b/ext/intck/sqlite3intck.h @@ -11,6 +11,50 @@ ************************************************************************* */ +/* +** Incremental Integrity-Check Extension +** ------------------------------------- +** +** This module contains code to check whether or not an SQLite database +** is well-formed or corrupt. This is the same task as performed by SQLite's +** built-in "PRAGMA integrity_check" command. This module differs from +** "PRAGMA integrity_check" in that: +** +** + It is less thorough - this module does not detect certain types +** of corruption that are detected by the PRAGMA command. However, +** it does detect all kinds of corruption that are likely to cause +** errors in SQLite applications. +** +** + It is slower. Sometimes up to three times slower. +** +** + It allows integrity-check operations to be split into multiple +** transactions, so that the database does not need to be read-locked +** for the duration of the integrity-check. +** +** One way to use the API to run integrity-check on the "main" database +** of handle db is: +** +** int rc = SQLITE_OK; +** sqlite3_intck *p = 0; +** +** sqlite3_intck_open(db, "main", &p); +** while( SQLITE_OK==sqlite3_intck_step(p) ){ +** const char *zMsg = sqlite3_intck_message(p); +** if( zMsg ) printf("corruption: %s\n", zMsg); +** } +** rc = sqlite3_intck_error(p, &zErr); +** if( rc!=SQLITE_OK ){ +** printf("error occured (rc=%d), (errmsg=%s)\n", rc, zErr); +** } +** sqlite3_intck_close(p); +** +** Usually, the sqlite3_intck object opens a read transaction within the +** first call to sqlite3_intck_step() and holds it open until the +** integrity-check is complete. However, if sqlite3_intck_unlock() is +** called, the read transaction is ended and a new read transaction opened +** by the subsequent call to sqlite3_intck_step(). +*/ + #ifndef _SQLITE_INTCK_H #define _SQLITE_INTCK_H @@ -20,23 +64,96 @@ extern "C" { #endif +/* +** An ongoing incremental integrity-check operation is represented by an +** opaque pointer of the following type. +*/ typedef struct sqlite3_intck sqlite3_intck; +/* +** Open a new incremental integrity-check object. If successful, populate +** output variable (*ppOut) with the new object handle and return SQLITE_OK. +** Or, if an error occurs, set (*ppOut) to NULL and return an SQLite error +** code (e.g. SQLITE_NOMEM). +** +** The integrity-check will be conducted on database zDb (which must be "main", +** "temp", or the name of an attached database) of database handle db. Once +** this function has been called successfully, the caller should not use +** database handle db until the integrity-check object has been destroyed +** using sqlite3_intck_close(). +*/ int sqlite3_intck_open( - sqlite3 *db, - const char *zDb, - sqlite3_intck **ppOut + sqlite3 *db, /* Database handle */ + const char *zDb, /* Database name ("main", "temp" etc.) */ + sqlite3_intck **ppOut /* OUT: New sqlite3_intck handle */ ); -void sqlite3_intck_close(sqlite3_intck*); +/* +** Close and release all resources associated with a handle opened by an +** earlier call to sqlite3_intck_open(). The results of using an +** integrity-check handle after it has been passed to this function are +** undefined. +*/ +void sqlite3_intck_close(sqlite3_intck *pCk); +/* +** Do the next step of the integrity-check operation specified by the handle +** passed as the only argument. This function returns SQLITE_DONE if the +** integrity-check operation is finished, or an SQLite error code if +** an error occurs, or SQLITE_OK if no error occurs but the integrity-check +** is not finished. It is not considered an error if database corruption +** is encountered. +** +** Following a successful call to sqlite3_intck_step() (one that returns +** SQLITE_OK), sqlite3_intck_message() returns a non-NULL value if +** corruption was detected in the db. +** +** If an error occurs and a value other than SQLITE_OK or SQLITE_DONE is +** returned, then the integrity-check handle is placed in an error state. +** In this state all subsequent calls to sqlite3_intck_step() or +** sqlite3_intck_unlock() will immediately return the same error. The +** sqlite3_intck_error() method may be used to obtain an English language +** error message in this case. +*/ int sqlite3_intck_step(sqlite3_intck *pCk); +/* +** If the previous call to sqlite3_intck_step() encountered corruption +** within the database, then this function returns a pointer to a buffer +** containing a nul-terminated string describing the corruption in +** English. If the previous call to sqlite3_intck_step() did not encounter +** corruption, or if there was no previous call, this function returns +** NULL. +*/ const char *sqlite3_intck_message(sqlite3_intck *pCk); -int sqlite3_intck_error(sqlite3_intck *pCk, const char **pzErr); +/* +** Close any read-transaction opened by an earlier call to +** sqlite3_intck_step(). Any subsequent call to sqlite3_intck_step() will +** open a new transaction. Return SQLITE_OK if successful, or an SQLite error +** code otherwise. +** +** If an error occurs, then the integrity-check handle is placed in an error +** state. In this state all subsequent calls to sqlite3_intck_step() or +** sqlite3_intck_unlock() will immediately return the same error. The +** sqlite3_intck_error() method may be used to obtain an English language +** error message in this case. +*/ +int sqlite3_intck_unlock(sqlite3_intck *pCk); -int sqlite3_intck_suspend(sqlite3_intck *pCk); +/* +** If an error has occurred in an earlier call to sqlite3_intck_step() +** or sqlite3_intck_unlock(), then this method returns the associated +** SQLite error code. Additionally, if pzErr is not NULL, then (*pzErr) +** may be set to point to a nul-terminated string containing an English +** language error message. Or, if no error message is available, to +** NULL. +** +** If no error has occurred within sqlite3_intck_step() or +** sqlite_intck_unlock() calls on the handle passed as the first argument, +** then SQLITE_OK is returned and (*pzErr) set to NULL. +*/ +int sqlite3_intck_error(sqlite3_intck *pCk, const char **pzErr); /* ** This API is used for testing only. It returns the full-text of an SQL diff --git a/ext/intck/test_intck.c b/ext/intck/test_intck.c index 0e2aebf056..4c34e2fd26 100644 --- a/ext/intck/test_intck.c +++ b/ext/intck/test_intck.c @@ -49,7 +49,7 @@ static int testIntckCmd( {"step", 0, ""}, /* 1 */ {"message", 0, ""}, /* 2 */ {"error", 0, ""}, /* 3 */ - {"suspend", 0, ""}, /* 4 */ + {"unlock", 0, ""}, /* 4 */ {"test_sql", 1, ""}, /* 5 */ {0 , 0} }; @@ -105,8 +105,8 @@ static int testIntckCmd( break; } - case 4: assert( 0==strcmp("suspend", aCmd[iIdx].zName) ); { - int rc = sqlite3_intck_suspend(p->intck); + case 4: assert( 0==strcmp("unlock", aCmd[iIdx].zName) ); { + int rc = sqlite3_intck_unlock(p->intck); Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1)); break; } @@ -146,7 +146,6 @@ static int test_sqlite3_intck( TestIntck *p = 0; sqlite3 *db = 0; const char *zDb = 0; - const char *zFile = 0; int rc = SQLITE_OK; if( objc!=3 ){ diff --git a/manifest b/manifest index 43263453a2..32648946aa 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Ensure\sintck\stests\sare\srun\sby\stestrunner.tcl. -D 2024-02-21T16:15:50.181 +C Add\sdocumentation\sto\sext/intck/sqlite3intck.h. +D 2024-02-21T19:17:45.632 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -250,11 +250,11 @@ F ext/icu/icu.c c074519b46baa484bb5396c7e01e051034da8884bad1a1cb7f09bbe6be3f0282 F ext/icu/sqliteicu.h fa373836ed5a1ee7478bdf8a1650689294e41d0c89c1daab26e9ae78a32075a8 F ext/intck/intck1.test 15e94ee868e939c6d3aef6884c5652d969b28d3eac940a15585511cf8aec71ad F ext/intck/intck2.test b65d7f627342f767e1d2c447d25619666cec36f516786dd56568bd741e5d7e67 -F ext/intck/intck_common.tcl 6e5df126e55d6a0d7e3be9757400d5fb87bd291726d01b72a9fe77a7201967e4 +F ext/intck/intck_common.tcl b8a63ae820dbcdf50ddaba474f130b43c26ba81f4b7720ff1d8b9a6592bdd420 F ext/intck/intckcorrupt.test 3211ef68ac53e83951b6c8f6a8d2396506d123fe5898f97f848a25837744ec56 -F ext/intck/sqlite3intck.c 0f9674952f84084741eb4e945d7532b02996ba8ea938b060228d7874f15557f9 -F ext/intck/sqlite3intck.h dec4a37584024154ad5734cf6ee98dc484ce25c26af88652a096be402fe51833 -F ext/intck/test_intck.c eb84e825a3a0fd43ae1e4280305280a19c9ef42c2bdef3110b282b115de60463 +F ext/intck/sqlite3intck.c edc93065be0e16394891bb6ad7d279cb54220db65cc66228ac6d6e0bc3919a63 +F ext/intck/sqlite3intck.h 2b40c38e7063ab822c974c0bd4aed97dabb579ccfe2e180a4639bb3bbef0f1c9 +F ext/intck/test_intck.c dec07fc82e2626a1450e58be474e351643627b04ee08ce8fae6495a533b87e85 F ext/jni/GNUmakefile 59eb05f2a363bdfac8d15d66bed624bfe1ff289229184f3861b95f98a19cf4b2 F ext/jni/README.md d899789a9082a07b99bf30b1bbb6204ae57c060efcaa634536fa669323918f42 F ext/jni/jar-dist.make 030aaa4ae71dd86e4ec5e7c1e6cd86f9dfa47c4592c070d2e35157e42498e1fa @@ -2169,8 +2169,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 63e8846ac1dc1cf1f7071c4634ccbfec3c13560db6afec376cd91515b62430d3 -R f5edb7901b3b1288ef6973d26c0dbe97 +P 11d6816c060b6edb9cd61f29297ab95e75e2b46f29c0a796820d94fc13586f6d +R f33ba3131334feff25011177f18e259c U dan -Z 5466cb7224bcaecbaa43fd497cf85347 +Z fc231a0a5c69f02b295f0b4d4c9d9f6a # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 1a133ba48f..12f0b62ab8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -11d6816c060b6edb9cd61f29297ab95e75e2b46f29c0a796820d94fc13586f6d \ No newline at end of file +4cc19bd74f05fe92658cc392a1d1afa173d93181a77303af6bc5684436ae832e \ No newline at end of file From 6a00f707e8dab995f652470283b7d766575210f1 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 21 Feb 2024 19:31:00 +0000 Subject: [PATCH 09/16] Simplify the way the restart key is saved internally by the intck extension. FossilOrigin-Name: 0e39962baae8a82a3021077676b792ac30c79426bcd8c075b5e92bee55e8c3a5 --- ext/intck/sqlite3intck.c | 101 +++++++++++++-------------------------- manifest | 12 ++--- manifest.uuid | 2 +- 3 files changed, 41 insertions(+), 74 deletions(-) diff --git a/ext/intck/sqlite3intck.c b/ext/intck/sqlite3intck.c index d00b5ef529..24bceb27d0 100644 --- a/ext/intck/sqlite3intck.c +++ b/ext/intck/sqlite3intck.c @@ -17,14 +17,9 @@ #include /* -** apKeyVal: -** If sqlite3_intck_unlock() is called when there is a running pCheck -** statement, this array is allocated and populated with the key values -** required to restart the check. If the intck object has not been -** suspended, this is set to NULL. -** ** nKeyVal: -** The size of the apKeyVal[] array, if it is allocated. +** The number of values that make up the 'key' for the current pCheck +** statement. */ struct sqlite3_intck { sqlite3 *db; @@ -32,8 +27,8 @@ struct sqlite3_intck { char *zObj; /* Current object. Or NULL. */ sqlite3_stmt *pCheck; /* Current check statement */ + char *zKey; int nKeyVal; - sqlite3_value **apKeyVal; char *zMessage; int bCorruptSchema; @@ -161,14 +156,8 @@ static char *intckMprintf(sqlite3_intck *p, const char *zFmt, ...){ ** Free the sqlite3_intck.apKeyVal, if it is allocated and populated. */ static void intckSavedKeyClear(sqlite3_intck *p){ - if( p->apKeyVal ){ - int ii; - for(ii=0; iinKeyVal; ii++){ - sqlite3_value_free(p->apKeyVal[ii]); - } - sqlite3_free(p->apKeyVal); - p->apKeyVal = 0; - } + sqlite3_free(p->zKey); + p->zKey = 0; } /* @@ -182,33 +171,33 @@ static void intckSavedKeyClear(sqlite3_intck *p){ ** occurs within this function, set sqlite3_intck.rc before returning ** and return NULL. */ -static char *intckSavedKeyToText(sqlite3_intck *p){ - char *zRet = 0; - if( p->apKeyVal ){ - int ii; - const char *zSep = "SELECT '(' || "; - char *zSql = 0; - sqlite3_stmt *pStmt = 0; +static void intckSavedKeySave(sqlite3_intck *p){ + int ii; + const char *zSep = "SELECT '(' || "; + char *zSql = 0; + sqlite3_stmt *pStmt = 0; - for(ii=0; iinKeyVal; ii++){ - zSql = intckMprintf(p, "%z%squote(?)", zSql, zSep); - zSep = " || ', ' || "; - } - zSql = intckMprintf(p, "%z || ')'", zSql); + assert( p->pCheck ); + assert( p->zKey==0 ); - pStmt = intckPrepare(p, "%s", zSql); - if( p->rc==SQLITE_OK ){ - for(ii=0; iinKeyVal; ii++){ - sqlite3_bind_value(pStmt, ii+1, p->apKeyVal[ii]); - } - if( SQLITE_ROW==sqlite3_step(pStmt) ){ - zRet = intckStrdup(p, (const char*)sqlite3_column_text(pStmt, 0)); - } - intckFinalize(p, pStmt); - } - sqlite3_free(zSql); + + for(ii=0; iinKeyVal; ii++){ + zSql = intckMprintf(p, "%z%squote(?)", zSql, zSep); + zSep = " || ', ' || "; } - return zRet; + zSql = intckMprintf(p, "%z || ')'", zSql); + + pStmt = intckPrepare(p, "%s", zSql); + if( p->rc==SQLITE_OK ){ + for(ii=0; iinKeyVal; ii++){ + sqlite3_bind_value(pStmt, ii+1, sqlite3_column_value(p->pCheck, ii+1)); + } + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + p->zKey = intckStrdup(p, (const char*)sqlite3_column_text(pStmt, 0)); + } + intckFinalize(p, pStmt); + } + sqlite3_free(zSql); } /* @@ -234,7 +223,7 @@ static void intckFindObject(sqlite3_intck *p){ "SELECT table_name FROM tables " "WHERE ?1 IS NULL OR table_name%s?1 " "ORDER BY 1" - , p->zDb, (p->apKeyVal ? ">=" : ">") + , p->zDb, (p->zKey ? ">=" : ">") ); if( p->rc==SQLITE_OK ){ @@ -733,12 +722,9 @@ static char *intckCheckObjectSql( static void intckCheckObject(sqlite3_intck *p){ char *zSql = 0; - char *zKey = 0; - zKey = intckSavedKeyToText(p); - zSql = intckCheckObjectSql(p, p->zObj, zKey, &p->nKeyVal); + zSql = intckCheckObjectSql(p, p->zObj, p->zKey, &p->nKeyVal); p->pCheck = intckPrepare(p, "%s", zSql); sqlite3_free(zSql); - sqlite3_free(zKey); intckSavedKeyClear(p); } @@ -849,27 +835,10 @@ int sqlite3_intck_error(sqlite3_intck *p, const char **pzErr){ return (p->rc==SQLITE_DONE ? SQLITE_OK : p->rc); } - -static sqlite3_value *intckValueDup(sqlite3_intck *p, sqlite3_value *pIn){ - sqlite3_value *pRet = 0; - if( p->rc==SQLITE_OK ){ - pRet = sqlite3_value_dup(pIn); - if( pRet==0 ){ - p->rc = SQLITE_NOMEM; - } - } - return pRet; -} - int sqlite3_intck_unlock(sqlite3_intck *p){ if( p->pCheck && p->rc==SQLITE_OK ){ - const int nByte = sizeof(sqlite3_value*) * p->nKeyVal; - int ii; - assert( p->apKeyVal==0 && p->nKeyVal>0 ); - p->apKeyVal = (sqlite3_value**)intckMalloc(p, nByte); - for(ii=0; p->rc==SQLITE_OK && iinKeyVal; ii++){ - p->apKeyVal[ii] = intckValueDup(p, sqlite3_column_value(p->pCheck, ii+1)); - } + assert( p->zKey==0 && p->nKeyVal>0 ); + intckSavedKeySave(p); intckFinalize(p, p->pCheck); p->pCheck = 0; } @@ -882,9 +851,7 @@ const char *sqlite3_intck_test_sql(sqlite3_intck *p, const char *zObj){ p->zTestSql = intckCheckObjectSql(p, zObj, 0, 0); }else{ if( p->zObj ){ - char *zKey = intckSavedKeyToText(p); - p->zTestSql = intckCheckObjectSql(p, p->zObj, zKey, 0); - sqlite3_free(zKey); + p->zTestSql = intckCheckObjectSql(p, p->zObj, p->zKey, 0); }else{ sqlite3_free(p->zTestSql); p->zTestSql = 0; diff --git a/manifest b/manifest index 32648946aa..172f8e8435 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sdocumentation\sto\sext/intck/sqlite3intck.h. -D 2024-02-21T19:17:45.632 +C Simplify\sthe\sway\sthe\srestart\skey\sis\ssaved\sinternally\sby\sthe\sintck\sextension. +D 2024-02-21T19:31:00.293 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -252,7 +252,7 @@ F ext/intck/intck1.test 15e94ee868e939c6d3aef6884c5652d969b28d3eac940a15585511cf F ext/intck/intck2.test b65d7f627342f767e1d2c447d25619666cec36f516786dd56568bd741e5d7e67 F ext/intck/intck_common.tcl b8a63ae820dbcdf50ddaba474f130b43c26ba81f4b7720ff1d8b9a6592bdd420 F ext/intck/intckcorrupt.test 3211ef68ac53e83951b6c8f6a8d2396506d123fe5898f97f848a25837744ec56 -F ext/intck/sqlite3intck.c edc93065be0e16394891bb6ad7d279cb54220db65cc66228ac6d6e0bc3919a63 +F ext/intck/sqlite3intck.c c437aaa1f4af77621ba984718a62c57722ad9d071151be79335fd87f2c2c2dde F ext/intck/sqlite3intck.h 2b40c38e7063ab822c974c0bd4aed97dabb579ccfe2e180a4639bb3bbef0f1c9 F ext/intck/test_intck.c dec07fc82e2626a1450e58be474e351643627b04ee08ce8fae6495a533b87e85 F ext/jni/GNUmakefile 59eb05f2a363bdfac8d15d66bed624bfe1ff289229184f3861b95f98a19cf4b2 @@ -2169,8 +2169,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 11d6816c060b6edb9cd61f29297ab95e75e2b46f29c0a796820d94fc13586f6d -R f33ba3131334feff25011177f18e259c +P 4cc19bd74f05fe92658cc392a1d1afa173d93181a77303af6bc5684436ae832e +R 735ea6d5058bc579028d0f5e808c7418 U dan -Z fc231a0a5c69f02b295f0b4d4c9d9f6a +Z b84fa961179c349189e714cd49d2cd34 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 12f0b62ab8..4d5fa2bca4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4cc19bd74f05fe92658cc392a1d1afa173d93181a77303af6bc5684436ae832e \ No newline at end of file +0e39962baae8a82a3021077676b792ac30c79426bcd8c075b5e92bee55e8c3a5 \ No newline at end of file From 639db50a026fb019cf08c167bd6534ddcf8d9033 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 21 Feb 2024 20:58:48 +0000 Subject: [PATCH 10/16] Fix various issues in sqlite3intck.c. FossilOrigin-Name: 8a7bfa74525a495f45b1ea212b1718633b637295090d514dd777f9263477d514 --- ext/intck/sqlite3intck.c | 248 ++++++++++++++++++++++----------------- manifest | 12 +- manifest.uuid | 2 +- 3 files changed, 145 insertions(+), 117 deletions(-) diff --git a/ext/intck/sqlite3intck.c b/ext/intck/sqlite3intck.c index 24bceb27d0..e683577bb7 100644 --- a/ext/intck/sqlite3intck.c +++ b/ext/intck/sqlite3intck.c @@ -14,12 +14,25 @@ #include "sqlite3intck.h" #include #include -#include /* ** nKeyVal: ** The number of values that make up the 'key' for the current pCheck ** statement. +** +** rc: +** Error code returned by most recent sqlite3_intck_step() or +** sqlite3_intck_unlock() call. This is set to SQLITE_DONE when +** the integrity-check operation is finished. +** +** zErr: +** If the object has entered the error state, this is the error message. +** Is freed using sqlite3_free() when the object is deleted. +** +** zTestSql: +** The value returned by the most recent call to sqlite3_intck_testsql(). +** Each call to testsql() frees the previous zTestSql value (using +** sqlite3_free()) and replaces it with the new value it will return. */ struct sqlite3_intck { sqlite3 *db; @@ -52,40 +65,56 @@ static void intckSaveErrmsg(sqlite3_intck *p){ } } -static sqlite3_stmt *intckPrepare(sqlite3_intck *p, const char *zFmt, ...){ +/* +** If the handle passed as the first argument is already in the error state, +** then this function is a no-op (returns NULL immediately). Otherwise, if an +** error occurs within this function, it leaves an error in said handle. +** +** Otherwise, this function attempts to prepare SQL statement zSql and +** return the resulting statement handle to the user. +*/ +static sqlite3_stmt *intckPrepare(sqlite3_intck *p, const char *zSql){ + sqlite3_stmt *pRet = 0; + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_prepare_v2(p->db, zSql, -1, &pRet, 0); + if( p->rc!=SQLITE_OK ){ + intckSaveErrmsg(p); + assert( pRet==0 ); + } + } + return pRet; +} + +/* +** If the handle passed as the first argument is already in the error state, +** then this function is a no-op (returns NULL immediately). Otherwise, if an +** error occurs within this function, it leaves an error in said handle. +** +** Otherwise, this function treats argument zFmt as a printf() style format +** string. It formats it according to the trailing arguments and then +** attempts to prepare the results and return the resulting prepared +** statement. +*/ +static sqlite3_stmt *intckPrepareFmt(sqlite3_intck *p, const char *zFmt, ...){ sqlite3_stmt *pRet = 0; va_list ap; char *zSql = 0; va_start(ap, zFmt); zSql = sqlite3_vmprintf(zFmt, ap); - if( p->rc==SQLITE_OK ){ - if( zSql==0 ){ - p->rc = SQLITE_NOMEM; - }else{ - p->rc = sqlite3_prepare_v2(p->db, zSql, -1, &pRet, 0); - fflush(stdout); - if( p->rc!=SQLITE_OK ){ -#if 0 - printf("ERROR: %s\n", zSql); - printf("MSG: %s\n", sqlite3_errmsg(p->db)); -#endif - if( sqlite3_error_offset(p->db)>=0 ){ -#if 0 - int iOff = sqlite3_error_offset(p->db); - printf("AT: %.40s\n", &zSql[iOff]); -#endif - } - fflush(stdout); - intckSaveErrmsg(p); - assert( pRet==0 ); - } - } + if( p->rc==SQLITE_OK && zSql==0 ){ + p->rc = SQLITE_NOMEM; } + pRet = intckPrepare(p, zSql); sqlite3_free(zSql); va_end(ap); return pRet; } +/* +** Finalize SQL statement pStmt. If an error occurs and the handle passed +** as the first argument does not already contain an error, store the +** error in the handle. +*/ static void intckFinalize(sqlite3_intck *p, sqlite3_stmt *pStmt){ int rc = sqlite3_finalize(pStmt); if( p->rc==SQLITE_OK && rc!=SQLITE_OK ){ @@ -93,6 +122,18 @@ static void intckFinalize(sqlite3_intck *p, sqlite3_stmt *pStmt){ } } +/* +** Execute SQL statement zSql. There is no way to obtain any results +** returned by the statement. This function uses the sqlite3_intck error +** code convention. +*/ +static void intckExec(sqlite3_intck *p, const char *zSql){ + sqlite3_stmt *pStmt = 0; + pStmt = intckPrepare(p, zSql); + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ); + intckFinalize(p, pStmt); +} + /* ** Wrapper around sqlite3_malloc64() that uses the sqlite3_intck error ** code convention. @@ -110,15 +151,8 @@ static void *intckMalloc(sqlite3_intck *p, sqlite3_int64 nByte){ } /* -** If p->rc is other than SQLITE_OK when this function is called, it -** immediately returns NULL. Otherwise, it attempts to create a copy of -** nul-terminated string zIn in a buffer obtained from sqlite3_malloc(). -** If successful, a pointer to this buffer is returned and it becomes -** the responsibility of the caller to release it using sqlite3_free() -** at some point in the future. -** -** Or, if an allocation fails within this function, p->rc is set to -** SQLITE_NOMEM and NULL is returned. +** Like strdup(), but uses the sqlite3_intck error code convention. Any +** returned buffer should eventually be freed using sqlite3_free(). */ static char *intckStrdup(sqlite3_intck *p, const char *zIn){ char *zOut = 0; @@ -131,10 +165,8 @@ static char *intckStrdup(sqlite3_intck *p, const char *zIn){ } /* -** A wrapper around sqlite3_mprintf() that: -** -** + Always returns 0 if p->rc is other than SQLITE_OK when it is called, and -** + Sets p->rc to SQLITE_NOMEM if an allocation fails. +** A wrapper around sqlite3_mprintf() that uses the sqlite3_intck error +** code convention. */ static char *intckMprintf(sqlite3_intck *p, const char *zFmt, ...){ va_list ap; @@ -153,25 +185,11 @@ static char *intckMprintf(sqlite3_intck *p, const char *zFmt, ...){ } /* -** Free the sqlite3_intck.apKeyVal, if it is allocated and populated. +** This is used by sqlite3_intck_unlock() to save the vector key value +** required to restart the current pCheck query as a nul-terminated string +** in p->zKey. */ -static void intckSavedKeyClear(sqlite3_intck *p){ - sqlite3_free(p->zKey); - p->zKey = 0; -} - -/* -** If the apKeyVal array is currently allocated and populated, return -** a pointer to a buffer containing a nul-terminated string representing -** the values as an SQL vector. e.g. -** -** "('abc', NULL, 2)" -** -** If apKeyVal is not allocated, return NULL. Or, if an error (e.g. OOM) -** occurs within this function, set sqlite3_intck.rc before returning -** and return NULL. -*/ -static void intckSavedKeySave(sqlite3_intck *p){ +static void intckSaveKey(sqlite3_intck *p){ int ii; const char *zSep = "SELECT '(' || "; char *zSql = 0; @@ -180,14 +198,13 @@ static void intckSavedKeySave(sqlite3_intck *p){ assert( p->pCheck ); assert( p->zKey==0 ); - for(ii=0; iinKeyVal; ii++){ zSql = intckMprintf(p, "%z%squote(?)", zSql, zSep); zSep = " || ', ' || "; } zSql = intckMprintf(p, "%z || ')'", zSql); - pStmt = intckPrepare(p, "%s", zSql); + pStmt = intckPrepare(p, zSql); if( p->rc==SQLITE_OK ){ for(ii=0; iinKeyVal; ii++){ sqlite3_bind_value(pStmt, ii+1, sqlite3_column_value(p->pCheck, ii+1)); @@ -213,7 +230,7 @@ static void intckFindObject(sqlite3_intck *p){ assert( p->rc==SQLITE_OK ); assert( p->pCheck==0 ); - pStmt = intckPrepare(p, + pStmt = intckPrepareFmt(p, "WITH tables(table_name) AS (" " SELECT name" " FROM %Q.sqlite_schema WHERE (type='table' OR type='index') AND rootpage" @@ -236,7 +253,8 @@ static void intckFindObject(sqlite3_intck *p){ /* If this is a new object, ensure the previous key value is cleared. */ if( sqlite3_stricmp(p->zObj, zPrev) ){ - intckSavedKeyClear(p); + sqlite3_free(p->zKey); + p->zKey = 0; } sqlite3_free(zPrev); @@ -274,22 +292,13 @@ static int intckGetToken(const char *z){ return iRet; } +/* +** Return true if argument c is an ascii whitespace character. +*/ static int intckIsSpace(char c){ return (c==' ' || c=='\t' || c=='\n' || c=='\r'); } -static int intckTokenMatch( - const char *zToken, - int nToken, - const char *z1, - const char *z2 -){ - return ( - (strlen(z1)==nToken && 0==sqlite3_strnicmp(zToken, z1, nToken)) - || (z2 && strlen(z2)==nToken && 0==sqlite3_strnicmp(zToken, z2, nToken)) - ); -} - /* ** Argument z points to the text of a CREATE INDEX statement. This function ** identifies the part of the text that contains either the index WHERE @@ -350,7 +359,9 @@ static const char *intckParseCreateIndex(const char *z, int iCol, int *pnByte){ if( z[iOff]==')' ) nOpen--; nToken = intckGetToken(zToken); - if( intckTokenMatch(zToken, nToken, "ASC", "DESC") ){ + if( (nToken==3 && 0==sqlite3_strnicmp(zToken, "ASC", nToken)) + || (nToken==4 && 0==sqlite3_strnicmp(zToken, "DESC", nToken)) + ){ iEndOfCol = iOff; }else if( 0==intckIsSpace(zToken[0]) ){ iEndOfCol = 0; @@ -383,7 +394,12 @@ static const char *intckParseCreateIndex(const char *z, int iCol, int *pnByte){ return zRet; } -static void parseCreateIndexFunc( +/* +** User-defined SQL function wrapper for intckParseCreateIndex(): +** +** SELECT parse_create_index(, ); +*/ +static void intckParseCreateIndexFunc( sqlite3_context *pCtx, int nVal, sqlite3_value **apVal @@ -421,7 +437,7 @@ static int intckGetAutoIndex(sqlite3_intck *p){ static int intckIsIndex(sqlite3_intck *p, const char *zObj){ int bRet = 0; sqlite3_stmt *pStmt = 0; - pStmt = intckPrepare(p, + pStmt = intckPrepareFmt(p, "SELECT 1 FROM %Q.sqlite_schema WHERE name=%Q AND type='index'", p->zDb, zObj ); @@ -432,17 +448,20 @@ static int intckIsIndex(sqlite3_intck *p, const char *zObj){ return bRet; } -static void intckExec(sqlite3_intck *p, const char *zSql){ - sqlite3_stmt *pStmt = 0; - pStmt = intckPrepare(p, "%s", zSql); - while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ); - intckFinalize(p, pStmt); -} - +/* +** Return a pointer to a nul-terminated buffer containing the SQL statement +** used to check database object zObj (a table or index) for corruption. +** If parameter zPrev is not NULL, then it must be a string containing the +** vector key required to restart the check where it left off last time. +** If pnKeyVal is not NULL, then (*pnKeyVal) is set to the number of +** columns in the vector key value for the specified object. +** +** This function uses the sqlite3_intck error code convention. +*/ static char *intckCheckObjectSql( - sqlite3_intck *p, - const char *zObj, - const char *zPrev, + sqlite3_intck *p, /* Integrity check object */ + const char *zObj, /* Object (table or index) to scan */ + const char *zPrev, /* Restart key vector, if any */ int *pnKeyVal /* OUT: Number of key-values for this scan */ ){ char *zRet = 0; @@ -563,7 +582,7 @@ static char *intckCheckObjectSql( bIsIndex = intckIsIndex(p, zObj); if( bIsIndex ){ - pStmt = intckPrepare(p, + pStmt = intckPrepareFmt(p, /* Table idxname contains a single row. The first column, "db", contains ** the name of the db containing the table (e.g. "main") and the second, ** "tab", the name of the table itself. */ @@ -616,7 +635,7 @@ static char *intckCheckObjectSql( , zPrev, zCommon ); }else{ - pStmt = intckPrepare(p, + pStmt = intckPrepareFmt(p, /* Table tabname contains a single row. The first column, "db", contains ** the name of the db containing the table (e.g. "main") and the second, ** "tab", the name of the table itself. */ @@ -697,22 +716,10 @@ static char *intckCheckObjectSql( } while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ -#if 0 - int nField = sqlite3_column_count(pStmt); - int ii; - for(ii=0; iizObj, p->zKey, &p->nKeyVal); - p->pCheck = intckPrepare(p, "%s", zSql); - sqlite3_free(zSql); - intckSavedKeyClear(p); -} - +/* +** Open a new integrity-check object. +*/ int sqlite3_intck_open( sqlite3 *db, /* Database handle to operate on */ const char *zDbArg, /* "main", "temp" etc. */ @@ -743,7 +745,7 @@ int sqlite3_intck_open( rc = SQLITE_NOMEM; }else{ sqlite3_create_function(db, "parse_create_index", - 2, SQLITE_UTF8, 0, parseCreateIndexFunc, 0, 0 + 2, SQLITE_UTF8, 0, intckParseCreateIndexFunc, 0, 0 ); memset(pNew, 0, sizeof(*pNew)); pNew->db = db; @@ -755,6 +757,9 @@ int sqlite3_intck_open( return rc; } +/* +** Free the integrity-check object. +*/ void sqlite3_intck_close(sqlite3_intck *p){ if( p ){ if( p->db ){ @@ -763,13 +768,16 @@ void sqlite3_intck_close(sqlite3_intck *p){ ); } sqlite3_free(p->zObj); - intckSavedKeyClear(p); + sqlite3_free(p->zKey); sqlite3_free(p->zTestSql); sqlite3_free(p->zErr); sqlite3_free(p); } } +/* +** Step the integrity-check object. +*/ int sqlite3_intck_step(sqlite3_intck *p){ if( p->rc==SQLITE_OK ){ @@ -785,7 +793,12 @@ int sqlite3_intck_step(sqlite3_intck *p){ intckFindObject(p); if( p->rc==SQLITE_OK ){ if( p->zObj ){ - intckCheckObject(p); + char *zSql = 0; + zSql = intckCheckObjectSql(p, p->zObj, p->zKey, &p->nKeyVal); + p->pCheck = intckPrepare(p, zSql); + sqlite3_free(zSql); + sqlite3_free(p->zKey); + p->zKey = 0; }else{ p->rc = SQLITE_DONE; } @@ -819,6 +832,10 @@ int sqlite3_intck_step(sqlite3_intck *p){ return p->rc; } +/* +** Return a message describing the corruption encountered by the most recent +** call to sqlite3_intck_step(), or NULL if no corruption was encountered. +*/ const char *sqlite3_intck_message(sqlite3_intck *p){ assert( p->pCheck==0 || p->zMessage==0 ); if( p->zMessage ){ @@ -830,21 +847,32 @@ const char *sqlite3_intck_message(sqlite3_intck *p){ return 0; } +/* +** Return the error code and message. +*/ int sqlite3_intck_error(sqlite3_intck *p, const char **pzErr){ if( pzErr ) *pzErr = p->zErr; return (p->rc==SQLITE_DONE ? SQLITE_OK : p->rc); } +/* +** Close any read transaction the integrity-check object is holding open +** on the database. +*/ int sqlite3_intck_unlock(sqlite3_intck *p){ if( p->pCheck && p->rc==SQLITE_OK ){ assert( p->zKey==0 && p->nKeyVal>0 ); - intckSavedKeySave(p); + intckSaveKey(p); intckFinalize(p, p->pCheck); p->pCheck = 0; } return p->rc; } +/* +** Return the SQL statement used to check object zObj. Or, if zObj is +** NULL, the current SQL statement. +*/ const char *sqlite3_intck_test_sql(sqlite3_intck *p, const char *zObj){ sqlite3_free(p->zTestSql); if( zObj ){ diff --git a/manifest b/manifest index 172f8e8435..316837de51 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Simplify\sthe\sway\sthe\srestart\skey\sis\ssaved\sinternally\sby\sthe\sintck\sextension. -D 2024-02-21T19:31:00.293 +C Fix\svarious\sissues\sin\ssqlite3intck.c. +D 2024-02-21T20:58:48.924 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -252,7 +252,7 @@ F ext/intck/intck1.test 15e94ee868e939c6d3aef6884c5652d969b28d3eac940a15585511cf F ext/intck/intck2.test b65d7f627342f767e1d2c447d25619666cec36f516786dd56568bd741e5d7e67 F ext/intck/intck_common.tcl b8a63ae820dbcdf50ddaba474f130b43c26ba81f4b7720ff1d8b9a6592bdd420 F ext/intck/intckcorrupt.test 3211ef68ac53e83951b6c8f6a8d2396506d123fe5898f97f848a25837744ec56 -F ext/intck/sqlite3intck.c c437aaa1f4af77621ba984718a62c57722ad9d071151be79335fd87f2c2c2dde +F ext/intck/sqlite3intck.c 3fa96647dde4c719a477f7dbdd1edf76b2f245549b1a4de7f7d60447b24a14df F ext/intck/sqlite3intck.h 2b40c38e7063ab822c974c0bd4aed97dabb579ccfe2e180a4639bb3bbef0f1c9 F ext/intck/test_intck.c dec07fc82e2626a1450e58be474e351643627b04ee08ce8fae6495a533b87e85 F ext/jni/GNUmakefile 59eb05f2a363bdfac8d15d66bed624bfe1ff289229184f3861b95f98a19cf4b2 @@ -2169,8 +2169,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4cc19bd74f05fe92658cc392a1d1afa173d93181a77303af6bc5684436ae832e -R 735ea6d5058bc579028d0f5e808c7418 +P 0e39962baae8a82a3021077676b792ac30c79426bcd8c075b5e92bee55e8c3a5 +R 2d481f6c889817bf845056ed7696633c U dan -Z b84fa961179c349189e714cd49d2cd34 +Z 13c318cbe203f9a86dd5c12df0ee017f # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 4d5fa2bca4..092b5c01d4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0e39962baae8a82a3021077676b792ac30c79426bcd8c075b5e92bee55e8c3a5 \ No newline at end of file +8a7bfa74525a495f45b1ea212b1718633b637295090d514dd777f9263477d514 \ No newline at end of file From 1545243368a908367f357dbf4e3c705574d645db Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 23 Feb 2024 15:13:53 +0000 Subject: [PATCH 11/16] Fix problems with resuming integrity-check operations on indexes with mixed ASC and DESC columns, and on indexes that contain NULL values. FossilOrigin-Name: 0f68b35a000ef9f4691c59797c66ed6c3435fc5c503e9d24f891afec6aceeada --- ext/intck/intck1.test | 20 +++--- ext/intck/intck2.test | 52 +++++++++++++++ ext/intck/intck_common.tcl | 7 +- ext/intck/sqlite3intck.c | 129 ++++++++++++++++++++++++++++--------- manifest | 18 +++--- manifest.uuid | 2 +- 6 files changed, 177 insertions(+), 51 deletions(-) diff --git a/ext/intck/intck1.test b/ext/intck/intck1.test index 4110ece049..4e86e4f2fc 100644 --- a/ext/intck/intck1.test +++ b/ext/intck/intck1.test @@ -181,16 +181,10 @@ do_test 3.2 { sqlite3 db test.db } {} -#puts "[intck_sql db x1a]" -#execsql_pp "EXPLAIN QUERY PLAN [intck_sql db x1a]" do_intck_test 3.3 { {entry (4,'six',5) missing from index x1a} } -#explain_i [intck_sql db x1] -#puts [intck_sql db x1] -#puts [intck_sql db x1a] - #------------------------------------------------------------------------- reset_db do_execsql_test 4.0 { @@ -199,13 +193,17 @@ do_execsql_test 4.0 { INSERT INTO www VALUES(1, 1, 1), (2, 2, 2); } -#puts [intck_sql db w1] -#execsql_pp [intck_sql db www] -#execsql_pp [intck_sql db w1] -#puts [intck_sql db w1] - do_intck_test 4.1 { } +#------------------------------------------------------------------------- +reset_db +do_execsql_test 5.0 { + CREATE TABLE t1(a, b); + CREATE INDEX i1 ON t1(a COLLATE NOCASE); +} + +#puts [intck_sql db i1] + finish_test diff --git a/ext/intck/intck2.test b/ext/intck/intck2.test index 90f263f880..bb57886968 100644 --- a/ext/intck/intck2.test +++ b/ext/intck/intck2.test @@ -61,6 +61,58 @@ do_execsql_test 2.0 { } do_intck_test 2.1 {} + +imposter_edit x1 { + CREATE TABLE imp(a, b, c); +} { + DELETE FROM imp WHERE c=7; +} +puts [intck_sql db x1b] +do_intck_test 2.2 { + {surplus entry ('ONE',6,3) in index x1a} + {surplus entry ('ONE6 "''" ',3) in index x1b} + {surplus entry ('ONE','7',3) in index x1c} +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 3.0 { + CREATE TABLE x1(a, b, c); + CREATE INDEX x1all ON x1(a DESC, b ASC, c DESC); + INSERT INTO x1 VALUES(2, 1, 2); + INSERT INTO x1 VALUES(2, 1, 1); + INSERT INTO x1 VALUES(2, 2, 2); + INSERT INTO x1 VALUES(2, 2, 1); + INSERT INTO x1 VALUES(1, 1, 2); + INSERT INTO x1 VALUES(1, 1, 1); + INSERT INTO x1 VALUES(1, 2, 2); + INSERT INTO x1 VALUES(1, 2, 1); +} + +do_intck_test 3.1 { +} + +imposter_edit x1 { + CREATE TABLE imp(a, b, c); +} { + DELETE FROM imp WHERE 1; +} + +db close +sqlite3 db test.db + +do_intck_test 3.2 { + {surplus entry (2,1,2,1) in index x1all} + {surplus entry (2,1,1,2) in index x1all} + {surplus entry (2,2,2,3) in index x1all} + {surplus entry (2,2,1,4) in index x1all} + {surplus entry (1,1,2,5) in index x1all} + {surplus entry (1,1,1,6) in index x1all} + {surplus entry (1,2,2,7) in index x1all} + {surplus entry (1,2,1,8) in index x1all} +} + +#puts [intck_sql db x1all] #puts [intck_sql db x1] finish_test diff --git a/ext/intck/intck_common.tcl b/ext/intck/intck_common.tcl index cea1a247a3..7d6579ae03 100644 --- a/ext/intck/intck_common.tcl +++ b/ext/intck/intck_common.tcl @@ -24,7 +24,12 @@ proc do_intck {db {bSuspend 0}} { if {$msg!=""} { lappend ret $msg } - if {$bSuspend} { $ic unlock } + if {$bSuspend} { + $ic unlock + #puts "SQL: [$ic test_sql {}]" + #execsql_pp "EXPLAIN query plan [$ic test_sql {}]" + #explain_i [$ic test_sql {}] + } } set err [$ic error] diff --git a/ext/intck/sqlite3intck.c b/ext/intck/sqlite3intck.c index e683577bb7..287ec157c3 100644 --- a/ext/intck/sqlite3intck.c +++ b/ext/intck/sqlite3intck.c @@ -15,6 +15,9 @@ #include #include +#include +#include + /* ** nKeyVal: ** The number of values that make up the 'key' for the current pCheck @@ -187,22 +190,87 @@ static char *intckMprintf(sqlite3_intck *p, const char *zFmt, ...){ /* ** This is used by sqlite3_intck_unlock() to save the vector key value ** required to restart the current pCheck query as a nul-terminated string -** in p->zKey. +** in p->zKey. */ static void intckSaveKey(sqlite3_intck *p){ int ii; - const char *zSep = "SELECT '(' || "; char *zSql = 0; sqlite3_stmt *pStmt = 0; + sqlite3_stmt *pXinfo = 0; + const char *zDir = 0; assert( p->pCheck ); assert( p->zKey==0 ); - for(ii=0; iinKeyVal; ii++){ - zSql = intckMprintf(p, "%z%squote(?)", zSql, zSep); - zSep = " || ', ' || "; + pXinfo = intckPrepareFmt(p, + "SELECT group_concat(desc, '') FROM %Q.sqlite_schema s, " + "pragma_index_xinfo(%Q, %Q) " + "WHERE s.type='index' AND s.name=%Q", + p->zDb, p->zObj, p->zDb, p->zObj + ); + if( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXinfo) ){ + zDir = (const char*)sqlite3_column_text(pXinfo, 0); + } + + if( zDir==0 ){ + /* Object is a table, not an index. This is the easy case,as there are + ** no DESC columns or NULL values in a primary key. */ + const char *zSep = "SELECT '(' || "; + for(ii=0; iinKeyVal; ii++){ + zSql = intckMprintf(p, "%z%squote(?)", zSql, zSep); + zSep = " || ', ' || "; + } + zSql = intckMprintf(p, "%z || ')'", zSql); + }else{ + + /* Object is an index. */ + assert( p->nKeyVal>1 ); + for(ii=p->nKeyVal; ii>0; ii--){ + int bLastIsDesc = zDir[ii-1]=='1'; + int bLastIsNull = sqlite3_column_type(p->pCheck, ii)==SQLITE_NULL; + const char *zLast = sqlite3_column_name(p->pCheck, ii); + char *zLhs = 0; + char *zRhs = 0; + char *zWhere = 0; + + if( bLastIsNull ){ + if( bLastIsDesc ) continue; + zWhere = intckMprintf(p, "'%s IS NOT NULL'", zLast); + }else{ + const char *zOp = bLastIsDesc ? "<" : ">"; + zWhere = intckMprintf(p, "'%s %s ' || quote(?%d)", zLast, zOp, ii); + } + + if( ii>1 ){ + const char *zLhsSep = ""; + const char *zRhsSep = ""; + int jj; + for(jj=0; jjpCheck,jj+1); + zLhs = intckMprintf(p, "%z%s%s", zLhs, zLhsSep, zAlias); + zRhs = intckMprintf(p, "%z%squote(?%d)", zRhs, zRhsSep, jj+1); + zLhsSep = ","; + zRhsSep = " || ',' || "; + } + + zWhere = intckMprintf(p, + "'(%z) IS (' || %z || ') AND ' || %z", + zLhs, zRhs, zWhere); + } + zWhere = intckMprintf(p, "'WHERE ' || %z", zWhere); + + zSql = intckMprintf(p, "%z%s(quote( %z ) )", + zSql, + (zSql==0 ? "VALUES" : ",\n "), + zWhere + ); + } + zSql = intckMprintf(p, + "WITH wc(q) AS (\n%z\n)" + "SELECT 'VALUES' || group_concat('(' || q || ')', ',\n ') FROM wc" + , zSql + ); } - zSql = intckMprintf(p, "%z || ')'", zSql); pStmt = intckPrepare(p, zSql); if( p->rc==SQLITE_OK ){ @@ -214,7 +282,9 @@ static void intckSaveKey(sqlite3_intck *p){ } intckFinalize(p, pStmt); } + sqlite3_free(zSql); + intckFinalize(p, pXinfo); } /* @@ -521,7 +591,7 @@ static char *intckCheckObjectSql( " SELECT i.col_name, i.col_alias FROM idx_cols i WHERE i.idx_ispk" " )" " SELECT t.db, t.tab, t.idx, " - " group_concat('o.'||a, ', '), " + " group_concat(a, ', '), " " group_concat('i.'||quote(f), ', '), " " group_concat('quote(o.'||a||')', ' || '','' || '), " " format('(%s)==(%s)'," @@ -586,24 +656,26 @@ static char *intckCheckObjectSql( /* Table idxname contains a single row. The first column, "db", contains ** the name of the db containing the table (e.g. "main") and the second, ** "tab", the name of the table itself. */ - "WITH tabname(db, tab, idx, prev) AS (" - " SELECT %Q, (SELECT tbl_name FROM %Q.sqlite_schema WHERE name=%Q), " - " %Q, %Q " + "WITH tabname(db, tab, idx) AS (" + " SELECT %Q, (SELECT tbl_name FROM %Q.sqlite_schema WHERE name=%Q), %Q " ")" + "" + ", whereclause(w_c) AS (%s)" + "" "%s" /* zCommon */ "" ", case_statement(c) AS (" " SELECT " - " 'CASE WHEN (' || group_concat(col_alias, ', ') || ') IS (\n ' " + " 'CASE WHEN (' || group_concat(col_alias, ', ') || ') IS (\n ' " " || 'SELECT ' || group_concat(col_expr, ', ') || ' FROM '" " || format('%%Q.%%Q NOT INDEXED WHERE %%s\n', t.db, t.tab, p.eq_pk)" - " || ' )\n THEN NULL\n '" + " || ' )\n THEN NULL\n '" " || 'ELSE format(''surplus entry ('" " || group_concat('%%s', ',') || ',' || p.ps_pk" " || ') in index ' || t.idx || ''', ' " " || group_concat('quote('||i.col_alias||')', ', ') || ', ' || p.pk_pk" " || ')'" - " || '\nEND AS error_message'" + " || '\n END AS error_message'" " FROM tabname t, tabpk p, idx_cols i WHERE i.idx_name=t.idx" ")" "" @@ -613,26 +685,25 @@ static char *intckCheckObjectSql( " FROM tabpk p, idx_cols i WHERE i.idx_name=p.idx" ")" "" - ", whereclause(w_c) AS (" - " SELECT CASE WHEN prev!='' THEN " - " '\nWHERE (' || group_concat(i.col_alias, ',') || ',' " - " || o_pk || ') > ' || prev" - " ELSE ''" - " END" - " FROM tabpk, tabname, idx_cols i WHERE i.idx_name=tabpk.idx" - ")" - "" ", main_select(m, n) AS (" " SELECT format(" - " 'WITH %%s\nSELECT %%s,\n%%s\nFROM intck_wrapper AS o%%s'," - " ww.s, c, t.k, whereclause.w_c" + " 'WITH %%s\n' ||" + " ', idx_checker AS (\n' ||" + " ' SELECT %%s,\n' ||" + " ' %%s\n' || " + " ' FROM intck_wrapper AS o\n' ||" + " ')\n'," + " ww.s, c, t.k" " ), t.n" - " FROM case_statement, wrapper_with ww, thiskey t, whereclause" + " FROM case_statement, wrapper_with ww, thiskey t" ")" - "SELECT m, n FROM main_select" + "SELECT m || " + " group_concat('SELECT * FROM idx_checker ' || w_c, ' UNION ALL '), n" + " FROM " + "main_select, whereclause " , p->zDb, p->zDb, zObj, zObj - , zPrev, zCommon + , zPrev ? zPrev : "VALUES('')", zCommon ); }else{ pStmt = intckPrepareFmt(p, @@ -689,7 +760,7 @@ static char *intckCheckObjectSql( ** format('(%d,%d)', _rowid_, n.ii) */ ", thiskey(k, n) AS (" - " SELECT o_pk || ', n.ii', n_pk+1 FROM tabpk" + " SELECT o_pk || ', ii', n_pk+1 FROM tabpk" ")" "" ", whereclause(w_c) AS (" @@ -702,7 +773,7 @@ static char *intckCheckObjectSql( "" ", main_select(m, n) AS (" " SELECT format(" - " '%%s, %%s\nSELECT %%s,\n%%s AS thiskey\nFROM intck_wrapper AS o" + " '%%s, %%s\nSELECT %%s,\n%%s\nFROM intck_wrapper AS o" ", intck_counter AS n%%s\nORDER BY %%s', " " w, ww.s, c, thiskey.k, whereclause.w_c, t.o_pk" " ), thiskey.n" diff --git a/manifest b/manifest index 316837de51..7508c5a2f6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\svarious\sissues\sin\ssqlite3intck.c. -D 2024-02-21T20:58:48.924 +C Fix\sproblems\swith\sresuming\sintegrity-check\soperations\son\sindexes\swith\smixed\sASC\sand\sDESC\scolumns,\sand\son\sindexes\sthat\scontain\sNULL\svalues. +D 2024-02-23T15:13:53.489 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -248,11 +248,11 @@ F ext/fts5/tool/showfts5.tcl d54da0e067306663e2d5d523965ca487698e722c F ext/icu/README.txt 7ab7ced8ae78e3a645b57e78570ff589d4c672b71370f5aa9e1cd7024f400fc9 F ext/icu/icu.c c074519b46baa484bb5396c7e01e051034da8884bad1a1cb7f09bbe6be3f0282 F ext/icu/sqliteicu.h fa373836ed5a1ee7478bdf8a1650689294e41d0c89c1daab26e9ae78a32075a8 -F ext/intck/intck1.test 15e94ee868e939c6d3aef6884c5652d969b28d3eac940a15585511cf8aec71ad -F ext/intck/intck2.test b65d7f627342f767e1d2c447d25619666cec36f516786dd56568bd741e5d7e67 -F ext/intck/intck_common.tcl b8a63ae820dbcdf50ddaba474f130b43c26ba81f4b7720ff1d8b9a6592bdd420 +F ext/intck/intck1.test e73c4f87b54176291b9a01b52dbc9dd88f42f3ec8aea48c8e0bd7f87a1440a40 +F ext/intck/intck2.test 9d083ccb06c9239400569846b077fb9867a19b12233ce9dcbc124540e12d38df +F ext/intck/intck_common.tcl 9e51458126576783f11051ac0fd25bea3f6b17f570a55884223737f3200b214b F ext/intck/intckcorrupt.test 3211ef68ac53e83951b6c8f6a8d2396506d123fe5898f97f848a25837744ec56 -F ext/intck/sqlite3intck.c 3fa96647dde4c719a477f7dbdd1edf76b2f245549b1a4de7f7d60447b24a14df +F ext/intck/sqlite3intck.c b7dd8354b4c3255cecc8cc190f7f980a667d3aec7409900703248296472b358f F ext/intck/sqlite3intck.h 2b40c38e7063ab822c974c0bd4aed97dabb579ccfe2e180a4639bb3bbef0f1c9 F ext/intck/test_intck.c dec07fc82e2626a1450e58be474e351643627b04ee08ce8fae6495a533b87e85 F ext/jni/GNUmakefile 59eb05f2a363bdfac8d15d66bed624bfe1ff289229184f3861b95f98a19cf4b2 @@ -2169,8 +2169,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0e39962baae8a82a3021077676b792ac30c79426bcd8c075b5e92bee55e8c3a5 -R 2d481f6c889817bf845056ed7696633c +P 8a7bfa74525a495f45b1ea212b1718633b637295090d514dd777f9263477d514 +R afa6cae1c52ecb818ac1cff0c8856821 U dan -Z 13c318cbe203f9a86dd5c12df0ee017f +Z d57af7b6bfe17ed12101d02b696a832f # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 092b5c01d4..391414ddf6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8a7bfa74525a495f45b1ea212b1718633b637295090d514dd777f9263477d514 \ No newline at end of file +0f68b35a000ef9f4691c59797c66ed6c3435fc5c503e9d24f891afec6aceeada \ No newline at end of file From 5956d1d94288cd816af1f1c1c667be472fdccdbd Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 23 Feb 2024 17:10:39 +0000 Subject: [PATCH 12/16] Fix detection of surplus index entries when all indexed fields are NULL. FossilOrigin-Name: 5f310fb7be450c96f01c21e6fd1377d4a274784377d0bd811922ab63e612bd11 --- ext/intck/intck2.test | 33 ++++++++++++++++++++++++++++++--- ext/intck/sqlite3intck.c | 4 ++-- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 40 insertions(+), 13 deletions(-) diff --git a/ext/intck/intck2.test b/ext/intck/intck2.test index bb57886968..54c392b0f5 100644 --- a/ext/intck/intck2.test +++ b/ext/intck/intck2.test @@ -67,7 +67,6 @@ imposter_edit x1 { } { DELETE FROM imp WHERE c=7; } -puts [intck_sql db x1b] do_intck_test 2.2 { {surplus entry ('ONE',6,3) in index x1a} {surplus entry ('ONE6 "''" ',3) in index x1b} @@ -112,8 +111,36 @@ do_intck_test 3.2 { {surplus entry (1,2,1,8) in index x1all} } -#puts [intck_sql db x1all] -#puts [intck_sql db x1] +do_execsql_test 3.3 { + DELETE FROM x1; + INSERT INTO x1 VALUES(NULL, NULL, NULL); + INSERT INTO x1 VALUES(NULL, NULL, NULL); + INSERT INTO x1 VALUES(NULL, NULL, NULL); + INSERT INTO x1 VALUES(NULL, NULL, NULL); +} + +do_intck_test 3.4 { +} + +imposter_edit x1 { + CREATE TABLE imp(a, b, c); +} { + DELETE FROM imp WHERE 1; + INSERT INTO imp(rowid) VALUES(-123); + INSERT INTO imp(rowid) VALUES(456); +} + +db close +sqlite3 db test.db + +do_intck_test 3.5 { + {entry (NULL,NULL,NULL,-123) missing from index x1all} + {entry (NULL,NULL,NULL,456) missing from index x1all} + {surplus entry (NULL,NULL,NULL,1) in index x1all} + {surplus entry (NULL,NULL,NULL,2) in index x1all} + {surplus entry (NULL,NULL,NULL,3) in index x1all} + {surplus entry (NULL,NULL,NULL,4) in index x1all} +} finish_test diff --git a/ext/intck/sqlite3intck.c b/ext/intck/sqlite3intck.c index 287ec157c3..23f93eb318 100644 --- a/ext/intck/sqlite3intck.c +++ b/ext/intck/sqlite3intck.c @@ -666,8 +666,8 @@ static char *intckCheckObjectSql( "" ", case_statement(c) AS (" " SELECT " - " 'CASE WHEN (' || group_concat(col_alias, ', ') || ') IS (\n ' " - " || 'SELECT ' || group_concat(col_expr, ', ') || ' FROM '" + " 'CASE WHEN (' || group_concat(col_alias, ', ') || ', 1) IS (\n' " + " || ' SELECT ' || group_concat(col_expr, ', ') || ', 1 FROM '" " || format('%%Q.%%Q NOT INDEXED WHERE %%s\n', t.db, t.tab, p.eq_pk)" " || ' )\n THEN NULL\n '" " || 'ELSE format(''surplus entry ('" diff --git a/manifest b/manifest index 7508c5a2f6..f702e591f9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sproblems\swith\sresuming\sintegrity-check\soperations\son\sindexes\swith\smixed\sASC\sand\sDESC\scolumns,\sand\son\sindexes\sthat\scontain\sNULL\svalues. -D 2024-02-23T15:13:53.489 +C Fix\sdetection\sof\ssurplus\sindex\sentries\swhen\sall\sindexed\sfields\sare\sNULL. +D 2024-02-23T17:10:39.995 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -249,10 +249,10 @@ F ext/icu/README.txt 7ab7ced8ae78e3a645b57e78570ff589d4c672b71370f5aa9e1cd7024f4 F ext/icu/icu.c c074519b46baa484bb5396c7e01e051034da8884bad1a1cb7f09bbe6be3f0282 F ext/icu/sqliteicu.h fa373836ed5a1ee7478bdf8a1650689294e41d0c89c1daab26e9ae78a32075a8 F ext/intck/intck1.test e73c4f87b54176291b9a01b52dbc9dd88f42f3ec8aea48c8e0bd7f87a1440a40 -F ext/intck/intck2.test 9d083ccb06c9239400569846b077fb9867a19b12233ce9dcbc124540e12d38df +F ext/intck/intck2.test 348a8c505bfe843eceaedd911f91ba4516b5e9c2d4aa83a300a6e84de6ff0955 F ext/intck/intck_common.tcl 9e51458126576783f11051ac0fd25bea3f6b17f570a55884223737f3200b214b F ext/intck/intckcorrupt.test 3211ef68ac53e83951b6c8f6a8d2396506d123fe5898f97f848a25837744ec56 -F ext/intck/sqlite3intck.c b7dd8354b4c3255cecc8cc190f7f980a667d3aec7409900703248296472b358f +F ext/intck/sqlite3intck.c c8515c1ed1bc725e156fc5053e8f9c8cff3c440f3bca47a8309e76d2411fc73d F ext/intck/sqlite3intck.h 2b40c38e7063ab822c974c0bd4aed97dabb579ccfe2e180a4639bb3bbef0f1c9 F ext/intck/test_intck.c dec07fc82e2626a1450e58be474e351643627b04ee08ce8fae6495a533b87e85 F ext/jni/GNUmakefile 59eb05f2a363bdfac8d15d66bed624bfe1ff289229184f3861b95f98a19cf4b2 @@ -2169,8 +2169,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8a7bfa74525a495f45b1ea212b1718633b637295090d514dd777f9263477d514 -R afa6cae1c52ecb818ac1cff0c8856821 +P 0f68b35a000ef9f4691c59797c66ed6c3435fc5c503e9d24f891afec6aceeada +R 67e774420e8bc9ef111ef98dbd2b2abc U dan -Z d57af7b6bfe17ed12101d02b696a832f +Z f383f29167401158528bd4b4e88fca49 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 391414ddf6..1b5128f2f2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0f68b35a000ef9f4691c59797c66ed6c3435fc5c503e9d24f891afec6aceeada \ No newline at end of file +5f310fb7be450c96f01c21e6fd1377d4a274784377d0bd811922ab63e612bd11 \ No newline at end of file From 645f21f15baf64486ac4c81543aa19476e10869e Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 23 Feb 2024 18:21:51 +0000 Subject: [PATCH 13/16] Add the ".intck ?STEPS_PER_UNLOCK?" command to the shell tool. FossilOrigin-Name: cfd051836b72f7d4e38cc9614f6ae5c003de4ce377359fd391adf06fe1ddf6b9 --- ext/intck/intck2.test | 33 +++++++++++++++++++++++++-- manifest | 14 ++++++------ manifest.uuid | 2 +- src/shell.c.in | 53 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 92 insertions(+), 10 deletions(-) diff --git a/ext/intck/intck2.test b/ext/intck/intck2.test index 54c392b0f5..c168118535 100644 --- a/ext/intck/intck2.test +++ b/ext/intck/intck2.test @@ -1,4 +1,4 @@ -# 2008 Feb 19 +# 2024 Feb 19 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: @@ -142,6 +142,35 @@ do_intck_test 3.5 { {surplus entry (NULL,NULL,NULL,4) in index x1all} } +reset_db + +do_execsql_test 3.6 { + CREATE TABLE w1(a PRIMARY KEY, b, c); + INSERT INTO w1 VALUES(1.0, NULL, NULL); + INSERT INTO w1 VALUES(33.0, NULL, NULL); + INSERT INTO w1 VALUES(100.0, NULL, NULL); + CREATE INDEX w1bc ON w1(b, c); +} + +do_intck_test 3.7 { +} + +imposter_edit w1 { + CREATE TABLE imp(a, b, c); +} { + DELETE FROM imp WHERE a=33; + INSERT INTO imp(a) VALUES(1234.5); + INSERT INTO imp(a) VALUES(-1234.5); +} + +do_intck_test 3.8 { + {surplus entry (33.0,2) in index sqlite_autoindex_w1_1} + {entry (1234.5,4) missing from index sqlite_autoindex_w1_1} + {entry (NULL,NULL,4) missing from index w1bc} + {entry (-1234.5,5) missing from index sqlite_autoindex_w1_1} + {entry (NULL,NULL,5) missing from index w1bc} + {surplus entry (NULL,NULL,2) in index w1bc} +} + finish_test - diff --git a/manifest b/manifest index f702e591f9..266c84bd6a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sdetection\sof\ssurplus\sindex\sentries\swhen\sall\sindexed\sfields\sare\sNULL. -D 2024-02-23T17:10:39.995 +C Add\sthe\s".intck\s?STEPS_PER_UNLOCK?"\scommand\sto\sthe\sshell\stool. +D 2024-02-23T18:21:51.131 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -249,7 +249,7 @@ F ext/icu/README.txt 7ab7ced8ae78e3a645b57e78570ff589d4c672b71370f5aa9e1cd7024f4 F ext/icu/icu.c c074519b46baa484bb5396c7e01e051034da8884bad1a1cb7f09bbe6be3f0282 F ext/icu/sqliteicu.h fa373836ed5a1ee7478bdf8a1650689294e41d0c89c1daab26e9ae78a32075a8 F ext/intck/intck1.test e73c4f87b54176291b9a01b52dbc9dd88f42f3ec8aea48c8e0bd7f87a1440a40 -F ext/intck/intck2.test 348a8c505bfe843eceaedd911f91ba4516b5e9c2d4aa83a300a6e84de6ff0955 +F ext/intck/intck2.test 97daaf43b8a3c870ba7c2fd421dc1887053f5b30188896fa64677ce8174c206f F ext/intck/intck_common.tcl 9e51458126576783f11051ac0fd25bea3f6b17f570a55884223737f3200b214b F ext/intck/intckcorrupt.test 3211ef68ac53e83951b6c8f6a8d2396506d123fe5898f97f848a25837744ec56 F ext/intck/sqlite3intck.c c8515c1ed1bc725e156fc5053e8f9c8cff3c440f3bca47a8309e76d2411fc73d @@ -747,7 +747,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c d77c6160bc8f249c2196fdd3e75f66a1dd70e37aa25c39aedc7b1f93c42b7c6d F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 F src/select.c 43fabfc01bf87addd15e39f112f1e2ade15b19594835ab8a9e5bd50839d4e1b1 -F src/shell.c.in e4a38496290d3979ea3d953fba0bc187d350d6e80f8e33c2e37b9a924850439b +F src/shell.c.in 0c13f7cc3bb8c31190efbd96f5c1d8f2fafdbcad549424b7e7850cb5617b115a F src/sqlite.h.in 020d7b7307dda51420dc48b47e5334eaface77baba6bd9818375d392eb3ab5b5 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54 @@ -2169,8 +2169,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0f68b35a000ef9f4691c59797c66ed6c3435fc5c503e9d24f891afec6aceeada -R 67e774420e8bc9ef111ef98dbd2b2abc +P 5f310fb7be450c96f01c21e6fd1377d4a274784377d0bd811922ab63e612bd11 +R 86a36a81ccd4a5a3ded74d8234b0250a U dan -Z f383f29167401158528bd4b4e88fca49 +Z 39f7083bcdb0c03c315a581938028ce8 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 1b5128f2f2..ff9880a342 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5f310fb7be450c96f01c21e6fd1377d4a274784377d0bd811922ab63e612bd11 \ No newline at end of file +cfd051836b72f7d4e38cc9614f6ae5c003de4ce377359fd391adf06fe1ddf6b9 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index da377982e1..07e4b201ce 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -1222,6 +1222,9 @@ INCLUDE ../ext/misc/sqlar.c INCLUDE ../ext/expert/sqlite3expert.h INCLUDE ../ext/expert/sqlite3expert.c +INCLUDE ../ext/intck/sqlite3intck.h +INCLUDE ../ext/intck/sqlite3intck.c + #if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) #define SQLITE_SHELL_HAVE_RECOVER 1 #else @@ -4741,6 +4744,7 @@ static const char *(azHelp[]) = { ".indexes ?TABLE? Show names of indexes", " If TABLE is specified, only show indexes for", " tables matching TABLE using the LIKE operator.", + ".intck ?STEPS_PER_UNLOCK? Run an incremental integrity check on the db", #ifdef SQLITE_ENABLE_IOTRACE ",iotrace FILE Enable I/O diagnostic logging to FILE", #endif @@ -7650,6 +7654,40 @@ static int recoverDatabaseCmd(ShellState *pState, int nArg, char **azArg){ } #endif /* SQLITE_SHELL_HAVE_RECOVER */ +/* +** Implementation of ".intck STEPS_PER_UNLOCK" command. +*/ +static int intckDatabaseCmd(ShellState *pState, i64 nStepPerUnlock){ + sqlite3_intck *p = 0; + int rc = SQLITE_OK; + + rc = sqlite3_intck_open(pState->db, "main", &p); + if( rc==SQLITE_OK ){ + i64 nStep = 0; + i64 nError = 0; + const char *zErr = 0; + while( SQLITE_OK==sqlite3_intck_step(p) ){ + const char *zMsg = sqlite3_intck_message(p); + if( zMsg ){ + oputf("%s\n", zMsg); + nError++; + } + nStep++; + if( nStepPerUnlock && (nStep % nStepPerUnlock)==0 ){ + sqlite3_intck_unlock(p); + } + } + rc = sqlite3_intck_error(p, &zErr); + if( zErr ){ + eputf("%s\n", zErr); + } + sqlite3_intck_close(p); + + oputf("%lld steps, %lld errors\n", nStep, nError); + } + + return rc; +} /* * zAutoColumn(zCol, &db, ?) => Maybe init db, add column zCol to it. @@ -9140,6 +9178,21 @@ static int do_meta_command(char *zLine, ShellState *p){ }else #endif /* !defined(SQLITE_OMIT_TEST_CONTROL) */ + if( c=='i' && cli_strncmp(azArg[0], "intck", n)==0 ){ + i64 iArg = 0; + if( nArg==2 ){ + iArg = integerValue(azArg[1]); + if( iArg==0 ) iArg = -1; + } + if( (nArg!=1 && nArg!=2) || iArg<0 ){ + eputf("Usage: .intck STEPS_PER_UNLOCK\n"); + rc = 1; + goto meta_command_exit; + } + open_db(p, 0); + rc = intckDatabaseCmd(p, iArg); + }else + #ifdef SQLITE_ENABLE_IOTRACE if( c=='i' && cli_strncmp(azArg[0], "iotrace", n)==0 ){ SQLITE_API extern void (SQLITE_CDECL *sqlite3IoTrace)(const char*, ...); From ee299cc7d4c5d72cf25cf59478212c3731848f5a Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 23 Feb 2024 20:51:06 +0000 Subject: [PATCH 14/16] Add tests for the new code on this branch. FossilOrigin-Name: 351d46b2373f08bc8033d0902d9f67cd6c8bcc16c0d9f787e4fb279c0a76da87 --- ext/intck/intckfault.test | 45 +++++++++++++++++++++++++++++++++++++++ ext/intck/sqlite3intck.c | 20 +++++++++++------ manifest | 13 +++++------ manifest.uuid | 2 +- 4 files changed, 66 insertions(+), 14 deletions(-) create mode 100644 ext/intck/intckfault.test diff --git a/ext/intck/intckfault.test b/ext/intck/intckfault.test new file mode 100644 index 0000000000..5c383681ac --- /dev/null +++ b/ext/intck/intckfault.test @@ -0,0 +1,45 @@ +# 2024 February 24 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +source [file join [file dirname [info script]] intck_common.tcl] +set testprefix intckfault + + + +do_execsql_test 1.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + INSERT INTO t1 VALUES(1, 2, 3); + INSERT INTO t1 VALUES(2, 'two', 'three'); + INSERT INTO t1 VALUES(3, NULL, NULL); + CREATE INDEX i1 ON t1(b, c); +} + +do_faultsim_test 1 -faults oom-t* -prep { +} -body { + set ::ic [sqlite3_intck db main] + set nStep 0 + while {"SQLITE_OK"==[$::ic step]} { + incr nStep + if {$nStep==3} { $::ic unlock } + } + set res [$::ic error] + $::ic close + set res +} -test { + catch { $::ic close } +puts $testresult +puts $testnfail + faultsim_test_result {0 {SQLITE_OK {}}} {0 {SQLITE_NOMEM {}}} {0 {SQLITE_NOMEM {out of memory}}} +} + +finish_test + diff --git a/ext/intck/sqlite3intck.c b/ext/intck/sqlite3intck.c index 23f93eb318..b61035f157 100644 --- a/ext/intck/sqlite3intck.c +++ b/ext/intck/sqlite3intck.c @@ -159,10 +159,12 @@ static void *intckMalloc(sqlite3_intck *p, sqlite3_int64 nByte){ */ static char *intckStrdup(sqlite3_intck *p, const char *zIn){ char *zOut = 0; - int nIn = strlen(zIn); - zOut = (char*)intckMalloc(p, nIn+1); - if( zOut ){ - memcpy(zOut, zIn, nIn+1); + if( zIn ){ + int nIn = strlen(zIn); + zOut = (char*)intckMalloc(p, nIn+1); + if( zOut ){ + memcpy(zOut, zIn, nIn+1); + } } return zOut; } @@ -815,13 +817,17 @@ int sqlite3_intck_open( if( pNew==0 ){ rc = SQLITE_NOMEM; }else{ - sqlite3_create_function(db, "parse_create_index", - 2, SQLITE_UTF8, 0, intckParseCreateIndexFunc, 0, 0 - ); memset(pNew, 0, sizeof(*pNew)); pNew->db = db; pNew->zDb = (const char*)&pNew[1]; memcpy(&pNew[1], zDb, nDb+1); + rc = sqlite3_create_function(db, "parse_create_index", + 2, SQLITE_UTF8, 0, intckParseCreateIndexFunc, 0, 0 + ); + if( rc!=SQLITE_OK ){ + sqlite3_intck_close(pNew); + pNew = 0; + } } *ppOut = pNew; diff --git a/manifest b/manifest index 266c84bd6a..068e592cd0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s".intck\s?STEPS_PER_UNLOCK?"\scommand\sto\sthe\sshell\stool. -D 2024-02-23T18:21:51.131 +C Add\stests\sfor\sthe\snew\scode\son\sthis\sbranch. +D 2024-02-23T20:51:06.837 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -252,7 +252,8 @@ F ext/intck/intck1.test e73c4f87b54176291b9a01b52dbc9dd88f42f3ec8aea48c8e0bd7f87 F ext/intck/intck2.test 97daaf43b8a3c870ba7c2fd421dc1887053f5b30188896fa64677ce8174c206f F ext/intck/intck_common.tcl 9e51458126576783f11051ac0fd25bea3f6b17f570a55884223737f3200b214b F ext/intck/intckcorrupt.test 3211ef68ac53e83951b6c8f6a8d2396506d123fe5898f97f848a25837744ec56 -F ext/intck/sqlite3intck.c c8515c1ed1bc725e156fc5053e8f9c8cff3c440f3bca47a8309e76d2411fc73d +F ext/intck/intckfault.test ba0213c9c8dce08d519d5251268a3bab076a184b4d07acdea23b65e89c9ae03c +F ext/intck/sqlite3intck.c 642f57a4604580513547df9d8489cdb49b8f5f3af1981c7ffb87bc37e5ce1439 F ext/intck/sqlite3intck.h 2b40c38e7063ab822c974c0bd4aed97dabb579ccfe2e180a4639bb3bbef0f1c9 F ext/intck/test_intck.c dec07fc82e2626a1450e58be474e351643627b04ee08ce8fae6495a533b87e85 F ext/jni/GNUmakefile 59eb05f2a363bdfac8d15d66bed624bfe1ff289229184f3861b95f98a19cf4b2 @@ -2169,8 +2170,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5f310fb7be450c96f01c21e6fd1377d4a274784377d0bd811922ab63e612bd11 -R 86a36a81ccd4a5a3ded74d8234b0250a +P cfd051836b72f7d4e38cc9614f6ae5c003de4ce377359fd391adf06fe1ddf6b9 +R aa9167c841a6524a2987fc0088c3a9be U dan -Z 39f7083bcdb0c03c315a581938028ce8 +Z 2a1b98d11090580141aae0cbd3e3021e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index ff9880a342..c043830657 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cfd051836b72f7d4e38cc9614f6ae5c003de4ce377359fd391adf06fe1ddf6b9 \ No newline at end of file +351d46b2373f08bc8033d0902d9f67cd6c8bcc16c0d9f787e4fb279c0a76da87 \ No newline at end of file From 9c59c87448c2c8a7411bb65b4a163a1f634db8ab Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 24 Feb 2024 16:26:15 +0000 Subject: [PATCH 15/16] Add further tests for the intck module. FossilOrigin-Name: c253e276b29de28a18270d01b60d95157ce3fc4b37e246d991f9119d26e718d7 --- ext/intck/intck1.test | 125 ++++++++++++++++++++++++++++++++++++++- ext/intck/intckbusy.test | 48 +++++++++++++++ ext/intck/sqlite3intck.c | 74 ++++++++--------------- ext/intck/test_intck.c | 8 ++- manifest | 17 +++--- manifest.uuid | 2 +- 6 files changed, 211 insertions(+), 63 deletions(-) create mode 100644 ext/intck/intckbusy.test diff --git a/ext/intck/intck1.test b/ext/intck/intck1.test index 4e86e4f2fc..e05ae06166 100644 --- a/ext/intck/intck1.test +++ b/ext/intck/intck1.test @@ -200,9 +200,132 @@ reset_db do_execsql_test 5.0 { CREATE TABLE t1(a, b); CREATE INDEX i1 ON t1(a COLLATE NOCASE); + INSERT INTO t1 VALUES(1, 1); + INSERT INTO t1 VALUES(2, 2); } -#puts [intck_sql db i1] +do_test 5.1 { + set ic [sqlite3_intck db nosuchdb] + $ic step +} {SQLITE_ERROR} + +do_test 5.2 { + $ic close + set ic [sqlite3_intck db {}] + while {[$ic step]=="SQLITE_OK"} {} + set res [$ic error] + $ic close + set res +} {SQLITE_OK {}} + +do_test 5.3 { test_do_intck db "main" } {} + +do_test 5.4 { + set ret {} + set ic [sqlite3_intck db main] + db eval [$ic test_sql t1] { + if {$error_message!=""} { lappend ret $error_message } + } + $ic close + set ret +} {} + +do_test 5.5 { + set ret {} + set ic [sqlite3_intck db main] + db eval [$ic test_sql {}] { + if {$error_message!=""} { lappend ret $error_message } + } + $ic close + set ret +} {} + +db cache flush + +do_test 5.6 { + set ret {} + set ic [sqlite3_intck db main] + $ic step + db eval [$ic test_sql {}] { + if {$error_message!=""} { lappend ret $error_message } + } + $ic close + set ret +} {} + +#------------------------------------------------------------------------- +reset_db + +do_execsql_test 6.0 { + CREATE TABLE t1(x, y, PRIMARY KEY(x)) WITHOUT ROWID; + CREATE INDEX i1 ON t1(y, x); + INSERT INTO t1 VALUES(X'0000', X'1111'); +} + +do_intck_test 6.1 {} + +do_execsql_test 6.2.1 { + PRAGMA writable_schema = 1; + UPDATE sqlite_schema SET sql = 'CREATE INDEX i1' WHERE name='i1'; +} {} +do_intck_test 6.2.2 {} + +do_execsql_test 6.3.1 { + UPDATE sqlite_schema SET sql = 'CREATE INDEX i1(y' WHERE name='i1'; +} {} +do_intck_test 6.3.2 {} + +do_execsql_test 6.4.1 { + UPDATE sqlite_schema + SET sql = 'CREATE INDEX i1(y) hello world' + WHERE name='i1'; +} {} +do_intck_test 6.4.2 {} + +do_execsql_test 6.5.1 { + UPDATE sqlite_schema + SET sql = 'CREATE INDEX i1(y, x) WHERE 1 ' + WHERE name='i1'; +} {} +do_intck_test 6.5.2 {} + +do_execsql_test 6.6.1 { + UPDATE sqlite_schema + SET sql = 'CREATE INDEX i1( , ) WHERE 1 ' + WHERE name='i1'; +} {} + +do_test 6.7.2 { + set ic [sqlite3_intck db main] + $ic step +} {SQLITE_ERROR} +do_test 6.5.3 { + $ic error +} {SQLITE_ERROR {near "AS": syntax error}} +$ic close + +do_execsql_test 6.6.1 { + UPDATE sqlite_schema + SET sql = 'CREATE INDEX i1([y' + WHERE name='i1'; +} {} +do_intck_test 6.6.2 {} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 7.0 { + CREATE TABLE x1("1", "22", "3333", four); + CREATE INDEX i1 ON x1( "1" , "22", NULL); + INSERT INTO x1 VALUES(1, 22, 3333, NULL); + INSERT INTO x1 VALUES(1, 22, 3333, NULL); +} +do_execsql_test 7.1 " CREATE INDEX i2 ON x1( \"1\"\r\n\t ) " +do_execsql_test 7.2 { CREATE INDEX i3 ON x1( "22" || 'abc''def' || `1` ) } +do_execsql_test 7.3 { CREATE INDEX i4 ON x1( [22] + [1] ) } +do_execsql_test 7.4 { CREATE INDEX i5 ON x1( four||'hello' ) } + +do_intck_test 7.5 {} + finish_test diff --git a/ext/intck/intckbusy.test b/ext/intck/intckbusy.test new file mode 100644 index 0000000000..edfedf5ae8 --- /dev/null +++ b/ext/intck/intckbusy.test @@ -0,0 +1,48 @@ +# 2024 February 24 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +source [file join [file dirname [info script]] intck_common.tcl] +set testprefix intckbusy + + + +do_execsql_test 1.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + INSERT INTO t1 VALUES(1, 2, 3); + INSERT INTO t1 VALUES(2, 'two', 'three'); + INSERT INTO t1 VALUES(3, NULL, NULL); + CREATE INDEX i1 ON t1(b, c); +} + +sqlite3 db2 test.db + +do_execsql_test -db db2 1.1 { + BEGIN EXCLUSIVE; + INSERT INTO t1 VALUES(4, 5, 6); +} + +do_test 1.2 { + set ic [sqlite3_intck db main] + $ic step +} {SQLITE_BUSY} +do_test 1.3 { + $ic unlock +} {SQLITE_BUSY} +do_test 1.4 { + $ic error +} {SQLITE_BUSY {database is locked}} +do_test 1.4 { + $ic close +} {} + +finish_test + diff --git a/ext/intck/sqlite3intck.c b/ext/intck/sqlite3intck.c index b61035f157..12d205e4d9 100644 --- a/ext/intck/sqlite3intck.c +++ b/ext/intck/sqlite3intck.c @@ -60,12 +60,9 @@ struct sqlite3_intck { ** and error code currently held by the database handle in p->rc and p->zErr. */ static void intckSaveErrmsg(sqlite3_intck *p){ - const char *zDberr = sqlite3_errmsg(p->db); p->rc = sqlite3_errcode(p->db); - if( zDberr ){ - sqlite3_free(p->zErr); - p->zErr = sqlite3_mprintf("%s", zDberr); - } + sqlite3_free(p->zErr); + p->zErr = sqlite3_mprintf("%s", sqlite3_errmsg(p->db)); } /* @@ -125,6 +122,15 @@ static void intckFinalize(sqlite3_intck *p, sqlite3_stmt *pStmt){ } } +/* +** If there is already an error in handle p, return it. Otherwise, call +** sqlite3_step() on the statement handle and return that value. +*/ +static int intckStep(sqlite3_intck *p, sqlite3_stmt *pStmt){ + if( p->rc ) return p->rc; + return sqlite3_step(pStmt); +} + /* ** Execute SQL statement zSql. There is no way to obtain any results ** returned by the statement. This function uses the sqlite3_intck error @@ -133,42 +139,10 @@ static void intckFinalize(sqlite3_intck *p, sqlite3_stmt *pStmt){ static void intckExec(sqlite3_intck *p, const char *zSql){ sqlite3_stmt *pStmt = 0; pStmt = intckPrepare(p, zSql); - while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ); + intckStep(p, pStmt); intckFinalize(p, pStmt); } -/* -** Wrapper around sqlite3_malloc64() that uses the sqlite3_intck error -** code convention. -*/ -static void *intckMalloc(sqlite3_intck *p, sqlite3_int64 nByte){ - void *pRet = 0; - assert( nByte>0 ); - if( p->rc==SQLITE_OK ){ - pRet = sqlite3_malloc64(nByte); - if( pRet==0 ){ - p->rc = SQLITE_NOMEM; - } - } - return pRet; -} - -/* -** Like strdup(), but uses the sqlite3_intck error code convention. Any -** returned buffer should eventually be freed using sqlite3_free(). -*/ -static char *intckStrdup(sqlite3_intck *p, const char *zIn){ - char *zOut = 0; - if( zIn ){ - int nIn = strlen(zIn); - zOut = (char*)intckMalloc(p, nIn+1); - if( zOut ){ - memcpy(zOut, zIn, nIn+1); - } - } - return zOut; -} - /* ** A wrapper around sqlite3_mprintf() that uses the sqlite3_intck error ** code convention. @@ -280,7 +254,7 @@ static void intckSaveKey(sqlite3_intck *p){ sqlite3_bind_value(pStmt, ii+1, sqlite3_column_value(p->pCheck, ii+1)); } if( SQLITE_ROW==sqlite3_step(pStmt) ){ - p->zKey = intckStrdup(p, (const char*)sqlite3_column_text(pStmt, 0)); + p->zKey = intckMprintf(p,"%s",(const char*)sqlite3_column_text(pStmt, 0)); } intckFinalize(p, pStmt); } @@ -318,7 +292,7 @@ static void intckFindObject(sqlite3_intck *p){ if( p->rc==SQLITE_OK ){ sqlite3_bind_text(pStmt, 1, zPrev, -1, SQLITE_TRANSIENT); if( sqlite3_step(pStmt)==SQLITE_ROW ){ - p->zObj = intckStrdup(p, (const char*)sqlite3_column_text(pStmt, 0)); + p->zObj = intckMprintf(p,"%s",(const char*)sqlite3_column_text(pStmt, 0)); } } intckFinalize(p, pStmt); @@ -347,7 +321,7 @@ static int intckGetToken(const char *z){ while( 1 ){ if( z[iRet]==c ){ iRet++; - if( z[iRet+1]!=c ) break; + if( z[iRet]!=c ) break; } iRet++; } @@ -496,7 +470,7 @@ static int intckGetAutoIndex(sqlite3_intck *p){ int bRet = 0; sqlite3_stmt *pStmt = 0; pStmt = intckPrepare(p, "PRAGMA automatic_index"); - if( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + if( SQLITE_ROW==intckStep(p, pStmt) ){ bRet = sqlite3_column_int(pStmt, 0); } intckFinalize(p, pStmt); @@ -789,7 +763,7 @@ static char *intckCheckObjectSql( } while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ - zRet = intckStrdup(p, (const char*)sqlite3_column_text(pStmt, 0)); + zRet = intckMprintf(p, "%s", (const char*)sqlite3_column_text(pStmt, 0)); if( pnKeyVal ){ *pnKeyVal = sqlite3_column_int(pStmt, 1); } @@ -839,15 +813,15 @@ int sqlite3_intck_open( */ void sqlite3_intck_close(sqlite3_intck *p){ if( p ){ - if( p->db ){ - sqlite3_create_function( - p->db, "parse_create_index", 1, SQLITE_UTF8, 0, 0, 0, 0 - ); - } + sqlite3_finalize(p->pCheck); + sqlite3_create_function( + p->db, "parse_create_index", 1, SQLITE_UTF8, 0, 0, 0, 0 + ); sqlite3_free(p->zObj); sqlite3_free(p->zKey); sqlite3_free(p->zTestSql); sqlite3_free(p->zErr); + sqlite3_free(p->zMessage); sqlite3_free(p); } } @@ -881,7 +855,7 @@ int sqlite3_intck_step(sqlite3_intck *p){ } }else if( p->rc==SQLITE_CORRUPT ){ p->rc = SQLITE_OK; - p->zMessage = intckStrdup(p, + p->zMessage = intckMprintf(p, "%s", "corruption found while reading database schema" ); p->bCorruptSchema = 1; @@ -937,7 +911,7 @@ int sqlite3_intck_error(sqlite3_intck *p, const char **pzErr){ ** on the database. */ int sqlite3_intck_unlock(sqlite3_intck *p){ - if( p->pCheck && p->rc==SQLITE_OK ){ + if( p->rc==SQLITE_OK && p->pCheck ){ assert( p->zKey==0 && p->nKeyVal>0 ); intckSaveKey(p); intckFinalize(p, p->pCheck); diff --git a/ext/intck/test_intck.c b/ext/intck/test_intck.c index 4c34e2fd26..72f72d8c13 100644 --- a/ext/intck/test_intck.c +++ b/ext/intck/test_intck.c @@ -92,12 +92,12 @@ static int testIntckCmd( case 3: assert( 0==strcmp("error", aCmd[iIdx].zName) ); { const char *zErr = 0; - int rc = sqlite3_intck_error(p->intck, &zErr); + int rc = sqlite3_intck_error(p->intck, 0); Tcl_Obj *pRes = Tcl_NewObj(); - Tcl_ListObjAppendElement( interp, pRes, Tcl_NewStringObj(sqlite3ErrName(rc), -1) ); + sqlite3_intck_error(p->intck, &zErr); Tcl_ListObjAppendElement( interp, pRes, Tcl_NewStringObj(zErr ? zErr : 0, -1) ); @@ -160,6 +160,7 @@ static int test_sqlite3_intck( return TCL_ERROR; } zDb = Tcl_GetString(objv[2]); + if( zDb[0]=='\0' ) zDb = 0; rc = sqlite3_intck_open(db, zDb, &p->intck); if( rc!=SQLITE_OK ){ @@ -169,7 +170,7 @@ static int test_sqlite3_intck( } do { - sprintf(zName, "intck%d", iName); + sprintf(zName, "intck%d", iName++); }while( Tcl_GetCommandInfo(interp, zName, &info)!=0 ); Tcl_CreateObjCommand(interp, zName, testIntckCmd, (void*)p, testIntckFree); Tcl_SetObjResult(interp, Tcl_NewStringObj(zName, -1)); @@ -226,6 +227,7 @@ static int test_do_intck( } Tcl_DecrRefCount(pRet); sqlite3_intck_close(pCk); + sqlite3_intck_close(0); return rc ? TCL_ERROR : TCL_OK; } diff --git a/manifest b/manifest index 068e592cd0..732ad1f72c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stests\sfor\sthe\snew\scode\son\sthis\sbranch. -D 2024-02-23T20:51:06.837 +C Add\sfurther\stests\sfor\sthe\sintck\smodule. +D 2024-02-24T16:26:15.674 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -248,14 +248,15 @@ F ext/fts5/tool/showfts5.tcl d54da0e067306663e2d5d523965ca487698e722c F ext/icu/README.txt 7ab7ced8ae78e3a645b57e78570ff589d4c672b71370f5aa9e1cd7024f400fc9 F ext/icu/icu.c c074519b46baa484bb5396c7e01e051034da8884bad1a1cb7f09bbe6be3f0282 F ext/icu/sqliteicu.h fa373836ed5a1ee7478bdf8a1650689294e41d0c89c1daab26e9ae78a32075a8 -F ext/intck/intck1.test e73c4f87b54176291b9a01b52dbc9dd88f42f3ec8aea48c8e0bd7f87a1440a40 +F ext/intck/intck1.test 866f0937911bf3a10491af6ce319b75bcd587c39dc8decf2444746b946aa4f3e F ext/intck/intck2.test 97daaf43b8a3c870ba7c2fd421dc1887053f5b30188896fa64677ce8174c206f F ext/intck/intck_common.tcl 9e51458126576783f11051ac0fd25bea3f6b17f570a55884223737f3200b214b +F ext/intck/intckbusy.test 0732fe3efbb9e0a53ffdcc240073f6ff2777ea82c3e08812b16494f650763fe1 F ext/intck/intckcorrupt.test 3211ef68ac53e83951b6c8f6a8d2396506d123fe5898f97f848a25837744ec56 F ext/intck/intckfault.test ba0213c9c8dce08d519d5251268a3bab076a184b4d07acdea23b65e89c9ae03c -F ext/intck/sqlite3intck.c 642f57a4604580513547df9d8489cdb49b8f5f3af1981c7ffb87bc37e5ce1439 +F ext/intck/sqlite3intck.c 52381a627637504a49e93400814b36e99afa0b972a9a24ef1732b8268bb27fa8 F ext/intck/sqlite3intck.h 2b40c38e7063ab822c974c0bd4aed97dabb579ccfe2e180a4639bb3bbef0f1c9 -F ext/intck/test_intck.c dec07fc82e2626a1450e58be474e351643627b04ee08ce8fae6495a533b87e85 +F ext/intck/test_intck.c d63f1707432802f5db125ee40b794923af77d4686869bd8d3a7eb43332344267 F ext/jni/GNUmakefile 59eb05f2a363bdfac8d15d66bed624bfe1ff289229184f3861b95f98a19cf4b2 F ext/jni/README.md d899789a9082a07b99bf30b1bbb6204ae57c060efcaa634536fa669323918f42 F ext/jni/jar-dist.make 030aaa4ae71dd86e4ec5e7c1e6cd86f9dfa47c4592c070d2e35157e42498e1fa @@ -2170,8 +2171,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P cfd051836b72f7d4e38cc9614f6ae5c003de4ce377359fd391adf06fe1ddf6b9 -R aa9167c841a6524a2987fc0088c3a9be +P 351d46b2373f08bc8033d0902d9f67cd6c8bcc16c0d9f787e4fb279c0a76da87 +R 38b420c887519ac9a48b5a886e5c9758 U dan -Z 2a1b98d11090580141aae0cbd3e3021e +Z 449a9c9353165606d5987b5f74af35a1 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index c043830657..af08de7704 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -351d46b2373f08bc8033d0902d9f67cd6c8bcc16c0d9f787e4fb279c0a76da87 \ No newline at end of file +c253e276b29de28a18270d01b60d95157ce3fc4b37e246d991f9119d26e718d7 \ No newline at end of file From 1103eb423e7c975ba77f8a9a10b0667636191be8 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 26 Feb 2024 11:51:21 +0000 Subject: [PATCH 16/16] Fix header comments on test scripts. FossilOrigin-Name: 9fe9670c977ce7ad2c19657783a63998769dbb7741a587889c72ecc04d895793 --- ext/intck/intck1.test | 5 ++--- ext/intck/intck2.test | 4 ++-- manifest | 16 ++++++++-------- manifest.uuid | 2 +- 4 files changed, 13 insertions(+), 14 deletions(-) diff --git a/ext/intck/intck1.test b/ext/intck/intck1.test index e05ae06166..1708406304 100644 --- a/ext/intck/intck1.test +++ b/ext/intck/intck1.test @@ -9,7 +9,8 @@ # #*********************************************************************** # -# The focus of this file is testing the r-tree extension. +# The focus of this file is testing the incremental integrity check +# (intck) extension. # source [file join [file dirname [info script]] intck_common.tcl] @@ -328,5 +329,3 @@ do_intck_test 7.5 {} finish_test - - diff --git a/ext/intck/intck2.test b/ext/intck/intck2.test index c168118535..c8503042c9 100644 --- a/ext/intck/intck2.test +++ b/ext/intck/intck2.test @@ -9,7 +9,8 @@ # #*********************************************************************** # -# The focus of this file is testing the r-tree extension. +# The focus of this file is testing the incremental integrity check +# (intck) extension. # source [file join [file dirname [info script]] intck_common.tcl] @@ -173,4 +174,3 @@ do_intck_test 3.8 { } finish_test - diff --git a/manifest b/manifest index f34154f890..912b02d07c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\slatest\strunk\schanges\sinto\sthis\sbranch. -D 2024-02-26T10:56:54.404 +C Fix\sheader\scomments\son\stest\sscripts. +D 2024-02-26T11:51:21.317 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -250,8 +250,8 @@ F ext/fts5/tool/showfts5.tcl d54da0e067306663e2d5d523965ca487698e722c F ext/icu/README.txt 7ab7ced8ae78e3a645b57e78570ff589d4c672b71370f5aa9e1cd7024f400fc9 F ext/icu/icu.c c074519b46baa484bb5396c7e01e051034da8884bad1a1cb7f09bbe6be3f0282 F ext/icu/sqliteicu.h fa373836ed5a1ee7478bdf8a1650689294e41d0c89c1daab26e9ae78a32075a8 -F ext/intck/intck1.test 866f0937911bf3a10491af6ce319b75bcd587c39dc8decf2444746b946aa4f3e -F ext/intck/intck2.test 97daaf43b8a3c870ba7c2fd421dc1887053f5b30188896fa64677ce8174c206f +F ext/intck/intck1.test 8a879640c90fdff5e91e6c2c41d509485ee634e8077fe0ca9f76be4cbd441fa3 +F ext/intck/intck2.test 47afb44681d13d11072cd8906e6aa877c967e65be788b01f6139922fd91474ef F ext/intck/intck_common.tcl 9e51458126576783f11051ac0fd25bea3f6b17f570a55884223737f3200b214b F ext/intck/intckbusy.test 0732fe3efbb9e0a53ffdcc240073f6ff2777ea82c3e08812b16494f650763fe1 F ext/intck/intckcorrupt.test 3211ef68ac53e83951b6c8f6a8d2396506d123fe5898f97f848a25837744ec56 @@ -2173,8 +2173,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c253e276b29de28a18270d01b60d95157ce3fc4b37e246d991f9119d26e718d7 7e4c743f9e6ef33500795543e6db9a77c533025bf00c2ee97abd433a3871b5a1 -R d91900a2b738ceadf979774120590e4a -U dan -Z 9b2321809e4d89643b001730f06f9a2c +P b6371ff9f5c3d4e87a5b1127a82970202f74c790b828dda20df88dd727bcc9b4 +R 3add559ac6b3cdeca1489f677e09e848 +U drh +Z 4a33c5af1c72f813c7944d5f457e89df # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b5e0bfba97..391e1cbab5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b6371ff9f5c3d4e87a5b1127a82970202f74c790b828dda20df88dd727bcc9b4 \ No newline at end of file +9fe9670c977ce7ad2c19657783a63998769dbb7741a587889c72ecc04d895793 \ No newline at end of file