diff --git a/Makefile.in b/Makefile.in index f78a7c31eb..862f397977 100644 --- a/Makefile.in +++ b/Makefile.in @@ -441,6 +441,7 @@ TESTSRC += \ $(TOP)/ext/misc/ieee754.c \ $(TOP)/ext/misc/mmapwarm.c \ $(TOP)/ext/misc/nextchar.c \ + $(TOP)/ext/misc/normalize.c \ $(TOP)/ext/misc/percentile.c \ $(TOP)/ext/misc/regexp.c \ $(TOP)/ext/misc/remember.c \ @@ -591,6 +592,7 @@ SHELL_OPT += -DSQLITE_INTROSPECTION_PRAGMAS FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1 FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ FUZZCHECK_OPT += -DSQLITE_MAX_MEMORY=50000000 +FUZZCHECK_OPT += -DSQLITE_PRINTF_PRECISION_LIMIT=1000 FUZZCHECK_SRC = $(TOP)/test/fuzzcheck.c $(TOP)/test/ossfuzz.c DBFUZZ_OPT = diff --git a/Makefile.msc b/Makefile.msc index 5d248bc9c6..0f906c80d5 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1501,6 +1501,7 @@ TESTEXT = \ $(TOP)\ext\misc\ieee754.c \ $(TOP)\ext\misc\mmapwarm.c \ $(TOP)\ext\misc\nextchar.c \ + $(TOP)\ext\misc\normalize.c \ $(TOP)\ext\misc\percentile.c \ $(TOP)\ext\misc\regexp.c \ $(TOP)\ext\misc\remember.c \ @@ -1615,7 +1616,7 @@ SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_RTREE # MPTESTER_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS5 FUZZERSHELL_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 -FUZZCHECK_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ -DSQLITE_MAX_MEMORY=50000000 +FUZZCHECK_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ -DSQLITE_MAX_MEMORY=50000000 -DSQLITE_PRINTF_PRECISION_LIMIT=1000 FUZZCHECK_SRC = $(TOP)\test\fuzzcheck.c $(TOP)\test\ossfuzz.c OSSSHELL_SRC = $(TOP)\test\ossshell.c $(TOP)\test\ossfuzz.c DBFUZZ_COMPILE_OPTS = -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION @@ -2255,6 +2256,7 @@ TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_SERIES_CONSTRAINT_VERIFY=1 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_DEFAULT_PAGE_SIZE=1024 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_DBPAGE_VTAB +TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_JSON1 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) $(TEST_CCONV_OPTS) TESTFIXTURE_SRC0 = $(TESTEXT) $(TESTSRC2) diff --git a/README.md b/README.md index e5fa2acb67..a92e5ae24e 100644 --- a/README.md +++ b/README.md @@ -14,15 +14,17 @@ SQLite sources are managed using the [Fossil](https://www.fossil-scm.org/), a distributed version control system that was specifically designed to support SQLite development. If you do not want to use Fossil, you can download tarballs or ZIP -archives as follows: +archives or [SQLite archives](https://sqlite.org/cli.html#sqlar) as follows: - * Lastest trunk check-in: - or - . + * Lastest trunk check-in as + [Tarball](https://www.sqlite.org/src/tarball/sqlite.tar.gz), + [ZIP-archive](https://www.sqlite.org/src/zip/sqlite.zip), or + [SQLite-archive](https://www.sqlite.org/src/sqlar/sqlite.sqlar). - * Latest release: - or - . + * Latest release as + [Tarball](https://www.sqlite.org/src/tarball/sqlite.tar.gz?r=release), + [ZIP-archive](https://www.sqlite.org/src/zip/sqlite.zip?r=release), or + [SQLite-archive](https://www.sqlite.org/src/sqlar/sqlite.sqlar?r=release). * For other check-ins, substitute an appropriate branch name or tag or hash prefix for "release" in the URLs of the previous diff --git a/autoconf/Makefile.am b/autoconf/Makefile.am index 0f0706873a..db246924fe 100644 --- a/autoconf/Makefile.am +++ b/autoconf/Makefile.am @@ -10,7 +10,7 @@ sqlite3_SOURCES = shell.c sqlite3.h EXTRA_sqlite3_SOURCES = sqlite3.c sqlite3_LDADD = @EXTRA_SHELL_OBJ@ @READLINE_LIBS@ sqlite3_DEPENDENCIES = @EXTRA_SHELL_OBJ@ -sqlite3_CFLAGS = $(AM_CFLAGS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_ENABLE_STMTVTAB -DSQLITE_ENABLE_DBSTAT_VTAB +sqlite3_CFLAGS = $(AM_CFLAGS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_ENABLE_STMTVTAB -DSQLITE_ENABLE_DBSTAT_VTAB $(SHELL_CFLAGS) include_HEADERS = sqlite3.h sqlite3ext.h diff --git a/autoconf/configure.ac b/autoconf/configure.ac index 19975e96c7..2680becbb0 100644 --- a/autoconf/configure.ac +++ b/autoconf/configure.ac @@ -169,6 +169,9 @@ AC_CHECK_HEADERS(zlib.h,[ ]) AC_SUBST(ZLIB_FLAGS) +AC_SEARCH_LIBS(system,,,[SHELL_CFLAGS="-DSQLITE_NOHAVE_SYSTEM"]) +AC_SUBST(SHELL_CFLAGS) + #----------------------------------------------------------------------- # UPDATE: Maybe it's better if users just set CFLAGS before invoking # configure. This option doesn't really add much... diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index daf3399a43..0baf82b76e 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -1908,6 +1908,7 @@ static int fts3WriteSegment( sqlite3_bind_blob(pStmt, 2, z, n, SQLITE_STATIC); sqlite3_step(pStmt); rc = sqlite3_reset(pStmt); + sqlite3_bind_null(pStmt, 2); } return rc; } @@ -1964,6 +1965,7 @@ static int fts3WriteSegdir( sqlite3_bind_blob(pStmt, 6, zRoot, nRoot, SQLITE_STATIC); sqlite3_step(pStmt); rc = sqlite3_reset(pStmt); + sqlite3_bind_null(pStmt, 6); } return rc; } @@ -3443,6 +3445,7 @@ static void fts3UpdateDocTotals( sqlite3_bind_blob(pStmt, 2, pBlob, nBlob, SQLITE_STATIC); sqlite3_step(pStmt); *pRC = sqlite3_reset(pStmt); + sqlite3_bind_null(pStmt, 2); sqlite3_free(a); } @@ -4631,6 +4634,7 @@ static int fts3TruncateSegment( sqlite3_bind_int(pChomp, 4, iIdx); sqlite3_step(pChomp); rc = sqlite3_reset(pChomp); + sqlite3_bind_null(pChomp, 2); } } @@ -4710,6 +4714,7 @@ static int fts3IncrmergeHintStore(Fts3Table *p, Blob *pHint){ sqlite3_bind_blob(pReplace, 2, pHint->a, pHint->n, SQLITE_STATIC); sqlite3_step(pReplace); rc = sqlite3_reset(pReplace); + sqlite3_bind_null(pReplace, 2); } return rc; @@ -5524,7 +5529,6 @@ int sqlite3Fts3UpdateMethod( ){ Fts3Table *p = (Fts3Table *)pVtab; int rc = SQLITE_OK; /* Return Code */ - int isRemove = 0; /* True for an UPDATE or DELETE */ u32 *aSzIns = 0; /* Sizes of inserted documents */ u32 *aSzDel = 0; /* Sizes of deleted documents */ int nChng = 0; /* Net change in number of documents */ @@ -5622,7 +5626,6 @@ int sqlite3Fts3UpdateMethod( if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){ assert( sqlite3_value_type(apVal[0])==SQLITE_INTEGER ); rc = fts3DeleteByRowid(p, apVal[0], &nChng, aSzDel); - isRemove = 1; } /* If this is an INSERT or UPDATE operation, insert the new record. */ @@ -5634,7 +5637,7 @@ int sqlite3Fts3UpdateMethod( rc = FTS_CORRUPT_VTAB; } } - if( rc==SQLITE_OK && (!isRemove || *pRowid!=p->iPrevDocid ) ){ + if( rc==SQLITE_OK ){ rc = fts3PendingTermsDocid(p, 0, iLangid, *pRowid); } if( rc==SQLITE_OK ){ diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index a75bf0fd42..412a04faca 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -758,6 +758,7 @@ static void fts5DataWrite(Fts5Index *p, i64 iRowid, const u8 *pData, int nData){ sqlite3_bind_blob(p->pWriter, 2, pData, nData, SQLITE_STATIC); sqlite3_step(p->pWriter); p->rc = sqlite3_reset(p->pWriter); + sqlite3_bind_null(p->pWriter, 2); } /* @@ -2386,6 +2387,7 @@ static void fts5SegIterSeekInit( bDlidx = (val & 0x0001); } p->rc = sqlite3_reset(pIdxSelect); + sqlite3_bind_null(pIdxSelect, 2); if( iPgpgnoFirst ){ iPg = pSeg->pgnoFirst; @@ -3598,6 +3600,7 @@ static int fts5AllocateSegid(Fts5Index *p, Fts5Structure *pStruct){ sqlite3_bind_blob(pIdxSelect, 2, aBlob, 2, SQLITE_STATIC); assert( sqlite3_step(pIdxSelect)!=SQLITE_ROW ); p->rc = sqlite3_reset(pIdxSelect); + sqlite3_bind_null(pIdxSelect, 2); } } #endif @@ -3724,6 +3727,7 @@ static void fts5WriteFlushBtree(Fts5Index *p, Fts5SegWriter *pWriter){ sqlite3_bind_int64(p->pIdxWriter, 3, bFlag + ((i64)pWriter->iBtPage<<1)); sqlite3_step(p->pIdxWriter); p->rc = sqlite3_reset(p->pIdxWriter); + sqlite3_bind_null(p->pIdxWriter, 2); } pWriter->iBtPage = 0; } diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c index 59336fc7ac..70d7135113 100644 --- a/ext/fts5/fts5_storage.c +++ b/ext/fts5/fts5_storage.c @@ -458,6 +458,7 @@ static int fts5StorageInsertDocsize( sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC); sqlite3_step(pReplace); rc = sqlite3_reset(pReplace); + sqlite3_bind_null(pReplace, 2); } } return rc; @@ -1118,6 +1119,7 @@ int sqlite3Fts5StorageConfigValue( } sqlite3_step(pReplace); rc = sqlite3_reset(pReplace); + sqlite3_bind_null(pReplace, 1); } if( rc==SQLITE_OK && pVal ){ int iNew = p->pConfig->iCookie + 1; diff --git a/ext/fts5/test/fts5aa.test b/ext/fts5/test/fts5aa.test index a3ea0afc28..67cb62012a 100644 --- a/ext/fts5/test/fts5aa.test +++ b/ext/fts5/test/fts5aa.test @@ -593,5 +593,5 @@ do_execsql_test 22.1 { } - +expand_all_sql db finish_test diff --git a/ext/misc/fileio.c b/ext/misc/fileio.c index e51b3e620b..7226ff18b3 100644 --- a/ext/misc/fileio.c +++ b/ext/misc/fileio.c @@ -491,6 +491,7 @@ static void fsdirResetCursor(fsdir_cursor *pCur){ sqlite3_free(pLvl->zDir); } sqlite3_free(pCur->zPath); + sqlite3_free(pCur->aLvl); pCur->aLvl = 0; pCur->zPath = 0; pCur->zBase = 0; @@ -506,7 +507,6 @@ static int fsdirClose(sqlite3_vtab_cursor *cur){ fsdir_cursor *pCur = (fsdir_cursor*)cur; fsdirResetCursor(pCur); - sqlite3_free(pCur->aLvl); sqlite3_free(pCur); return SQLITE_OK; } diff --git a/ext/misc/normalize.c b/ext/misc/normalize.c new file mode 100644 index 0000000000..fd656f1303 --- /dev/null +++ b/ext/misc/normalize.c @@ -0,0 +1,707 @@ +/* +** 2018-01-08 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains code to implement the sqlite3_normalize() function. +** +** char *sqlite3_normalize(const char *zSql); +** +** This function takes an SQL string as input and returns a "normalized" +** version of that string in memory obtained from sqlite3_malloc64(). The +** caller is responsible for ensuring that the returned memory is freed. +** +** If a memory allocation error occurs, this routine returns NULL. +** +** The normalization consists of the following transformations: +** +** (1) Convert every literal (string, blob literal, numeric constant, +** or "NULL" constant) into a ? +** +** (2) Remove all superfluous whitespace, including comments. Change +** all required whitespace to a single space character. +** +** (3) Lowercase all ASCII characters. +** +** (4) If an IN or NOT IN operator is followed by a list of 1 or more +** values, convert that list into "(?,?,?)". +** +** The purpose of normalization is two-fold: +** +** (1) Sanitize queries by removing potentially private or sensitive +** information contained in literals. +** +** (2) Identify structurally identical queries by comparing their +** normalized forms. +** +** Command-Line Utility +** -------------------- +** +** This file also contains code for a command-line utility that converts +** SQL queries in text files into their normalized forms. To build the +** command-line program, compile this file with -DSQLITE_NORMALIZE_CLI +** and link it against the SQLite library. +*/ +#include +#include + +/* +** Implementation note: +** +** Much of the tokenizer logic is copied out of the tokenize.c source file +** of SQLite. That logic could be simplified for this particular application, +** but that would impose a risk of introducing subtle errors. It is best to +** keep the code as close to the original as possible. +** +** The tokenize code is in sync with the SQLite core as of 2018-01-08. +** Any future changes to the core tokenizer might require corresponding +** adjustments to the tokenizer logic in this module. +*/ + + +/* Character classes for tokenizing +** +** In the sqlite3GetToken() function, a switch() on aiClass[c] is implemented +** using a lookup table, whereas a switch() directly on c uses a binary search. +** The lookup table is much faster. To maximize speed, and to ensure that +** a lookup table is used, all of the classes need to be small integers and +** all of them need to be used within the switch. +*/ +#define CC_X 0 /* The letter 'x', or start of BLOB literal */ +#define CC_KYWD 1 /* Alphabetics or '_'. Usable in a keyword */ +#define CC_ID 2 /* unicode characters usable in IDs */ +#define CC_DIGIT 3 /* Digits */ +#define CC_DOLLAR 4 /* '$' */ +#define CC_VARALPHA 5 /* '@', '#', ':'. Alphabetic SQL variables */ +#define CC_VARNUM 6 /* '?'. Numeric SQL variables */ +#define CC_SPACE 7 /* Space characters */ +#define CC_QUOTE 8 /* '"', '\'', or '`'. String literals, quoted ids */ +#define CC_QUOTE2 9 /* '['. [...] style quoted ids */ +#define CC_PIPE 10 /* '|'. Bitwise OR or concatenate */ +#define CC_MINUS 11 /* '-'. Minus or SQL-style comment */ +#define CC_LT 12 /* '<'. Part of < or <= or <> */ +#define CC_GT 13 /* '>'. Part of > or >= */ +#define CC_EQ 14 /* '='. Part of = or == */ +#define CC_BANG 15 /* '!'. Part of != */ +#define CC_SLASH 16 /* '/'. / or c-style comment */ +#define CC_LP 17 /* '(' */ +#define CC_RP 18 /* ')' */ +#define CC_SEMI 19 /* ';' */ +#define CC_PLUS 20 /* '+' */ +#define CC_STAR 21 /* '*' */ +#define CC_PERCENT 22 /* '%' */ +#define CC_COMMA 23 /* ',' */ +#define CC_AND 24 /* '&' */ +#define CC_TILDA 25 /* '~' */ +#define CC_DOT 26 /* '.' */ +#define CC_ILLEGAL 27 /* Illegal character */ + +static const unsigned char aiClass[] = { +/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xa xb xc xd xe xf */ +/* 0x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 7, 7, 27, 7, 7, 27, 27, +/* 1x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, +/* 2x */ 7, 15, 8, 5, 4, 22, 24, 8, 17, 18, 21, 20, 23, 11, 26, 16, +/* 3x */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 5, 19, 12, 14, 13, 6, +/* 4x */ 5, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, +/* 5x */ 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 9, 27, 27, 27, 1, +/* 6x */ 8, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, +/* 7x */ 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 27, 10, 27, 25, 27, +/* 8x */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* 9x */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* Ax */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* Bx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* Cx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* Dx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* Ex */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* Fx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 +}; + +/* An array to map all upper-case characters into their corresponding +** lower-case character. +** +** SQLite only considers US-ASCII (or EBCDIC) characters. We do not +** handle case conversions for the UTF character set since the tables +** involved are nearly as big or bigger than SQLite itself. +*/ +static const unsigned char sqlite3UpperToLower[] = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, + 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, + 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, + 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 97, 98, 99,100,101,102,103, + 104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121, + 122, 91, 92, 93, 94, 95, 96, 97, 98, 99,100,101,102,103,104,105,106,107, + 108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125, + 126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143, + 144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161, + 162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179, + 180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197, + 198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215, + 216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233, + 234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251, + 252,253,254,255 +}; + +/* +** The following 256 byte lookup table is used to support SQLites built-in +** equivalents to the following standard library functions: +** +** isspace() 0x01 +** isalpha() 0x02 +** isdigit() 0x04 +** isalnum() 0x06 +** isxdigit() 0x08 +** toupper() 0x20 +** SQLite identifier character 0x40 +** Quote character 0x80 +** +** Bit 0x20 is set if the mapped character requires translation to upper +** case. i.e. if the character is a lower-case ASCII character. +** If x is a lower-case ASCII character, then its upper-case equivalent +** is (x - 0x20). Therefore toupper() can be implemented as: +** +** (x & ~(map[x]&0x20)) +** +** The equivalent of tolower() is implemented using the sqlite3UpperToLower[] +** array. tolower() is used more often than toupper() by SQLite. +** +** Bit 0x40 is set if the character is non-alphanumeric and can be used in an +** SQLite identifier. Identifiers are alphanumerics, "_", "$", and any +** non-ASCII UTF character. Hence the test for whether or not a character is +** part of an identifier is 0x46. +*/ +static const unsigned char sqlite3CtypeMap[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 00..07 ........ */ + 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00, /* 08..0f ........ */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 10..17 ........ */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 18..1f ........ */ + 0x01, 0x00, 0x80, 0x00, 0x40, 0x00, 0x00, 0x80, /* 20..27 !"#$%&' */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 28..2f ()*+,-./ */ + 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, /* 30..37 01234567 */ + 0x0c, 0x0c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 38..3f 89:;<=>? */ + + 0x00, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x02, /* 40..47 @ABCDEFG */ + 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 48..4f HIJKLMNO */ + 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 50..57 PQRSTUVW */ + 0x02, 0x02, 0x02, 0x80, 0x00, 0x00, 0x00, 0x40, /* 58..5f XYZ[\]^_ */ + 0x80, 0x2a, 0x2a, 0x2a, 0x2a, 0x2a, 0x2a, 0x22, /* 60..67 `abcdefg */ + 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, /* 68..6f hijklmno */ + 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, /* 70..77 pqrstuvw */ + 0x22, 0x22, 0x22, 0x00, 0x00, 0x00, 0x00, 0x00, /* 78..7f xyz{|}~. */ + + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* 80..87 ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* 88..8f ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* 90..97 ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* 98..9f ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* a0..a7 ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* a8..af ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* b0..b7 ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* b8..bf ........ */ + + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* c0..c7 ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* c8..cf ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* d0..d7 ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* d8..df ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* e0..e7 ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* e8..ef ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* f0..f7 ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40 /* f8..ff ........ */ +}; +#define sqlite3Toupper(x) ((x)&~(sqlite3CtypeMap[(unsigned char)(x)]&0x20)) +#define sqlite3Isspace(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x01) +#define sqlite3Isalnum(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x06) +#define sqlite3Isalpha(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x02) +#define sqlite3Isdigit(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x04) +#define sqlite3Isxdigit(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x08) +#define sqlite3Tolower(x) (sqlite3UpperToLower[(unsigned char)(x)]) +#define sqlite3Isquote(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x80) + + +/* +** If X is a character that can be used in an identifier then +** IdChar(X) will be true. Otherwise it is false. +** +** For ASCII, any character with the high-order bit set is +** allowed in an identifier. For 7-bit characters, +** sqlite3IsIdChar[X] must be 1. +** +** For EBCDIC, the rules are more complex but have the same +** end result. +** +** Ticket #1066. the SQL standard does not allow '$' in the +** middle of identifiers. But many SQL implementations do. +** SQLite will allow '$' in identifiers for compatibility. +** But the feature is undocumented. +*/ +#define IdChar(C) ((sqlite3CtypeMap[(unsigned char)C]&0x46)!=0) + +/* +** Ignore testcase() macros +*/ +#define testcase(X) + +/* +** Token values +*/ +#define TK_SPACE 0 +#define TK_NAME 1 +#define TK_LITERAL 2 +#define TK_PUNCT 3 +#define TK_ERROR 4 + +#define TK_MINUS TK_PUNCT +#define TK_LP TK_PUNCT +#define TK_RP TK_PUNCT +#define TK_SEMI TK_PUNCT +#define TK_PLUS TK_PUNCT +#define TK_STAR TK_PUNCT +#define TK_SLASH TK_PUNCT +#define TK_REM TK_PUNCT +#define TK_EQ TK_PUNCT +#define TK_LE TK_PUNCT +#define TK_NE TK_PUNCT +#define TK_LSHIFT TK_PUNCT +#define TK_LT TK_PUNCT +#define TK_GE TK_PUNCT +#define TK_RSHIFT TK_PUNCT +#define TK_GT TK_PUNCT +#define TK_GE TK_PUNCT +#define TK_BITOR TK_PUNCT +#define TK_CONCAT TK_PUNCT +#define TK_COMMA TK_PUNCT +#define TK_BITAND TK_PUNCT +#define TK_BITNOT TK_PUNCT +#define TK_STRING TK_LITERAL +#define TK_ID TK_NAME +#define TK_ILLEGAL TK_ERROR +#define TK_DOT TK_PUNCT +#define TK_INTEGER TK_LITERAL +#define TK_FLOAT TK_LITERAL +#define TK_VARIABLE TK_LITERAL +#define TK_BLOB TK_LITERAL + +/* +** Return the length (in bytes) of the token that begins at z[0]. +** Store the token type in *tokenType before returning. +*/ +static int sqlite3GetToken(const unsigned char *z, int *tokenType){ + int i, c; + switch( aiClass[*z] ){ /* Switch on the character-class of the first byte + ** of the token. See the comment on the CC_ defines + ** above. */ + case CC_SPACE: { + for(i=1; sqlite3Isspace(z[i]); i++){} + *tokenType = TK_SPACE; + return i; + } + case CC_MINUS: { + if( z[1]=='-' ){ + for(i=2; (c=z[i])!=0 && c!='\n'; i++){} + *tokenType = TK_SPACE; + return i; + } + *tokenType = TK_MINUS; + return 1; + } + case CC_LP: { + *tokenType = TK_LP; + return 1; + } + case CC_RP: { + *tokenType = TK_RP; + return 1; + } + case CC_SEMI: { + *tokenType = TK_SEMI; + return 1; + } + case CC_PLUS: { + *tokenType = TK_PLUS; + return 1; + } + case CC_STAR: { + *tokenType = TK_STAR; + return 1; + } + case CC_SLASH: { + if( z[1]!='*' || z[2]==0 ){ + *tokenType = TK_SLASH; + return 1; + } + for(i=3, c=z[2]; (c!='*' || z[i]!='/') && (c=z[i])!=0; i++){} + if( c ) i++; + *tokenType = TK_SPACE; + return i; + } + case CC_PERCENT: { + *tokenType = TK_REM; + return 1; + } + case CC_EQ: { + *tokenType = TK_EQ; + return 1 + (z[1]=='='); + } + case CC_LT: { + if( (c=z[1])=='=' ){ + *tokenType = TK_LE; + return 2; + }else if( c=='>' ){ + *tokenType = TK_NE; + return 2; + }else if( c=='<' ){ + *tokenType = TK_LSHIFT; + return 2; + }else{ + *tokenType = TK_LT; + return 1; + } + } + case CC_GT: { + if( (c=z[1])=='=' ){ + *tokenType = TK_GE; + return 2; + }else if( c=='>' ){ + *tokenType = TK_RSHIFT; + return 2; + }else{ + *tokenType = TK_GT; + return 1; + } + } + case CC_BANG: { + if( z[1]!='=' ){ + *tokenType = TK_ILLEGAL; + return 1; + }else{ + *tokenType = TK_NE; + return 2; + } + } + case CC_PIPE: { + if( z[1]!='|' ){ + *tokenType = TK_BITOR; + return 1; + }else{ + *tokenType = TK_CONCAT; + return 2; + } + } + case CC_COMMA: { + *tokenType = TK_COMMA; + return 1; + } + case CC_AND: { + *tokenType = TK_BITAND; + return 1; + } + case CC_TILDA: { + *tokenType = TK_BITNOT; + return 1; + } + case CC_QUOTE: { + int delim = z[0]; + testcase( delim=='`' ); + testcase( delim=='\'' ); + testcase( delim=='"' ); + for(i=1; (c=z[i])!=0; i++){ + if( c==delim ){ + if( z[i+1]==delim ){ + i++; + }else{ + break; + } + } + } + if( c=='\'' ){ + *tokenType = TK_STRING; + return i+1; + }else if( c!=0 ){ + *tokenType = TK_ID; + return i+1; + }else{ + *tokenType = TK_ILLEGAL; + return i; + } + } + case CC_DOT: { + if( !sqlite3Isdigit(z[1]) ){ + *tokenType = TK_DOT; + return 1; + } + /* If the next character is a digit, this is a floating point + ** number that begins with ".". Fall thru into the next case */ + } + case CC_DIGIT: { + *tokenType = TK_INTEGER; + if( z[0]=='0' && (z[1]=='x' || z[1]=='X') && sqlite3Isxdigit(z[2]) ){ + for(i=3; sqlite3Isxdigit(z[i]); i++){} + return i; + } + for(i=0; sqlite3Isdigit(z[i]); i++){} + if( z[i]=='.' ){ + i++; + while( sqlite3Isdigit(z[i]) ){ i++; } + *tokenType = TK_FLOAT; + } + if( (z[i]=='e' || z[i]=='E') && + ( sqlite3Isdigit(z[i+1]) + || ((z[i+1]=='+' || z[i+1]=='-') && sqlite3Isdigit(z[i+2])) + ) + ){ + i += 2; + while( sqlite3Isdigit(z[i]) ){ i++; } + *tokenType = TK_FLOAT; + } + while( IdChar(z[i]) ){ + *tokenType = TK_ILLEGAL; + i++; + } + return i; + } + case CC_QUOTE2: { + for(i=1, c=z[0]; c!=']' && (c=z[i])!=0; i++){} + *tokenType = c==']' ? TK_ID : TK_ILLEGAL; + return i; + } + case CC_VARNUM: { + *tokenType = TK_VARIABLE; + for(i=1; sqlite3Isdigit(z[i]); i++){} + return i; + } + case CC_DOLLAR: + case CC_VARALPHA: { + int n = 0; + testcase( z[0]=='$' ); testcase( z[0]=='@' ); + testcase( z[0]==':' ); testcase( z[0]=='#' ); + *tokenType = TK_VARIABLE; + for(i=1; (c=z[i])!=0; i++){ + if( IdChar(c) ){ + n++; + }else if( c=='(' && n>0 ){ + do{ + i++; + }while( (c=z[i])!=0 && !sqlite3Isspace(c) && c!=')' ); + if( c==')' ){ + i++; + }else{ + *tokenType = TK_ILLEGAL; + } + break; + }else if( c==':' && z[i+1]==':' ){ + i++; + }else{ + break; + } + } + if( n==0 ) *tokenType = TK_ILLEGAL; + return i; + } + case CC_KYWD: { + for(i=1; aiClass[z[i]]<=CC_KYWD; i++){} + if( IdChar(z[i]) ){ + /* This token started out using characters that can appear in keywords, + ** but z[i] is a character not allowed within keywords, so this must + ** be an identifier instead */ + i++; + break; + } + *tokenType = TK_ID; + return i; + } + case CC_X: { + testcase( z[0]=='x' ); testcase( z[0]=='X' ); + if( z[1]=='\'' ){ + *tokenType = TK_BLOB; + for(i=2; sqlite3Isxdigit(z[i]); i++){} + if( z[i]!='\'' || i%2 ){ + *tokenType = TK_ILLEGAL; + while( z[i] && z[i]!='\'' ){ i++; } + } + if( z[i] ) i++; + return i; + } + /* If it is not a BLOB literal, then it must be an ID, since no + ** SQL keywords start with the letter 'x'. Fall through */ + } + case CC_ID: { + i = 1; + break; + } + default: { + *tokenType = TK_ILLEGAL; + return 1; + } + } + while( IdChar(z[i]) ){ i++; } + *tokenType = TK_ID; + return i; +} + +char *sqlite3_normalize(const char *zSql){ + char *z; /* The output string */ + sqlite3_int64 nZ; /* Size of the output string in bytes */ + sqlite3_int64 nSql; /* Size of the input string in bytes */ + int i; /* Next character to read from zSql[] */ + int j; /* Next slot to fill in on z[] */ + int tokenType; /* Type of the next token */ + int n; /* Size of the next token */ + int k; /* Loop counter */ + + nSql = strlen(zSql); + nZ = nSql; + z = sqlite3_malloc64( nZ+2 ); + if( z==0 ) return 0; + for(i=j=0; zSql[i]; i += n){ + n = sqlite3GetToken((unsigned char*)zSql+i, &tokenType); + switch( tokenType ){ + case TK_SPACE: { + break; + } + case TK_ERROR: { + sqlite3_free(z); + return 0; + } + case TK_LITERAL: { + z[j++] = '?'; + break; + } + case TK_PUNCT: + case TK_NAME: { + if( n==4 && sqlite3_strnicmp(zSql+i,"NULL",4)==0 ){ + if( (j>=3 && strncmp(z+j-2,"is",2)==0 && !IdChar(z[j-3])) + || (j>=4 && strncmp(z+j-3,"not",3)==0 && !IdChar(z[j-4])) + ){ + /* NULL is a keyword in this case, not a literal value */ + }else{ + /* Here the NULL is a literal value */ + z[j++] = '?'; + break; + } + } + if( j>0 && IdChar(z[j-1]) && IdChar(zSql[i]) ) z[j++] = ' '; + for(k=0; k0 && z[j-1]==' ' ){ j--; } + if( i>0 && z[j-1]!=';' ){ z[j++] = ';'; } + z[j] = 0; + + /* Make a second pass converting "in(...)" where the "..." is not a + ** SELECT statement into "in(?,?,?)" */ + for(i=0; i5 ){ + memmove(z+n+5, z+n+k, j-(n+k)); + } + j = j-k+5; + z[j] = 0; + memcpy(z+n, "?,?,?", 5); + } + return z; +} + +/* +** For testing purposes, or to build a stand-alone SQL normalizer program, +** compile this one source file with the -DSQLITE_NORMALIZE_CLI and link +** it against any SQLite library. The resulting command-line program will +** run sqlite3_normalize() over the text of all files named on the command- +** line and show the result on standard output. +*/ +#ifdef SQLITE_NORMALIZE_CLI +#include +#include + +/* +** Break zIn up into separate SQL statements and run sqlite3_normalize() +** on each one. Print the result of each run. +*/ +static void normalizeFile(char *zIn){ + int i; + if( zIn==0 ) return; + for(i=0; zIn[i]; i++){ + char cSaved; + if( zIn[i]!=';' ) continue; + cSaved = zIn[i+1]; + zIn[i+1] = 0; + if( sqlite3_complete(zIn) ){ + char *zOut = sqlite3_normalize(zIn); + if( zOut ){ + printf("%s\n", zOut); + sqlite3_free(zOut); + }else{ + fprintf(stderr, "ERROR: %s\n", zIn); + } + zIn[i+1] = cSaved; + zIn += i+1; + i = -1; + }else{ + zIn[i+1] = cSaved; + } + } +} + +/* +** The main routine for "sql_normalize". Read files named on the +** command-line and run the text of each through sqlite3_normalize(). +*/ +int main(int argc, char **argv){ + int i; + FILE *in; + char *zBuf = 0; + sqlite3_int64 sz, got; + + for(i=1; i # include # include @@ -652,6 +658,79 @@ static void editDist3ConfigDelete(void *pIn){ sqlite3_free(p); } +/* Compare the FROM values of two EditDist3Cost objects, for sorting. +** Return negative, zero, or positive if the A is less than, equal to, +** or greater than B. +*/ +static int editDist3CostCompare(EditDist3Cost *pA, EditDist3Cost *pB){ + int n = pA->nFrom; + int rc; + if( n>pB->nFrom ) n = pB->nFrom; + rc = strncmp(pA->a, pB->a, n); + if( rc==0 ) rc = pA->nFrom - pB->nFrom; + return rc; +} + +/* +** Merge together two sorted lists of EditDist3Cost objects, in order +** of increasing FROM. +*/ +static EditDist3Cost *editDist3CostMerge( + EditDist3Cost *pA, + EditDist3Cost *pB +){ + EditDist3Cost *pHead = 0; + EditDist3Cost **ppTail = &pHead; + EditDist3Cost *p; + while( pA && pB ){ + if( editDist3CostCompare(pA,pB)<=0 ){ + p = pA; + pA = pA->pNext; + }else{ + p = pB; + pB = pB->pNext; + } + *ppTail = p; + ppTail = &p->pNext; + } + if( pA ){ + *ppTail = pA; + }else{ + *ppTail = pB; + } + return pHead; +} + +/* +** Sort a list of EditDist3Cost objects into order of increasing FROM +*/ +static EditDist3Cost *editDist3CostSort(EditDist3Cost *pList){ + EditDist3Cost *ap[60], *p; + int i; + int mx = 0; + ap[0] = 0; + ap[1] = 0; + while( pList ){ + p = pList; + pList = p->pNext; + p->pNext = 0; + for(i=0; ap[i]; i++){ + p = editDist3CostMerge(ap[i],p); + ap[i] = 0; + } + ap[i] = p; + if( i>mx ){ + mx = i; + ap[i+1] = 0; + } + } + p = 0; + for(i=0; i<=mx; i++){ + if( ap[i] ) p = editDist3CostMerge(p,ap[i]); + } + return p; +} + /* ** Load all edit-distance weights from a table. */ @@ -685,6 +764,7 @@ static int editDist3ConfigLoad( assert( zTo!=0 || nTo==0 ); if( nFrom>100 || nTo>100 ) continue; if( iCost<0 ) continue; + if( iCost>10000 ) continue; /* Costs above 10K are considered infinite */ if( pLang==0 || iLang!=iLangPrev ){ EditDist3Lang *pNew; pNew = sqlite3_realloc64(p->a, (p->nLang+1)*sizeof(p->a[0])); @@ -722,6 +802,12 @@ static int editDist3ConfigLoad( } rc2 = sqlite3_finalize(pStmt); if( rc==SQLITE_OK ) rc = rc2; + if( rc==SQLITE_OK ){ + int iLang; + for(iLang=0; iLangnLang; iLang++){ + p->a[iLang].pCost = editDist3CostSort(p->a[iLang].pCost); + } + } return rc; } @@ -749,6 +835,7 @@ static int utf8Len(unsigned char c, int N){ ** the given string. */ static int matchTo(EditDist3Cost *p, const char *z, int n){ + if( p->a[p->nFrom]!=z[0] ) return 0; if( p->nTo>n ) return 0; if( strncmp(p->a+p->nFrom, z, p->nTo)!=0 ) return 0; return 1; @@ -760,6 +847,7 @@ static int matchTo(EditDist3Cost *p, const char *z, int n){ */ static int matchFrom(EditDist3Cost *p, const char *z, int n){ assert( p->nFrom<=n ); + if( p->a[0]!=z[0] ) return 0; if( strncmp(p->a, z, p->nFrom)!=0 ) return 0; return 1; } @@ -776,7 +864,8 @@ static int matchFromTo( ){ int b1 = pStr->a[n1].nByte; if( b1>n2 ) return 0; - if( memcmp(pStr->z+n1, z2, b1)!=0 ) return 0; + if( pStr->z[n1]!=z2[0] ) return 0; + if( strncmp(pStr->z+n1, z2, b1)!=0 ) return 0; return 1; } @@ -858,9 +947,6 @@ static EditDist3FromString *editDist3FromStringNew( /* ** Update entry m[i] such that it is the minimum of its current value ** and m[j]+iCost. -** -** If the iCost is 1,000,000 or greater, then consider the cost to be -** infinite and skip the update. */ static void updateCost( unsigned int *m, @@ -868,11 +954,11 @@ static void updateCost( int j, int iCost ){ + unsigned int b; assert( iCost>=0 ); - if( iCost<10000 ){ - unsigned int b = m[j] + iCost; - if( bpCost; p; p=p->pNext){ EditDist3Cost **apNew; - if( p->nFrom>0 ) continue; + if( p->nFrom>0 ) break; if( i2+p->nTo>n2 ) continue; + if( p->a[0]>z2[i2] ) break; if( matchTo(p, z2+i2, n2-i2)==0 ) continue; a2[i2].nIns++; apNew = sqlite3_realloc64(a2[i2].apIns, sizeof(*apNew)*a2[i2].nIns); @@ -1200,404 +1287,412 @@ static int utf8Charlen(const char *zIn, int nIn){ return nChar; } +typedef struct Transliteration Transliteration; +struct Transliteration { + unsigned short int cFrom; + unsigned char cTo0, cTo1, cTo2, cTo3; +}; + /* ** Table of translations from unicode characters into ASCII. */ -static const struct { - unsigned short int cFrom; - unsigned char cTo0, cTo1; -} translit[] = { - { 0x00A0, 0x20, 0x00 }, /*   to */ - { 0x00B5, 0x75, 0x00 }, /* µ to u */ - { 0x00C0, 0x41, 0x00 }, /* À to A */ - { 0x00C1, 0x41, 0x00 }, /* Á to A */ - { 0x00C2, 0x41, 0x00 }, /*  to A */ - { 0x00C3, 0x41, 0x00 }, /* à to A */ - { 0x00C4, 0x41, 0x65 }, /* Ä to Ae */ - { 0x00C5, 0x41, 0x61 }, /* Å to Aa */ - { 0x00C6, 0x41, 0x45 }, /* Æ to AE */ - { 0x00C7, 0x43, 0x00 }, /* Ç to C */ - { 0x00C8, 0x45, 0x00 }, /* È to E */ - { 0x00C9, 0x45, 0x00 }, /* É to E */ - { 0x00CA, 0x45, 0x00 }, /* Ê to E */ - { 0x00CB, 0x45, 0x00 }, /* Ë to E */ - { 0x00CC, 0x49, 0x00 }, /* Ì to I */ - { 0x00CD, 0x49, 0x00 }, /* Í to I */ - { 0x00CE, 0x49, 0x00 }, /* Î to I */ - { 0x00CF, 0x49, 0x00 }, /* Ï to I */ - { 0x00D0, 0x44, 0x00 }, /* Ð to D */ - { 0x00D1, 0x4E, 0x00 }, /* Ñ to N */ - { 0x00D2, 0x4F, 0x00 }, /* Ò to O */ - { 0x00D3, 0x4F, 0x00 }, /* Ó to O */ - { 0x00D4, 0x4F, 0x00 }, /* Ô to O */ - { 0x00D5, 0x4F, 0x00 }, /* Õ to O */ - { 0x00D6, 0x4F, 0x65 }, /* Ö to Oe */ - { 0x00D7, 0x78, 0x00 }, /* × to x */ - { 0x00D8, 0x4F, 0x00 }, /* Ø to O */ - { 0x00D9, 0x55, 0x00 }, /* Ù to U */ - { 0x00DA, 0x55, 0x00 }, /* Ú to U */ - { 0x00DB, 0x55, 0x00 }, /* Û to U */ - { 0x00DC, 0x55, 0x65 }, /* Ü to Ue */ - { 0x00DD, 0x59, 0x00 }, /* Ý to Y */ - { 0x00DE, 0x54, 0x68 }, /* Þ to Th */ - { 0x00DF, 0x73, 0x73 }, /* ß to ss */ - { 0x00E0, 0x61, 0x00 }, /* à to a */ - { 0x00E1, 0x61, 0x00 }, /* á to a */ - { 0x00E2, 0x61, 0x00 }, /* â to a */ - { 0x00E3, 0x61, 0x00 }, /* ã to a */ - { 0x00E4, 0x61, 0x65 }, /* ä to ae */ - { 0x00E5, 0x61, 0x61 }, /* å to aa */ - { 0x00E6, 0x61, 0x65 }, /* æ to ae */ - { 0x00E7, 0x63, 0x00 }, /* ç to c */ - { 0x00E8, 0x65, 0x00 }, /* è to e */ - { 0x00E9, 0x65, 0x00 }, /* é to e */ - { 0x00EA, 0x65, 0x00 }, /* ê to e */ - { 0x00EB, 0x65, 0x00 }, /* ë to e */ - { 0x00EC, 0x69, 0x00 }, /* ì to i */ - { 0x00ED, 0x69, 0x00 }, /* í to i */ - { 0x00EE, 0x69, 0x00 }, /* î to i */ - { 0x00EF, 0x69, 0x00 }, /* ï to i */ - { 0x00F0, 0x64, 0x00 }, /* ð to d */ - { 0x00F1, 0x6E, 0x00 }, /* ñ to n */ - { 0x00F2, 0x6F, 0x00 }, /* ò to o */ - { 0x00F3, 0x6F, 0x00 }, /* ó to o */ - { 0x00F4, 0x6F, 0x00 }, /* ô to o */ - { 0x00F5, 0x6F, 0x00 }, /* õ to o */ - { 0x00F6, 0x6F, 0x65 }, /* ö to oe */ - { 0x00F7, 0x3A, 0x00 }, /* ÷ to : */ - { 0x00F8, 0x6F, 0x00 }, /* ø to o */ - { 0x00F9, 0x75, 0x00 }, /* ù to u */ - { 0x00FA, 0x75, 0x00 }, /* ú to u */ - { 0x00FB, 0x75, 0x00 }, /* û to u */ - { 0x00FC, 0x75, 0x65 }, /* ü to ue */ - { 0x00FD, 0x79, 0x00 }, /* ý to y */ - { 0x00FE, 0x74, 0x68 }, /* þ to th */ - { 0x00FF, 0x79, 0x00 }, /* ÿ to y */ - { 0x0100, 0x41, 0x00 }, /* Ā to A */ - { 0x0101, 0x61, 0x00 }, /* ā to a */ - { 0x0102, 0x41, 0x00 }, /* Ă to A */ - { 0x0103, 0x61, 0x00 }, /* ă to a */ - { 0x0104, 0x41, 0x00 }, /* Ą to A */ - { 0x0105, 0x61, 0x00 }, /* ą to a */ - { 0x0106, 0x43, 0x00 }, /* Ć to C */ - { 0x0107, 0x63, 0x00 }, /* ć to c */ - { 0x0108, 0x43, 0x68 }, /* Ĉ to Ch */ - { 0x0109, 0x63, 0x68 }, /* ĉ to ch */ - { 0x010A, 0x43, 0x00 }, /* Ċ to C */ - { 0x010B, 0x63, 0x00 }, /* ċ to c */ - { 0x010C, 0x43, 0x00 }, /* Č to C */ - { 0x010D, 0x63, 0x00 }, /* č to c */ - { 0x010E, 0x44, 0x00 }, /* Ď to D */ - { 0x010F, 0x64, 0x00 }, /* ď to d */ - { 0x0110, 0x44, 0x00 }, /* Đ to D */ - { 0x0111, 0x64, 0x00 }, /* đ to d */ - { 0x0112, 0x45, 0x00 }, /* Ē to E */ - { 0x0113, 0x65, 0x00 }, /* ē to e */ - { 0x0114, 0x45, 0x00 }, /* Ĕ to E */ - { 0x0115, 0x65, 0x00 }, /* ĕ to e */ - { 0x0116, 0x45, 0x00 }, /* Ė to E */ - { 0x0117, 0x65, 0x00 }, /* ė to e */ - { 0x0118, 0x45, 0x00 }, /* Ę to E */ - { 0x0119, 0x65, 0x00 }, /* ę to e */ - { 0x011A, 0x45, 0x00 }, /* Ě to E */ - { 0x011B, 0x65, 0x00 }, /* ě to e */ - { 0x011C, 0x47, 0x68 }, /* Ĝ to Gh */ - { 0x011D, 0x67, 0x68 }, /* ĝ to gh */ - { 0x011E, 0x47, 0x00 }, /* Ğ to G */ - { 0x011F, 0x67, 0x00 }, /* ğ to g */ - { 0x0120, 0x47, 0x00 }, /* Ġ to G */ - { 0x0121, 0x67, 0x00 }, /* ġ to g */ - { 0x0122, 0x47, 0x00 }, /* Ģ to G */ - { 0x0123, 0x67, 0x00 }, /* ģ to g */ - { 0x0124, 0x48, 0x68 }, /* Ĥ to Hh */ - { 0x0125, 0x68, 0x68 }, /* ĥ to hh */ - { 0x0126, 0x48, 0x00 }, /* Ħ to H */ - { 0x0127, 0x68, 0x00 }, /* ħ to h */ - { 0x0128, 0x49, 0x00 }, /* Ĩ to I */ - { 0x0129, 0x69, 0x00 }, /* ĩ to i */ - { 0x012A, 0x49, 0x00 }, /* Ī to I */ - { 0x012B, 0x69, 0x00 }, /* ī to i */ - { 0x012C, 0x49, 0x00 }, /* Ĭ to I */ - { 0x012D, 0x69, 0x00 }, /* ĭ to i */ - { 0x012E, 0x49, 0x00 }, /* Į to I */ - { 0x012F, 0x69, 0x00 }, /* į to i */ - { 0x0130, 0x49, 0x00 }, /* İ to I */ - { 0x0131, 0x69, 0x00 }, /* ı to i */ - { 0x0132, 0x49, 0x4A }, /* IJ to IJ */ - { 0x0133, 0x69, 0x6A }, /* ij to ij */ - { 0x0134, 0x4A, 0x68 }, /* Ĵ to Jh */ - { 0x0135, 0x6A, 0x68 }, /* ĵ to jh */ - { 0x0136, 0x4B, 0x00 }, /* Ķ to K */ - { 0x0137, 0x6B, 0x00 }, /* ķ to k */ - { 0x0138, 0x6B, 0x00 }, /* ĸ to k */ - { 0x0139, 0x4C, 0x00 }, /* Ĺ to L */ - { 0x013A, 0x6C, 0x00 }, /* ĺ to l */ - { 0x013B, 0x4C, 0x00 }, /* Ļ to L */ - { 0x013C, 0x6C, 0x00 }, /* ļ to l */ - { 0x013D, 0x4C, 0x00 }, /* Ľ to L */ - { 0x013E, 0x6C, 0x00 }, /* ľ to l */ - { 0x013F, 0x4C, 0x2E }, /* Ŀ to L. */ - { 0x0140, 0x6C, 0x2E }, /* ŀ to l. */ - { 0x0141, 0x4C, 0x00 }, /* Ł to L */ - { 0x0142, 0x6C, 0x00 }, /* ł to l */ - { 0x0143, 0x4E, 0x00 }, /* Ń to N */ - { 0x0144, 0x6E, 0x00 }, /* ń to n */ - { 0x0145, 0x4E, 0x00 }, /* Ņ to N */ - { 0x0146, 0x6E, 0x00 }, /* ņ to n */ - { 0x0147, 0x4E, 0x00 }, /* Ň to N */ - { 0x0148, 0x6E, 0x00 }, /* ň to n */ - { 0x0149, 0x27, 0x6E }, /* ʼn to 'n */ - { 0x014A, 0x4E, 0x47 }, /* Ŋ to NG */ - { 0x014B, 0x6E, 0x67 }, /* ŋ to ng */ - { 0x014C, 0x4F, 0x00 }, /* Ō to O */ - { 0x014D, 0x6F, 0x00 }, /* ō to o */ - { 0x014E, 0x4F, 0x00 }, /* Ŏ to O */ - { 0x014F, 0x6F, 0x00 }, /* ŏ to o */ - { 0x0150, 0x4F, 0x00 }, /* Ő to O */ - { 0x0151, 0x6F, 0x00 }, /* ő to o */ - { 0x0152, 0x4F, 0x45 }, /* Œ to OE */ - { 0x0153, 0x6F, 0x65 }, /* œ to oe */ - { 0x0154, 0x52, 0x00 }, /* Ŕ to R */ - { 0x0155, 0x72, 0x00 }, /* ŕ to r */ - { 0x0156, 0x52, 0x00 }, /* Ŗ to R */ - { 0x0157, 0x72, 0x00 }, /* ŗ to r */ - { 0x0158, 0x52, 0x00 }, /* Ř to R */ - { 0x0159, 0x72, 0x00 }, /* ř to r */ - { 0x015A, 0x53, 0x00 }, /* Ś to S */ - { 0x015B, 0x73, 0x00 }, /* ś to s */ - { 0x015C, 0x53, 0x68 }, /* Ŝ to Sh */ - { 0x015D, 0x73, 0x68 }, /* ŝ to sh */ - { 0x015E, 0x53, 0x00 }, /* Ş to S */ - { 0x015F, 0x73, 0x00 }, /* ş to s */ - { 0x0160, 0x53, 0x00 }, /* Š to S */ - { 0x0161, 0x73, 0x00 }, /* š to s */ - { 0x0162, 0x54, 0x00 }, /* Ţ to T */ - { 0x0163, 0x74, 0x00 }, /* ţ to t */ - { 0x0164, 0x54, 0x00 }, /* Ť to T */ - { 0x0165, 0x74, 0x00 }, /* ť to t */ - { 0x0166, 0x54, 0x00 }, /* Ŧ to T */ - { 0x0167, 0x74, 0x00 }, /* ŧ to t */ - { 0x0168, 0x55, 0x00 }, /* Ũ to U */ - { 0x0169, 0x75, 0x00 }, /* ũ to u */ - { 0x016A, 0x55, 0x00 }, /* Ū to U */ - { 0x016B, 0x75, 0x00 }, /* ū to u */ - { 0x016C, 0x55, 0x00 }, /* Ŭ to U */ - { 0x016D, 0x75, 0x00 }, /* ŭ to u */ - { 0x016E, 0x55, 0x00 }, /* Ů to U */ - { 0x016F, 0x75, 0x00 }, /* ů to u */ - { 0x0170, 0x55, 0x00 }, /* Ű to U */ - { 0x0171, 0x75, 0x00 }, /* ű to u */ - { 0x0172, 0x55, 0x00 }, /* Ų to U */ - { 0x0173, 0x75, 0x00 }, /* ų to u */ - { 0x0174, 0x57, 0x00 }, /* Ŵ to W */ - { 0x0175, 0x77, 0x00 }, /* ŵ to w */ - { 0x0176, 0x59, 0x00 }, /* Ŷ to Y */ - { 0x0177, 0x79, 0x00 }, /* ŷ to y */ - { 0x0178, 0x59, 0x00 }, /* Ÿ to Y */ - { 0x0179, 0x5A, 0x00 }, /* Ź to Z */ - { 0x017A, 0x7A, 0x00 }, /* ź to z */ - { 0x017B, 0x5A, 0x00 }, /* Ż to Z */ - { 0x017C, 0x7A, 0x00 }, /* ż to z */ - { 0x017D, 0x5A, 0x00 }, /* Ž to Z */ - { 0x017E, 0x7A, 0x00 }, /* ž to z */ - { 0x017F, 0x73, 0x00 }, /* ſ to s */ - { 0x0192, 0x66, 0x00 }, /* ƒ to f */ - { 0x0218, 0x53, 0x00 }, /* Ș to S */ - { 0x0219, 0x73, 0x00 }, /* ș to s */ - { 0x021A, 0x54, 0x00 }, /* Ț to T */ - { 0x021B, 0x74, 0x00 }, /* ț to t */ - { 0x0386, 0x41, 0x00 }, /* Ά to A */ - { 0x0388, 0x45, 0x00 }, /* Έ to E */ - { 0x0389, 0x49, 0x00 }, /* Ή to I */ - { 0x038A, 0x49, 0x00 }, /* Ί to I */ - { 0x038C, 0x4f, 0x00 }, /* Ό to O */ - { 0x038E, 0x59, 0x00 }, /* Ύ to Y */ - { 0x038F, 0x4f, 0x00 }, /* Ώ to O */ - { 0x0390, 0x69, 0x00 }, /* ΐ to i */ - { 0x0391, 0x41, 0x00 }, /* Α to A */ - { 0x0392, 0x42, 0x00 }, /* Β to B */ - { 0x0393, 0x47, 0x00 }, /* Γ to G */ - { 0x0394, 0x44, 0x00 }, /* Δ to D */ - { 0x0395, 0x45, 0x00 }, /* Ε to E */ - { 0x0396, 0x5a, 0x00 }, /* Ζ to Z */ - { 0x0397, 0x49, 0x00 }, /* Η to I */ - { 0x0398, 0x54, 0x68 }, /* Θ to Th */ - { 0x0399, 0x49, 0x00 }, /* Ι to I */ - { 0x039A, 0x4b, 0x00 }, /* Κ to K */ - { 0x039B, 0x4c, 0x00 }, /* Λ to L */ - { 0x039C, 0x4d, 0x00 }, /* Μ to M */ - { 0x039D, 0x4e, 0x00 }, /* Ν to N */ - { 0x039E, 0x58, 0x00 }, /* Ξ to X */ - { 0x039F, 0x4f, 0x00 }, /* Ο to O */ - { 0x03A0, 0x50, 0x00 }, /* Π to P */ - { 0x03A1, 0x52, 0x00 }, /* Ρ to R */ - { 0x03A3, 0x53, 0x00 }, /* Σ to S */ - { 0x03A4, 0x54, 0x00 }, /* Τ to T */ - { 0x03A5, 0x59, 0x00 }, /* Υ to Y */ - { 0x03A6, 0x46, 0x00 }, /* Φ to F */ - { 0x03A7, 0x43, 0x68 }, /* Χ to Ch */ - { 0x03A8, 0x50, 0x73 }, /* Ψ to Ps */ - { 0x03A9, 0x4f, 0x00 }, /* Ω to O */ - { 0x03AA, 0x49, 0x00 }, /* Ϊ to I */ - { 0x03AB, 0x59, 0x00 }, /* Ϋ to Y */ - { 0x03AC, 0x61, 0x00 }, /* ά to a */ - { 0x03AD, 0x65, 0x00 }, /* έ to e */ - { 0x03AE, 0x69, 0x00 }, /* ή to i */ - { 0x03AF, 0x69, 0x00 }, /* ί to i */ - { 0x03B1, 0x61, 0x00 }, /* α to a */ - { 0x03B2, 0x62, 0x00 }, /* β to b */ - { 0x03B3, 0x67, 0x00 }, /* γ to g */ - { 0x03B4, 0x64, 0x00 }, /* δ to d */ - { 0x03B5, 0x65, 0x00 }, /* ε to e */ - { 0x03B6, 0x7a, 0x00 }, /* ζ to z */ - { 0x03B7, 0x69, 0x00 }, /* η to i */ - { 0x03B8, 0x74, 0x68 }, /* θ to th */ - { 0x03B9, 0x69, 0x00 }, /* ι to i */ - { 0x03BA, 0x6b, 0x00 }, /* κ to k */ - { 0x03BB, 0x6c, 0x00 }, /* λ to l */ - { 0x03BC, 0x6d, 0x00 }, /* μ to m */ - { 0x03BD, 0x6e, 0x00 }, /* ν to n */ - { 0x03BE, 0x78, 0x00 }, /* ξ to x */ - { 0x03BF, 0x6f, 0x00 }, /* ο to o */ - { 0x03C0, 0x70, 0x00 }, /* π to p */ - { 0x03C1, 0x72, 0x00 }, /* ρ to r */ - { 0x03C3, 0x73, 0x00 }, /* σ to s */ - { 0x03C4, 0x74, 0x00 }, /* τ to t */ - { 0x03C5, 0x79, 0x00 }, /* υ to y */ - { 0x03C6, 0x66, 0x00 }, /* φ to f */ - { 0x03C7, 0x63, 0x68 }, /* χ to ch */ - { 0x03C8, 0x70, 0x73 }, /* ψ to ps */ - { 0x03C9, 0x6f, 0x00 }, /* ω to o */ - { 0x03CA, 0x69, 0x00 }, /* ϊ to i */ - { 0x03CB, 0x79, 0x00 }, /* ϋ to y */ - { 0x03CC, 0x6f, 0x00 }, /* ό to o */ - { 0x03CD, 0x79, 0x00 }, /* ύ to y */ - { 0x03CE, 0x69, 0x00 }, /* ώ to i */ - { 0x0400, 0x45, 0x00 }, /* Ѐ to E */ - { 0x0401, 0x45, 0x00 }, /* Ё to E */ - { 0x0402, 0x44, 0x00 }, /* Ђ to D */ - { 0x0403, 0x47, 0x00 }, /* Ѓ to G */ - { 0x0404, 0x45, 0x00 }, /* Є to E */ - { 0x0405, 0x5a, 0x00 }, /* Ѕ to Z */ - { 0x0406, 0x49, 0x00 }, /* І to I */ - { 0x0407, 0x49, 0x00 }, /* Ї to I */ - { 0x0408, 0x4a, 0x00 }, /* Ј to J */ - { 0x0409, 0x49, 0x00 }, /* Љ to I */ - { 0x040A, 0x4e, 0x00 }, /* Њ to N */ - { 0x040B, 0x44, 0x00 }, /* Ћ to D */ - { 0x040C, 0x4b, 0x00 }, /* Ќ to K */ - { 0x040D, 0x49, 0x00 }, /* Ѝ to I */ - { 0x040E, 0x55, 0x00 }, /* Ў to U */ - { 0x040F, 0x44, 0x00 }, /* Џ to D */ - { 0x0410, 0x41, 0x00 }, /* А to A */ - { 0x0411, 0x42, 0x00 }, /* Б to B */ - { 0x0412, 0x56, 0x00 }, /* В to V */ - { 0x0413, 0x47, 0x00 }, /* Г to G */ - { 0x0414, 0x44, 0x00 }, /* Д to D */ - { 0x0415, 0x45, 0x00 }, /* Е to E */ - { 0x0416, 0x5a, 0x68 }, /* Ж to Zh */ - { 0x0417, 0x5a, 0x00 }, /* З to Z */ - { 0x0418, 0x49, 0x00 }, /* И to I */ - { 0x0419, 0x49, 0x00 }, /* Й to I */ - { 0x041A, 0x4b, 0x00 }, /* К to K */ - { 0x041B, 0x4c, 0x00 }, /* Л to L */ - { 0x041C, 0x4d, 0x00 }, /* М to M */ - { 0x041D, 0x4e, 0x00 }, /* Н to N */ - { 0x041E, 0x4f, 0x00 }, /* О to O */ - { 0x041F, 0x50, 0x00 }, /* П to P */ - { 0x0420, 0x52, 0x00 }, /* Р to R */ - { 0x0421, 0x53, 0x00 }, /* С to S */ - { 0x0422, 0x54, 0x00 }, /* Т to T */ - { 0x0423, 0x55, 0x00 }, /* У to U */ - { 0x0424, 0x46, 0x00 }, /* Ф to F */ - { 0x0425, 0x4b, 0x68 }, /* Х to Kh */ - { 0x0426, 0x54, 0x63 }, /* Ц to Tc */ - { 0x0427, 0x43, 0x68 }, /* Ч to Ch */ - { 0x0428, 0x53, 0x68 }, /* Ш to Sh */ - { 0x0429, 0x53, 0x68 }, /* Щ to Shch */ - { 0x042A, 0x61, 0x00 }, /* to A */ - { 0x042B, 0x59, 0x00 }, /* Ы to Y */ - { 0x042C, 0x59, 0x00 }, /* to Y */ - { 0x042D, 0x45, 0x00 }, /* Э to E */ - { 0x042E, 0x49, 0x75 }, /* Ю to Iu */ - { 0x042F, 0x49, 0x61 }, /* Я to Ia */ - { 0x0430, 0x61, 0x00 }, /* а to a */ - { 0x0431, 0x62, 0x00 }, /* б to b */ - { 0x0432, 0x76, 0x00 }, /* в to v */ - { 0x0433, 0x67, 0x00 }, /* г to g */ - { 0x0434, 0x64, 0x00 }, /* д to d */ - { 0x0435, 0x65, 0x00 }, /* е to e */ - { 0x0436, 0x7a, 0x68 }, /* ж to zh */ - { 0x0437, 0x7a, 0x00 }, /* з to z */ - { 0x0438, 0x69, 0x00 }, /* и to i */ - { 0x0439, 0x69, 0x00 }, /* й to i */ - { 0x043A, 0x6b, 0x00 }, /* к to k */ - { 0x043B, 0x6c, 0x00 }, /* л to l */ - { 0x043C, 0x6d, 0x00 }, /* м to m */ - { 0x043D, 0x6e, 0x00 }, /* н to n */ - { 0x043E, 0x6f, 0x00 }, /* о to o */ - { 0x043F, 0x70, 0x00 }, /* п to p */ - { 0x0440, 0x72, 0x00 }, /* р to r */ - { 0x0441, 0x73, 0x00 }, /* с to s */ - { 0x0442, 0x74, 0x00 }, /* т to t */ - { 0x0443, 0x75, 0x00 }, /* у to u */ - { 0x0444, 0x66, 0x00 }, /* ф to f */ - { 0x0445, 0x6b, 0x68 }, /* х to kh */ - { 0x0446, 0x74, 0x63 }, /* ц to tc */ - { 0x0447, 0x63, 0x68 }, /* ч to ch */ - { 0x0448, 0x73, 0x68 }, /* ш to sh */ - { 0x0449, 0x73, 0x68 }, /* щ to shch */ - { 0x044A, 0x61, 0x00 }, /* to a */ - { 0x044B, 0x79, 0x00 }, /* ы to y */ - { 0x044C, 0x79, 0x00 }, /* to y */ - { 0x044D, 0x65, 0x00 }, /* э to e */ - { 0x044E, 0x69, 0x75 }, /* ю to iu */ - { 0x044F, 0x69, 0x61 }, /* я to ia */ - { 0x0450, 0x65, 0x00 }, /* ѐ to e */ - { 0x0451, 0x65, 0x00 }, /* ё to e */ - { 0x0452, 0x64, 0x00 }, /* ђ to d */ - { 0x0453, 0x67, 0x00 }, /* ѓ to g */ - { 0x0454, 0x65, 0x00 }, /* є to e */ - { 0x0455, 0x7a, 0x00 }, /* ѕ to z */ - { 0x0456, 0x69, 0x00 }, /* і to i */ - { 0x0457, 0x69, 0x00 }, /* ї to i */ - { 0x0458, 0x6a, 0x00 }, /* ј to j */ - { 0x0459, 0x69, 0x00 }, /* љ to i */ - { 0x045A, 0x6e, 0x00 }, /* њ to n */ - { 0x045B, 0x64, 0x00 }, /* ћ to d */ - { 0x045C, 0x6b, 0x00 }, /* ќ to k */ - { 0x045D, 0x69, 0x00 }, /* ѝ to i */ - { 0x045E, 0x75, 0x00 }, /* ў to u */ - { 0x045F, 0x64, 0x00 }, /* џ to d */ - { 0x1E02, 0x42, 0x00 }, /* Ḃ to B */ - { 0x1E03, 0x62, 0x00 }, /* ḃ to b */ - { 0x1E0A, 0x44, 0x00 }, /* Ḋ to D */ - { 0x1E0B, 0x64, 0x00 }, /* ḋ to d */ - { 0x1E1E, 0x46, 0x00 }, /* Ḟ to F */ - { 0x1E1F, 0x66, 0x00 }, /* ḟ to f */ - { 0x1E40, 0x4D, 0x00 }, /* Ṁ to M */ - { 0x1E41, 0x6D, 0x00 }, /* ṁ to m */ - { 0x1E56, 0x50, 0x00 }, /* Ṗ to P */ - { 0x1E57, 0x70, 0x00 }, /* ṗ to p */ - { 0x1E60, 0x53, 0x00 }, /* Ṡ to S */ - { 0x1E61, 0x73, 0x00 }, /* ṡ to s */ - { 0x1E6A, 0x54, 0x00 }, /* Ṫ to T */ - { 0x1E6B, 0x74, 0x00 }, /* ṫ to t */ - { 0x1E80, 0x57, 0x00 }, /* Ẁ to W */ - { 0x1E81, 0x77, 0x00 }, /* ẁ to w */ - { 0x1E82, 0x57, 0x00 }, /* Ẃ to W */ - { 0x1E83, 0x77, 0x00 }, /* ẃ to w */ - { 0x1E84, 0x57, 0x00 }, /* Ẅ to W */ - { 0x1E85, 0x77, 0x00 }, /* ẅ to w */ - { 0x1EF2, 0x59, 0x00 }, /* Ỳ to Y */ - { 0x1EF3, 0x79, 0x00 }, /* ỳ to y */ - { 0xFB00, 0x66, 0x66 }, /* ff to ff */ - { 0xFB01, 0x66, 0x69 }, /* fi to fi */ - { 0xFB02, 0x66, 0x6C }, /* fl to fl */ - { 0xFB05, 0x73, 0x74 }, /* ſt to st */ - { 0xFB06, 0x73, 0x74 }, /* st to st */ +static const Transliteration translit[] = { + { 0x00A0, 0x20, 0x00, 0x00, 0x00 }, /*   to */ + { 0x00B5, 0x75, 0x00, 0x00, 0x00 }, /* µ to u */ + { 0x00C0, 0x41, 0x00, 0x00, 0x00 }, /* À to A */ + { 0x00C1, 0x41, 0x00, 0x00, 0x00 }, /* Á to A */ + { 0x00C2, 0x41, 0x00, 0x00, 0x00 }, /*  to A */ + { 0x00C3, 0x41, 0x00, 0x00, 0x00 }, /* à to A */ + { 0x00C4, 0x41, 0x65, 0x00, 0x00 }, /* Ä to Ae */ + { 0x00C5, 0x41, 0x61, 0x00, 0x00 }, /* Å to Aa */ + { 0x00C6, 0x41, 0x45, 0x00, 0x00 }, /* Æ to AE */ + { 0x00C7, 0x43, 0x00, 0x00, 0x00 }, /* Ç to C */ + { 0x00C8, 0x45, 0x00, 0x00, 0x00 }, /* È to E */ + { 0x00C9, 0x45, 0x00, 0x00, 0x00 }, /* É to E */ + { 0x00CA, 0x45, 0x00, 0x00, 0x00 }, /* Ê to E */ + { 0x00CB, 0x45, 0x00, 0x00, 0x00 }, /* Ë to E */ + { 0x00CC, 0x49, 0x00, 0x00, 0x00 }, /* Ì to I */ + { 0x00CD, 0x49, 0x00, 0x00, 0x00 }, /* Í to I */ + { 0x00CE, 0x49, 0x00, 0x00, 0x00 }, /* Î to I */ + { 0x00CF, 0x49, 0x00, 0x00, 0x00 }, /* Ï to I */ + { 0x00D0, 0x44, 0x00, 0x00, 0x00 }, /* Ð to D */ + { 0x00D1, 0x4E, 0x00, 0x00, 0x00 }, /* Ñ to N */ + { 0x00D2, 0x4F, 0x00, 0x00, 0x00 }, /* Ò to O */ + { 0x00D3, 0x4F, 0x00, 0x00, 0x00 }, /* Ó to O */ + { 0x00D4, 0x4F, 0x00, 0x00, 0x00 }, /* Ô to O */ + { 0x00D5, 0x4F, 0x00, 0x00, 0x00 }, /* Õ to O */ + { 0x00D6, 0x4F, 0x65, 0x00, 0x00 }, /* Ö to Oe */ + { 0x00D7, 0x78, 0x00, 0x00, 0x00 }, /* × to x */ + { 0x00D8, 0x4F, 0x00, 0x00, 0x00 }, /* Ø to O */ + { 0x00D9, 0x55, 0x00, 0x00, 0x00 }, /* Ù to U */ + { 0x00DA, 0x55, 0x00, 0x00, 0x00 }, /* Ú to U */ + { 0x00DB, 0x55, 0x00, 0x00, 0x00 }, /* Û to U */ + { 0x00DC, 0x55, 0x65, 0x00, 0x00 }, /* Ü to Ue */ + { 0x00DD, 0x59, 0x00, 0x00, 0x00 }, /* Ý to Y */ + { 0x00DE, 0x54, 0x68, 0x00, 0x00 }, /* Þ to Th */ + { 0x00DF, 0x73, 0x73, 0x00, 0x00 }, /* ß to ss */ + { 0x00E0, 0x61, 0x00, 0x00, 0x00 }, /* à to a */ + { 0x00E1, 0x61, 0x00, 0x00, 0x00 }, /* á to a */ + { 0x00E2, 0x61, 0x00, 0x00, 0x00 }, /* â to a */ + { 0x00E3, 0x61, 0x00, 0x00, 0x00 }, /* ã to a */ + { 0x00E4, 0x61, 0x65, 0x00, 0x00 }, /* ä to ae */ + { 0x00E5, 0x61, 0x61, 0x00, 0x00 }, /* å to aa */ + { 0x00E6, 0x61, 0x65, 0x00, 0x00 }, /* æ to ae */ + { 0x00E7, 0x63, 0x00, 0x00, 0x00 }, /* ç to c */ + { 0x00E8, 0x65, 0x00, 0x00, 0x00 }, /* è to e */ + { 0x00E9, 0x65, 0x00, 0x00, 0x00 }, /* é to e */ + { 0x00EA, 0x65, 0x00, 0x00, 0x00 }, /* ê to e */ + { 0x00EB, 0x65, 0x00, 0x00, 0x00 }, /* ë to e */ + { 0x00EC, 0x69, 0x00, 0x00, 0x00 }, /* ì to i */ + { 0x00ED, 0x69, 0x00, 0x00, 0x00 }, /* í to i */ + { 0x00EE, 0x69, 0x00, 0x00, 0x00 }, /* î to i */ + { 0x00EF, 0x69, 0x00, 0x00, 0x00 }, /* ï to i */ + { 0x00F0, 0x64, 0x00, 0x00, 0x00 }, /* ð to d */ + { 0x00F1, 0x6E, 0x00, 0x00, 0x00 }, /* ñ to n */ + { 0x00F2, 0x6F, 0x00, 0x00, 0x00 }, /* ò to o */ + { 0x00F3, 0x6F, 0x00, 0x00, 0x00 }, /* ó to o */ + { 0x00F4, 0x6F, 0x00, 0x00, 0x00 }, /* ô to o */ + { 0x00F5, 0x6F, 0x00, 0x00, 0x00 }, /* õ to o */ + { 0x00F6, 0x6F, 0x65, 0x00, 0x00 }, /* ö to oe */ + { 0x00F7, 0x3A, 0x00, 0x00, 0x00 }, /* ÷ to : */ + { 0x00F8, 0x6F, 0x00, 0x00, 0x00 }, /* ø to o */ + { 0x00F9, 0x75, 0x00, 0x00, 0x00 }, /* ù to u */ + { 0x00FA, 0x75, 0x00, 0x00, 0x00 }, /* ú to u */ + { 0x00FB, 0x75, 0x00, 0x00, 0x00 }, /* û to u */ + { 0x00FC, 0x75, 0x65, 0x00, 0x00 }, /* ü to ue */ + { 0x00FD, 0x79, 0x00, 0x00, 0x00 }, /* ý to y */ + { 0x00FE, 0x74, 0x68, 0x00, 0x00 }, /* þ to th */ + { 0x00FF, 0x79, 0x00, 0x00, 0x00 }, /* ÿ to y */ + { 0x0100, 0x41, 0x00, 0x00, 0x00 }, /* Ā to A */ + { 0x0101, 0x61, 0x00, 0x00, 0x00 }, /* ā to a */ + { 0x0102, 0x41, 0x00, 0x00, 0x00 }, /* Ă to A */ + { 0x0103, 0x61, 0x00, 0x00, 0x00 }, /* ă to a */ + { 0x0104, 0x41, 0x00, 0x00, 0x00 }, /* Ą to A */ + { 0x0105, 0x61, 0x00, 0x00, 0x00 }, /* ą to a */ + { 0x0106, 0x43, 0x00, 0x00, 0x00 }, /* Ć to C */ + { 0x0107, 0x63, 0x00, 0x00, 0x00 }, /* ć to c */ + { 0x0108, 0x43, 0x68, 0x00, 0x00 }, /* Ĉ to Ch */ + { 0x0109, 0x63, 0x68, 0x00, 0x00 }, /* ĉ to ch */ + { 0x010A, 0x43, 0x00, 0x00, 0x00 }, /* Ċ to C */ + { 0x010B, 0x63, 0x00, 0x00, 0x00 }, /* ċ to c */ + { 0x010C, 0x43, 0x00, 0x00, 0x00 }, /* Č to C */ + { 0x010D, 0x63, 0x00, 0x00, 0x00 }, /* č to c */ + { 0x010E, 0x44, 0x00, 0x00, 0x00 }, /* Ď to D */ + { 0x010F, 0x64, 0x00, 0x00, 0x00 }, /* ď to d */ + { 0x0110, 0x44, 0x00, 0x00, 0x00 }, /* Đ to D */ + { 0x0111, 0x64, 0x00, 0x00, 0x00 }, /* đ to d */ + { 0x0112, 0x45, 0x00, 0x00, 0x00 }, /* Ē to E */ + { 0x0113, 0x65, 0x00, 0x00, 0x00 }, /* ē to e */ + { 0x0114, 0x45, 0x00, 0x00, 0x00 }, /* Ĕ to E */ + { 0x0115, 0x65, 0x00, 0x00, 0x00 }, /* ĕ to e */ + { 0x0116, 0x45, 0x00, 0x00, 0x00 }, /* Ė to E */ + { 0x0117, 0x65, 0x00, 0x00, 0x00 }, /* ė to e */ + { 0x0118, 0x45, 0x00, 0x00, 0x00 }, /* Ę to E */ + { 0x0119, 0x65, 0x00, 0x00, 0x00 }, /* ę to e */ + { 0x011A, 0x45, 0x00, 0x00, 0x00 }, /* Ě to E */ + { 0x011B, 0x65, 0x00, 0x00, 0x00 }, /* ě to e */ + { 0x011C, 0x47, 0x68, 0x00, 0x00 }, /* Ĝ to Gh */ + { 0x011D, 0x67, 0x68, 0x00, 0x00 }, /* ĝ to gh */ + { 0x011E, 0x47, 0x00, 0x00, 0x00 }, /* Ğ to G */ + { 0x011F, 0x67, 0x00, 0x00, 0x00 }, /* ğ to g */ + { 0x0120, 0x47, 0x00, 0x00, 0x00 }, /* Ġ to G */ + { 0x0121, 0x67, 0x00, 0x00, 0x00 }, /* ġ to g */ + { 0x0122, 0x47, 0x00, 0x00, 0x00 }, /* Ģ to G */ + { 0x0123, 0x67, 0x00, 0x00, 0x00 }, /* ģ to g */ + { 0x0124, 0x48, 0x68, 0x00, 0x00 }, /* Ĥ to Hh */ + { 0x0125, 0x68, 0x68, 0x00, 0x00 }, /* ĥ to hh */ + { 0x0126, 0x48, 0x00, 0x00, 0x00 }, /* Ħ to H */ + { 0x0127, 0x68, 0x00, 0x00, 0x00 }, /* ħ to h */ + { 0x0128, 0x49, 0x00, 0x00, 0x00 }, /* Ĩ to I */ + { 0x0129, 0x69, 0x00, 0x00, 0x00 }, /* ĩ to i */ + { 0x012A, 0x49, 0x00, 0x00, 0x00 }, /* Ī to I */ + { 0x012B, 0x69, 0x00, 0x00, 0x00 }, /* ī to i */ + { 0x012C, 0x49, 0x00, 0x00, 0x00 }, /* Ĭ to I */ + { 0x012D, 0x69, 0x00, 0x00, 0x00 }, /* ĭ to i */ + { 0x012E, 0x49, 0x00, 0x00, 0x00 }, /* Į to I */ + { 0x012F, 0x69, 0x00, 0x00, 0x00 }, /* į to i */ + { 0x0130, 0x49, 0x00, 0x00, 0x00 }, /* İ to I */ + { 0x0131, 0x69, 0x00, 0x00, 0x00 }, /* ı to i */ + { 0x0132, 0x49, 0x4A, 0x00, 0x00 }, /* IJ to IJ */ + { 0x0133, 0x69, 0x6A, 0x00, 0x00 }, /* ij to ij */ + { 0x0134, 0x4A, 0x68, 0x00, 0x00 }, /* Ĵ to Jh */ + { 0x0135, 0x6A, 0x68, 0x00, 0x00 }, /* ĵ to jh */ + { 0x0136, 0x4B, 0x00, 0x00, 0x00 }, /* Ķ to K */ + { 0x0137, 0x6B, 0x00, 0x00, 0x00 }, /* ķ to k */ + { 0x0138, 0x6B, 0x00, 0x00, 0x00 }, /* ĸ to k */ + { 0x0139, 0x4C, 0x00, 0x00, 0x00 }, /* Ĺ to L */ + { 0x013A, 0x6C, 0x00, 0x00, 0x00 }, /* ĺ to l */ + { 0x013B, 0x4C, 0x00, 0x00, 0x00 }, /* Ļ to L */ + { 0x013C, 0x6C, 0x00, 0x00, 0x00 }, /* ļ to l */ + { 0x013D, 0x4C, 0x00, 0x00, 0x00 }, /* Ľ to L */ + { 0x013E, 0x6C, 0x00, 0x00, 0x00 }, /* ľ to l */ + { 0x013F, 0x4C, 0x2E, 0x00, 0x00 }, /* Ŀ to L. */ + { 0x0140, 0x6C, 0x2E, 0x00, 0x00 }, /* ŀ to l. */ + { 0x0141, 0x4C, 0x00, 0x00, 0x00 }, /* Ł to L */ + { 0x0142, 0x6C, 0x00, 0x00, 0x00 }, /* ł to l */ + { 0x0143, 0x4E, 0x00, 0x00, 0x00 }, /* Ń to N */ + { 0x0144, 0x6E, 0x00, 0x00, 0x00 }, /* ń to n */ + { 0x0145, 0x4E, 0x00, 0x00, 0x00 }, /* Ņ to N */ + { 0x0146, 0x6E, 0x00, 0x00, 0x00 }, /* ņ to n */ + { 0x0147, 0x4E, 0x00, 0x00, 0x00 }, /* Ň to N */ + { 0x0148, 0x6E, 0x00, 0x00, 0x00 }, /* ň to n */ + { 0x0149, 0x27, 0x6E, 0x00, 0x00 }, /* ʼn to 'n */ + { 0x014A, 0x4E, 0x47, 0x00, 0x00 }, /* Ŋ to NG */ + { 0x014B, 0x6E, 0x67, 0x00, 0x00 }, /* ŋ to ng */ + { 0x014C, 0x4F, 0x00, 0x00, 0x00 }, /* Ō to O */ + { 0x014D, 0x6F, 0x00, 0x00, 0x00 }, /* ō to o */ + { 0x014E, 0x4F, 0x00, 0x00, 0x00 }, /* Ŏ to O */ + { 0x014F, 0x6F, 0x00, 0x00, 0x00 }, /* ŏ to o */ + { 0x0150, 0x4F, 0x00, 0x00, 0x00 }, /* Ő to O */ + { 0x0151, 0x6F, 0x00, 0x00, 0x00 }, /* ő to o */ + { 0x0152, 0x4F, 0x45, 0x00, 0x00 }, /* Œ to OE */ + { 0x0153, 0x6F, 0x65, 0x00, 0x00 }, /* œ to oe */ + { 0x0154, 0x52, 0x00, 0x00, 0x00 }, /* Ŕ to R */ + { 0x0155, 0x72, 0x00, 0x00, 0x00 }, /* ŕ to r */ + { 0x0156, 0x52, 0x00, 0x00, 0x00 }, /* Ŗ to R */ + { 0x0157, 0x72, 0x00, 0x00, 0x00 }, /* ŗ to r */ + { 0x0158, 0x52, 0x00, 0x00, 0x00 }, /* Ř to R */ + { 0x0159, 0x72, 0x00, 0x00, 0x00 }, /* ř to r */ + { 0x015A, 0x53, 0x00, 0x00, 0x00 }, /* Ś to S */ + { 0x015B, 0x73, 0x00, 0x00, 0x00 }, /* ś to s */ + { 0x015C, 0x53, 0x68, 0x00, 0x00 }, /* Ŝ to Sh */ + { 0x015D, 0x73, 0x68, 0x00, 0x00 }, /* ŝ to sh */ + { 0x015E, 0x53, 0x00, 0x00, 0x00 }, /* Ş to S */ + { 0x015F, 0x73, 0x00, 0x00, 0x00 }, /* ş to s */ + { 0x0160, 0x53, 0x00, 0x00, 0x00 }, /* Š to S */ + { 0x0161, 0x73, 0x00, 0x00, 0x00 }, /* š to s */ + { 0x0162, 0x54, 0x00, 0x00, 0x00 }, /* Ţ to T */ + { 0x0163, 0x74, 0x00, 0x00, 0x00 }, /* ţ to t */ + { 0x0164, 0x54, 0x00, 0x00, 0x00 }, /* Ť to T */ + { 0x0165, 0x74, 0x00, 0x00, 0x00 }, /* ť to t */ + { 0x0166, 0x54, 0x00, 0x00, 0x00 }, /* Ŧ to T */ + { 0x0167, 0x74, 0x00, 0x00, 0x00 }, /* ŧ to t */ + { 0x0168, 0x55, 0x00, 0x00, 0x00 }, /* Ũ to U */ + { 0x0169, 0x75, 0x00, 0x00, 0x00 }, /* ũ to u */ + { 0x016A, 0x55, 0x00, 0x00, 0x00 }, /* Ū to U */ + { 0x016B, 0x75, 0x00, 0x00, 0x00 }, /* ū to u */ + { 0x016C, 0x55, 0x00, 0x00, 0x00 }, /* Ŭ to U */ + { 0x016D, 0x75, 0x00, 0x00, 0x00 }, /* ŭ to u */ + { 0x016E, 0x55, 0x00, 0x00, 0x00 }, /* Ů to U */ + { 0x016F, 0x75, 0x00, 0x00, 0x00 }, /* ů to u */ + { 0x0170, 0x55, 0x00, 0x00, 0x00 }, /* Ű to U */ + { 0x0171, 0x75, 0x00, 0x00, 0x00 }, /* ű to u */ + { 0x0172, 0x55, 0x00, 0x00, 0x00 }, /* Ų to U */ + { 0x0173, 0x75, 0x00, 0x00, 0x00 }, /* ų to u */ + { 0x0174, 0x57, 0x00, 0x00, 0x00 }, /* Ŵ to W */ + { 0x0175, 0x77, 0x00, 0x00, 0x00 }, /* ŵ to w */ + { 0x0176, 0x59, 0x00, 0x00, 0x00 }, /* Ŷ to Y */ + { 0x0177, 0x79, 0x00, 0x00, 0x00 }, /* ŷ to y */ + { 0x0178, 0x59, 0x00, 0x00, 0x00 }, /* Ÿ to Y */ + { 0x0179, 0x5A, 0x00, 0x00, 0x00 }, /* Ź to Z */ + { 0x017A, 0x7A, 0x00, 0x00, 0x00 }, /* ź to z */ + { 0x017B, 0x5A, 0x00, 0x00, 0x00 }, /* Ż to Z */ + { 0x017C, 0x7A, 0x00, 0x00, 0x00 }, /* ż to z */ + { 0x017D, 0x5A, 0x00, 0x00, 0x00 }, /* Ž to Z */ + { 0x017E, 0x7A, 0x00, 0x00, 0x00 }, /* ž to z */ + { 0x017F, 0x73, 0x00, 0x00, 0x00 }, /* ſ to s */ + { 0x0192, 0x66, 0x00, 0x00, 0x00 }, /* ƒ to f */ + { 0x0218, 0x53, 0x00, 0x00, 0x00 }, /* Ș to S */ + { 0x0219, 0x73, 0x00, 0x00, 0x00 }, /* ș to s */ + { 0x021A, 0x54, 0x00, 0x00, 0x00 }, /* Ț to T */ + { 0x021B, 0x74, 0x00, 0x00, 0x00 }, /* ț to t */ + { 0x0386, 0x41, 0x00, 0x00, 0x00 }, /* Ά to A */ + { 0x0388, 0x45, 0x00, 0x00, 0x00 }, /* Έ to E */ + { 0x0389, 0x49, 0x00, 0x00, 0x00 }, /* Ή to I */ + { 0x038A, 0x49, 0x00, 0x00, 0x00 }, /* Ί to I */ + { 0x038C, 0x4f, 0x00, 0x00, 0x00 }, /* Ό to O */ + { 0x038E, 0x59, 0x00, 0x00, 0x00 }, /* Ύ to Y */ + { 0x038F, 0x4f, 0x00, 0x00, 0x00 }, /* Ώ to O */ + { 0x0390, 0x69, 0x00, 0x00, 0x00 }, /* ΐ to i */ + { 0x0391, 0x41, 0x00, 0x00, 0x00 }, /* Α to A */ + { 0x0392, 0x42, 0x00, 0x00, 0x00 }, /* Β to B */ + { 0x0393, 0x47, 0x00, 0x00, 0x00 }, /* Γ to G */ + { 0x0394, 0x44, 0x00, 0x00, 0x00 }, /* Δ to D */ + { 0x0395, 0x45, 0x00, 0x00, 0x00 }, /* Ε to E */ + { 0x0396, 0x5a, 0x00, 0x00, 0x00 }, /* Ζ to Z */ + { 0x0397, 0x49, 0x00, 0x00, 0x00 }, /* Η to I */ + { 0x0398, 0x54, 0x68, 0x00, 0x00 }, /* Θ to Th */ + { 0x0399, 0x49, 0x00, 0x00, 0x00 }, /* Ι to I */ + { 0x039A, 0x4b, 0x00, 0x00, 0x00 }, /* Κ to K */ + { 0x039B, 0x4c, 0x00, 0x00, 0x00 }, /* Λ to L */ + { 0x039C, 0x4d, 0x00, 0x00, 0x00 }, /* Μ to M */ + { 0x039D, 0x4e, 0x00, 0x00, 0x00 }, /* Ν to N */ + { 0x039E, 0x58, 0x00, 0x00, 0x00 }, /* Ξ to X */ + { 0x039F, 0x4f, 0x00, 0x00, 0x00 }, /* Ο to O */ + { 0x03A0, 0x50, 0x00, 0x00, 0x00 }, /* Π to P */ + { 0x03A1, 0x52, 0x00, 0x00, 0x00 }, /* Ρ to R */ + { 0x03A3, 0x53, 0x00, 0x00, 0x00 }, /* Σ to S */ + { 0x03A4, 0x54, 0x00, 0x00, 0x00 }, /* Τ to T */ + { 0x03A5, 0x59, 0x00, 0x00, 0x00 }, /* Υ to Y */ + { 0x03A6, 0x46, 0x00, 0x00, 0x00 }, /* Φ to F */ + { 0x03A7, 0x43, 0x68, 0x00, 0x00 }, /* Χ to Ch */ + { 0x03A8, 0x50, 0x73, 0x00, 0x00 }, /* Ψ to Ps */ + { 0x03A9, 0x4f, 0x00, 0x00, 0x00 }, /* Ω to O */ + { 0x03AA, 0x49, 0x00, 0x00, 0x00 }, /* Ϊ to I */ + { 0x03AB, 0x59, 0x00, 0x00, 0x00 }, /* Ϋ to Y */ + { 0x03AC, 0x61, 0x00, 0x00, 0x00 }, /* ά to a */ + { 0x03AD, 0x65, 0x00, 0x00, 0x00 }, /* έ to e */ + { 0x03AE, 0x69, 0x00, 0x00, 0x00 }, /* ή to i */ + { 0x03AF, 0x69, 0x00, 0x00, 0x00 }, /* ί to i */ + { 0x03B1, 0x61, 0x00, 0x00, 0x00 }, /* α to a */ + { 0x03B2, 0x62, 0x00, 0x00, 0x00 }, /* β to b */ + { 0x03B3, 0x67, 0x00, 0x00, 0x00 }, /* γ to g */ + { 0x03B4, 0x64, 0x00, 0x00, 0x00 }, /* δ to d */ + { 0x03B5, 0x65, 0x00, 0x00, 0x00 }, /* ε to e */ + { 0x03B6, 0x7a, 0x00, 0x00, 0x00 }, /* ζ to z */ + { 0x03B7, 0x69, 0x00, 0x00, 0x00 }, /* η to i */ + { 0x03B8, 0x74, 0x68, 0x00, 0x00 }, /* θ to th */ + { 0x03B9, 0x69, 0x00, 0x00, 0x00 }, /* ι to i */ + { 0x03BA, 0x6b, 0x00, 0x00, 0x00 }, /* κ to k */ + { 0x03BB, 0x6c, 0x00, 0x00, 0x00 }, /* λ to l */ + { 0x03BC, 0x6d, 0x00, 0x00, 0x00 }, /* μ to m */ + { 0x03BD, 0x6e, 0x00, 0x00, 0x00 }, /* ν to n */ + { 0x03BE, 0x78, 0x00, 0x00, 0x00 }, /* ξ to x */ + { 0x03BF, 0x6f, 0x00, 0x00, 0x00 }, /* ο to o */ + { 0x03C0, 0x70, 0x00, 0x00, 0x00 }, /* π to p */ + { 0x03C1, 0x72, 0x00, 0x00, 0x00 }, /* ρ to r */ + { 0x03C3, 0x73, 0x00, 0x00, 0x00 }, /* σ to s */ + { 0x03C4, 0x74, 0x00, 0x00, 0x00 }, /* τ to t */ + { 0x03C5, 0x79, 0x00, 0x00, 0x00 }, /* υ to y */ + { 0x03C6, 0x66, 0x00, 0x00, 0x00 }, /* φ to f */ + { 0x03C7, 0x63, 0x68, 0x00, 0x00 }, /* χ to ch */ + { 0x03C8, 0x70, 0x73, 0x00, 0x00 }, /* ψ to ps */ + { 0x03C9, 0x6f, 0x00, 0x00, 0x00 }, /* ω to o */ + { 0x03CA, 0x69, 0x00, 0x00, 0x00 }, /* ϊ to i */ + { 0x03CB, 0x79, 0x00, 0x00, 0x00 }, /* ϋ to y */ + { 0x03CC, 0x6f, 0x00, 0x00, 0x00 }, /* ό to o */ + { 0x03CD, 0x79, 0x00, 0x00, 0x00 }, /* ύ to y */ + { 0x03CE, 0x69, 0x00, 0x00, 0x00 }, /* ώ to i */ + { 0x0400, 0x45, 0x00, 0x00, 0x00 }, /* Ѐ to E */ + { 0x0401, 0x45, 0x00, 0x00, 0x00 }, /* Ё to E */ + { 0x0402, 0x44, 0x00, 0x00, 0x00 }, /* Ђ to D */ + { 0x0403, 0x47, 0x00, 0x00, 0x00 }, /* Ѓ to G */ + { 0x0404, 0x45, 0x00, 0x00, 0x00 }, /* Є to E */ + { 0x0405, 0x5a, 0x00, 0x00, 0x00 }, /* Ѕ to Z */ + { 0x0406, 0x49, 0x00, 0x00, 0x00 }, /* І to I */ + { 0x0407, 0x49, 0x00, 0x00, 0x00 }, /* Ї to I */ + { 0x0408, 0x4a, 0x00, 0x00, 0x00 }, /* Ј to J */ + { 0x0409, 0x49, 0x00, 0x00, 0x00 }, /* Љ to I */ + { 0x040A, 0x4e, 0x00, 0x00, 0x00 }, /* Њ to N */ + { 0x040B, 0x44, 0x00, 0x00, 0x00 }, /* Ћ to D */ + { 0x040C, 0x4b, 0x00, 0x00, 0x00 }, /* Ќ to K */ + { 0x040D, 0x49, 0x00, 0x00, 0x00 }, /* Ѝ to I */ + { 0x040E, 0x55, 0x00, 0x00, 0x00 }, /* Ў to U */ + { 0x040F, 0x44, 0x00, 0x00, 0x00 }, /* Џ to D */ + { 0x0410, 0x41, 0x00, 0x00, 0x00 }, /* А to A */ + { 0x0411, 0x42, 0x00, 0x00, 0x00 }, /* Б to B */ + { 0x0412, 0x56, 0x00, 0x00, 0x00 }, /* В to V */ + { 0x0413, 0x47, 0x00, 0x00, 0x00 }, /* Г to G */ + { 0x0414, 0x44, 0x00, 0x00, 0x00 }, /* Д to D */ + { 0x0415, 0x45, 0x00, 0x00, 0x00 }, /* Е to E */ + { 0x0416, 0x5a, 0x68, 0x00, 0x00 }, /* Ж to Zh */ + { 0x0417, 0x5a, 0x00, 0x00, 0x00 }, /* З to Z */ + { 0x0418, 0x49, 0x00, 0x00, 0x00 }, /* И to I */ + { 0x0419, 0x49, 0x00, 0x00, 0x00 }, /* Й to I */ + { 0x041A, 0x4b, 0x00, 0x00, 0x00 }, /* К to K */ + { 0x041B, 0x4c, 0x00, 0x00, 0x00 }, /* Л to L */ + { 0x041C, 0x4d, 0x00, 0x00, 0x00 }, /* М to M */ + { 0x041D, 0x4e, 0x00, 0x00, 0x00 }, /* Н to N */ + { 0x041E, 0x4f, 0x00, 0x00, 0x00 }, /* О to O */ + { 0x041F, 0x50, 0x00, 0x00, 0x00 }, /* П to P */ + { 0x0420, 0x52, 0x00, 0x00, 0x00 }, /* Р to R */ + { 0x0421, 0x53, 0x00, 0x00, 0x00 }, /* С to S */ + { 0x0422, 0x54, 0x00, 0x00, 0x00 }, /* Т to T */ + { 0x0423, 0x55, 0x00, 0x00, 0x00 }, /* У to U */ + { 0x0424, 0x46, 0x00, 0x00, 0x00 }, /* Ф to F */ + { 0x0425, 0x4b, 0x68, 0x00, 0x00 }, /* Х to Kh */ + { 0x0426, 0x54, 0x63, 0x00, 0x00 }, /* Ц to Tc */ + { 0x0427, 0x43, 0x68, 0x00, 0x00 }, /* Ч to Ch */ + { 0x0428, 0x53, 0x68, 0x00, 0x00 }, /* Ш to Sh */ + { 0x0429, 0x53, 0x68, 0x63, 0x68 }, /* Щ to Shch */ + { 0x042A, 0x61, 0x00, 0x00, 0x00 }, /* to A */ + { 0x042B, 0x59, 0x00, 0x00, 0x00 }, /* Ы to Y */ + { 0x042C, 0x59, 0x00, 0x00, 0x00 }, /* to Y */ + { 0x042D, 0x45, 0x00, 0x00, 0x00 }, /* Э to E */ + { 0x042E, 0x49, 0x75, 0x00, 0x00 }, /* Ю to Iu */ + { 0x042F, 0x49, 0x61, 0x00, 0x00 }, /* Я to Ia */ + { 0x0430, 0x61, 0x00, 0x00, 0x00 }, /* а to a */ + { 0x0431, 0x62, 0x00, 0x00, 0x00 }, /* б to b */ + { 0x0432, 0x76, 0x00, 0x00, 0x00 }, /* в to v */ + { 0x0433, 0x67, 0x00, 0x00, 0x00 }, /* г to g */ + { 0x0434, 0x64, 0x00, 0x00, 0x00 }, /* д to d */ + { 0x0435, 0x65, 0x00, 0x00, 0x00 }, /* е to e */ + { 0x0436, 0x7a, 0x68, 0x00, 0x00 }, /* ж to zh */ + { 0x0437, 0x7a, 0x00, 0x00, 0x00 }, /* з to z */ + { 0x0438, 0x69, 0x00, 0x00, 0x00 }, /* и to i */ + { 0x0439, 0x69, 0x00, 0x00, 0x00 }, /* й to i */ + { 0x043A, 0x6b, 0x00, 0x00, 0x00 }, /* к to k */ + { 0x043B, 0x6c, 0x00, 0x00, 0x00 }, /* л to l */ + { 0x043C, 0x6d, 0x00, 0x00, 0x00 }, /* м to m */ + { 0x043D, 0x6e, 0x00, 0x00, 0x00 }, /* н to n */ + { 0x043E, 0x6f, 0x00, 0x00, 0x00 }, /* о to o */ + { 0x043F, 0x70, 0x00, 0x00, 0x00 }, /* п to p */ + { 0x0440, 0x72, 0x00, 0x00, 0x00 }, /* р to r */ + { 0x0441, 0x73, 0x00, 0x00, 0x00 }, /* с to s */ + { 0x0442, 0x74, 0x00, 0x00, 0x00 }, /* т to t */ + { 0x0443, 0x75, 0x00, 0x00, 0x00 }, /* у to u */ + { 0x0444, 0x66, 0x00, 0x00, 0x00 }, /* ф to f */ + { 0x0445, 0x6b, 0x68, 0x00, 0x00 }, /* х to kh */ + { 0x0446, 0x74, 0x63, 0x00, 0x00 }, /* ц to tc */ + { 0x0447, 0x63, 0x68, 0x00, 0x00 }, /* ч to ch */ + { 0x0448, 0x73, 0x68, 0x00, 0x00 }, /* ш to sh */ + { 0x0449, 0x73, 0x68, 0x63, 0x68 }, /* щ to shch */ + { 0x044A, 0x61, 0x00, 0x00, 0x00 }, /* to a */ + { 0x044B, 0x79, 0x00, 0x00, 0x00 }, /* ы to y */ + { 0x044C, 0x79, 0x00, 0x00, 0x00 }, /* to y */ + { 0x044D, 0x65, 0x00, 0x00, 0x00 }, /* э to e */ + { 0x044E, 0x69, 0x75, 0x00, 0x00 }, /* ю to iu */ + { 0x044F, 0x69, 0x61, 0x00, 0x00 }, /* я to ia */ + { 0x0450, 0x65, 0x00, 0x00, 0x00 }, /* ѐ to e */ + { 0x0451, 0x65, 0x00, 0x00, 0x00 }, /* ё to e */ + { 0x0452, 0x64, 0x00, 0x00, 0x00 }, /* ђ to d */ + { 0x0453, 0x67, 0x00, 0x00, 0x00 }, /* ѓ to g */ + { 0x0454, 0x65, 0x00, 0x00, 0x00 }, /* є to e */ + { 0x0455, 0x7a, 0x00, 0x00, 0x00 }, /* ѕ to z */ + { 0x0456, 0x69, 0x00, 0x00, 0x00 }, /* і to i */ + { 0x0457, 0x69, 0x00, 0x00, 0x00 }, /* ї to i */ + { 0x0458, 0x6a, 0x00, 0x00, 0x00 }, /* ј to j */ + { 0x0459, 0x69, 0x00, 0x00, 0x00 }, /* љ to i */ + { 0x045A, 0x6e, 0x00, 0x00, 0x00 }, /* њ to n */ + { 0x045B, 0x64, 0x00, 0x00, 0x00 }, /* ћ to d */ + { 0x045C, 0x6b, 0x00, 0x00, 0x00 }, /* ќ to k */ + { 0x045D, 0x69, 0x00, 0x00, 0x00 }, /* ѝ to i */ + { 0x045E, 0x75, 0x00, 0x00, 0x00 }, /* ў to u */ + { 0x045F, 0x64, 0x00, 0x00, 0x00 }, /* џ to d */ + { 0x1E02, 0x42, 0x00, 0x00, 0x00 }, /* Ḃ to B */ + { 0x1E03, 0x62, 0x00, 0x00, 0x00 }, /* ḃ to b */ + { 0x1E0A, 0x44, 0x00, 0x00, 0x00 }, /* Ḋ to D */ + { 0x1E0B, 0x64, 0x00, 0x00, 0x00 }, /* ḋ to d */ + { 0x1E1E, 0x46, 0x00, 0x00, 0x00 }, /* Ḟ to F */ + { 0x1E1F, 0x66, 0x00, 0x00, 0x00 }, /* ḟ to f */ + { 0x1E40, 0x4D, 0x00, 0x00, 0x00 }, /* Ṁ to M */ + { 0x1E41, 0x6D, 0x00, 0x00, 0x00 }, /* ṁ to m */ + { 0x1E56, 0x50, 0x00, 0x00, 0x00 }, /* Ṗ to P */ + { 0x1E57, 0x70, 0x00, 0x00, 0x00 }, /* ṗ to p */ + { 0x1E60, 0x53, 0x00, 0x00, 0x00 }, /* Ṡ to S */ + { 0x1E61, 0x73, 0x00, 0x00, 0x00 }, /* ṡ to s */ + { 0x1E6A, 0x54, 0x00, 0x00, 0x00 }, /* Ṫ to T */ + { 0x1E6B, 0x74, 0x00, 0x00, 0x00 }, /* ṫ to t */ + { 0x1E80, 0x57, 0x00, 0x00, 0x00 }, /* Ẁ to W */ + { 0x1E81, 0x77, 0x00, 0x00, 0x00 }, /* ẁ to w */ + { 0x1E82, 0x57, 0x00, 0x00, 0x00 }, /* Ẃ to W */ + { 0x1E83, 0x77, 0x00, 0x00, 0x00 }, /* ẃ to w */ + { 0x1E84, 0x57, 0x00, 0x00, 0x00 }, /* Ẅ to W */ + { 0x1E85, 0x77, 0x00, 0x00, 0x00 }, /* ẅ to w */ + { 0x1EF2, 0x59, 0x00, 0x00, 0x00 }, /* Ỳ to Y */ + { 0x1EF3, 0x79, 0x00, 0x00, 0x00 }, /* ỳ to y */ + { 0xFB00, 0x66, 0x66, 0x00, 0x00 }, /* ff to ff */ + { 0xFB01, 0x66, 0x69, 0x00, 0x00 }, /* fi to fi */ + { 0xFB02, 0x66, 0x6C, 0x00, 0x00 }, /* fl to fl */ + { 0xFB05, 0x73, 0x74, 0x00, 0x00 }, /* ſt to st */ + { 0xFB06, 0x73, 0x74, 0x00, 0x00 }, /* st to st */ }; +static const Transliteration *spellfixFindTranslit(int c, int *pxTop){ + *pxTop = (sizeof(translit)/sizeof(translit[0])) - 1; + return translit; +} + /* ** Convert the input string from UTF-8 into pure ASCII by converting ** all non-ASCII characters to some combination of characters in the @@ -1621,23 +1716,24 @@ static unsigned char *transliterate(const unsigned char *zIn, int nIn){ zOut[nOut++] = (unsigned char)c; }else{ int xTop, xBtm, x; - xTop = sizeof(translit)/sizeof(translit[0]) - 1; + const Transliteration *tbl = spellfixFindTranslit(c, &xTop); xBtm = 0; while( xTop>=xBtm ){ x = (xTop + xBtm)/2; - if( translit[x].cFrom==c ){ - zOut[nOut++] = translit[x].cTo0; - if( translit[x].cTo1 ){ - zOut[nOut++] = translit[x].cTo1; - /* Add an extra "ch" after the "sh" for Щ and щ */ - if( c==0x0429 || c== 0x0449 ){ - zOut[nOut++] = 'c'; - zOut[nOut++] = 'h'; + if( tbl[x].cFrom==c ){ + zOut[nOut++] = tbl[x].cTo0; + if( tbl[x].cTo1 ){ + zOut[nOut++] = tbl[x].cTo1; + if( tbl[x].cTo2 ){ + zOut[nOut++] = tbl[x].cTo2; + if( tbl[x].cTo3 ){ + zOut[nOut++] = tbl[x].cTo3; + } } } c = 0; break; - }else if( translit[x].cFrom>c ){ + }else if( tbl[x].cFrom>c ){ xTop = x-1; }else{ xBtm = x+1; @@ -1668,15 +1764,22 @@ static int translen_to_charlen(const char *zIn, int nIn, int nTrans){ nOut++; if( c>=128 ){ int xTop, xBtm, x; - xTop = sizeof(translit)/sizeof(translit[0]) - 1; + const Transliteration *tbl = spellfixFindTranslit(c, &xTop); xBtm = 0; while( xTop>=xBtm ){ x = (xTop + xBtm)/2; - if( translit[x].cFrom==c ){ - if( translit[x].cTo1 ) nOut++; - if( c==0x0429 || c== 0x0449 ) nOut += 2; + if( tbl[x].cFrom==c ){ + if( tbl[x].cTo1 ){ + nOut++; + if( tbl[x].cTo2 ){ + nOut++; + if( tbl[x].cTo3 ){ + nOut++; + } + } + } break; - }else if( translit[x].cFrom>c ){ + }else if( tbl[x].cFrom>c ){ xTop = x-1; }else{ xBtm = x+1; @@ -2476,7 +2579,7 @@ static int spellfix1FilterForMatch( nPattern = (int)strlen(zPattern); if( zPattern[nPattern-1]=='*' ) nPattern--; zSql = sqlite3_mprintf( - "SELECT id, word, rank, k1" + "SELECT id, word, rank, coalesce(k1,word)" " FROM \"%w\".\"%w_vocab\"" " WHERE langid=%d AND k2>=?1 AND k2zDbName, p->zTableName, iLang @@ -2810,17 +2913,17 @@ static int spellfix1Update( if( sqlite3_value_type(argv[1])==SQLITE_NULL ){ spellfix1DbExec(&rc, db, "INSERT INTO \"%w\".\"%w_vocab\"(rank,langid,word,k1,k2) " - "VALUES(%d,%d,%Q,%Q,%Q)", + "VALUES(%d,%d,%Q,nullif(%Q,%Q),%Q)", p->zDbName, p->zTableName, - iRank, iLang, zWord, zK1, zK2 + iRank, iLang, zWord, zK1, zWord, zK2 ); }else{ newRowid = sqlite3_value_int64(argv[1]); spellfix1DbExec(&rc, db, "INSERT OR %s INTO \"%w\".\"%w_vocab\"(id,rank,langid,word,k1,k2) " - "VALUES(%lld,%d,%d,%Q,%Q,%Q)", + "VALUES(%lld,%d,%d,%Q,nullif(%Q,%Q),%Q)", zConflict, p->zDbName, p->zTableName, - newRowid, iRank, iLang, zWord, zK1, zK2 + newRowid, iRank, iLang, zWord, zK1, zWord, zK2 ); } *pRowid = sqlite3_last_insert_rowid(db); @@ -2829,9 +2932,9 @@ static int spellfix1Update( newRowid = *pRowid = sqlite3_value_int64(argv[1]); spellfix1DbExec(&rc, db, "UPDATE OR %s \"%w\".\"%w_vocab\" SET id=%lld, rank=%d, langid=%d," - " word=%Q, k1=%Q, k2=%Q WHERE id=%lld", + " word=%Q, k1=nullif(%Q,%Q), k2=%Q WHERE id=%lld", zConflict, p->zDbName, p->zTableName, newRowid, iRank, iLang, - zWord, zK1, zK2, rowid + zWord, zK1, zWord, zK2, rowid ); } sqlite3_free(zK1); diff --git a/ext/misc/zipfile.c b/ext/misc/zipfile.c index fa31d62d11..a400ae30e1 100644 --- a/ext/misc/zipfile.c +++ b/ext/misc/zipfile.c @@ -30,29 +30,48 @@ SQLITE_EXTENSION_INIT1 #include #include -#include -#include -#include -#if !defined(_WIN32) && !defined(WIN32) -# include -# include -# include -#else -# include -#endif -#include -#include - #include #ifndef SQLITE_OMIT_VIRTUALTABLE #ifndef SQLITE_AMALGAMATION + typedef sqlite3_int64 i64; typedef unsigned char u8; typedef unsigned short u16; typedef unsigned long u32; #define MIN(a,b) ((a)<(b) ? (a) : (b)) + +#if defined(SQLITE_COVERAGE_TEST) || defined(SQLITE_MUTATION_TEST) +# define ALWAYS(X) (1) +# define NEVER(X) (0) +#elif !defined(NDEBUG) +# define ALWAYS(X) ((X)?1:(assert(0),0)) +# define NEVER(X) ((X)?(assert(0),1):0) +#else +# define ALWAYS(X) (X) +# define NEVER(X) (X) +#endif + +#endif /* SQLITE_AMALGAMATION */ + +/* +** Definitions for mode bitmasks S_IFDIR, S_IFREG and S_IFLNK. +** +** In some ways it would be better to obtain these values from system +** header files. But, the dependency is undesirable and (a) these +** have been stable for decades, (b) the values are part of POSIX and +** are also made explicit in [man stat], and (c) are part of the +** file format for zip archives. +*/ +#ifndef S_IFDIR +# define S_IFDIR 0040000 +#endif +#ifndef S_IFREG +# define S_IFREG 0100000 +#endif +#ifndef S_IFLNK +# define S_IFLNK 0120000 #endif static const char ZIPFILE_SCHEMA[] = @@ -93,6 +112,9 @@ static const char ZIPFILE_SCHEMA[] = ** ** ZIPFILE_SIGNATURE_LFH: ** First 4 bytes of a valid LFH record. +** +** ZIPFILE_SIGNATURE_EOCD +** First 4 bytes of a valid EOCD record. */ #define ZIPFILE_EXTRA_TIMESTAMP 0x5455 #define ZIPFILE_NEWENTRY_MADEBY ((3<<8) + 30) @@ -101,22 +123,14 @@ static const char ZIPFILE_SCHEMA[] = #define ZIPFILE_SIGNATURE_CDS 0x02014b50 #define ZIPFILE_SIGNATURE_LFH 0x04034b50 #define ZIPFILE_SIGNATURE_EOCD 0x06054b50 -#define ZIPFILE_LFH_FIXED_SZ 30 /* -** Set the error message contained in context ctx to the results of -** vprintf(zFmt, ...). +** The sizes of the fixed-size part of each of the three main data +** structures in a zip archive. */ -static void zipfileCtxErrorMsg(sqlite3_context *ctx, const char *zFmt, ...){ - char *zMsg = 0; - va_list ap; - va_start(ap, zFmt); - zMsg = sqlite3_vmprintf(zFmt, ap); - sqlite3_result_error(ctx, zMsg, -1); - sqlite3_free(zMsg); - va_end(ap); -} - +#define ZIPFILE_LFH_FIXED_SZ 30 +#define ZIPFILE_EOCD_FIXED_SZ 22 +#define ZIPFILE_CDS_FIXED_SZ 46 /* *** 4.3.16 End of central directory record: @@ -222,43 +236,34 @@ struct ZipfileLFH { typedef struct ZipfileEntry ZipfileEntry; struct ZipfileEntry { - char *zPath; /* Path of zipfile entry */ - u8 *aCdsEntry; /* Buffer containing entire CDS entry */ - int nCdsEntry; /* Size of buffer aCdsEntry[] in bytes */ - int bDeleted; /* True if entry has been deleted */ + ZipfileCDS cds; /* Parsed CDS record */ + u32 mUnixTime; /* Modification time, in UNIX format */ + u8 *aExtra; /* cds.nExtra+cds.nComment bytes of extra data */ + i64 iDataOff; /* Offset to data in file (if aData==0) */ + u8 *aData; /* cds.szCompressed bytes of compressed data */ ZipfileEntry *pNext; /* Next element in in-memory CDS */ }; /* -** Cursor type for recursively iterating through a directory structure. +** Cursor type for zipfile tables. */ typedef struct ZipfileCsr ZipfileCsr; struct ZipfileCsr { sqlite3_vtab_cursor base; /* Base class - must be first */ i64 iId; /* Cursor ID */ - int bEof; /* True when at EOF */ + u8 bEof; /* True when at EOF */ + u8 bNoop; /* If next xNext() call is no-op */ /* Used outside of write transactions */ FILE *pFile; /* Zip file */ i64 iNextOff; /* Offset of next record in central directory */ ZipfileEOCD eocd; /* Parse of central directory record */ - /* Used inside write transactions */ - ZipfileEntry *pCurrent; - - ZipfileCDS cds; /* Central Directory Structure */ - ZipfileLFH lfh; /* Local File Header for current entry */ - i64 iDataOff; /* Offset in zipfile to data */ - u32 mTime; /* Extended mtime value */ - int flags; /* Flags byte (see below for bits) */ + ZipfileEntry *pFreeEntry; /* Free this list when cursor is closed or reset */ + ZipfileEntry *pCurrent; /* Current entry */ ZipfileCsr *pCsrNext; /* Next cursor on same virtual table */ }; -/* -** Values for ZipfileCsr.flags. -*/ -#define ZIPFILE_MTIME_VALID 0x0001 - typedef struct ZipfileTab ZipfileTab; struct ZipfileTab { sqlite3_vtab base; /* Base class - must be first */ @@ -276,17 +281,33 @@ struct ZipfileTab { i64 szOrig; /* Size of archive at start of transaction */ }; +/* +** Set the error message contained in context ctx to the results of +** vprintf(zFmt, ...). +*/ +static void zipfileCtxErrorMsg(sqlite3_context *ctx, const char *zFmt, ...){ + char *zMsg = 0; + va_list ap; + va_start(ap, zFmt); + zMsg = sqlite3_vmprintf(zFmt, ap); + sqlite3_result_error(ctx, zMsg, -1); + sqlite3_free(zMsg); + va_end(ap); +} + +/* +** If string zIn is quoted, dequote it in place. Otherwise, if the string +** is not quoted, do nothing. +*/ static void zipfileDequote(char *zIn){ char q = zIn[0]; if( q=='"' || q=='\'' || q=='`' || q=='[' ){ - char c; int iIn = 1; int iOut = 0; if( q=='[' ) q = ']'; - while( (c = zIn[iIn++]) ){ - if( c==q ){ - if( zIn[iIn++]!=q ) break; - } + while( ALWAYS(zIn[iIn]) ){ + char c = zIn[iIn++]; + if( c==q && zIn[iIn++]!=q ) break; zIn[iOut++] = c; } zIn[iOut] = '\0'; @@ -314,6 +335,21 @@ static int zipfileConnect( ZipfileTab *pNew = 0; int rc; + /* If the table name is not "zipfile", require that the argument be + ** specified. This stops zipfile tables from being created as: + ** + ** CREATE VIRTUAL TABLE zzz USING zipfile(); + ** + ** It does not prevent: + ** + ** CREATE VIRTUAL TABLE zipfile USING zipfile(); + */ + assert( 0==sqlite3_stricmp(argv[0], "zipfile") ); + if( (0!=sqlite3_stricmp(argv[2], "zipfile") && argc<4) || argc>4 ){ + *pzErr = sqlite3_mprintf("zipfile constructor requires one argument"); + return SQLITE_ERROR; + } + if( argc>3 ){ zFile = argv[3]; nFile = (int)strlen(zFile)+1; @@ -335,10 +371,43 @@ static int zipfileConnect( return rc; } +/* +** Free the ZipfileEntry structure indicated by the only argument. +*/ +static void zipfileEntryFree(ZipfileEntry *p){ + if( p ){ + sqlite3_free(p->cds.zFile); + sqlite3_free(p); + } +} + +/* +** Release resources that should be freed at the end of a write +** transaction. +*/ +static void zipfileCleanupTransaction(ZipfileTab *pTab){ + ZipfileEntry *pEntry; + ZipfileEntry *pNext; + + if( pTab->pWriteFd ){ + fclose(pTab->pWriteFd); + pTab->pWriteFd = 0; + } + for(pEntry=pTab->pFirstEntry; pEntry; pEntry=pNext){ + pNext = pEntry->pNext; + zipfileEntryFree(pEntry); + } + pTab->pFirstEntry = 0; + pTab->pLastEntry = 0; + pTab->szCurrent = 0; + pTab->szOrig = 0; +} + /* ** This method is the destructor for zipfile vtab objects. */ static int zipfileDisconnect(sqlite3_vtab *pVtab){ + zipfileCleanupTransaction((ZipfileTab*)pVtab); sqlite3_free(pVtab); return SQLITE_OK; } @@ -366,12 +435,20 @@ static int zipfileOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCsr){ ** by zipfileOpen(). */ static void zipfileResetCursor(ZipfileCsr *pCsr){ - sqlite3_free(pCsr->cds.zFile); - pCsr->cds.zFile = 0; + ZipfileEntry *p; + ZipfileEntry *pNext; + pCsr->bEof = 0; if( pCsr->pFile ){ fclose(pCsr->pFile); pCsr->pFile = 0; + zipfileEntryFree(pCsr->pCurrent); + pCsr->pCurrent = 0; + } + + for(p=pCsr->pFreeEntry; p; p=pNext){ + pNext = p->pNext; + zipfileEntryFree(p); } } @@ -385,12 +462,8 @@ static int zipfileClose(sqlite3_vtab_cursor *cur){ zipfileResetCursor(pCsr); /* Remove this cursor from the ZipfileTab.pCsrList list. */ - for(pp=&pTab->pCsrList; *pp; pp=&((*pp)->pCsrNext)){ - if( *pp==pCsr ){ - *pp = pCsr->pCsrNext; - break; - } - } + for(pp=&pTab->pCsrList; *pp!=pCsr; pp=&((*pp)->pCsrNext)); + *pp = pCsr->pCsrNext; sqlite3_free(pCsr); return SQLITE_OK; @@ -407,6 +480,16 @@ static void zipfileSetErrmsg(ZipfileCsr *pCsr, const char *zFmt, ...){ va_end(ap); } +/* +** Read nRead bytes of data from offset iOff of file pFile into buffer +** aRead[]. Return SQLITE_OK if successful, or an SQLite error code +** otherwise. +** +** If an error does occur, output variable (*pzErrmsg) may be set to point +** to an English language error message. It is the responsibility of the +** caller to eventually free this buffer using +** sqlite3_free(). +*/ static int zipfileReadData( FILE *pFile, /* Read from this file */ u8 *aRead, /* Read into this buffer */ @@ -440,9 +523,16 @@ static int zipfileAppendData( return SQLITE_OK; } +/* +** Read and return a 16-bit little-endian unsigned integer from buffer aBuf. +*/ static u16 zipfileGetU16(const u8 *aBuf){ return (aBuf[1] << 8) + aBuf[0]; } + +/* +** Read and return a 32-bit little-endian unsigned integer from buffer aBuf. +*/ static u32 zipfileGetU32(const u8 *aBuf){ return ((u32)(aBuf[3]) << 24) + ((u32)(aBuf[2]) << 16) @@ -450,10 +540,17 @@ static u32 zipfileGetU32(const u8 *aBuf){ + ((u32)(aBuf[0]) << 0); } +/* +** Write a 16-bit little endiate integer into buffer aBuf. +*/ static void zipfilePutU16(u8 *aBuf, u16 val){ aBuf[0] = val & 0xFF; aBuf[1] = (val>>8) & 0xFF; } + +/* +** Write a 32-bit little endiate integer into buffer aBuf. +*/ static void zipfilePutU32(u8 *aBuf, u32 val){ aBuf[0] = val & 0xFF; aBuf[1] = (val>>8) & 0xFF; @@ -467,15 +564,11 @@ static void zipfilePutU32(u8 *aBuf, u32 val){ #define zipfileWrite32(aBuf,val) { zipfilePutU32(aBuf,val); aBuf+=4; } #define zipfileWrite16(aBuf,val) { zipfilePutU16(aBuf,val); aBuf+=2; } -static u8* zipfileCsrBuffer(ZipfileCsr *pCsr){ - return ((ZipfileTab*)(pCsr->base.pVtab))->aBuffer; -} - /* ** Magic numbers used to read CDS records. */ -#define ZIPFILE_CDS_FIXED_SZ 46 #define ZIPFILE_CDS_NFILE_OFF 28 +#define ZIPFILE_CDS_SZCOMPRESSED_OFF 20 /* ** Decode the CDS record in buffer aBuf into (*pCDS). Return SQLITE_ERROR @@ -512,168 +605,80 @@ static int zipfileReadCDS(u8 *aBuf, ZipfileCDS *pCDS){ } /* -** Read the CDS record for the current entry from disk into pCsr->cds. +** Decode the LFH record in buffer aBuf into (*pLFH). Return SQLITE_ERROR +** if the record is not well-formed, or SQLITE_OK otherwise. */ -static int zipfileCsrReadCDS(ZipfileCsr *pCsr){ - char **pzErr = &pCsr->base.pVtab->zErrMsg; - u8 *aRead; - int rc = SQLITE_OK; - - sqlite3_free(pCsr->cds.zFile); - pCsr->cds.zFile = 0; - - if( pCsr->pCurrent==0 ){ - aRead = zipfileCsrBuffer(pCsr); - rc = zipfileReadData( - pCsr->pFile, aRead, ZIPFILE_CDS_FIXED_SZ, pCsr->iNextOff, pzErr - ); - }else{ - aRead = pCsr->pCurrent->aCdsEntry; - } - - if( rc==SQLITE_OK ){ - rc = zipfileReadCDS(aRead, &pCsr->cds); - if( rc!=SQLITE_OK ){ - assert( pCsr->pCurrent==0 ); - zipfileSetErrmsg(pCsr,"failed to read CDS at offset %lld",pCsr->iNextOff); - }else{ - int nRead; - if( pCsr->pCurrent==0 ){ - nRead = pCsr->cds.nFile + pCsr->cds.nExtra; - aRead = zipfileCsrBuffer(pCsr); - pCsr->iNextOff += ZIPFILE_CDS_FIXED_SZ; - rc = zipfileReadData(pCsr->pFile, aRead, nRead, pCsr->iNextOff, pzErr); - }else{ - aRead = &aRead[ZIPFILE_CDS_FIXED_SZ]; - } - - if( rc==SQLITE_OK ){ - pCsr->cds.zFile = sqlite3_mprintf("%.*s", (int)pCsr->cds.nFile, aRead); - pCsr->iNextOff += pCsr->cds.nFile; - pCsr->iNextOff += pCsr->cds.nExtra; - pCsr->iNextOff += pCsr->cds.nComment; - } - - /* Scan the cds.nExtra bytes of "extra" fields for any that can - ** be interpreted. The general format of an extra field is: - ** - ** Header ID 2 bytes - ** Data Size 2 bytes - ** Data N bytes - ** - */ - if( rc==SQLITE_OK ){ - u8 *p = &aRead[pCsr->cds.nFile]; - u8 *pEnd = &p[pCsr->cds.nExtra]; - - while( p modtime is present */ - pCsr->mTime = zipfileGetU32(&p[1]); - pCsr->flags |= ZIPFILE_MTIME_VALID; - } - break; - } - } - - p += nByte; - } - } - } - } - - return rc; -} - -static FILE *zipfileGetFd(ZipfileCsr *pCsr){ - if( pCsr->pFile ) return pCsr->pFile; - return ((ZipfileTab*)(pCsr->base.pVtab))->pWriteFd; -} - static int zipfileReadLFH( - FILE *pFd, - i64 iOffset, - u8 *aTmp, - ZipfileLFH *pLFH, - char **pzErr + u8 *aBuffer, + ZipfileLFH *pLFH ){ - u8 *aRead = aTmp; - static const int szFix = ZIPFILE_LFH_FIXED_SZ; - int rc; - - rc = zipfileReadData(pFd, aRead, szFix, iOffset, pzErr); - if( rc==SQLITE_OK ){ - u32 sig = zipfileRead32(aRead); - if( sig!=ZIPFILE_SIGNATURE_LFH ){ - *pzErr = sqlite3_mprintf("failed to read LFH at offset %d", (int)iOffset); - rc = SQLITE_ERROR; - }else{ - pLFH->iVersionExtract = zipfileRead16(aRead); - pLFH->flags = zipfileRead16(aRead); - pLFH->iCompression = zipfileRead16(aRead); - pLFH->mTime = zipfileRead16(aRead); - pLFH->mDate = zipfileRead16(aRead); - pLFH->crc32 = zipfileRead32(aRead); - pLFH->szCompressed = zipfileRead32(aRead); - pLFH->szUncompressed = zipfileRead32(aRead); - pLFH->nFile = zipfileRead16(aRead); - pLFH->nExtra = zipfileRead16(aRead); - assert( aRead==&aTmp[szFix] ); - } - } - return rc; -} - -static int zipfileCsrReadLFH(ZipfileCsr *pCsr){ - FILE *pFile = zipfileGetFd(pCsr); - char **pzErr = &pCsr->base.pVtab->zErrMsg; - u8 *aRead = zipfileCsrBuffer(pCsr); - int rc = zipfileReadLFH(pFile, pCsr->cds.iOffset, aRead, &pCsr->lfh, pzErr); - pCsr->iDataOff = pCsr->cds.iOffset + ZIPFILE_LFH_FIXED_SZ; - pCsr->iDataOff += pCsr->lfh.nFile+pCsr->lfh.nExtra; - return rc; -} - - -/* -** Advance an ZipfileCsr to its next row of output. -*/ -static int zipfileNext(sqlite3_vtab_cursor *cur){ - ZipfileCsr *pCsr = (ZipfileCsr*)cur; + u8 *aRead = aBuffer; int rc = SQLITE_OK; - pCsr->flags = 0; - if( pCsr->pCurrent==0 ){ - i64 iEof = pCsr->eocd.iOffset + pCsr->eocd.nSize; - if( pCsr->iNextOff>=iEof ){ - pCsr->bEof = 1; - } + u32 sig = zipfileRead32(aRead); + if( sig!=ZIPFILE_SIGNATURE_LFH ){ + rc = SQLITE_ERROR; }else{ - assert( pCsr->pFile==0 ); - do { - pCsr->pCurrent = pCsr->pCurrent->pNext; - }while( pCsr->pCurrent && pCsr->pCurrent->bDeleted ); - if( pCsr->pCurrent==0 ){ - pCsr->bEof = 1; - } + pLFH->iVersionExtract = zipfileRead16(aRead); + pLFH->flags = zipfileRead16(aRead); + pLFH->iCompression = zipfileRead16(aRead); + pLFH->mTime = zipfileRead16(aRead); + pLFH->mDate = zipfileRead16(aRead); + pLFH->crc32 = zipfileRead32(aRead); + pLFH->szCompressed = zipfileRead32(aRead); + pLFH->szUncompressed = zipfileRead32(aRead); + pLFH->nFile = zipfileRead16(aRead); + pLFH->nExtra = zipfileRead16(aRead); } - - if( pCsr->bEof==0 ){ - rc = zipfileCsrReadCDS(pCsr); - if( rc==SQLITE_OK ){ - rc = zipfileCsrReadLFH(pCsr); - } - } - return rc; } + /* +** Buffer aExtra (size nExtra bytes) contains zip archive "extra" fields. +** Scan through this buffer to find an "extra-timestamp" field. If one +** exists, extract the 32-bit modification-timestamp from it and store +** the value in output parameter *pmTime. +** +** Zero is returned if no extra-timestamp record could be found (and so +** *pmTime is left unchanged), or non-zero otherwise. +** +** The general format of an extra field is: +** +** Header ID 2 bytes +** Data Size 2 bytes +** Data N bytes +*/ +static int zipfileScanExtra(u8 *aExtra, int nExtra, u32 *pmTime){ + int ret = 0; + u8 *p = aExtra; + u8 *pEnd = &aExtra[nExtra]; + + while( p modtime is present */ + *pmTime = zipfileGetU32(&p[1]); + ret = 1; + } + break; + } + } + + p += nByte; + } + return ret; +} + +/* +** Convert the standard MS-DOS timestamp stored in the mTime and mDate +** fields of the CDS structure passed as the only argument to a 32-bit +** UNIX seconds-since-the-epoch timestamp. Return the result. +** ** "Standard" MS-DOS time format: ** ** File modification time: @@ -684,44 +689,233 @@ static int zipfileNext(sqlite3_vtab_cursor *cur){ ** Bits 00-04: day ** Bits 05-08: month (1-12) ** Bits 09-15: years from 1980 +** +** https://msdn.microsoft.com/en-us/library/9kkf9tah.aspx */ -static time_t zipfileMtime(ZipfileCsr *pCsr){ - struct tm t; - memset(&t, 0, sizeof(t)); - t.tm_sec = (pCsr->cds.mTime & 0x1F)*2; - t.tm_min = (pCsr->cds.mTime >> 5) & 0x2F; - t.tm_hour = (pCsr->cds.mTime >> 11) & 0x1F; +static u32 zipfileMtime(ZipfileCDS *pCDS){ + int Y = (1980 + ((pCDS->mDate >> 9) & 0x7F)); + int M = ((pCDS->mDate >> 5) & 0x0F); + int D = (pCDS->mDate & 0x1F); + int B = -13; - t.tm_mday = (pCsr->cds.mDate & 0x1F); - t.tm_mon = ((pCsr->cds.mDate >> 5) & 0x0F) - 1; - t.tm_year = 80 + ((pCsr->cds.mDate >> 9) & 0x7F); + int sec = (pCDS->mTime & 0x1F)*2; + int min = (pCDS->mTime >> 5) & 0x3F; + int hr = (pCDS->mTime >> 11) & 0x1F; + i64 JD; - return mktime(&t); + /* JD = INT(365.25 * (Y+4716)) + INT(30.6001 * (M+1)) + D + B - 1524.5 */ + + /* Calculate the JD in seconds for noon on the day in question */ + if( M<3 ){ + Y = Y-1; + M = M+12; + } + JD = (i64)(24*60*60) * ( + (int)(365.25 * (Y + 4716)) + + (int)(30.6001 * (M + 1)) + + D + B - 1524 + ); + + /* Correct the JD for the time within the day */ + JD += (hr-12) * 3600 + min * 60 + sec; + + /* Convert JD to unix timestamp (the JD epoch is 2440587.5) */ + return (u32)(JD - (i64)(24405875) * 24*60*6); } -static void zipfileMtimeToDos(ZipfileCDS *pCds, u32 mTime){ - time_t t = (time_t)mTime; - struct tm res; +/* +** The opposite of zipfileMtime(). This function populates the mTime and +** mDate fields of the CDS structure passed as the first argument according +** to the UNIX timestamp value passed as the second. +*/ +static void zipfileMtimeToDos(ZipfileCDS *pCds, u32 mUnixTime){ + /* Convert unix timestamp to JD (2440588 is noon on 1/1/1970) */ + i64 JD = (i64)2440588 + mUnixTime / (24*60*60); -#if !defined(_WIN32) && !defined(WIN32) - localtime_r(&t, &res); -#else - memcpy(&res, localtime(&t), sizeof(struct tm)); -#endif + int A, B, C, D, E; + int yr, mon, day; + int hr, min, sec; - pCds->mTime = (u16)( - (res.tm_sec / 2) + - (res.tm_min << 5) + - (res.tm_hour << 11)); + A = (int)((JD - 1867216.25)/36524.25); + A = (int)(JD + 1 + A - (A/4)); + B = A + 1524; + C = (int)((B - 122.1)/365.25); + D = (36525*(C&32767))/100; + E = (int)((B-D)/30.6001); - pCds->mDate = (u16)( - (res.tm_mday-1) + - ((res.tm_mon+1) << 5) + - ((res.tm_year-80) << 9)); + day = B - D - (int)(30.6001*E); + mon = (E<14 ? E-1 : E-13); + yr = mon>2 ? C-4716 : C-4715; + + hr = (mUnixTime % (24*60*60)) / (60*60); + min = (mUnixTime % (60*60)) / 60; + sec = (mUnixTime % 60); + + pCds->mDate = (u16)(day + (mon << 5) + ((yr-1980) << 9)); + pCds->mTime = (u16)(sec/2 + (min<<5) + (hr<<11)); + + assert( mUnixTime<315507600 + || mUnixTime==zipfileMtime(pCds) + || ((mUnixTime % 2) && mUnixTime-1==zipfileMtime(pCds)) + /* || (mUnixTime % 2) */ + ); } +/* +** If aBlob is not NULL, then it is a pointer to a buffer (nBlob bytes in +** size) containing an entire zip archive image. Or, if aBlob is NULL, +** then pFile is a file-handle open on a zip file. In either case, this +** function creates a ZipfileEntry object based on the zip archive entry +** for which the CDS record is at offset iOff. +** +** If successful, SQLITE_OK is returned and (*ppEntry) set to point to +** the new object. Otherwise, an SQLite error code is returned and the +** final value of (*ppEntry) undefined. +*/ +static int zipfileGetEntry( + ZipfileTab *pTab, /* Store any error message here */ + const u8 *aBlob, /* Pointer to in-memory file image */ + int nBlob, /* Size of aBlob[] in bytes */ + FILE *pFile, /* If aBlob==0, read from this file */ + i64 iOff, /* Offset of CDS record */ + ZipfileEntry **ppEntry /* OUT: Pointer to new object */ +){ + u8 *aRead; + char **pzErr = &pTab->base.zErrMsg; + int rc = SQLITE_OK; + + if( aBlob==0 ){ + aRead = pTab->aBuffer; + rc = zipfileReadData(pFile, aRead, ZIPFILE_CDS_FIXED_SZ, iOff, pzErr); + }else{ + aRead = (u8*)&aBlob[iOff]; + } + + if( rc==SQLITE_OK ){ + int nAlloc; + ZipfileEntry *pNew; + + int nFile = zipfileGetU16(&aRead[ZIPFILE_CDS_NFILE_OFF]); + int nExtra = zipfileGetU16(&aRead[ZIPFILE_CDS_NFILE_OFF+2]); + nExtra += zipfileGetU16(&aRead[ZIPFILE_CDS_NFILE_OFF+4]); + + nAlloc = sizeof(ZipfileEntry) + nExtra; + if( aBlob ){ + nAlloc += zipfileGetU32(&aRead[ZIPFILE_CDS_SZCOMPRESSED_OFF]); + } + + pNew = (ZipfileEntry*)sqlite3_malloc(nAlloc); + if( pNew==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(pNew, 0, sizeof(ZipfileEntry)); + rc = zipfileReadCDS(aRead, &pNew->cds); + if( rc!=SQLITE_OK ){ + *pzErr = sqlite3_mprintf("failed to read CDS at offset %lld", iOff); + }else if( aBlob==0 ){ + rc = zipfileReadData( + pFile, aRead, nExtra+nFile, iOff+ZIPFILE_CDS_FIXED_SZ, pzErr + ); + }else{ + aRead = (u8*)&aBlob[iOff + ZIPFILE_CDS_FIXED_SZ]; + } + } + + if( rc==SQLITE_OK ){ + u32 *pt = &pNew->mUnixTime; + pNew->cds.zFile = sqlite3_mprintf("%.*s", nFile, aRead); + pNew->aExtra = (u8*)&pNew[1]; + memcpy(pNew->aExtra, &aRead[nFile], nExtra); + if( pNew->cds.zFile==0 ){ + rc = SQLITE_NOMEM; + }else if( 0==zipfileScanExtra(&aRead[nFile], pNew->cds.nExtra, pt) ){ + pNew->mUnixTime = zipfileMtime(&pNew->cds); + } + } + + if( rc==SQLITE_OK ){ + static const int szFix = ZIPFILE_LFH_FIXED_SZ; + ZipfileLFH lfh; + if( pFile ){ + rc = zipfileReadData(pFile, aRead, szFix, pNew->cds.iOffset, pzErr); + }else{ + aRead = (u8*)&aBlob[pNew->cds.iOffset]; + } + + rc = zipfileReadLFH(aRead, &lfh); + if( rc==SQLITE_OK ){ + pNew->iDataOff = pNew->cds.iOffset + ZIPFILE_LFH_FIXED_SZ; + pNew->iDataOff += lfh.nFile + lfh.nExtra; + if( aBlob && pNew->cds.szCompressed ){ + pNew->aData = &pNew->aExtra[nExtra]; + memcpy(pNew->aData, &aBlob[pNew->iDataOff], pNew->cds.szCompressed); + } + }else{ + *pzErr = sqlite3_mprintf("failed to read LFH at offset %d", + (int)pNew->cds.iOffset + ); + } + } + + if( rc!=SQLITE_OK ){ + zipfileEntryFree(pNew); + }else{ + *ppEntry = pNew; + } + } + + return rc; +} + +/* +** Advance an ZipfileCsr to its next row of output. +*/ +static int zipfileNext(sqlite3_vtab_cursor *cur){ + ZipfileCsr *pCsr = (ZipfileCsr*)cur; + int rc = SQLITE_OK; + + if( pCsr->pFile ){ + i64 iEof = pCsr->eocd.iOffset + pCsr->eocd.nSize; + zipfileEntryFree(pCsr->pCurrent); + pCsr->pCurrent = 0; + if( pCsr->iNextOff>=iEof ){ + pCsr->bEof = 1; + }else{ + ZipfileEntry *p = 0; + ZipfileTab *pTab = (ZipfileTab*)(cur->pVtab); + rc = zipfileGetEntry(pTab, 0, 0, pCsr->pFile, pCsr->iNextOff, &p); + if( rc==SQLITE_OK ){ + pCsr->iNextOff += ZIPFILE_CDS_FIXED_SZ; + pCsr->iNextOff += (int)p->cds.nExtra + p->cds.nFile + p->cds.nComment; + } + pCsr->pCurrent = p; + } + }else{ + if( !pCsr->bNoop ){ + pCsr->pCurrent = pCsr->pCurrent->pNext; + } + if( pCsr->pCurrent==0 ){ + pCsr->bEof = 1; + } + } + + pCsr->bNoop = 0; + return rc; +} + +static void zipfileFree(void *p) { + sqlite3_free(p); +} + +/* +** Buffer aIn (size nIn bytes) contains compressed data. Uncompressed, the +** size is nOut bytes. This function uncompresses the data and sets the +** return value in context pCtx to the result (a blob). +** +** If an error occurs, an error code is left in pCtx instead. +*/ static void zipfileInflate( - sqlite3_context *pCtx, /* Store error here, if any */ + sqlite3_context *pCtx, /* Store result here */ const u8 *aIn, /* Compressed data */ int nIn, /* Size of buffer aIn[] in bytes */ int nOut /* Expected output size */ @@ -747,7 +941,8 @@ static void zipfileInflate( if( err!=Z_STREAM_END ){ zipfileCtxErrorMsg(pCtx, "inflate() failed (%d)", err); }else{ - sqlite3_result_blob(pCtx, aRes, nOut, SQLITE_TRANSIENT); + sqlite3_result_blob(pCtx, aRes, nOut, zipfileFree); + aRes = 0; } } sqlite3_free(aRes); @@ -755,10 +950,22 @@ static void zipfileInflate( } } +/* +** Buffer aIn (size nIn bytes) contains uncompressed data. This function +** compresses it and sets (*ppOut) to point to a buffer containing the +** compressed data. The caller is responsible for eventually calling +** sqlite3_free() to release buffer (*ppOut). Before returning, (*pnOut) +** is set to the size of buffer (*ppOut) in bytes. +** +** If no error occurs, SQLITE_OK is returned. Otherwise, an SQLite error +** code is returned and an error message left in virtual-table handle +** pTab. The values of (*ppOut) and (*pnOut) are left unchanged in this +** case. +*/ static int zipfileDeflate( - ZipfileTab *pTab, /* Set error message here */ const u8 *aIn, int nIn, /* Input */ - u8 **ppOut, int *pnOut /* Output */ + u8 **ppOut, int *pnOut, /* Output */ + char **pzErr /* OUT: Error message */ ){ int nAlloc = (int)compressBound(nIn); u8 *aOut; @@ -784,7 +991,7 @@ static int zipfileDeflate( *pnOut = (int)str.total_out; }else{ sqlite3_free(aOut); - pTab->base.zErrMsg = sqlite3_mprintf("zipfile: deflate() error"); + *pzErr = sqlite3_mprintf("zipfile: deflate() error"); rc = SQLITE_ERROR; } deflateEnd(&str); @@ -804,60 +1011,66 @@ static int zipfileColumn( int i /* Which column to return */ ){ ZipfileCsr *pCsr = (ZipfileCsr*)cur; + ZipfileCDS *pCDS = &pCsr->pCurrent->cds; int rc = SQLITE_OK; switch( i ){ case 0: /* name */ - sqlite3_result_text(ctx, pCsr->cds.zFile, -1, SQLITE_TRANSIENT); + sqlite3_result_text(ctx, pCDS->zFile, -1, SQLITE_TRANSIENT); break; case 1: /* mode */ /* TODO: Whether or not the following is correct surely depends on ** the platform on which the archive was created. */ - sqlite3_result_int(ctx, pCsr->cds.iExternalAttr >> 16); + sqlite3_result_int(ctx, pCDS->iExternalAttr >> 16); break; case 2: { /* mtime */ - if( pCsr->flags & ZIPFILE_MTIME_VALID ){ - sqlite3_result_int64(ctx, pCsr->mTime); - }else{ - sqlite3_result_int64(ctx, zipfileMtime(pCsr)); - } + sqlite3_result_int64(ctx, pCsr->pCurrent->mUnixTime); break; } case 3: { /* sz */ if( sqlite3_vtab_nochange(ctx)==0 ){ - sqlite3_result_int64(ctx, pCsr->cds.szUncompressed); + sqlite3_result_int64(ctx, pCDS->szUncompressed); } break; } case 4: /* rawdata */ if( sqlite3_vtab_nochange(ctx) ) break; case 5: { /* data */ - if( i==4 || pCsr->cds.iCompression==0 || pCsr->cds.iCompression==8 ){ - int sz = pCsr->cds.szCompressed; - int szFinal = pCsr->cds.szUncompressed; + if( i==4 || pCDS->iCompression==0 || pCDS->iCompression==8 ){ + int sz = pCDS->szCompressed; + int szFinal = pCDS->szUncompressed; if( szFinal>0 ){ - u8 *aBuf = sqlite3_malloc(sz); - if( aBuf==0 ){ - rc = SQLITE_NOMEM; + u8 *aBuf; + u8 *aFree = 0; + if( pCsr->pCurrent->aData ){ + aBuf = pCsr->pCurrent->aData; }else{ - FILE *pFile = zipfileGetFd(pCsr); - rc = zipfileReadData(pFile, aBuf, sz, pCsr->iDataOff, - &pCsr->base.pVtab->zErrMsg - ); + aBuf = aFree = sqlite3_malloc(sz); + if( aBuf==0 ){ + rc = SQLITE_NOMEM; + }else{ + FILE *pFile = pCsr->pFile; + if( pFile==0 ){ + pFile = ((ZipfileTab*)(pCsr->base.pVtab))->pWriteFd; + } + rc = zipfileReadData(pFile, aBuf, sz, pCsr->pCurrent->iDataOff, + &pCsr->base.pVtab->zErrMsg + ); + } } if( rc==SQLITE_OK ){ - if( i==5 && pCsr->cds.iCompression ){ + if( i==5 && pCDS->iCompression ){ zipfileInflate(ctx, aBuf, sz, szFinal); }else{ sqlite3_result_blob(ctx, aBuf, sz, SQLITE_TRANSIENT); } - sqlite3_free(aBuf); } + sqlite3_free(aFree); }else{ /* Figure out if this is a directory or a zero-sized file. Consider ** it to be a directory either if the mode suggests so, or if ** the final character in the name is '/'. */ - u32 mode = pCsr->cds.iExternalAttr >> 16; - if( !(mode & S_IFDIR) && pCsr->cds.zFile[pCsr->cds.nFile-1]!='/' ){ + u32 mode = pCDS->iExternalAttr >> 16; + if( !(mode & S_IFDIR) && pCDS->zFile[pCDS->nFile-1]!='/' ){ sqlite3_result_blob(ctx, "", 0, SQLITE_STATIC); } } @@ -865,9 +1078,10 @@ static int zipfileColumn( break; } case 6: /* method */ - sqlite3_result_int(ctx, pCsr->cds.iCompression); + sqlite3_result_int(ctx, pCDS->iCompression); break; - case 7: /* z */ + default: /* z */ + assert( i==7 ); sqlite3_result_int64(ctx, pCsr->iId); break; } @@ -876,16 +1090,7 @@ static int zipfileColumn( } /* -** Return the rowid for the current row. -*/ -static int zipfileRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ - assert( 0 ); - return SQLITE_OK; -} - -/* -** Return TRUE if the cursor has been moved off of the last -** row of output. +** Return TRUE if the cursor is at EOF. */ static int zipfileEof(sqlite3_vtab_cursor *cur){ ZipfileCsr *pCsr = (ZipfileCsr*)cur; @@ -893,28 +1098,43 @@ static int zipfileEof(sqlite3_vtab_cursor *cur){ } /* +** If aBlob is not NULL, then it points to a buffer nBlob bytes in size +** containing an entire zip archive image. Or, if aBlob is NULL, then pFile +** is guaranteed to be a file-handle open on a zip file. +** +** This function attempts to locate the EOCD record within the zip archive +** and populate *pEOCD with the results of decoding it. SQLITE_OK is +** returned if successful. Otherwise, an SQLite error code is returned and +** an English language error message may be left in virtual-table pTab. */ static int zipfileReadEOCD( ZipfileTab *pTab, /* Return errors here */ - FILE *pFile, /* Read from this file */ + const u8 *aBlob, /* Pointer to in-memory file image */ + int nBlob, /* Size of aBlob[] in bytes */ + FILE *pFile, /* Read from this file if aBlob==0 */ ZipfileEOCD *pEOCD /* Object to populate */ ){ u8 *aRead = pTab->aBuffer; /* Temporary buffer */ - i64 szFile; /* Total size of file in bytes */ int nRead; /* Bytes to read from file */ - i64 iOff; /* Offset to read from */ - int rc; + int rc = SQLITE_OK; - fseek(pFile, 0, SEEK_END); - szFile = (i64)ftell(pFile); - if( szFile==0 ){ - memset(pEOCD, 0, sizeof(ZipfileEOCD)); - return SQLITE_OK; + if( aBlob==0 ){ + i64 iOff; /* Offset to read from */ + i64 szFile; /* Total size of file in bytes */ + fseek(pFile, 0, SEEK_END); + szFile = (i64)ftell(pFile); + if( szFile==0 ){ + memset(pEOCD, 0, sizeof(ZipfileEOCD)); + return SQLITE_OK; + } + nRead = (int)(MIN(szFile, ZIPFILE_BUFFER_SIZE)); + iOff = szFile - nRead; + rc = zipfileReadData(pFile, aRead, nRead, iOff, &pTab->base.zErrMsg); + }else{ + nRead = (int)(MIN(nBlob, ZIPFILE_BUFFER_SIZE)); + aRead = (u8*)&aBlob[nBlob-nRead]; } - nRead = (int)(MIN(szFile, ZIPFILE_BUFFER_SIZE)); - iOff = szFile - nRead; - rc = zipfileReadData(pFile, aRead, nRead, iOff, &pTab->base.zErrMsg); if( rc==SQLITE_OK ){ int i; @@ -940,17 +1160,59 @@ static int zipfileReadEOCD( pEOCD->nEntryTotal = zipfileRead16(aRead); pEOCD->nSize = zipfileRead32(aRead); pEOCD->iOffset = zipfileRead32(aRead); - -#if 0 - printf("iDisk=%d iFirstDisk=%d nEntry=%d " - "nEntryTotal=%d nSize=%d iOffset=%d", - (int)pEOCD->iDisk, (int)pEOCD->iFirstDisk, (int)pEOCD->nEntry, - (int)pEOCD->nEntryTotal, (int)pEOCD->nSize, (int)pEOCD->iOffset - ); -#endif } - return SQLITE_OK; + return rc; +} + +/* +** Add object pNew to the linked list that begins at ZipfileTab.pFirstEntry +** and ends with pLastEntry. If argument pBefore is NULL, then pNew is added +** to the end of the list. Otherwise, it is added to the list immediately +** before pBefore (which is guaranteed to be a part of said list). +*/ +static void zipfileAddEntry( + ZipfileTab *pTab, + ZipfileEntry *pBefore, + ZipfileEntry *pNew +){ + assert( (pTab->pFirstEntry==0)==(pTab->pLastEntry==0) ); + assert( pNew->pNext==0 ); + if( pBefore==0 ){ + if( pTab->pFirstEntry==0 ){ + pTab->pFirstEntry = pTab->pLastEntry = pNew; + }else{ + assert( pTab->pLastEntry->pNext==0 ); + pTab->pLastEntry->pNext = pNew; + pTab->pLastEntry = pNew; + } + }else{ + ZipfileEntry **pp; + for(pp=&pTab->pFirstEntry; *pp!=pBefore; pp=&((*pp)->pNext)); + pNew->pNext = pBefore; + *pp = pNew; + } +} + +static int zipfileLoadDirectory(ZipfileTab *pTab, const u8 *aBlob, int nBlob){ + ZipfileEOCD eocd; + int rc; + int i; + i64 iOff; + + rc = zipfileReadEOCD(pTab, aBlob, nBlob, pTab->pWriteFd, &eocd); + iOff = eocd.iOffset; + for(i=0; rc==SQLITE_OK && ipWriteFd, iOff, &pNew); + + if( rc==SQLITE_OK ){ + zipfileAddEntry(pTab, 0, pNew); + iOff += ZIPFILE_CDS_FIXED_SZ; + iOff += (int)pNew->cds.nExtra + pNew->cds.nFile + pNew->cds.nComment; + } + } + return rc; } /* @@ -963,29 +1225,37 @@ static int zipfileFilter( ){ ZipfileTab *pTab = (ZipfileTab*)cur->pVtab; ZipfileCsr *pCsr = (ZipfileCsr*)cur; - const char *zFile; /* Zip file to scan */ + const char *zFile = 0; /* Zip file to scan */ int rc = SQLITE_OK; /* Return Code */ + int bInMemory = 0; /* True for an in-memory zipfile */ zipfileResetCursor(pCsr); if( pTab->zFile ){ zFile = pTab->zFile; }else if( idxNum==0 ){ - /* Error. This is an eponymous virtual table and the user has not - ** supplied a file name. */ - zipfileSetErrmsg(pCsr, "table function zipfile() requires an argument"); + zipfileSetErrmsg(pCsr, "zipfile() function requires an argument"); return SQLITE_ERROR; + }else if( sqlite3_value_type(argv[0])==SQLITE_BLOB ){ + const u8 *aBlob = (const u8*)sqlite3_value_blob(argv[0]); + int nBlob = sqlite3_value_bytes(argv[0]); + assert( pTab->pFirstEntry==0 ); + rc = zipfileLoadDirectory(pTab, aBlob, nBlob); + pCsr->pFreeEntry = pTab->pFirstEntry; + pTab->pFirstEntry = pTab->pLastEntry = 0; + if( rc!=SQLITE_OK ) return rc; + bInMemory = 1; }else{ zFile = (const char*)sqlite3_value_text(argv[0]); } - if( pTab->pWriteFd==0 ){ + if( 0==pTab->pWriteFd && 0==bInMemory ){ pCsr->pFile = fopen(zFile, "rb"); if( pCsr->pFile==0 ){ zipfileSetErrmsg(pCsr, "cannot open file: %s", zFile); rc = SQLITE_ERROR; }else{ - rc = zipfileReadEOCD(pTab, pCsr->pFile, &pCsr->eocd); + rc = zipfileReadEOCD(pTab, 0, 0, pCsr->pFile, &pCsr->eocd); if( rc==SQLITE_OK ){ if( pCsr->eocd.nEntry==0 ){ pCsr->bEof = 1; @@ -996,12 +1266,9 @@ static int zipfileFilter( } } }else{ - ZipfileEntry e; - memset(&e, 0, sizeof(e)); - e.pNext = pTab->pFirstEntry; - pCsr->pCurrent = &e; + pCsr->bNoop = 1; + pCsr->pCurrent = pCsr->pFreeEntry ? pCsr->pFreeEntry : pTab->pFirstEntry; rc = zipfileNext(cur); - assert( pCsr->pCurrent!=&e ); } return rc; @@ -1037,182 +1304,67 @@ static int zipfileBestIndex( return SQLITE_OK; } -/* -** Add object pNew to the end of the linked list that begins at -** ZipfileTab.pFirstEntry and ends with pLastEntry. -*/ -static void zipfileAddEntry( - ZipfileTab *pTab, - ZipfileEntry *pBefore, - ZipfileEntry *pNew -){ - assert( (pTab->pFirstEntry==0)==(pTab->pLastEntry==0) ); - assert( pNew->pNext==0 ); - if( pBefore==0 ){ - if( pTab->pFirstEntry==0 ){ - pTab->pFirstEntry = pTab->pLastEntry = pNew; - }else{ - assert( pTab->pLastEntry->pNext==0 ); - pTab->pLastEntry->pNext = pNew; - pTab->pLastEntry = pNew; - } - }else{ - ZipfileEntry **pp; - for(pp=&pTab->pFirstEntry; *pp!=pBefore; pp=&((*pp)->pNext)); - pNew->pNext = pBefore; - *pp = pNew; - } -} - -static int zipfileLoadDirectory(ZipfileTab *pTab){ - ZipfileEOCD eocd; - int rc; - - rc = zipfileReadEOCD(pTab, pTab->pWriteFd, &eocd); - if( rc==SQLITE_OK && eocd.nEntry>0 ){ - int i; - int iOff = 0; - u8 *aBuf = sqlite3_malloc(eocd.nSize); - if( aBuf==0 ){ - rc = SQLITE_NOMEM; - }else{ - rc = zipfileReadData( - pTab->pWriteFd, aBuf, eocd.nSize, eocd.iOffset, &pTab->base.zErrMsg - ); - } - - for(i=0; rc==SQLITE_OK && izPath = (char*)&pNew[1]; - memcpy(pNew->zPath, &aRec[ZIPFILE_CDS_FIXED_SZ], nFile); - pNew->zPath[nFile] = '\0'; - pNew->aCdsEntry = (u8*)&pNew->zPath[nFile+1]; - pNew->nCdsEntry = ZIPFILE_CDS_FIXED_SZ+nFile+nExtra+nComment; - memcpy(pNew->aCdsEntry, aRec, pNew->nCdsEntry); - zipfileAddEntry(pTab, 0, pNew); - } - - iOff += ZIPFILE_CDS_FIXED_SZ+nFile+nExtra+nComment; - } - - sqlite3_free(aBuf); - } - - return rc; -} - -static ZipfileEntry *zipfileNewEntry( - ZipfileCDS *pCds, /* Values for fixed size part of CDS */ - const char *zPath, /* Path for new entry */ - int nPath, /* strlen(zPath) */ - u32 mTime /* Modification time (or 0) */ -){ - u8 *aWrite; +static ZipfileEntry *zipfileNewEntry(const char *zPath){ ZipfileEntry *pNew; - pCds->nFile = (u16)nPath; - pCds->nExtra = mTime ? 9 : 0; - pNew = (ZipfileEntry*)sqlite3_malloc( - sizeof(ZipfileEntry) + - nPath+1 + - ZIPFILE_CDS_FIXED_SZ + nPath + pCds->nExtra - ); - + pNew = sqlite3_malloc(sizeof(ZipfileEntry)); if( pNew ){ memset(pNew, 0, sizeof(ZipfileEntry)); - pNew->zPath = (char*)&pNew[1]; - pNew->aCdsEntry = (u8*)&pNew->zPath[nPath+1]; - pNew->nCdsEntry = ZIPFILE_CDS_FIXED_SZ + nPath + pCds->nExtra; - memcpy(pNew->zPath, zPath, nPath+1); - - aWrite = pNew->aCdsEntry; - zipfileWrite32(aWrite, ZIPFILE_SIGNATURE_CDS); - zipfileWrite16(aWrite, pCds->iVersionMadeBy); - zipfileWrite16(aWrite, pCds->iVersionExtract); - zipfileWrite16(aWrite, pCds->flags); - zipfileWrite16(aWrite, pCds->iCompression); - zipfileWrite16(aWrite, pCds->mTime); - zipfileWrite16(aWrite, pCds->mDate); - zipfileWrite32(aWrite, pCds->crc32); - zipfileWrite32(aWrite, pCds->szCompressed); - zipfileWrite32(aWrite, pCds->szUncompressed); - zipfileWrite16(aWrite, pCds->nFile); - zipfileWrite16(aWrite, pCds->nExtra); - zipfileWrite16(aWrite, pCds->nComment); assert( pCds->nComment==0 ); - zipfileWrite16(aWrite, pCds->iDiskStart); - zipfileWrite16(aWrite, pCds->iInternalAttr); - zipfileWrite32(aWrite, pCds->iExternalAttr); - zipfileWrite32(aWrite, pCds->iOffset); - assert( aWrite==&pNew->aCdsEntry[ZIPFILE_CDS_FIXED_SZ] ); - memcpy(aWrite, zPath, nPath); - if( pCds->nExtra ){ - aWrite += nPath; - zipfileWrite16(aWrite, ZIPFILE_EXTRA_TIMESTAMP); - zipfileWrite16(aWrite, 5); - *aWrite++ = 0x01; - zipfileWrite32(aWrite, mTime); + pNew->cds.zFile = sqlite3_mprintf("%s", zPath); + if( pNew->cds.zFile==0 ){ + sqlite3_free(pNew); + pNew = 0; } } - return pNew; } +static int zipfileSerializeLFH(ZipfileEntry *pEntry, u8 *aBuf){ + ZipfileCDS *pCds = &pEntry->cds; + u8 *a = aBuf; + + pCds->nExtra = 9; + + /* Write the LFH itself */ + zipfileWrite32(a, ZIPFILE_SIGNATURE_LFH); + zipfileWrite16(a, pCds->iVersionExtract); + zipfileWrite16(a, pCds->flags); + zipfileWrite16(a, pCds->iCompression); + zipfileWrite16(a, pCds->mTime); + zipfileWrite16(a, pCds->mDate); + zipfileWrite32(a, pCds->crc32); + zipfileWrite32(a, pCds->szCompressed); + zipfileWrite32(a, pCds->szUncompressed); + zipfileWrite16(a, (u16)pCds->nFile); + zipfileWrite16(a, pCds->nExtra); + assert( a==&aBuf[ZIPFILE_LFH_FIXED_SZ] ); + + /* Add the file name */ + memcpy(a, pCds->zFile, (int)pCds->nFile); + a += (int)pCds->nFile; + + /* The "extra" data */ + zipfileWrite16(a, ZIPFILE_EXTRA_TIMESTAMP); + zipfileWrite16(a, 5); + *a++ = 0x01; + zipfileWrite32(a, pEntry->mUnixTime); + + return a-aBuf; +} + static int zipfileAppendEntry( ZipfileTab *pTab, - ZipfileCDS *pCds, - const char *zPath, /* Path for new entry */ - int nPath, /* strlen(zPath) */ + ZipfileEntry *pEntry, const u8 *pData, - int nData, - u32 mTime + int nData ){ u8 *aBuf = pTab->aBuffer; + int nBuf; int rc; - zipfileWrite32(aBuf, ZIPFILE_SIGNATURE_LFH); - zipfileWrite16(aBuf, pCds->iVersionExtract); - zipfileWrite16(aBuf, pCds->flags); - zipfileWrite16(aBuf, pCds->iCompression); - zipfileWrite16(aBuf, pCds->mTime); - zipfileWrite16(aBuf, pCds->mDate); - zipfileWrite32(aBuf, pCds->crc32); - zipfileWrite32(aBuf, pCds->szCompressed); - zipfileWrite32(aBuf, pCds->szUncompressed); - zipfileWrite16(aBuf, (u16)nPath); - zipfileWrite16(aBuf, pCds->nExtra); - assert( aBuf==&pTab->aBuffer[ZIPFILE_LFH_FIXED_SZ] ); - rc = zipfileAppendData(pTab, pTab->aBuffer, (int)(aBuf - pTab->aBuffer)); - if( rc==SQLITE_OK ){ - rc = zipfileAppendData(pTab, (const u8*)zPath, nPath); - } - - if( rc==SQLITE_OK && pCds->nExtra ){ - aBuf = pTab->aBuffer; - zipfileWrite16(aBuf, ZIPFILE_EXTRA_TIMESTAMP); - zipfileWrite16(aBuf, 5); - *aBuf++ = 0x01; - zipfileWrite32(aBuf, mTime); - rc = zipfileAppendData(pTab, pTab->aBuffer, 9); - } - + nBuf = zipfileSerializeLFH(pEntry, aBuf); + rc = zipfileAppendData(pTab, aBuf, nBuf); if( rc==SQLITE_OK ){ + pEntry->iDataOff = pTab->szCurrent; rc = zipfileAppendData(pTab, pData, nData); } @@ -1220,15 +1372,15 @@ static int zipfileAppendEntry( } static int zipfileGetMode( - ZipfileTab *pTab, sqlite3_value *pVal, - u32 defaultMode, /* Value to use if pVal IS NULL */ - u32 *pMode + int bIsDir, /* If true, default to directory */ + u32 *pMode, /* OUT: Mode value */ + char **pzErr /* OUT: Error message */ ){ const char *z = (const char*)sqlite3_value_text(pVal); u32 mode = 0; if( z==0 ){ - mode = defaultMode; + mode = (bIsDir ? (S_IFDIR + 0755) : (S_IFREG + 0644)); }else if( z[0]>='0' && z[0]<='9' ){ mode = (unsigned int)sqlite3_value_int(pVal); }else{ @@ -1248,11 +1400,16 @@ static int zipfileGetMode( else if( z[i]!='-' ) goto parse_error; } } + if( (bIsDir == ((mode & S_IFDIR)==0)) ){ + /* The "mode" attribute is a directory, but data has been specified. + ** Or vice-versa - no data but "mode" is a file or symlink. */ + return SQLITE_CONSTRAINT; + } *pMode = mode; return SQLITE_OK; parse_error: - pTab->base.zErrMsg = sqlite3_mprintf("zipfile: parse error in mode: %s", z); + *pzErr = sqlite3_mprintf("zipfile: parse error in mode: %s", z); return SQLITE_ERROR; } @@ -1268,6 +1425,68 @@ static int zipfileComparePath(const char *zA, const char *zB, int nB){ return 1; } +static int zipfileBegin(sqlite3_vtab *pVtab){ + ZipfileTab *pTab = (ZipfileTab*)pVtab; + int rc = SQLITE_OK; + + assert( pTab->pWriteFd==0 ); + + /* Open a write fd on the file. Also load the entire central directory + ** structure into memory. During the transaction any new file data is + ** appended to the archive file, but the central directory is accumulated + ** in main-memory until the transaction is committed. */ + pTab->pWriteFd = fopen(pTab->zFile, "ab+"); + if( pTab->pWriteFd==0 ){ + pTab->base.zErrMsg = sqlite3_mprintf( + "zipfile: failed to open file %s for writing", pTab->zFile + ); + rc = SQLITE_ERROR; + }else{ + fseek(pTab->pWriteFd, 0, SEEK_END); + pTab->szCurrent = pTab->szOrig = (i64)ftell(pTab->pWriteFd); + rc = zipfileLoadDirectory(pTab, 0, 0); + } + + if( rc!=SQLITE_OK ){ + zipfileCleanupTransaction(pTab); + } + + return rc; +} + +/* +** Return the current time as a 32-bit timestamp in UNIX epoch format (like +** time(2)). +*/ +static u32 zipfileTime(void){ + sqlite3_vfs *pVfs = sqlite3_vfs_find(0); + u32 ret; + if( pVfs->iVersion>=2 && pVfs->xCurrentTimeInt64 ){ + i64 ms; + pVfs->xCurrentTimeInt64(pVfs, &ms); + ret = (u32)((ms/1000) - ((i64)24405875 * 8640)); + }else{ + double day; + pVfs->xCurrentTime(pVfs, &day); + ret = (u32)((day - 2440587.5) * 86400); + } + return ret; +} + +/* +** Return a 32-bit timestamp in UNIX epoch format. +** +** If the value passed as the only argument is either NULL or an SQL NULL, +** return the current time. Otherwise, return the value stored in (*pVal) +** cast to a 32-bit unsigned integer. +*/ +static u32 zipfileGetTime(sqlite3_value *pVal){ + if( pVal==0 || sqlite3_value_type(pVal)==SQLITE_NULL ){ + return zipfileTime(); + } + return (u32)sqlite3_value_int64(pVal); +} + /* ** xUpdate method. */ @@ -1282,7 +1501,7 @@ static int zipfileUpdate( ZipfileEntry *pNew = 0; /* New in-memory CDS entry */ u32 mode = 0; /* Mode for new entry */ - i64 mTime = 0; /* Modification time for new entry */ + u32 mTime = 0; /* Modification time for new entry */ i64 sz = 0; /* Uncompressed size */ const char *zPath = 0; /* Path for new entry */ int nPath = 0; /* strlen(zPath) */ @@ -1291,217 +1510,213 @@ static int zipfileUpdate( int iMethod = 0; /* Compression method for new entry */ u8 *pFree = 0; /* Free this */ char *zFree = 0; /* Also free this */ - ZipfileCDS cds; /* New Central Directory Structure entry */ ZipfileEntry *pOld = 0; int bIsDir = 0; u32 iCrc32 = 0; - assert( pTab->zFile ); - assert( pTab->pWriteFd ); + if( pTab->pWriteFd==0 ){ + rc = zipfileBegin(pVtab); + if( rc!=SQLITE_OK ) return rc; + } + /* If this is a DELETE or UPDATE, find the archive entry to delete. */ if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){ const char *zDelete = (const char*)sqlite3_value_text(apVal[0]); int nDelete = (int)strlen(zDelete); for(pOld=pTab->pFirstEntry; 1; pOld=pOld->pNext){ - if( pOld->bDeleted ) continue; - if( zipfileComparePath(pOld->zPath, zDelete, nDelete)==0 ){ - pOld->bDeleted = 1; + if( zipfileComparePath(pOld->cds.zFile, zDelete, nDelete)==0 ){ break; } assert( pOld->pNext ); } - if( nVal==1 ) return SQLITE_OK; } - /* Check that "sz" and "rawdata" are both NULL: */ - if( sqlite3_value_type(apVal[5])!=SQLITE_NULL - || sqlite3_value_type(apVal[6])!=SQLITE_NULL - ){ - rc = SQLITE_CONSTRAINT; - } - - if( rc==SQLITE_OK ){ - if( sqlite3_value_type(apVal[7])==SQLITE_NULL ){ - /* data=NULL. A directory */ - bIsDir = 1; - }else{ - /* Value specified for "data", and possibly "method". This must be - ** a regular file or a symlink. */ - const u8 *aIn = sqlite3_value_blob(apVal[7]); - int nIn = sqlite3_value_bytes(apVal[7]); - int bAuto = sqlite3_value_type(apVal[8])==SQLITE_NULL; - - iMethod = sqlite3_value_int(apVal[8]); - sz = nIn; - pData = aIn; - nData = nIn; - if( iMethod!=0 && iMethod!=8 ){ - rc = SQLITE_CONSTRAINT; - }else{ - if( bAuto || iMethod ){ - int nCmp; - rc = zipfileDeflate(pTab, aIn, nIn, &pFree, &nCmp); - if( rc==SQLITE_OK ){ - if( iMethod || nCmp1 ){ + /* Check that "sz" and "rawdata" are both NULL: */ + if( sqlite3_value_type(apVal[5])!=SQLITE_NULL + || sqlite3_value_type(apVal[6])!=SQLITE_NULL + ){ rc = SQLITE_CONSTRAINT; } - } - if( rc==SQLITE_OK ){ - zPath = (const char*)sqlite3_value_text(apVal[2]); - nPath = (int)strlen(zPath); - if( sqlite3_value_type(apVal[4])==SQLITE_NULL ){ - mTime = (sqlite3_int64)time(0); - }else{ - mTime = sqlite3_value_int64(apVal[4]); + if( rc==SQLITE_OK ){ + if( sqlite3_value_type(apVal[7])==SQLITE_NULL ){ + /* data=NULL. A directory */ + bIsDir = 1; + }else{ + /* Value specified for "data", and possibly "method". This must be + ** a regular file or a symlink. */ + const u8 *aIn = sqlite3_value_blob(apVal[7]); + int nIn = sqlite3_value_bytes(apVal[7]); + int bAuto = sqlite3_value_type(apVal[8])==SQLITE_NULL; + + iMethod = sqlite3_value_int(apVal[8]); + sz = nIn; + pData = aIn; + nData = nIn; + if( iMethod!=0 && iMethod!=8 ){ + rc = SQLITE_CONSTRAINT; + }else{ + if( bAuto || iMethod ){ + int nCmp; + rc = zipfileDeflate(aIn, nIn, &pFree, &nCmp, &pTab->base.zErrMsg); + if( rc==SQLITE_OK ){ + if( iMethod || nCmpbase.zErrMsg); } - } - /* Check that we're not inserting a duplicate entry */ - if( rc==SQLITE_OK ){ - ZipfileEntry *p; - for(p=pTab->pFirstEntry; p; p=p->pNext){ - if( p->bDeleted ) continue; - if( zipfileComparePath(p->zPath, zPath, nPath)==0 ){ - rc = SQLITE_CONSTRAINT; - break; + if( rc==SQLITE_OK ){ + zPath = (const char*)sqlite3_value_text(apVal[2]); + nPath = (int)strlen(zPath); + mTime = zipfileGetTime(apVal[4]); + } + + if( rc==SQLITE_OK && bIsDir ){ + /* For a directory, check that the last character in the path is a + ** '/'. This appears to be required for compatibility with info-zip + ** (the unzip command on unix). It does not create directories + ** otherwise. */ + if( zPath[nPath-1]!='/' ){ + zFree = sqlite3_mprintf("%s/", zPath); + if( zFree==0 ){ rc = SQLITE_NOMEM; } + zPath = (const char*)zFree; + nPath++; + } + } + + /* Check that we're not inserting a duplicate entry */ + if( pOld==0 && rc==SQLITE_OK ){ + ZipfileEntry *p; + for(p=pTab->pFirstEntry; p; p=p->pNext){ + if( zipfileComparePath(p->cds.zFile, zPath, nPath)==0 ){ + rc = SQLITE_CONSTRAINT; + break; + } + } + } + + if( rc==SQLITE_OK ){ + /* Create the new CDS record. */ + pNew = zipfileNewEntry(zPath); + if( pNew==0 ){ + rc = SQLITE_NOMEM; + }else{ + pNew->cds.iVersionMadeBy = ZIPFILE_NEWENTRY_MADEBY; + pNew->cds.iVersionExtract = ZIPFILE_NEWENTRY_REQUIRED; + pNew->cds.flags = ZIPFILE_NEWENTRY_FLAGS; + pNew->cds.iCompression = (u16)iMethod; + zipfileMtimeToDos(&pNew->cds, mTime); + pNew->cds.crc32 = iCrc32; + pNew->cds.szCompressed = nData; + pNew->cds.szUncompressed = (u32)sz; + pNew->cds.iExternalAttr = (mode<<16); + pNew->cds.iOffset = (u32)pTab->szCurrent; + pNew->cds.nFile = (u16)nPath; + pNew->mUnixTime = (u32)mTime; + rc = zipfileAppendEntry(pTab, pNew, pData, nData); + zipfileAddEntry(pTab, pOld, pNew); } } } - if( rc==SQLITE_OK ){ - /* Create the new CDS record. */ - memset(&cds, 0, sizeof(cds)); - cds.iVersionMadeBy = ZIPFILE_NEWENTRY_MADEBY; - cds.iVersionExtract = ZIPFILE_NEWENTRY_REQUIRED; - cds.flags = ZIPFILE_NEWENTRY_FLAGS; - cds.iCompression = (u16)iMethod; - zipfileMtimeToDos(&cds, (u32)mTime); - cds.crc32 = iCrc32; - cds.szCompressed = nData; - cds.szUncompressed = (u32)sz; - cds.iExternalAttr = (mode<<16); - cds.iOffset = (u32)pTab->szCurrent; - pNew = zipfileNewEntry(&cds, zPath, nPath, (u32)mTime); - if( pNew==0 ){ - rc = SQLITE_NOMEM; - }else{ - zipfileAddEntry(pTab, pOld, pNew); + if( rc==SQLITE_OK && pOld ){ + ZipfileEntry **pp; + ZipfileCsr *pCsr; + for(pCsr=pTab->pCsrList; pCsr; pCsr=pCsr->pCsrNext){ + if( pCsr->pCurrent==pOld ){ + pCsr->pCurrent = pOld->pNext; + pCsr->bNoop = 1; + } } + for(pp=&pTab->pFirstEntry; (*pp)!=pOld; pp=&((*pp)->pNext)); + *pp = (*pp)->pNext; + zipfileEntryFree(pOld); } - /* Append the new header+file to the archive */ - if( rc==SQLITE_OK ){ - rc = zipfileAppendEntry(pTab, &cds, zPath, nPath, pData, nData, (u32)mTime); - } - - if( rc!=SQLITE_OK && pOld ){ - pOld->bDeleted = 0; - } sqlite3_free(pFree); sqlite3_free(zFree); return rc; } +static int zipfileSerializeEOCD(ZipfileEOCD *p, u8 *aBuf){ + u8 *a = aBuf; + zipfileWrite32(a, ZIPFILE_SIGNATURE_EOCD); + zipfileWrite16(a, p->iDisk); + zipfileWrite16(a, p->iFirstDisk); + zipfileWrite16(a, p->nEntry); + zipfileWrite16(a, p->nEntryTotal); + zipfileWrite32(a, p->nSize); + zipfileWrite32(a, p->iOffset); + zipfileWrite16(a, 0); /* Size of trailing comment in bytes*/ + + return a-aBuf; +} + static int zipfileAppendEOCD(ZipfileTab *pTab, ZipfileEOCD *p){ - u8 *aBuf = pTab->aBuffer; - - zipfileWrite32(aBuf, ZIPFILE_SIGNATURE_EOCD); - zipfileWrite16(aBuf, p->iDisk); - zipfileWrite16(aBuf, p->iFirstDisk); - zipfileWrite16(aBuf, p->nEntry); - zipfileWrite16(aBuf, p->nEntryTotal); - zipfileWrite32(aBuf, p->nSize); - zipfileWrite32(aBuf, p->iOffset); - zipfileWrite16(aBuf, 0); /* Size of trailing comment in bytes*/ - - assert( (aBuf-pTab->aBuffer)==22 ); - return zipfileAppendData(pTab, pTab->aBuffer, (int)(aBuf - pTab->aBuffer)); + int nBuf = zipfileSerializeEOCD(p, pTab->aBuffer); + assert( nBuf==ZIPFILE_EOCD_FIXED_SZ ); + return zipfileAppendData(pTab, pTab->aBuffer, nBuf); } -static void zipfileCleanupTransaction(ZipfileTab *pTab){ - ZipfileEntry *pEntry; - ZipfileEntry *pNext; +/* +** Serialize the CDS structure into buffer aBuf[]. Return the number +** of bytes written. +*/ +static int zipfileSerializeCDS(ZipfileEntry *pEntry, u8 *aBuf){ + u8 *a = aBuf; + ZipfileCDS *pCDS = &pEntry->cds; - for(pEntry=pTab->pFirstEntry; pEntry; pEntry=pNext){ - pNext = pEntry->pNext; - sqlite3_free(pEntry); - } - pTab->pFirstEntry = 0; - pTab->pLastEntry = 0; - fclose(pTab->pWriteFd); - pTab->pWriteFd = 0; - pTab->szCurrent = 0; - pTab->szOrig = 0; -} - -static int zipfileBegin(sqlite3_vtab *pVtab){ - ZipfileTab *pTab = (ZipfileTab*)pVtab; - int rc = SQLITE_OK; - - assert( pTab->pWriteFd==0 ); - - /* This table is only writable if a default archive path was specified - ** as part of the CREATE VIRTUAL TABLE statement. */ - if( pTab->zFile==0 ){ - pTab->base.zErrMsg = sqlite3_mprintf( - "zipfile: writing requires a default archive" - ); - return SQLITE_ERROR; + if( pEntry->aExtra==0 ){ + pCDS->nExtra = 9; } - /* Open a write fd on the file. Also load the entire central directory - ** structure into memory. During the transaction any new file data is - ** appended to the archive file, but the central directory is accumulated - ** in main-memory until the transaction is committed. */ - pTab->pWriteFd = fopen(pTab->zFile, "ab+"); - if( pTab->pWriteFd==0 ){ - pTab->base.zErrMsg = sqlite3_mprintf( - "zipfile: failed to open file %s for writing", pTab->zFile - ); - rc = SQLITE_ERROR; + zipfileWrite32(a, ZIPFILE_SIGNATURE_CDS); + zipfileWrite16(a, pCDS->iVersionMadeBy); + zipfileWrite16(a, pCDS->iVersionExtract); + zipfileWrite16(a, pCDS->flags); + zipfileWrite16(a, pCDS->iCompression); + zipfileWrite16(a, pCDS->mTime); + zipfileWrite16(a, pCDS->mDate); + zipfileWrite32(a, pCDS->crc32); + zipfileWrite32(a, pCDS->szCompressed); + zipfileWrite32(a, pCDS->szUncompressed); + assert( a==&aBuf[ZIPFILE_CDS_NFILE_OFF] ); + zipfileWrite16(a, pCDS->nFile); + zipfileWrite16(a, pCDS->nExtra); + zipfileWrite16(a, pCDS->nComment); + zipfileWrite16(a, pCDS->iDiskStart); + zipfileWrite16(a, pCDS->iInternalAttr); + zipfileWrite32(a, pCDS->iExternalAttr); + zipfileWrite32(a, pCDS->iOffset); + + memcpy(a, pCDS->zFile, pCDS->nFile); + a += pCDS->nFile; + + if( pEntry->aExtra ){ + int n = (int)pCDS->nExtra + (int)pCDS->nComment; + memcpy(a, pEntry->aExtra, n); + a += n; }else{ - fseek(pTab->pWriteFd, 0, SEEK_END); - pTab->szCurrent = pTab->szOrig = (i64)ftell(pTab->pWriteFd); - rc = zipfileLoadDirectory(pTab); + assert( pCDS->nExtra==9 ); + zipfileWrite16(a, ZIPFILE_EXTRA_TIMESTAMP); + zipfileWrite16(a, 5); + *a++ = 0x01; + zipfileWrite32(a, pEntry->mUnixTime); } - if( rc!=SQLITE_OK ){ - zipfileCleanupTransaction(pTab); - } - - return rc; + return a-aBuf; } static int zipfileCommit(sqlite3_vtab *pVtab){ @@ -1513,10 +1728,10 @@ static int zipfileCommit(sqlite3_vtab *pVtab){ ZipfileEOCD eocd; int nEntry = 0; - /* Write out all undeleted entries */ + /* Write out all entries */ for(p=pTab->pFirstEntry; rc==SQLITE_OK && p; p=p->pNext){ - if( p->bDeleted ) continue; - rc = zipfileAppendData(pTab, p->aCdsEntry, p->nCdsEntry); + int n = zipfileSerializeCDS(p, pTab->aBuffer); + rc = zipfileAppendData(pTab, pTab->aBuffer, n); nEntry++; } @@ -1557,7 +1772,7 @@ static void zipfileFunctionCds( pCsr = zipfileFindCursor(pTab, sqlite3_value_int64(argv[0])); if( pCsr ){ - ZipfileCDS *p = &pCsr->cds; + ZipfileCDS *p = &pCsr->pCurrent->cds; char *zRes = sqlite3_mprintf("{" "\"version-made-by\" : %u, " "\"version-to-extract\" : %u, " @@ -1594,7 +1809,6 @@ static void zipfileFunctionCds( } } - /* ** xFindFunction method. */ @@ -1605,17 +1819,256 @@ static int zipfileFindFunction( void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), /* OUT: Result */ void **ppArg /* OUT: User data for *pxFunc */ ){ - if( nArg>0 ){ - if( sqlite3_stricmp("zipfile_cds", zName)==0 ){ - *pxFunc = zipfileFunctionCds; - *ppArg = (void*)pVtab; - return 1; + if( sqlite3_stricmp("zipfile_cds", zName)==0 ){ + *pxFunc = zipfileFunctionCds; + *ppArg = (void*)pVtab; + return 1; + } + return 0; +} + +typedef struct ZipfileBuffer ZipfileBuffer; +struct ZipfileBuffer { + u8 *a; /* Pointer to buffer */ + int n; /* Size of buffer in bytes */ + int nAlloc; /* Byte allocated at a[] */ +}; + +typedef struct ZipfileCtx ZipfileCtx; +struct ZipfileCtx { + int nEntry; + ZipfileBuffer body; + ZipfileBuffer cds; +}; + +static int zipfileBufferGrow(ZipfileBuffer *pBuf, int nByte){ + if( pBuf->n+nByte>pBuf->nAlloc ){ + u8 *aNew; + int nNew = pBuf->n ? pBuf->n*2 : 512; + int nReq = pBuf->n + nByte; + + while( nNewa, nNew); + if( aNew==0 ) return SQLITE_NOMEM; + pBuf->a = aNew; + pBuf->nAlloc = nNew; + } + return SQLITE_OK; +} + +/* +** xStep() callback for the zipfile() aggregate. This can be called in +** any of the following ways: +** +** SELECT zipfile(name,data) ... +** SELECT zipfile(name,mode,mtime,data) ... +** SELECT zipfile(name,mode,mtime,data,method) ... +*/ +void zipfileStep(sqlite3_context *pCtx, int nVal, sqlite3_value **apVal){ + ZipfileCtx *p; /* Aggregate function context */ + ZipfileEntry e; /* New entry to add to zip archive */ + + sqlite3_value *pName = 0; + sqlite3_value *pMode = 0; + sqlite3_value *pMtime = 0; + sqlite3_value *pData = 0; + sqlite3_value *pMethod = 0; + + int bIsDir = 0; + u32 mode; + int rc = SQLITE_OK; + char *zErr = 0; + + int iMethod = -1; /* Compression method to use (0 or 8) */ + + const u8 *aData = 0; /* Possibly compressed data for new entry */ + int nData = 0; /* Size of aData[] in bytes */ + int szUncompressed = 0; /* Size of data before compression */ + u8 *aFree = 0; /* Free this before returning */ + u32 iCrc32 = 0; /* crc32 of uncompressed data */ + + char *zName = 0; /* Path (name) of new entry */ + int nName = 0; /* Size of zName in bytes */ + char *zFree = 0; /* Free this before returning */ + int nByte; + + memset(&e, 0, sizeof(e)); + p = (ZipfileCtx*)sqlite3_aggregate_context(pCtx, sizeof(ZipfileCtx)); + if( p==0 ) return; + + /* Martial the arguments into stack variables */ + if( nVal!=2 && nVal!=4 && nVal!=5 ){ + zErr = sqlite3_mprintf("wrong number of arguments to function zipfile()"); + rc = SQLITE_ERROR; + goto zipfile_step_out; + } + pName = apVal[0]; + if( nVal==2 ){ + pData = apVal[1]; + }else{ + pMode = apVal[1]; + pMtime = apVal[2]; + pData = apVal[3]; + if( nVal==5 ){ + pMethod = apVal[4]; } } - return 0; + /* Check that the 'name' parameter looks ok. */ + zName = (char*)sqlite3_value_text(pName); + nName = sqlite3_value_bytes(pName); + if( zName==0 ){ + zErr = sqlite3_mprintf("first argument to zipfile() must be non-NULL"); + rc = SQLITE_ERROR; + goto zipfile_step_out; + } + + /* Inspect the 'method' parameter. This must be either 0 (store), 8 (use + ** deflate compression) or NULL (choose automatically). */ + if( pMethod && SQLITE_NULL!=sqlite3_value_type(pMethod) ){ + iMethod = (int)sqlite3_value_int64(pMethod); + if( iMethod!=0 && iMethod!=8 ){ + zErr = sqlite3_mprintf("illegal method value: %d", iMethod); + rc = SQLITE_ERROR; + goto zipfile_step_out; + } + } + + /* Now inspect the data. If this is NULL, then the new entry must be a + ** directory. Otherwise, figure out whether or not the data should + ** be deflated or simply stored in the zip archive. */ + if( sqlite3_value_type(pData)==SQLITE_NULL ){ + bIsDir = 1; + iMethod = 0; + }else{ + aData = sqlite3_value_blob(pData); + szUncompressed = nData = sqlite3_value_bytes(pData); + iCrc32 = crc32(0, aData, nData); + if( iMethod<0 || iMethod==8 ){ + int nOut = 0; + rc = zipfileDeflate(aData, nData, &aFree, &nOut, &zErr); + if( rc!=SQLITE_OK ){ + goto zipfile_step_out; + } + if( iMethod==8 || nOut1 && zName[nName-2]=='/' ) nName--; + } + } + + /* Assemble the ZipfileEntry object for the new zip archive entry */ + e.cds.iVersionMadeBy = ZIPFILE_NEWENTRY_MADEBY; + e.cds.iVersionExtract = ZIPFILE_NEWENTRY_REQUIRED; + e.cds.flags = ZIPFILE_NEWENTRY_FLAGS; + e.cds.iCompression = (u16)iMethod; + zipfileMtimeToDos(&e.cds, (u32)e.mUnixTime); + e.cds.crc32 = iCrc32; + e.cds.szCompressed = nData; + e.cds.szUncompressed = szUncompressed; + e.cds.iExternalAttr = (mode<<16); + e.cds.iOffset = p->body.n; + e.cds.nFile = (u16)nName; + e.cds.zFile = zName; + + /* Append the LFH to the body of the new archive */ + nByte = ZIPFILE_LFH_FIXED_SZ + e.cds.nFile + 9; + if( (rc = zipfileBufferGrow(&p->body, nByte)) ) goto zipfile_step_out; + p->body.n += zipfileSerializeLFH(&e, &p->body.a[p->body.n]); + + /* Append the data to the body of the new archive */ + if( (rc = zipfileBufferGrow(&p->body, nData)) ) goto zipfile_step_out; + memcpy(&p->body.a[p->body.n], aData, nData); + p->body.n += nData; + + /* Append the CDS record to the directory of the new archive */ + nByte = ZIPFILE_CDS_FIXED_SZ + e.cds.nFile + 9; + if( (rc = zipfileBufferGrow(&p->cds, nByte)) ) goto zipfile_step_out; + p->cds.n += zipfileSerializeCDS(&e, &p->cds.a[p->cds.n]); + + /* Increment the count of entries in the archive */ + p->nEntry++; + + zipfile_step_out: + sqlite3_free(aFree); + sqlite3_free(zFree); + if( rc ){ + if( zErr ){ + sqlite3_result_error(pCtx, zErr, -1); + }else{ + sqlite3_result_error_code(pCtx, rc); + } + } + sqlite3_free(zErr); } +/* +** xFinalize() callback for zipfile aggregate function. +*/ +void zipfileFinal(sqlite3_context *pCtx){ + ZipfileCtx *p; + ZipfileEOCD eocd; + int nZip; + u8 *aZip; + + p = (ZipfileCtx*)sqlite3_aggregate_context(pCtx, sizeof(ZipfileCtx)); + if( p==0 ) return; + if( p->nEntry>0 ){ + memset(&eocd, 0, sizeof(eocd)); + eocd.nEntry = (u16)p->nEntry; + eocd.nEntryTotal = (u16)p->nEntry; + eocd.nSize = p->cds.n; + eocd.iOffset = p->body.n; + + nZip = p->body.n + p->cds.n + ZIPFILE_EOCD_FIXED_SZ; + aZip = (u8*)sqlite3_malloc(nZip); + if( aZip==0 ){ + sqlite3_result_error_nomem(pCtx); + }else{ + memcpy(aZip, p->body.a, p->body.n); + memcpy(&aZip[p->body.n], p->cds.a, p->cds.n); + zipfileSerializeEOCD(&eocd, &aZip[p->body.n + p->cds.n]); + sqlite3_result_blob(pCtx, aZip, nZip, zipfileFree); + } + } + + sqlite3_free(p->body.a); + sqlite3_free(p->cds.a); +} + + /* ** Register the "zipfile" virtual table. */ @@ -1633,7 +2086,7 @@ static int zipfileRegister(sqlite3 *db){ zipfileNext, /* xNext - advance a cursor */ zipfileEof, /* xEof - check for end of scan */ zipfileColumn, /* xColumn - read data */ - zipfileRowid, /* xRowid - read data */ + 0, /* xRowid - read data */ zipfileUpdate, /* xUpdate */ zipfileBegin, /* xBegin */ 0, /* xSync */ @@ -1644,8 +2097,11 @@ static int zipfileRegister(sqlite3 *db){ }; int rc = sqlite3_create_module(db, "zipfile" , &zipfileModule, 0); + if( rc==SQLITE_OK ) rc = sqlite3_overload_function(db, "zipfile_cds", -1); if( rc==SQLITE_OK ){ - rc = sqlite3_overload_function(db, "zipfile_cds", -1); + rc = sqlite3_create_function(db, "zipfile", -1, SQLITE_UTF8, 0, 0, + zipfileStep, zipfileFinal + ); } return rc; } diff --git a/ext/misc/zorder.c b/ext/misc/zorder.c new file mode 100644 index 0000000000..c385d3c3c3 --- /dev/null +++ b/ext/misc/zorder.c @@ -0,0 +1,102 @@ +/* +** 2018-02-09 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** SQL functions for z-order (Morton code) transformations. +** +** zorder(X0,X0,..,xN) Generate an N+1 dimension Morton code +** +** unzorder(Z,N,I) Extract the I-th dimension from N-dimensional +** Morton code Z. +*/ +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 +#include +#include + +/* +** Functions: zorder(X0,X1,....) +** +** Convert integers X0, X1, ... into morton code. +** +** The output is a signed 64-bit integer. If any argument is too large, +** an error is thrown. +*/ +static void zorderFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + sqlite3_int64 z, x[63]; + int i, j; + z = 0; + for(i=0; i0 ){ + for(i=0; i<63; i++){ + j = i%argc; + z |= (x[j]&1)<>= 1; + } + } + sqlite3_result_int64(context, z); + for(i=0; i>j)&1)<isDirty = 0; rc = sqlite3_reset(p); + sqlite3_bind_null(p, 2); if( pNode->iNode==0 && rc==SQLITE_OK ){ pNode->iNode = sqlite3_last_insert_rowid(pRtree->db); nodeHashInsert(pRtree, pNode); diff --git a/ext/rtree/rtree1.test b/ext/rtree/rtree1.test index 0deee6635b..ac6e8d9d96 100644 --- a/ext/rtree/rtree1.test +++ b/ext/rtree/rtree1.test @@ -609,4 +609,5 @@ do_execsql_test 15.2 { COMMIT; } +expand_all_sql db finish_test diff --git a/ext/rtree/rtree4.test b/ext/rtree/rtree4.test index af3f8d3995..a73921d8d5 100644 --- a/ext/rtree/rtree4.test +++ b/ext/rtree/rtree4.test @@ -250,4 +250,5 @@ for {set nDim 1} {$nDim<=5} {incr nDim} { do_rtree_integrity_test rtree4-$nDim.3 rx } +expand_all_sql db finish_test diff --git a/ext/rtree/rtree5.test b/ext/rtree/rtree5.test index 749385e882..92bb6905c7 100644 --- a/ext/rtree/rtree5.test +++ b/ext/rtree/rtree5.test @@ -79,4 +79,5 @@ do_test rtree5-1.13 { } {2 2147483643 2147483647 -2147483648 -2147483643} do_rtree_integrity_test rtree5-1.14 t1 +expand_all_sql db finish_test diff --git a/ext/rtree/rtree6.test b/ext/rtree/rtree6.test index c9c87e8ad9..406604810b 100644 --- a/ext/rtree/rtree6.test +++ b/ext/rtree/rtree6.test @@ -158,5 +158,5 @@ do_execsql_test rtree6-3.5 { x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>1.1 } {} - +expand_all_sql db finish_test diff --git a/ext/rtree/rtreeG.test b/ext/rtree/rtreeG.test index 3bef89c8e7..12225d5832 100644 --- a/ext/rtree/rtreeG.test +++ b/ext/rtree/rtreeG.test @@ -59,6 +59,7 @@ do_test rtreeG-1.4log { set ::log } {} +expand_all_sql db db close sqlite3_shutdown test_sqlite3_log diff --git a/ext/session/session4.test b/ext/session/session4.test index 120a230b57..d07c57ffd6 100644 --- a/ext/session/session4.test +++ b/ext/session/session4.test @@ -64,4 +64,29 @@ do_test 1.3 { list [catch { sqlite3changeset_apply db $x xConflict } msg] $msg } {1 SQLITE_CORRUPT} +#------------------------------------------------------------------------- +reset_db +do_execsql_test 2.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY,b,c,d); + CREATE TABLE t2(e TEXT PRIMARY KEY NOT NULL,f,g); + CREATE TABLE t3(w REAL PRIMARY KEY NOT NULL,x,y); + CREATE TABLE t4(z PRIMARY KEY) WITHOUT ROWID; +} + +foreach {tn blob} { + 1 {54010174340012000000} + 2 {54fefe8bcb0012000300} + 3 {5480809280808001017434001200fb} + 4 {50af9c939c9c9cb09c9c6400b09c9c6400} + 5 {12000300} + 6 {09847304} + 7 {5401017434001208} +} { +if {$tn==7} breakpoint + do_test 2.$tn { + set changeset [binary decode hex $blob] + list [catch { sqlite3changeset_apply db $changeset xConflict } msg] $msg + } {1 SQLITE_CORRUPT} +} + finish_test diff --git a/ext/session/sqlite3session.c b/ext/session/sqlite3session.c index bdecdd1031..2a130c7e9e 100644 --- a/ext/session/sqlite3session.c +++ b/ext/session/sqlite3session.c @@ -2718,15 +2718,18 @@ static int sessionReadRecord( for(i=0; iiNext>=pIn->nData ){ + rc = SQLITE_CORRUPT; + }else{ + rc = sessionInputBuffer(pIn, 9); + } if( rc==SQLITE_OK ){ eType = pIn->aData[pIn->iNext++]; - } - - assert( apOut[i]==0 ); - if( eType ){ - apOut[i] = sqlite3ValueNew(0); - if( !apOut[i] ) rc = SQLITE_NOMEM; + assert( apOut[i]==0 ); + if( eType ){ + apOut[i] = sqlite3ValueNew(0); + if( !apOut[i] ) rc = SQLITE_NOMEM; + } } if( rc==SQLITE_OK ){ @@ -2857,11 +2860,15 @@ static int sessionChangesetReadTblhdr(sqlite3_changeset_iter *p){ int nByte; int nVarint; nVarint = sessionVarintGet(&p->in.aData[p->in.iNext], &p->nCol); - nCopy -= nVarint; - p->in.iNext += nVarint; - nByte = p->nCol * sizeof(sqlite3_value*) * 2 + nCopy; - p->tblhdr.nBuf = 0; - sessionBufferGrow(&p->tblhdr, nByte, &rc); + if( p->nCol>0 ){ + nCopy -= nVarint; + p->in.iNext += nVarint; + nByte = p->nCol * sizeof(sqlite3_value*) * 2 + nCopy; + p->tblhdr.nBuf = 0; + sessionBufferGrow(&p->tblhdr, nByte, &rc); + }else{ + rc = SQLITE_CORRUPT; + } } if( rc==SQLITE_OK ){ @@ -2939,6 +2946,13 @@ static int sessionChangesetNext( op = p->in.aData[p->in.iNext++]; } + if( p->zTab==0 ){ + /* The first record in the changeset is not a table header. Must be a + ** corrupt changeset. */ + assert( p->in.iNext==1 ); + return (p->rc = SQLITE_CORRUPT_BKPT); + } + p->op = op; p->bIndirect = p->in.aData[p->in.iNext++]; if( p->op!=SQLITE_UPDATE && p->op!=SQLITE_DELETE && p->op!=SQLITE_INSERT ){ @@ -3708,7 +3722,13 @@ static int sessionBindRow( if( !abPK || abPK[i] ){ sqlite3_value *pVal; (void)xValue(pIter, i, &pVal); - rc = sessionBindValue(pStmt, i+1, pVal); + if( pVal==0 ){ + /* The value in the changeset was "undefined". This indicates a + ** corrupt changeset blob. */ + rc = SQLITE_CORRUPT; + }else{ + rc = sessionBindValue(pStmt, i+1, pVal); + } } } return rc; diff --git a/ext/session/sqlite3session.h b/ext/session/sqlite3session.h index 9f33855df0..81b1ed06d6 100644 --- a/ext/session/sqlite3session.h +++ b/ext/session/sqlite3session.h @@ -13,16 +13,23 @@ extern "C" { /* ** CAPI3REF: Session Object Handle +** +** An instance of this object is a [session] that can be used to +** record changes to a database. */ typedef struct sqlite3_session sqlite3_session; /* ** CAPI3REF: Changeset Iterator Handle +** +** An instance of this object is as as a cursor for iterating +** over the elements of a [changeset] or [patchset]. */ typedef struct sqlite3_changeset_iter sqlite3_changeset_iter; /* ** CAPI3REF: Create A New Session Object +** CONSTRUCTOR: sqlite3_session ** ** Create a new session object attached to database handle db. If successful, ** a pointer to the new object is written to *ppSession and SQLITE_OK is @@ -59,6 +66,7 @@ int sqlite3session_create( /* ** CAPI3REF: Delete A Session Object +** DESTRUCTOR: sqlite3_session ** ** Delete a session object previously allocated using ** [sqlite3session_create()]. Once a session object has been deleted, the @@ -74,6 +82,7 @@ void sqlite3session_delete(sqlite3_session *pSession); /* ** CAPI3REF: Enable Or Disable A Session Object +** METHOD: sqlite3_session ** ** Enable or disable the recording of changes by a session object. When ** enabled, a session object records changes made to the database. When @@ -93,6 +102,7 @@ int sqlite3session_enable(sqlite3_session *pSession, int bEnable); /* ** CAPI3REF: Set Or Clear the Indirect Change Flag +** METHOD: sqlite3_session ** ** Each change recorded by a session object is marked as either direct or ** indirect. A change is marked as indirect if either: @@ -122,6 +132,7 @@ int sqlite3session_indirect(sqlite3_session *pSession, int bIndirect); /* ** CAPI3REF: Attach A Table To A Session Object +** METHOD: sqlite3_session ** ** If argument zTab is not NULL, then it is the name of a table to attach ** to the session object passed as the first argument. All subsequent changes @@ -184,6 +195,7 @@ int sqlite3session_attach( /* ** CAPI3REF: Set a table filter on a Session Object. +** METHOD: sqlite3_session ** ** The second argument (xFilter) is the "filter callback". For changes to rows ** in tables that are not attached to the Session object, the filter is called @@ -202,6 +214,7 @@ void sqlite3session_table_filter( /* ** CAPI3REF: Generate A Changeset From A Session Object +** METHOD: sqlite3_session ** ** Obtain a changeset containing changes to the tables attached to the ** session object passed as the first argument. If successful, @@ -311,7 +324,8 @@ int sqlite3session_changeset( ); /* -** CAPI3REF: Load The Difference Between Tables Into A Session +** CAPI3REF: Load The Difference Between Tables Into A Session +** METHOD: sqlite3_session ** ** If it is not already attached to the session object passed as the first ** argument, this function attaches table zTbl in the same manner as the @@ -376,6 +390,7 @@ int sqlite3session_diff( /* ** CAPI3REF: Generate A Patchset From A Session Object +** METHOD: sqlite3_session ** ** The differences between a patchset and a changeset are that: ** @@ -427,6 +442,7 @@ int sqlite3session_isempty(sqlite3_session *pSession); /* ** CAPI3REF: Create An Iterator To Traverse A Changeset +** CONSTRUCTOR: sqlite3_changeset_iter ** ** Create an iterator used to iterate through the contents of a changeset. ** If successful, *pp is set to point to the iterator handle and SQLITE_OK @@ -467,6 +483,7 @@ int sqlite3changeset_start( /* ** CAPI3REF: Advance A Changeset Iterator +** METHOD: sqlite3_changeset_iter ** ** This function may only be used with iterators created by function ** [sqlite3changeset_start()]. If it is called on an iterator passed to @@ -491,6 +508,7 @@ int sqlite3changeset_next(sqlite3_changeset_iter *pIter); /* ** CAPI3REF: Obtain The Current Operation From A Changeset Iterator +** METHOD: sqlite3_changeset_iter ** ** The pIter argument passed to this function may either be an iterator ** passed to a conflict-handler by [sqlite3changeset_apply()], or an iterator @@ -525,6 +543,7 @@ int sqlite3changeset_op( /* ** CAPI3REF: Obtain The Primary Key Definition Of A Table +** METHOD: sqlite3_changeset_iter ** ** For each modified table, a changeset includes the following: ** @@ -556,6 +575,7 @@ int sqlite3changeset_pk( /* ** CAPI3REF: Obtain old.* Values From A Changeset Iterator +** METHOD: sqlite3_changeset_iter ** ** The pIter argument passed to this function may either be an iterator ** passed to a conflict-handler by [sqlite3changeset_apply()], or an iterator @@ -586,6 +606,7 @@ int sqlite3changeset_old( /* ** CAPI3REF: Obtain new.* Values From A Changeset Iterator +** METHOD: sqlite3_changeset_iter ** ** The pIter argument passed to this function may either be an iterator ** passed to a conflict-handler by [sqlite3changeset_apply()], or an iterator @@ -619,6 +640,7 @@ int sqlite3changeset_new( /* ** CAPI3REF: Obtain Conflicting Row Values From A Changeset Iterator +** METHOD: sqlite3_changeset_iter ** ** This function should only be used with iterator objects passed to a ** conflict-handler callback by [sqlite3changeset_apply()] with either @@ -646,6 +668,7 @@ int sqlite3changeset_conflict( /* ** CAPI3REF: Determine The Number Of Foreign Key Constraint Violations +** METHOD: sqlite3_changeset_iter ** ** This function may only be called with an iterator passed to an ** SQLITE_CHANGESET_FOREIGN_KEY conflict handler callback. In this case @@ -662,6 +685,7 @@ int sqlite3changeset_fk_conflicts( /* ** CAPI3REF: Finalize A Changeset Iterator +** METHOD: sqlite3_changeset_iter ** ** This function is used to finalize an iterator allocated with ** [sqlite3changeset_start()]. @@ -678,6 +702,7 @@ int sqlite3changeset_fk_conflicts( ** to that error is returned by this function. Otherwise, SQLITE_OK is ** returned. This is to allow the following pattern (pseudo-code): ** +** 
 **   sqlite3changeset_start();
 **   while( SQLITE_ROW==sqlite3changeset_next() ){
 **     // Do something with change.
@@ -686,6 +711,7 @@ int sqlite3changeset_fk_conflicts(
 **   if( rc!=SQLITE_OK ){
 **     // An error has occurred 
 **   }
+**
*/ int sqlite3changeset_finalize(sqlite3_changeset_iter *pIter); @@ -733,6 +759,7 @@ int sqlite3changeset_invert( ** sqlite3_changegroup object. Calling it produces similar results as the ** following code fragment: ** +** 
 **   sqlite3_changegroup *pGrp;
 **   rc = sqlite3_changegroup_new(&pGrp);
 **   if( rc==SQLITE_OK ) rc = sqlite3changegroup_add(pGrp, nA, pA);
@@ -743,6 +770,7 @@ int sqlite3changeset_invert(
 **     *ppOut = 0;
 **     *pnOut = 0;
 **   }
+**
** ** Refer to the sqlite3_changegroup documentation below for details. */ @@ -758,11 +786,15 @@ int sqlite3changeset_concat( /* ** CAPI3REF: Changegroup Handle +** +** A changegroup is an object used to combine two or more +** [changesets] or [patchsets] */ typedef struct sqlite3_changegroup sqlite3_changegroup; /* ** CAPI3REF: Create A New Changegroup Object +** CONSTRUCTOR: sqlite3_changegroup ** ** An sqlite3_changegroup object is used to combine two or more changesets ** (or patchsets) into a single changeset (or patchset). A single changegroup @@ -800,6 +832,7 @@ int sqlite3changegroup_new(sqlite3_changegroup **pp); /* ** CAPI3REF: Add A Changeset To A Changegroup +** METHOD: sqlite3_changegroup ** ** Add all changes within the changeset (or patchset) in buffer pData (size ** nData bytes) to the changegroup. @@ -877,6 +910,7 @@ int sqlite3changegroup_add(sqlite3_changegroup*, int nData, void *pData); /* ** CAPI3REF: Obtain A Composite Changeset From A Changegroup +** METHOD: sqlite3_changegroup ** ** Obtain a buffer containing a changeset (or patchset) representing the ** current contents of the changegroup. If the inputs to the changegroup @@ -907,6 +941,7 @@ int sqlite3changegroup_output( /* ** CAPI3REF: Delete A Changegroup Object +** DESTRUCTOR: sqlite3_changegroup */ void sqlite3changegroup_delete(sqlite3_changegroup*); diff --git a/main.mk b/main.mk index 43cfaf1a2c..cdbef4e44e 100644 --- a/main.mk +++ b/main.mk @@ -362,6 +362,7 @@ TESTSRC += \ $(TOP)/ext/misc/ieee754.c \ $(TOP)/ext/misc/mmapwarm.c \ $(TOP)/ext/misc/nextchar.c \ + $(TOP)/ext/misc/normalize.c \ $(TOP)/ext/misc/percentile.c \ $(TOP)/ext/misc/regexp.c \ $(TOP)/ext/misc/remember.c \ @@ -519,6 +520,7 @@ SHELL_OPT += -DSQLITE_INTROSPECTION_PRAGMAS FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1 FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 FUZZCHECK_OPT += -DSQLITE_MAX_MEMORY=50000000 +FUZZCHECK_OPT += -DSQLITE_PRINTF_PRECISION_LIMIT=1000 DBFUZZ_OPT = KV_OPT = -DSQLITE_THREADSAFE=0 -DSQLITE_DIRECT_OVERFLOW_READ ST_OPT = -DSQLITE_THREADSAFE=0 diff --git a/manifest b/manifest index eefcc19af2..737fe75fb0 100644 --- a/manifest +++ b/manifest @@ -1,22 +1,22 @@ -C Merge\schanges\sfrom\strunk. -D 2018-01-24T15:02:53.185 +C Merge\sthe\slatest\senhancements\sfrom\strunk. +D 2018-03-01T13:44:40.762 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea -F Makefile.in 2974b995b10238cb009db0b89edafb8f5a3a309ef8e50ceb1e988ae687d5f50d +F Makefile.in 1d5a68043cc4d8a6e45b37e2639b148cdd7973aa75e90ec71e12d55cd95e32c0 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 -F Makefile.msc ce9db774520fb2e2d82ce32cefa4e16db37105ab4b28ed32c6c0f90212dd4a83 -F README.md d748f58e3ab0fe0307fb4ae0942b415d93dcc4288756e366cc9e7cf8260c093f +F Makefile.msc 28df1e20dd77b9613ce5b65659a4ab9ab19a1f0d885f1eae2399baf0ebc43f46 +F README.md 1d5342ebda97420f114283e604e5fe99b0da939d63b76d492eabbaae23488276 F VERSION cdf91ac446255ecf3d8f6d8c3ee40d64123235ae5b3cef29d344e61b45ec3759 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2 F art/sqlite370.ico af56c1d00fee7cd4753e8631ed60703ed0fc6e90 F art/sqlite370.jpg d512473dae7e378a67e28ff96a34da7cb331def2 F autoconf/INSTALL 83e4a25da9fd053c7b3665eaaaf7919707915903 -F autoconf/Makefile.am 6cca3f797c649b40c762484ce26491839fec54de72d376d774969e76ed13931f +F autoconf/Makefile.am 2c274948734e03c51790ff51468f91db8d570bcca864284d9c6d6e777264cd7e F autoconf/Makefile.msc 2c50a59319af7da4eaca8c13e3240881b1bc245fd175845a055faab7d03d6e67 F autoconf/README.first 6c4f34fe115ff55d4e8dbfa3cecf04a0188292f7 F autoconf/README.txt 4f04b0819303aabaa35fff5f7b257fb0c1ef95f1 -F autoconf/configure.ac aeeed858e5e54e79052ae44ba774e56595dcb787f23a2155aa98a8aa27327b66 +F autoconf/configure.ac 18fca06f884213be062dd5e07c5297079cc45893d9cd3f522ce426e715033e3d F autoconf/tea/Makefile.in b438a7020446c8a8156e8d97c8914a04833da6fd F autoconf/tea/README 3e9a3c060f29a44344ab50aec506f4db903fb873 F autoconf/tea/aclocal.m4 52c47aac44ce0ddb1f918b6993e8beb8eee88f43 @@ -96,7 +96,7 @@ F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 F ext/fts3/fts3_unicode.c 525a3bd9a7564603c5c061b7de55403a565307758a94600e8a2f6b00d1c40d9d F ext/fts3/fts3_unicode2.c cc04fc672bfd42b1e650398cb0bf71f64f9aae032cfe75bbcfe75b9cf966029c -F ext/fts3/fts3_write.c a3f7bf869622d1d0aa66661ba71d88e6f9646d69a2c335f40a0addf25974db47 +F ext/fts3/fts3_write.c b583dede85eb0c3c3026f8d7ccb781ea4e845ae583754fecb2ca425b5907d87d F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/fts3/tool/fts3cov.sh c331d006359456cf6f8f953e37f2b9c7d568f3863f00bb5f7eb87fea4ac01b73 @@ -113,9 +113,9 @@ F ext/fts5/fts5_buffer.c 1dd1ec0446b3acfc2d7d407eb894762a461613e2695273f48e449bf F ext/fts5/fts5_config.c 5af9c360e99669d29f06492c370892394aba0857 F ext/fts5/fts5_expr.c 01048018d21524e2c302b063ff5c3cdcf546e03297215e577205d85b47499deb F ext/fts5/fts5_hash.c 32be400cf761868c9db33efe81a06eb19a17c5402ad477ee9efb51301546dd55 -F ext/fts5/fts5_index.c 5fe14375a29e8a7aa8f3e863babe180a19269206c254c8f47b216821d4ac1e15 +F ext/fts5/fts5_index.c 22b71d0e9e4b3ddd123a39ae27174e0012da2806f91b64087a68584f13f189de F ext/fts5/fts5_main.c 24868f88ab2a865defbba7a92eebeb726cc991eb092b71b5f5508f180c72605b -F ext/fts5/fts5_storage.c fb5ef3c27073f67ade2e1bea08405f9e43f68f5f3676ed0ab7013bce5ba10be6 +F ext/fts5/fts5_storage.c 4bec8a1b3905978b22a67bca5f4a3cfdb94af234cf51efb36f4f2d733d278634 F ext/fts5/fts5_tcl.c 39bcbae507f594aad778172fa914cad0f585bf92fd3b078c686e249282db0d95 F ext/fts5/fts5_test_mi.c 65864ba1e5c34a61d409c4c587e0bbe0466eb4f8f478d85dc42a92caad1338e6 F ext/fts5/fts5_test_tok.c ffd657dd67e7fcdb31bf63fb60b6d867299a581d0f46e97086abacd66c2a9b26 @@ -126,7 +126,7 @@ F ext/fts5/fts5_vocab.c 1cd79854cb21543e66507b25b0578bc1b20aa6a1349b7feceb8e8fed F ext/fts5/fts5parse.y eb526940f892ade5693f22ffd6c4f2702543a9059942772526eac1fde256bb05 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba F ext/fts5/test/fts5_common.tcl b01c584144b5064f30e6c648145a2dd6bc440841 -F ext/fts5/test/fts5aa.test cba3fae6466446980caf1b9f5f26df77f95a999d35db7d932d6e82ae7ba0ede9 +F ext/fts5/test/fts5aa.test 6e2fdb0ee667c05f41921e7ec345cae874be651670900918e9ccc539514b9356 F ext/fts5/test/fts5ab.test 9205c839332c908aaad2b01ab8670ece8b161e8f2ec8a9fabf18ca9385880bb7 F ext/fts5/test/fts5ac.test a7aa7e1fefc6e1918aa4d3111d5c44a09177168e962c5fd2cca9620de8a7ed6d F ext/fts5/test/fts5ad.test e8cf959dfcd57c8e46d6f5f25665686f3b6627130a9a981371dafdf6482790de @@ -277,13 +277,14 @@ F ext/misc/compress.c dd4f8a6d0baccff3c694757db5b430f3bbd821d8686d1fc24df55cf9f0 F ext/misc/csv.c 1a009b93650732e22334edc92459c4630b9fa703397cbb3c8ca279921a36ca11 F ext/misc/dbdump.c 3509fa6b8932d04e932d6b6b827b6a82ca362781b8e8f3c77336f416793e215e F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2 -F ext/misc/fileio.c bd2f717be63a9ae9ff85977a756c43a96d47a7763d98ae913636c64b714c232b +F ext/misc/fileio.c 4cad3a78bfbbea9b1414b4405fd704ee57fbe2621254b011888dae7b0ba8c513 F ext/misc/fuzzer.c 7c64b8197bb77b7d64eff7cac7848870235d4c25 F ext/misc/ieee754.c f190d0cc5182529acb15babd177781be1ac1718c F ext/misc/json1.c dbe086615b9546c156bf32b9378fc09383b58bd17513b866cfd24c1e15281984 F ext/misc/memvfs.c ab36f49e02ebcdf85a1e08dc4d8599ea8f343e073ac9e0bca18a98b7e1ec9567 F ext/misc/mmapwarm.c 70b618f2d0bde43fae288ad0b7498a629f2b6f61b50a27e06fae3cd23c83af29 F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 +F ext/misc/normalize.c 19262ef3ef29d4de2f281b423326865c8916c63d0cb09f1dc98d24d5c1e8ba64 F ext/misc/percentile.c 92699c8cd7d517ff610e6037e56506f8904dae2e F ext/misc/regexp.c a68d25c659bd2d893cd1215667bbf75ecb9dc7d4 F ext/misc/remember.c add730f0f7e7436cd15ea3fd6a90fd83c3f706ab44169f7f048438b7d6baa69c @@ -293,7 +294,7 @@ F ext/misc/series.c f3c0dba5c5c749ce1782b53076108f87cf0b71041eb6023f727a9c50681d F ext/misc/sha1.c 0b9e9b855354910d3ca467bf39099d570e73db56 F ext/misc/shathree.c 9e960ba50483214c6a7a4b1517f8d8cef799e9db381195178c3fd3ad207e10c0 F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52 -F ext/misc/spellfix.c 41cf26c6b89fcaa8798ae10ae64d39c1f1d9d6995152e545bd491c13058b8fac +F ext/misc/spellfix.c b3a644285cb008f3c10ed4cf04e17adcbc7d62c3911c79d786dfc91c177534f0 F ext/misc/sqlar.c 57d5bc45cd5492208e451f697404be88f8612527d64c9d42f96b325b64983d74 F ext/misc/stmt.c 6f16443abb3551e3f5813bb13ba19a30e7032830015b0f92fe0c0453045c0a11 F ext/misc/totype.c 4a167594e791abeed95e0a8db028822b5e8fe512 @@ -303,7 +304,8 @@ F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178 F ext/misc/vtablog.c 31d0d8f4406795679dcd3a67917c213d3a2a5fb3ea5de35f6e773491ed7e13c9 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 -F ext/misc/zipfile.c d99efb67ecdfcae7e1855984c218c8c33d0d46a833eaa4b5a5c3d7a4f6690ce4 +F ext/misc/zipfile.c bc88dbf9aa26c24595d158c1398208786ef34e7aee19ea29e8f0110c752fca03 +F ext/misc/zorder.c b0ff58fa643afa1d846786d51ea8d5c4b6b35aa0254ab5a82617db92f3adda64 F ext/rbu/rbu.c ea7d1b7eb44c123a2a619332e19fe5313500705c4a58aaa1887905c0d83ffc2e F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842 F ext/rbu/rbu10.test 1846519a438697f45e9dcb246908af81b551c29e1078d0304fae83f1fed7e9ee @@ -350,14 +352,14 @@ F ext/repair/test/checkfreelist01.test 3e8aa6aeb4007680c94a8d07b41c339aa635cc782 F ext/repair/test/checkindex01.test 6945d0ffc0c1dc993b2ce88036b26e0f5d6fcc65da70fc9df27c2647bb358b0f F ext/repair/test/test.tcl 686d76d888dffd021f64260abf29a55c57b2cedfa7fc69150b42b1d6119aac3c F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 -F ext/rtree/rtree.c d941e44ad901da039caebb9f9fa99d81f2a4fc822e67cafe33fa4f6f789074a0 +F ext/rtree/rtree.c bc61010e978b5b8ae6dbb90274a2fbb5db5ff5e2880b5c6e8abd48eea77264db F ext/rtree/rtree.h 4a690463901cb5e6127cf05eb8e642f127012fd5003830dbc974eca5802d9412 -F ext/rtree/rtree1.test 82a353747fcab1083d114b2ac84723dfefdbf86c1a6e1df57bf588c7d4285436 +F ext/rtree/rtree1.test 47e2095bebea6813754fd7afa6a20e2b7b4ebcd5cb7dbcb6932b6c9f86bbf972 F ext/rtree/rtree2.test 5f25b01acd03470067a2d52783b2eb0a50bf836803d4342d20ca39e541220fe2 F ext/rtree/rtree3.test 2cafe8265d1ff28f206fce88d114f208349df482 -F ext/rtree/rtree4.test 67b021858ba4334c8d49b3449476942c2ce0e5ef7123538f2e9dd508ed03a12d -F ext/rtree/rtree5.test 8aaa4bcdc42f718fe165572f5623e4732831aca95a2bc32482d33d4d2cf1325d -F ext/rtree/rtree6.test 773a90db2dce6a8353dd0d5b64bca69b29761196 +F ext/rtree/rtree4.test 304de65d484540111b896827e4261815e5dca4ce28eeecd58be648cd73452c4b +F ext/rtree/rtree5.test 49c9041d713d54560b315c2c7ef7207ee287eba1b20f8266968a06f2e55d3142 +F ext/rtree/rtree6.test 916a641d2beac01b9880871ff07612d56c1e466190a27c82ab36ffd58be03b9f F ext/rtree/rtree7.test c8fb2e555b128dd0f0bdb520c61380014f497f8a23c40f2e820acc9f9e4fdce5 F ext/rtree/rtree8.test 649f5a37ec656028a4a32674b9b1183104285a7625a09d2a8f52a1cef72c93f2 F ext/rtree/rtree9.test c646f12c8c1c68ef015c6c043d86a0c42488e2e68ed1bb1b0771a7ca246cbabf @@ -367,7 +369,7 @@ F ext/rtree/rtreeC.test d9d06dda1aee68b4dc227dfcc899f335f8b621e9d1920ee3d4e5dab8 F ext/rtree/rtreeD.test fe46aa7f012e137bd58294409b16c0d43976c3bb92c8f710481e577c4a1100dc F ext/rtree/rtreeE.test e65d3fc625da1800b412fc8785817327d43ccfec5f5973912d8c9e471928caa9 F ext/rtree/rtreeF.test 81ffa7ef51c4e4618d497a57328c265bf576990c7070633b623b23cd450ed331 -F ext/rtree/rtreeG.test fd3af1ca944a0bdb0cbb5455a4905c9f012e2fffcab6b791f07afa0dcbbcae0e +F ext/rtree/rtreeG.test 1b9ca6e3effb48f4161edaa463ddeaa8fca4b2526d084f9cbf5dbe4e0184939c F ext/rtree/rtree_perf.tcl 6c18c1f23cd48e0f948930c98dfdd37dfccb5195 F ext/rtree/rtree_util.tcl db734b4c5e75fed6acc56d9701f2235345acfdec750b5fc7b587936f5f6bceed F ext/rtree/rtreecheck.test 4d29103d1e16fcbf90135d1c637b833688492b063b2971dfb5dc6ba76555cfee @@ -379,7 +381,7 @@ F ext/session/changeset.c 4ccbaa4531944c24584bf6a61ba3a39c62b6267a F ext/session/session1.test 736d7ff178662f0b717c37f46531b84a5ce0210ccb0c4edf629c55dbcbbc3ea1 F ext/session/session2.test 284de45abae4cc1082bc52012ee81521d5ac58e0 F ext/session/session3.test ce9ce3dfa489473987f899e9f6a0f2db9bde3479 -F ext/session/session4.test 457b02bdc349eb01151e54de014df77abd3c08c8 +F ext/session/session4.test 488539cee1d2510b415236fac2727575f4110e7609a500eb21c8d9e517dfff38 F ext/session/session5.test 716bc6fafd625ce60dfa62ae128971628c1a1169 F ext/session/session6.test 443789bc2fca12e4f7075cf692c60b8a2bea1a26 F ext/session/session8.test 8e194b3f655d861ca36de5d4de53f702751bab3b @@ -400,8 +402,8 @@ F ext/session/sessionfault.test da273f2712b6411e85e71465a1733b8501dbf6f7 F ext/session/sessionfault2.test 04aa0bc9aa70ea43d8de82c4f648db4de1e990b0 F ext/session/sessionstat1.test 41cd97c2e48619a41cdf8ae749e1b25f34719de638689221aa43971be693bf4e F ext/session/sessionwor.test 2f3744236dc8b170a695b7d8ddc8c743c7e79fdc -F ext/session/sqlite3session.c a5b7aed647abe7e366254b755597fce3f2719d82c98990cb0e1e07a3d203fe2b -F ext/session/sqlite3session.h 01774161cbd328fe3d496323655b9cc142317ff1fb1ae15c1232075ea240e3a4 +F ext/session/sqlite3session.c bd8e52f8b4adef1d886564655030d5a7675baf59d52542c8f7eda99971048e13 +F ext/session/sqlite3session.h b4de978c24a48a0d9b3b92ddfb749f4b07461766325ee950f5ecb8384c10606f F ext/session/test_session.c eb0bd6c1ea791c1d66ee4ef94c16500dad936386 F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 @@ -409,7 +411,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk 05ae4da060eaf044c33d40fdaf99299fc55cde4e168c5f955a8f42da944731d2 +F main.mk 761e48fd24449071b8f8480449122fa59c052904666f72d803c64ae09b139298 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 @@ -423,15 +425,15 @@ F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c cf7a8af45cb0ace672f47a1b29ab24092a9e8cd8d945a9974e3b5d925f548594 F src/analyze.c 6b42e36a5dcc2703a771f2411bd5e99524bd62c7ecde209bb88dfb04c72f046e -F src/attach.c 8158394251d62a6513d6e395f14c9b8ad3f3447d8a1249c242c940cde5c1453e +F src/attach.c e79ef463dc0a457281512c5a67c02d886a1a4502001bc9b7eb008e9b7c0f7bf3 F src/auth.c 6277d63837357549fe14e723490d6dc1a38768d71c795c5eb5c0f8a99f918f73 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca -F src/btree.c 1beceb1c5f9563271241fd0c294484668e4ad28cf6aa970eab70e50ff6f75e25 +F src/btree.c 18a53540aa35dbdf77f715ea928422a4ed9011dc16ea7b50f803fd1617fcc4f5 F src/btree.h 0866c0a08255142ea0e754aabd211c843cab32045c978a592a43152405ed0c84 -F src/btreeInt.h 6e70d15435b60a29386ae9864875618ca225c31bf815038bbd8f510a66bb9cb1 -F src/build.c 9f9647454f236cab097f266ae970f899b53c71cadab6756c47e2b2e81392c2a1 +F src/btreeInt.h 620ab4c7235f43572cf3ac2ac8723cbdf68073be4d29da24897c7b77dda5fd96 +F src/build.c 672022c06e1a5c2653f80c77a687de11f7e65ce81d20fe2825aadfa13a875c33 F src/callback.c fe677cb5f5abb02f7a772a62a98c2f516426081df68856e8f2d5f950929b966a F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c ff1be3eed7bdd75aaca61ca8dc848f7c9f850ef2fb9cb56f2734e922a098f9c0 @@ -439,11 +441,11 @@ F src/date.c ebe1dc7c8a347117bb02570f1a931c62dd78f4a2b1b516f4837d45b7d6426957 F src/dbpage.c 8db4c97f630e7d83f884ea75caf1ffd0988c160e9d530194d93721c80821e0f6 F src/dbstat.c 7a4ba8518b6369ef3600c49cf9c918ad979acba610b2aebef1b656d649b96720 F src/delete.c 20c8788451dc737a967c87ea53ad43544d617f5b57d32ccce8bd52a0daf9e89b -F src/expr.c 9e06de431c09f144438aa6895ea4d4290fa3c6875bfcc3ba331012ca78deadf0 +F src/expr.c 904f85e5f5fa232488cbb662fcf3306ae203728751468670fb9840a9b1886abc F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c d617daf66b5515e2b42c1405b2b4984c30ca50fb705ab164271a9bf66c69e331 -F src/func.c 9970db37cc004136996a5c9b966b86f06583bcf3f275449b977fbb06d75e7300 -F src/global.c ac3094f1dc59fbeb919aef7cc0cc827a8459d1fb1adb7972ef75bd9e0c10b75b +F src/func.c 94f42cba2cc1c34aeaa441022ba0170ec3fec4bba54db4e0ded085c6dc0fdc51 +F src/global.c 01506976bd75e5e7b977207a6a05062e2dd0050012f8071be06bbea22ec6d69a F src/hash.c a12580e143f10301ed5166ea4964ae2853d3905a511d4e0c44497245c7ce1f7a F src/hash.h ab34c5c54a9e9de2e790b24349ba5aab3dbb4fd4 F src/hwtime.h 747c1bbe9df21a92e9c50f3bbec1de841dc5e5da @@ -451,7 +453,7 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 14686083cedc198540b15a79586cdd4be2acf6d5fa97627e355f817ab07e9fee F src/legacy.c 134ab3e3fae00a0f67a5187981d6935b24b337bcf0f4b3e5c9fa5763da95bf4e F src/loadext.c f6e4e416a736369f9e80eba609f0acda97148a8b0453784d670c78d3eed2f302 -F src/main.c a3e15ef7ffe64ccc468fe83acc669536ee45ded148efeac2ed522422beb4ac91 +F src/main.c d45d07a1c707f88cf734ec7699d9c393247f18682d748ab53065fb669231f66a F src/malloc.c 07295435093ce354c6d9063ac05a2eeae28bd251d2e63c48b3d67c12c76f7e18 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de @@ -464,40 +466,40 @@ F src/msvc.h 4942752b6a253116baaa8de75256c51a459a5e81 F src/mutex.c b021263554c8a3995e9d53193b8194b96d1ed28e06c3b532dd7f7d29cf0c7d53 F src/mutex.h 779d588e3b7756ec3ecf7d78cde1d84aba414f85 F src/mutex_noop.c 9d4309c075ba9cc7249e19412d3d62f7f94839c4 -F src/mutex_unix.c 27bb6cc49485ee46711a6580ab7b3f1402211d23 -F src/mutex_w32.c a898fa969823b100c0f5fdc57e54c9a1e419ab4d +F src/mutex_unix.c aaf9ebc3f89df28483c52208497a99a02cc3650011422fc9d4c57e4392f7fe58 +F src/mutex_w32.c 7670d770c94bbfe8289bec9d7f1394c5a00a57c37f892aab6b6612d085255235 F src/notify.c 9711a7575036f0d3040ba61bc6e217f13a9888e7 F src/os.c 22d31db3ca5a96a408fbf1ceeaaebcaf64c87024d2ff9fe1cf2ddbec3e75c104 F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 -F src/os_unix.c a82505be158d8ce42b38dcc9b426187d776904c12cdc68dc8925e1dfcc5cb6ce -F src/os_win.c 501dde1ee770f4ffa458bfe1cf376a556de3ab00bb8320d659c5984403991d62 +F src/os_unix.c 928586cbde31890538854f1657463392d63b2d7794f577774573ae53ec283d7b +F src/os_win.c eb03c6d52f893bcd7fdd4c6006674c13c1b5e49543fec98d605201af2997171c F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a -F src/pager.c 20e9757e5b428f9624f82b13a32e0d68f646be4d6dbb0e4687467bd657df81d3 +F src/pager.c 0dc92667c803ed2e9566c35521bed2c53717197b011bc7b64d17cd552a241a7b F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a -F src/parse.y 4e750e1b261ff9f1d0b6b5d40a829c66d691899f48953fde839d8b52d41aa148 +F src/parse.y 3be4b2b5e33ef7dab3128b765898ee786fc6cd698a7f053f756a68f2f238f0aa F src/pcache.c 7ae91a4557a43d77d449accbfdc68846e6516f8e2eda46e8bbe4536fb669b201 F src/pcache.h 072f94d29281cffd99e46c1539849f248c4b56ae7684c1f36626797fee375170 F src/pcache1.c 716975564c15eb6679e97f734cec1bfd6c16ac3d4010f05f1f8e509fc7d19880 F src/pragma.c bea56df3ae0637768c0da4fbbb8f2492f780980d95000034a105ff291bf7ca69 F src/pragma.h bb83728944b42f6d409c77f5838a8edbdb0fe83046c5496ffc9602b40340a324 F src/prepare.c 259f4e7960c47082c9653f3d5f0c294abd68bb9c3aab86de7630700cba1c20fb -F src/printf.c 9506b4b96e59c0467047155f09015750cb2878aeda3d39e5610c1192ddc3c41c +F src/printf.c d3b7844ddeb11fbbdd38dd84d09c9c1ac171d21fb038473c3aa97981201cc660 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 -F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730 +F src/resolve.c 66c73fcb7719b8ff0e841b58338f13604ff3e2b50a723f9b8f383595735262f6 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac -F src/select.c bebe7cce45d899d2237c76bce059d525abf5b861f2fce92f6b53914a961c01ba -F src/shell.c.in 7cea439c3f7f2e4ed6eb4b3a633cd93dccb1349241400de4da0c1291285ed514 -F src/sqlite.h.in 9d0ef749ff6492d13686d153e359e07226564be4f70d0cbc46385c8d8d0b96fc +F src/select.c f02352ac5cbb6fad3804add825743b924cfb6c742ba2e8666d726828a9985d73 +F src/shell.c.in 47dac081e15544f4d4373a837215391459971b312c8bed15903e7e9045c3e652 +F src/sqlite.h.in 82eb2b02de6dda3ca9797a5a5418c8409e2aaf5b4f20530a9036722faa0cc43c F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 83a3c4ce93d650bedfd1aa558cb85a516bd6d094445ee989740827d0d944368d -F src/sqliteInt.h fc800ea6bb5897d2c33ee19f8b08e631309cd0fb5096c6ae3db002a8d1b551b6 +F src/sqliteInt.h 574a0dd6978197cc1f9b8d571744c9a30e256d29a3092050bd8a95e37c7229b9 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 F src/tclsqlite.c d05dde2bb9534ee39a5095c4c4a43625997e0f3ab6889b55e76393b05b6895b1 -F src/test1.c b52f9e7fe62016d357c3266fcfa0793cc1883d3cb2b11dfa39fcba2e70b0305c +F src/test1.c 1ab7cbbb6693e08364c1a9241e2aee17f8c4925e4cc52396be77ae6845a05828 F src/test2.c 3efb99ab7f1fc8d154933e02ae1378bac9637da5 F src/test3.c b8434949dfb8aff8dfa082c8b592109e77844c2135ed3c492113839b6956255b F src/test4.c 18ec393bb4d0ad1de729f0b94da7267270f3d8e6 @@ -512,7 +514,7 @@ F src/test_backup.c bf5da90c9926df0a4b941f2d92825a01bbe090a0 F src/test_bestindex.c 78809f11026f18a93fcfd798d9479cba37e1201c830260bf1edc674b2fa9b857 F src/test_blob.c ae4a0620b478548afb67963095a7417cd06a4ec0a56adb453542203bfdcb31ce F src/test_btree.c 8b2dc8b8848cf3a4db93f11578f075e82252a274 -F src/test_config.c 6b749332d98924c52d5b08d1fd847bbe8ff049fc4446af738e64e2532a804348 +F src/test_config.c 09bd69f7e59bad08b67dd7bd8e7de6d992f1a668466b4500c870139fe1792126 F src/test_delete.c e2fe07646dff6300b48d49b2fee2fe192ed389e834dd635e3b3bac0ce0bf9f8f F src/test_demovfs.c a0c3bdd45ed044115c2c9f7779e56eafff18741e F src/test_devsym.c 1960abbb234b97e9b920f07e99503fc04b443f62bbc3c6ff2c2cea2133e3b8a2 @@ -550,30 +552,30 @@ F src/test_windirent.h 90dfbe95442c9762357fe128dc7ae3dc199d006de93eb33ba3972e0a9 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c F src/tokenize.c 5b0c661a85f783d35b9883830736eeb63be4aefc4f6b7d9cd081d48782c041e2 -F src/treeview.c eae35972ff44f67064de2eaf35f04afe94e7aea3271a8b3bcebb3f954880fec3 +F src/treeview.c e0d62678314abf0e1e6d09ea405f4de3f404a17b4641c2169c304b5edf509320 F src/trigger.c a34539c69433276d37b0da9a89c117726ff2d292c0902895af1f393a983cd3a1 F src/update.c a90a32ffc0100265b0693dbbdbe490756447af181f5ea2c138cce515b08c8795 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5 -F src/util.c ef4a5f904d942e660abade7fbf3e6bdb402dabe9e7c27f3361ecf40b945538b5 -F src/vacuum.c 90839322fd5f00df9617eb21b68beda9b6e2a2937576b0d65985e4aeb1c53739 -F src/vdbe.c d4684bd5efad095d9e4405e5ca2e1168dc7aff643cae04c424e4bade70641d70 +F src/util.c d9eb0a6c4aae1b00a7369eadd7ca0bbe946cb4c953b6751aa20d357c2f482157 +F src/vacuum.c 762ee9bbf8733d87d8cd06f58d950e881982e416f8c767334a40ffd341b6bff5 +F src/vdbe.c 99a0f275c83c01332b9af6179f2ee1e5d24ad81f90f17f8364397f8a3d20fb40 F src/vdbe.h 134beb7a12a6213c00eba58febaede33447cc4441bc568a0d9c144b33fc3720a -F src/vdbeInt.h 8d7d07f13cb3c4cbca91e22ba4a1920e542dda7c5d9299920432a0b3d5b009f5 +F src/vdbeInt.h 95f7adfdc5c8f1353321f55a6c5ec00a90877e3b85af5159e393afb41ff54110 F src/vdbeapi.c fea41171884a4de119f8b10ab514c788674eeeb7f27218bb6d008e1310bfd07f F src/vdbeaux.c 2756ac68ac259c416554100598fc291870063288cd7e1af22847f57b3e130e56 F src/vdbeblob.c f5c70f973ea3a9e915d1693278a5f890dc78594300cf4d54e64f2b0917c94191 -F src/vdbemem.c 943e41881e6317c9f93c77c1d60d3b37ddc8d26a3f852233ce7423d3e581523e +F src/vdbemem.c 414e28d3a7e2a8bee2bb247de115dcbc68e3cbac284d5862d077002f7a93bce1 F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2f F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c 5a3f464edd64596f601683ed321d12e6fd93c5fb9afdfb3653d6ffd0fee9c48f +F src/wal.c 0ba9af2589b67c4bc60c773c7d2080491099037af21e73545f932bb29bf70b65 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a F src/walker.c da987a20d40145c0a03c07d8fefcb2ed363becc7680d0500d9c79915591f5b1f -F src/where.c caf0b6c9d31f22f0b2c91aba723858de52b5d665aaa89034099015aaf9bb8219 +F src/where.c 7cae47e813393d70c6d327fdf000fcb30f76b1b0b5a5b52ff6402e0c658de32c F src/whereInt.h 82c04c5075308abbac59180c8bad5ecb45b07453981f60a53f3c7dee21e1e971 -F src/wherecode.c cb360c511f69294ddf00340d7b390e6b1d601a1cfb77b42c4d316fe2f4cd01c3 -F src/whereexpr.c 427ea8e96ec24f2a7814c67b8024ad664a9c7656264c4566c34743cb23186e46 +F src/wherecode.c e1aaadd8fec650037cfbf27d1b3470338fb3b58fec34d11082df16fe9a08fbd7 +F src/whereexpr.c 53532be687e12f3cd314f1e204cd4fbdac7ad250e918a182b048121e16e828ae F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test a6d901b436328bd67a79b41bb0ac2663918fe3bd F test/affinity3.test 6a101af2fc945ce2912f6fe54dd646018551710d @@ -673,7 +675,7 @@ F test/capi3b.test efb2b9cfd127efa84433cd7a2d72ce0454ae0dc4 F test/capi3c.test 7ebed1d8fa2f3190149d556fe8cff5a006be62af437c5c4640db614470126098 F test/capi3d.test 485048dc5cd07bc68011e4917ad035ad6047ab82 F test/capi3e.test 3d49c01ef2a1a55f41d73cba2b23b5059ec460fe -F test/cast.test 4c275cbdc8202d6f9c54a3596701719868ac7dc3 +F test/cast.test 5ceb920718d280b61163500a7d29e0e0a86458b1cbd92d96f962c9d970aa3857 F test/cffault.test 9d6b20606afe712374952eec4f8fd74b1a8097ef F test/check.test 33a698e8c63613449d85d624a38ef669bf20331daabebe3891c9405dd6df463a F test/close.test 799ea4599d2f5704b0a30f477d17c2c760d8523fa5d0c8be4a7df2a8cad787d8 @@ -791,7 +793,7 @@ F test/exclusive.test 1206b87e192497d78c7f35552e86a9d05421498da300fb1cce5ca5351c F test/exclusive2.test 984090e8e9d1b331d2e8111daf6e5d61dda0bef7 F test/exec.test e949714dc127eaa5ecc7d723efec1ec27118fdd7 F test/exists.test 79a75323c78f02bbe9c251ea502a092f9ef63dac -F test/expr.test 66a2c9ac34f74f036faa4092f5402c7d3162fc93 +F test/expr.test 7cb55e80aeb41d65fec968c08212505123063fea60bdc355d764d747670e9eea F test/extension01.test 00d13cec817f331a687a243e0e5a2d87b0e358c9 F test/extraquick.test cb254400bd42bfb777ff675356aabf3287978f79 F test/fallocate.test 07416bd593a116d5893cb244f45a94d5c6fe030561df3bd972e6135f8106e509 @@ -845,7 +847,7 @@ F test/fts2r.test b154c30b63061d8725e320fba1a39e2201cadd5e F test/fts2token.test d8070b241a15ff13592a9ae4a8b7c171af6f445a F test/fts3.test 672a040ea57036fb4b6fdc09027c18d7d24ab654 F test/fts3_common.tcl 99cf6659b87c0f74f55963c2aea03b3a7d66ceb0 -F test/fts3aa.test 39b65c11913d277c91d7426c62cfc1d147d1b4e9a48fecd9e38f60d0b5a5f505 +F test/fts3aa.test f267fcd6aca30fc70b81e5d82b68b34b38f581896020b57ed49e9777c7ebd85f F test/fts3ab.test 7f6cf260ae80dda064023df8e8e503e9a412b91f F test/fts3ac.test 636ed7486043055d4f126a0e385f2d5a82ebbf63 F test/fts3ad.test e40570cb6f74f059129ad48bcef3d7cbc20dda49 @@ -917,16 +919,16 @@ F test/fts4merge2.test 5faa558d1b672f82b847d2a337465fa745e46891 F test/fts4merge3.test 8d9ccb4a3d41c4c617a149d6c4b13ad02de797d0 F test/fts4merge4.test d895b1057a7798b67e03455d0fa50e9ea836c47b F test/fts4noti.test 5553d7bb2e20bf4a06b23e849352efc022ce6309 -F test/fts4onepass.test 7319d61a2ed1325fc54afd0c060a0513b462303a +F test/fts4onepass.test d69ddc4ee3415e40b0c5d1d0408488a87614d4f63ba9c44f3e52db541d6b7cc7 F test/fts4opt.test fd6a11684b965e1999564ae763797b7fb9e34c96 F test/fts4unicode.test ceca76422abc251818cb25dabe33d3c3970da5f7c90e1540f190824e6b3a7c95 F test/full.test 6b3c8fb43c6beab6b95438c1675374b95fab245d -F test/func.test ae97561957aba6ca9e3a7b8a13aac41830d701ef +F test/func.test 09dda479bcfc568f99f3070413e9672a8eeedc1be9c5d819bf55d4788c2583b7 F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f F test/func3.test d202a7606d23f90988a664e88e268aed1087c11c F test/func4.test 6beacdfcb0e18c358e6c2dcacf1b65d1fa80955f F test/func5.test cdd224400bc3e48d891827cc913a57051a426fa4 -F test/func6.test a4281c8fcd42b56f7a60f28e8e4d444e8b2256f9e82658b7ab87699f8318f564 +F test/func6.test 90e42b64c4f9fb6f04f44cb8a1da586c8542502e926b19c76504fe74ff2a9b7c F test/fuzz-oss1.test e58330d01cbbd8215ee636b17a03fe220b37dbfa F test/fuzz.test 96083052bf5765e4518c1ba686ce2bab785670d1 F test/fuzz2.test 76dc35b32b6d6f965259508508abce75a6c4d7e1 @@ -978,7 +980,7 @@ F test/index7.test 7feababe16f2091b229c22aff2bcc1d4d6b9d2bb F test/index8.test bc2e3db70e8e62459aaa1bd7e4a9b39664f8f9d7 F test/index9.test 0aa3e509dddf81f93380396e40e9bb386904c1054924ba8fa9bcdfe85a8e7721 F test/indexedby.test faa585e315e868f09bce0eb39c41d6134649b13d2801638294d3ae616edf1609 -F test/indexexpr1.test ace1ad489adc25325ad298434f13b1a515b36bf5dca9fe2a4b66cdf17aea3fa0 +F test/indexexpr1.test 635261197bcdc19b9b2c59bbfa7227d525c00e9587faddb2d293c44d287ce60e F test/indexexpr2.test 13247bac49143196556eb3f65e97ef301bd3e993f4511558b5db322ddc370ea6 F test/indexfault.test 31d4ab9a7d2f6e9616933eb079722362a883eb1d F test/init.test 15c823093fdabbf7b531fe22cf037134d09587a7 @@ -1000,8 +1002,9 @@ F test/ioerr3.test d3cec5e1a11ad6d27527d0d38573fbff14c71bdd F test/ioerr4.test f130fe9e71008577b342b8874d52984bd04ede2c F test/ioerr5.test 2edfa4fb0f896f733071303b42224df8bedd9da4 F test/ioerr6.test a395a6ab144b26a9e3e21059a1ab6a7149cca65b +F test/istrue.test 2d89873d72248b7a0ef7b5457120100cce75a19181bcf4b30e90a3f96a5c5558 F test/join.test 442c462eea85cf065d70a663c626b780a95af6e11585d909bb63b87598afe678 -F test/join2.test 1a0c26399910b015d9f8f95b884e9a079fd2cfdccd65f7b1603846508cae0dc6 +F test/join2.test f1b5ba94ba70a791df2d99e61adcebf62edacad1eca57881c97eb3e2ca718595 F test/join3.test 6f0c774ff1ba0489e6c88a3e77b9d3528fb4fda0 F test/join4.test 1a352e4e267114444c29266ce79e941af5885916 F test/join5.test bc98ea4b4e5003f5b1453701ebb8cd7d1c01a550 @@ -1012,7 +1015,7 @@ F test/journal3.test c9c29883f5bf535ae82ae21c472df6263806a22e467b6db7cd0d6d54530 F test/jrnlmode.test a6693f2bed4541a21e703aaa37bb3e10de154130645952933b82b2dec0a8b539 F test/jrnlmode2.test 8759a1d4657c064637f8b079592651530db738419e1d649c6df7048cd724363d F test/jrnlmode3.test 556b447a05be0e0963f4311e95ab1632b11c9eaa -F test/json101.test d7cdf3e6731d41e0c4bde1c88806abd17f1f478486a1409933c1d8eac9120095 +F test/json101.test 24e97954e3bd6404f3715888c7f8f835e36e19c7ae6513b5d9ab2d381498962d F test/json102.test eeb54efa221e50b74a2d6fb9259963b48d7414dca3ce2fdfdeed45cb28487bc1 F test/json103.test c5f6b85e69de05f6b3195f9f9d5ce9cd179099a0 F test/json104.test 877d5845f6303899b7889ea5dd1bea99076e3100574d5c536082245c5805dcaa @@ -1095,12 +1098,13 @@ F test/multiplex4.test e8ae4c4bd70606a5727743241f13b5701990abe4 F test/mutex1.test ea2cc74d97f077b9e74c84cbd024f14d79a8126f F test/mutex2.test bfeaeac2e73095b2ac32285d2756e3a65e681660 F test/nan.test 437d40e6d0778b050d7750726c0cbd2c9936b81962926e8f8c48ca698f00f4d1 -F test/nockpt.test 9a436a7213ba5ef7a32304998d386d3ea3f76c9d +F test/nockpt.test fd5473e30a84848b25ab06524750c05095383e191be83ccb4e6951c8beddfb5b F test/nolock.test f196cf8b8fbea4e2ca345140a2b3f3b0da45c76e +F test/normalize.test 501630ab49b0b26b65c74124bf03e3374c1b57fa97aae750f84803609141d167 F test/notify1.test 669b2b743618efdc18ca4b02f45423d5d2304abf F test/notify2.test 2ecabaa1305083856b7c39cf32816b612740c161 F test/notify3.test 10ff25cde502e72a92053a2f215d64bece4ef934 -F test/notnull.test f8fcf58669ddba79274daa2770d61dfad8274f62 +F test/notnull.test b6999231221df3534827e45e2005dd7a815fdd5f2c2e1afb9be21ead410816f8 F test/null.test 0dcce4f04284ec66108c503327ad6d224c0752b3 F test/numcast.test 5d126f7f581432e86a90d1e35cac625164aec4a1 F test/numindex1.test 20a5450d4b056e48cd5db30e659f13347a099823 @@ -1140,7 +1144,7 @@ F test/pragma4.test 3046501bee2f652dc2a4f9c87781e2741361d6864439c8381aba6c3b774b F test/pragma5.test 824ce6ced5d6b7ec71abe37fc6005ff836fe39d638273dc5192b39864b9ee983 F test/pragmafault.test 275edaf3161771d37de60e5c2b412627ac94cef11739236bec12ed1258b240f8 F test/printf.test b3ff34e73d59124140eaf89f7672e21bc2ca5fcc -F test/printf2.test 9e6db85f81c63f2367c34a9d7db384088bd374ad +F test/printf2.test 30b5dd0b4b992dc5626496846ecce17ff592cacbcb11c3e589f3ac4d7e129dae F test/progress.test ebab27f670bd0d4eb9d20d49cef96e68141d92fb F test/ptrchng.test ef1aa72d6cf35a2bbd0869a649b744e9d84977fc F test/pushdown.test 5e72c51c5e33253ed639ccee1e01ce62d62b6eee5ca893cd82334e4ee7b1d7fc @@ -1165,7 +1169,7 @@ F test/rollbackfault.test 0e646aeab8840c399cfbfa43daab46fd609cf04a F test/rowallock.test 3f88ec6819489d0b2341c7a7528ae17c053ab7cc F test/rowhash.test 0bc1d31415e4575d10cacf31e1a66b5cc0f8be81 F test/rowid.test 5b7509f384f4f6fae1af3c8c104c8ca299fea18d -F test/rowvalue.test e1bd1690d891abff6138f9612241615d2de7671cb28eb58ebdd591a5b01089a4 +F test/rowvalue.test 32861d6a933ded868035f2ec79aeb993a2a46eb7a6d282ae13415a4c2e369463 F test/rowvalue2.test 060d238b7e5639a7c5630cb5e63e311b44efef2b F test/rowvalue3.test 3068f508753af69884b12125995f023da0dbb256 F test/rowvalue4.test 4b556d7de161a0dd8cff095c336e913986398bea @@ -1261,9 +1265,10 @@ F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715 F test/speed4p.explain 6b5f104ebeb34a038b2f714150f51d01143e59aa F test/speed4p.test 0e51908951677de5a969b723e03a27a1c45db38b F test/speedtest1.c a5faf4cbe5769eee4b721b3875cb3f12520a9b99d9026b1063b47c39603375b8 -F test/spellfix.test f9c1f431e2c096c8775fec032952320c0e4700db +F test/spellfix.test 951a6405d49d1a23d6b78027d3877b4a33eeb8221dcab5704b499755bb4f552e F test/spellfix2.test dfc8f519a3fc204cb2dfa8b4f29821ae90f6f8c3 F test/spellfix3.test 0f9efaaa502a0e0a09848028518a6fb096c8ad33 +F test/spellfix4.test 51c7c26514ade169855c66bcf130bd5acfb4d7fd090cc624645ab275ae6a41fb F test/sqldiff1.test 28cd737cf1b0078b1ec1bbf425e674c47785835e F test/sqllimits1.test a74ee2a3740b9f9c2437c246d8fb77354862a142 F test/sqllog.test 6af6cb0b09f4e44e1917e06ce85be7670302517a @@ -1299,7 +1304,7 @@ F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30 F test/temptable2.test cd396beb41117a5302fff61767c35fa4270a0d5e F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc -F test/tester.tcl 3ed81b9e1d9718a8d9603596c8a877793d054294053c4277a3d3897eabab3866 +F test/tester.tcl 94901a4625d9a2229666dd5c44120ddf7f0fb639470710ef74a4cefc7b039e07 F test/thread001.test 9f22fd3525a307ff42a326b6bc7b0465be1745a5 F test/thread002.test e630504f8a06c00bf8bbe68528774dd96aeb2e58 F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7 @@ -1555,7 +1560,7 @@ F test/waloverwrite.test dad2f26567f1b45174e54fbf9a8dc1cb876a7f03 F test/walpersist.test 8c6b7e3ec1ba91b5e4dc4e0921d6d3f87cd356a6 F test/walprotocol.test a112aba0b79e3adeaa485fed09484b32c654e97df58e454aa8489ac2cd57bf84 F test/walro.test cb438d05ba0d191f10b688e39c4f0cd5b71569a1d1f4440e5bdf3c6880e08c20 -F test/walro2.test 6c73e8e4b5ccc55f907f4603ba36458b45c085fb6dfb04f30e3c0babbc1c2f41 +F test/walro2.test 0e79dd15cbdb4f482c01ea248373669c732414a726b357d04846a816afafb768 F test/walrofault.test c70cb6e308c443867701856cce92ad8288cd99488fa52afab77cca6cfd51af68 F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417 F test/walslow.test c05c68d4dc2700a982f89133ce103a1a84cc285f @@ -1574,7 +1579,7 @@ F test/whereB.test 0def95db3bdec220a731c7e4bec5930327c1d8c5 F test/whereC.test cae295158703cb3fc23bf1a108a9ab730efff0f6 F test/whereD.test 711d4df58d6d4fb9b3f5ce040b818564198be002 F test/whereE.test b3a055eef928c992b0a33198a7b8dc10eea5ad2f -F test/whereF.test d44b58338fe5ddd7286023e9bedb255aa264a6c4d2168b49591b167371c675c7 +F test/whereF.test 3d9412b1199d3e2bed34fcb76b4c48d0bf4df95d27e3f8dd27b6f8b4716d0d89 F test/whereG.test dde4c52a97385a55be6a7cd46be8373f0cf35501 F test/whereH.test e4b07f7a3c2f5d31195cd33710054c78667573b2 F test/whereI.test eab5b226bbc344ac70d7dc09b963a064860ae6d7 @@ -1592,6 +1597,7 @@ F test/win32nolock.test ac4f08811a562e45a5755e661f45ca85892bdbbc F test/with1.test ca08e291249a810a2ec9b72ceef5575e07d5925b360fcf6652ae6fe06ac4dced F test/with2.test e0030e2f0267a910d6c0e4f46f2dfe941c1cc0d4f659ba69b3597728e7e8f1ab F test/with3.test e71604a0e53cba82bc04c703987cb1d6751ec0b6 +F test/with4.test 257be66c0c67fee1defbbac0f685c3465e2cad037f21ce65f23f86084f198205 F test/withM.test 693b61765f2b387b5e3e24a4536e2e82de15ff64 F test/without_rowid1.test 06b7215130882d6a072233820dd364c874c4fd69221e8fc756ec471009192874 F test/without_rowid2.test af260339f79d13cb220288b67cd287fbcf81ad99 @@ -1603,11 +1609,13 @@ F test/wordcount.c cb589cec469a1d90add05b1f8cee75c7210338d87a5afd65260ed5c0f4bbf F test/writecrash.test f1da7f7adfe8d7f09ea79b42e5ca6dcc41102f27f8e334ad71539501ddd910cc F test/zeroblob.test 3857870fe681b8185654414a9bccfde80b62a0fa F test/zerodamage.test 9c41628db7e8d9e8a0181e59ea5f189df311a9f6ce99cc376dc461f66db6f8dc -F test/zipfile.test cb42e8fa6ba5db4a03ce6baa4401fc6236baf6eb5e62b44f3e463bf6aafd631d +F test/zipfile.test 44aa8af115cc3e8c905468768dc761260650a8fdfca57e10f9818f5f8008d340 +F test/zipfile2.test 67d5f08a202796d4b7a71dfa4b8dcb74aa7a9d1f42c5f17bedff9855c1ba7aa5 +F test/zipfilefault.test 050be76778d2ec83566a542800fc3967f65ce719671a819016ada8b654bc14f7 F tool/GetFile.cs a15e08acb5dd7539b75ba23501581d7c2b462cb5 F tool/GetTclKit.bat 8995df40c4209808b31f24de0b58f90930239a234f7591e3675d45bfbb990c5d F tool/Replace.cs 02c67258801c2fb5f63231e0ac0f220b4b36ba91 -F tool/addopcodes.tcl 7181c041d495e3f26acc36d15c86923ed722285f9015f017f41a3efdb9a0dab4 +F tool/addopcodes.tcl 0288d5b26b9b35f4cb5affb76eec63f1dfce117bbc2020066708069ef60b86ff F tool/build-all-msvc.bat c12328d06c45fec8baada5949e3d5af54bf8c887 x F tool/build-shell.sh 950f47c6174f1eea171319438b93ba67ff5bf367 F tool/cg_anno.tcl f95b0006c52cf7f0496b506343415b6ee3cdcdd3 x @@ -1622,7 +1630,7 @@ F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce F tool/kvtest-speed.sh 4761a9c4b3530907562314d7757995787f7aef8f F tool/lemon.c 7f7735326ca9c3b48327b241063cee52d35d44e20ebe1b3624a81658052a4d39 -F tool/lempar.c da840fc8a6fbac23599a65ff075e6e3d01320417c794ff577088e09f5d74b689 +F tool/lempar.c 468a155e8729cfbccfe1d85bf60d064f1dab76167a51149ec5c7928a2de63953 F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9 F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862 F tool/logest.c 11346aa019e2e77a00902aa7d0cabd27bd2e8cca @@ -1660,7 +1668,7 @@ F tool/showstat4.c 0682ebea7abf4d3657f53c4a243f2e7eab48eab344ed36a94bb75dcd19a5c F tool/showwal.c ad9d768f96ca6199ad3a8c9562d679680bd032dd01204ea3e5ea6fb931d81847 F tool/soak1.tcl 8d407956e1a45b485a8e072470a3e629a27037fe F tool/spaceanal.tcl 4bfd19aad7eb3ce0372ef0255f58035e0bba4ff5e9acfd763a10c6fb365c8dec -F tool/speed-check.sh 9ae425da8819e54e780cf494fc6d8175dfb16e109ae3214a45a5c9bb2b74e2c4 +F tool/speed-check.sh 4ff9b095cf1a7643f0264e7fb7d23f0b12b7cce587a9de315877c378e90eeaf4 F tool/speedtest.tcl 06c76698485ccf597b9e7dbb1ac70706eb873355 F tool/speedtest16.c ecb6542862151c3e6509bbc00509b234562ae81e F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff @@ -1702,7 +1710,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 512b8e40c4ce2066d230b5f007bd704dc3577e21c55131ef07d90bf2fcf91b40 e0f192ea6dda4fa0b243d58c8ce41932519141bcae0689a90318b4f866f54edd -R d69faf4c612ebfba1638ce54002893c8 +P 6ef3de810d54563c227045b16197b8011ce285ea867261946f93b2de55344f29 745a9a7fef0f28a57ea3f44899058993f6ecdedda52c81a09a4a9ce09c9004d6 +R 130ade233524cab0a0641e76e7f210be U drh -Z 027f1b62b36e246537cbc0170b90eb31 +Z 8606a5979f84a0a30cbed3690217d5a2 diff --git a/manifest.uuid b/manifest.uuid index ba38829ee2..cfa0aa6b38 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6ef3de810d54563c227045b16197b8011ce285ea867261946f93b2de55344f29 \ No newline at end of file +c8083de14b54fba1071385f8530d03d5684efd4963fde0f83ddc53ae3118a75b \ No newline at end of file diff --git a/src/attach.c b/src/attach.c index 597351fa39..9b07f89b79 100644 --- a/src/attach.c +++ b/src/attach.c @@ -529,6 +529,14 @@ int sqlite3FixSelect( if( sqlite3FixExpr(pFix, pSelect->pLimit) ){ return 1; } + if( pSelect->pWith ){ + int i; + for(i=0; ipWith->nCte; i++){ + if( sqlite3FixSelect(pFix, pSelect->pWith->a[i].pSelect) ){ + return 1; + } + } + } pSelect = pSelect->pPrior; } return 0; diff --git a/src/btree.c b/src/btree.c index 168f3105fb..22f8a50e59 100644 --- a/src/btree.c +++ b/src/btree.c @@ -4345,7 +4345,7 @@ int sqlite3BtreeCursorSize(void){ ** of run-time by skipping the initialization of those elements. */ void sqlite3BtreeCursorZero(BtCursor *p){ - memset(p, 0, offsetof(BtCursor, iPage)); + memset(p, 0, offsetof(BtCursor, BTCURSOR_FIRST_UNINIT)); } /* @@ -4388,11 +4388,19 @@ int sqlite3BtreeCloseCursor(BtCursor *pCur){ ** Using this cache reduces the number of calls to btreeParseCell(). */ #ifndef NDEBUG + static int cellInfoEqual(CellInfo *a, CellInfo *b){ + if( a->nKey!=b->nKey ) return 0; + if( a->pPayload!=b->pPayload ) return 0; + if( a->nPayload!=b->nPayload ) return 0; + if( a->nLocal!=b->nLocal ) return 0; + if( a->nSize!=b->nSize ) return 0; + return 1; + } static void assertCellInfo(BtCursor *pCur){ CellInfo info; memset(&info, 0, sizeof(info)); btreeParseCell(pCur->pPage, pCur->ix, &info); - assert( CORRUPT_DB || memcmp(&info, &pCur->info, sizeof(info))==0 ); + assert( CORRUPT_DB || cellInfoEqual(&info, &pCur->info) ); } #else #define assertCellInfo(x) @@ -4668,14 +4676,15 @@ static int accessPayload( */ if( (pCur->curFlags & BTCF_ValidOvfl)==0 ){ int nOvfl = (pCur->info.nPayload-pCur->info.nLocal+ovflSize-1)/ovflSize; - if( nOvfl>pCur->nOvflAlloc ){ + if( pCur->aOverflow==0 + || nOvfl*(int)sizeof(Pgno) > sqlite3MallocSize(pCur->aOverflow) + ){ Pgno *aNew = (Pgno*)sqlite3Realloc( pCur->aOverflow, nOvfl*2*sizeof(Pgno) ); if( aNew==0 ){ return SQLITE_NOMEM_BKPT; }else{ - pCur->nOvflAlloc = nOvfl*2; pCur->aOverflow = aNew; } } diff --git a/src/btreeInt.h b/src/btreeInt.h index 87d3247e13..d1e2c08d18 100644 --- a/src/btreeInt.h +++ b/src/btreeInt.h @@ -503,20 +503,20 @@ struct BtCursor { u8 curFlags; /* zero or more BTCF_* flags defined below */ u8 curPagerFlags; /* Flags to send to sqlite3PagerGet() */ u8 hints; /* As configured by CursorSetHints() */ - int nOvflAlloc; /* Allocated size of aOverflow[] array */ - Btree *pBtree; /* The Btree to which this cursor belongs */ - BtShared *pBt; /* The BtShared this cursor points to */ - BtCursor *pNext; /* Forms a linked list of all cursors */ - Pgno *aOverflow; /* Cache of overflow page locations */ - CellInfo info; /* A parse of the cell we are pointing at */ - i64 nKey; /* Size of pKey, or last integer key */ - void *pKey; /* Saved key that was cursor last known position */ - Pgno pgnoRoot; /* The root page of this tree */ int skipNext; /* Prev() is noop if negative. Next() is noop if positive. ** Error code if eState==CURSOR_FAULT */ + Btree *pBtree; /* The Btree to which this cursor belongs */ + Pgno *aOverflow; /* Cache of overflow page locations */ + void *pKey; /* Saved key that was cursor last known position */ /* All fields above are zeroed when the cursor is allocated. See ** sqlite3BtreeCursorZero(). Fields that follow must be manually ** initialized. */ +#define BTCURSOR_FIRST_UNINIT pBt /* Name of first uninitialized field */ + BtShared *pBt; /* The BtShared this cursor points to */ + BtCursor *pNext; /* Forms a linked list of all cursors */ + CellInfo info; /* A parse of the cell we are pointing at */ + i64 nKey; /* Size of pKey, or last integer key */ + Pgno pgnoRoot; /* The root page of this tree */ i8 iPage; /* Index of current page in apPage */ u8 curIntKey; /* Value of apPage[0]->intKey */ u16 ix; /* Current index for apPage[iPage] */ diff --git a/src/build.c b/src/build.c index 58b39d6475..3a656def3e 100644 --- a/src/build.c +++ b/src/build.c @@ -1118,10 +1118,24 @@ void sqlite3AddColumn(Parse *pParse, Token *pName, Token *pType){ */ void sqlite3AddNotNull(Parse *pParse, int onError){ Table *p; + Column *pCol; p = pParse->pNewTable; if( p==0 || NEVER(p->nCol<1) ) return; - p->aCol[p->nCol-1].notNull = (u8)onError; + pCol = &p->aCol[p->nCol-1]; + pCol->notNull = (u8)onError; p->tabFlags |= TF_HasNotNull; + + /* Set the uniqNotNull flag on any UNIQUE or PK indexes already created + ** on this column. */ + if( pCol->colFlags & COLFLAG_UNIQUE ){ + Index *pIdx; + for(pIdx=p->pIndex; pIdx; pIdx=pIdx->pNext){ + assert( pIdx->nKeyCol==1 && pIdx->onError!=OE_None ); + if( pIdx->aiColumn[0]==p->nCol-1 ){ + pIdx->uniqNotNull = 1; + } + } + } } /* @@ -3085,7 +3099,9 @@ void sqlite3CreateIndex( */ if( pList==0 ){ Token prevCol; - sqlite3TokenInit(&prevCol, pTab->aCol[pTab->nCol-1].zName); + Column *pCol = &pTab->aCol[pTab->nCol-1]; + pCol->colFlags |= COLFLAG_UNIQUE; + sqlite3TokenInit(&prevCol, pCol->zName); pList = sqlite3ExprListAppend(pParse, 0, sqlite3ExprAlloc(db, TK_ID, &prevCol, 0)); if( pList==0 ) goto exit_create_index; diff --git a/src/expr.c b/src/expr.c index 1b87734923..d2fad0ffc3 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1732,6 +1732,34 @@ int sqlite3SelectWalkFail(Walker *pWalker, Select *NotUsed){ return WRC_Abort; } +/* +** If the input expression is an ID with the name "true" or "false" +** then convert it into an TK_TRUEFALSE term. Return non-zero if +** the conversion happened, and zero if the expression is unaltered. +*/ +int sqlite3ExprIdToTrueFalse(Expr *pExpr){ + assert( pExpr->op==TK_ID || pExpr->op==TK_STRING ); + if( sqlite3StrICmp(pExpr->u.zToken, "true")==0 + || sqlite3StrICmp(pExpr->u.zToken, "false")==0 + ){ + pExpr->op = TK_TRUEFALSE; + return 1; + } + return 0; +} + +/* +** The argument must be a TK_TRUEFALSE Expr node. Return 1 if it is TRUE +** and 0 if it is FALSE. +*/ +int sqlite3ExprTruthValue(const Expr *pExpr){ + assert( pExpr->op==TK_TRUEFALSE ); + assert( sqlite3StrICmp(pExpr->u.zToken,"true")==0 + || sqlite3StrICmp(pExpr->u.zToken,"false")==0 ); + return pExpr->u.zToken[4]==0; +} + + /* ** These routines are Walker callbacks used to check expressions to ** see if they are "constant" for some definition of constant. The @@ -1779,6 +1807,12 @@ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){ return WRC_Abort; } case TK_ID: + /* Convert "true" or "false" in a DEFAULT clause into the + ** appropriate TK_TRUEFALSE operator */ + if( sqlite3ExprIdToTrueFalse(pExpr) ){ + return WRC_Prune; + } + /* Fall thru */ case TK_COLUMN: case TK_AGG_FUNCTION: case TK_AGG_COLUMN: @@ -3543,6 +3577,10 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ codeInteger(pParse, pExpr, 0, target); return target; } + case TK_TRUEFALSE: { + sqlite3VdbeAddOp2(v, OP_Integer, sqlite3ExprTruthValue(pExpr), target); + return target; + } #ifndef SQLITE_OMIT_FLOATING_POINT case TK_FLOAT: { assert( !ExprHasProperty(pExpr, EP_IntValue) ); @@ -3698,6 +3736,18 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ sqlite3VdbeAddOp2(v, op, r1, inReg); break; } + case TK_TRUTH: { + int isTrue; /* IS TRUE or IS NOT TRUE */ + int bNormal; /* IS TRUE or IS FALSE */ + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + testcase( regFree1==0 ); + isTrue = sqlite3ExprTruthValue(pExpr->pRight); + bNormal = pExpr->op2==TK_IS; + testcase( isTrue && bNormal); + testcase( !isTrue && bNormal); + sqlite3VdbeAddOp4Int(v, OP_IsTrue, r1, inReg, !isTrue, isTrue ^ bNormal); + break; + } case TK_ISNULL: case TK_NOTNULL: { int addr; @@ -4473,6 +4523,23 @@ void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ sqlite3ExprIfFalse(pParse, pExpr->pLeft, dest, jumpIfNull); break; } + case TK_TRUTH: { + int isNot; /* IS NOT TRUE or IS NOT FALSE */ + int isTrue; /* IS TRUE or IS NOT TRUE */ + testcase( jumpIfNull==0 ); + isNot = pExpr->op2==TK_ISNOT; + isTrue = sqlite3ExprTruthValue(pExpr->pRight); + testcase( isTrue && isNot ); + testcase( !isTrue && isNot ); + if( isTrue ^ isNot ){ + sqlite3ExprIfTrue(pParse, pExpr->pLeft, dest, + isNot ? SQLITE_JUMPIFNULL : 0); + }else{ + sqlite3ExprIfFalse(pParse, pExpr->pLeft, dest, + isNot ? SQLITE_JUMPIFNULL : 0); + } + break; + } case TK_IS: case TK_ISNOT: testcase( op==TK_IS ); @@ -4627,6 +4694,26 @@ void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ sqlite3ExprIfTrue(pParse, pExpr->pLeft, dest, jumpIfNull); break; } + case TK_TRUTH: { + int isNot; /* IS NOT TRUE or IS NOT FALSE */ + int isTrue; /* IS TRUE or IS NOT TRUE */ + testcase( jumpIfNull==0 ); + isNot = pExpr->op2==TK_ISNOT; + isTrue = sqlite3ExprTruthValue(pExpr->pRight); + testcase( isTrue && isNot ); + testcase( !isTrue && isNot ); + if( isTrue ^ isNot ){ + /* IS TRUE and IS NOT FALSE */ + sqlite3ExprIfFalse(pParse, pExpr->pLeft, dest, + isNot ? 0 : SQLITE_JUMPIFNULL); + + }else{ + /* IS FALSE and IS NOT TRUE */ + sqlite3ExprIfTrue(pParse, pExpr->pLeft, dest, + isNot ? 0 : SQLITE_JUMPIFNULL); + } + break; + } case TK_IS: case TK_ISNOT: testcase( pExpr->op==TK_IS ); diff --git a/src/func.c b/src/func.c index 4e80535e81..deb7e74928 100644 --- a/src/func.c +++ b/src/func.c @@ -1199,6 +1199,8 @@ static void replaceFunc( i64 nOut; /* Maximum size of zOut */ int loopLimit; /* Last zStr[] that might match zPattern[] */ int i, j; /* Loop counters */ + unsigned cntExpand; /* Number zOut expansions */ + sqlite3 *db = sqlite3_context_db_handle(context); assert( argc==3 ); UNUSED_PARAMETER(argc); @@ -1230,33 +1232,40 @@ static void replaceFunc( return; } loopLimit = nStr - nPattern; + cntExpand = 0; for(i=j=0; i<=loopLimit; i++){ if( zStr[i]!=zPattern[0] || memcmp(&zStr[i], zPattern, nPattern) ){ zOut[j++] = zStr[i]; }else{ - u8 *zOld; - sqlite3 *db = sqlite3_context_db_handle(context); - nOut += nRep - nPattern; - testcase( nOut-1==db->aLimit[SQLITE_LIMIT_LENGTH] ); - testcase( nOut-2==db->aLimit[SQLITE_LIMIT_LENGTH] ); - if( nOut-1>db->aLimit[SQLITE_LIMIT_LENGTH] ){ - sqlite3_result_error_toobig(context); - sqlite3_free(zOut); - return; - } - zOld = zOut; - zOut = sqlite3_realloc64(zOut, (int)nOut); - if( zOut==0 ){ - sqlite3_result_error_nomem(context); - sqlite3_free(zOld); - return; + if( nRep>nPattern ){ + nOut += nRep - nPattern; + testcase( nOut-1==db->aLimit[SQLITE_LIMIT_LENGTH] ); + testcase( nOut-2==db->aLimit[SQLITE_LIMIT_LENGTH] ); + if( nOut-1>db->aLimit[SQLITE_LIMIT_LENGTH] ){ + sqlite3_result_error_toobig(context); + sqlite3_free(zOut); + return; + } + cntExpand++; + if( (cntExpand&(cntExpand-1))==0 ){ + /* Grow the size of the output buffer only on substitutions + ** whose index is a power of two: 1, 2, 4, 8, 16, 32, ... */ + u8 *zOld; + zOld = zOut; + zOut = sqlite3_realloc64(zOut, (int)nOut + (nOut - nStr - 1)); + if( zOut==0 ){ + sqlite3_result_error_nomem(context); + sqlite3_free(zOld); + return; + } + } } memcpy(&zOut[j], zRep, nRep); j += nRep; i += nPattern-1; } } - assert( j+nStr-i+1==nOut ); + assert( j+nStr-i+1<=nOut ); memcpy(&zOut[j], &zStr[i], nStr-i); j += nStr - i; assert( j<=nOut ); diff --git a/src/global.c b/src/global.c index e66be81495..04a3d185a8 100644 --- a/src/global.c +++ b/src/global.c @@ -258,6 +258,13 @@ const Token sqlite3IntTokens[] = { { "1", 1 } }; +#ifdef VDBE_PROFILE +/* +** The following performance counter can be used in place of +** sqlite3Hwtime() for profiling. This is a no-op on standard builds. +*/ +sqlite3_uint64 sqlite3NProfileCnt = 0; +#endif /* ** The value of the "pending" byte must be 0x40000000 (1 byte past the diff --git a/src/main.c b/src/main.c index cf2be6022f..2fef116868 100644 --- a/src/main.c +++ b/src/main.c @@ -1443,6 +1443,8 @@ const char *sqlite3ErrStr(int rc){ /* SQLITE_FORMAT */ 0, /* SQLITE_RANGE */ "column index out of range", /* SQLITE_NOTADB */ "file is not a database", + /* SQLITE_NOTICE */ "notification message", + /* SQLITE_WARNING */ "warning message", }; const char *zErr = "unknown error"; switch( rc ){ @@ -1450,6 +1452,14 @@ const char *sqlite3ErrStr(int rc){ zErr = "abort due to ROLLBACK"; break; } + case SQLITE_ROW: { + zErr = "another row available"; + break; + } + case SQLITE_DONE: { + zErr = "no more rows available"; + break; + } default: { rc &= 0xff; if( ALWAYS(rc>=0) && rcmutex, &recursiveAttr); pthread_mutexattr_destroy(&recursiveAttr); +#endif +#if SQLITE_MUTEX_NREF || defined(SQLITE_ENABLE_API_ARMOR) + p->id = SQLITE_MUTEX_RECURSIVE; #endif } break; @@ -188,6 +192,9 @@ static sqlite3_mutex *pthreadMutexAlloc(int iType){ p = sqlite3MallocZero( sizeof(*p) ); if( p ){ pthread_mutex_init(&p->mutex, 0); +#if SQLITE_MUTEX_NREF || defined(SQLITE_ENABLE_API_ARMOR) + p->id = SQLITE_MUTEX_FAST; +#endif } break; } @@ -203,7 +210,7 @@ static sqlite3_mutex *pthreadMutexAlloc(int iType){ } } #if SQLITE_MUTEX_NREF || defined(SQLITE_ENABLE_API_ARMOR) - if( p ) p->id = iType; + assert( p==0 || p->id==iType ); #endif return p; } diff --git a/src/mutex_w32.c b/src/mutex_w32.c index 9da93cf319..8a8ae289ba 100644 --- a/src/mutex_w32.c +++ b/src/mutex_w32.c @@ -40,7 +40,7 @@ struct sqlite3_mutex { #ifdef SQLITE_DEBUG volatile int nRef; /* Number of enterances */ volatile DWORD owner; /* Thread holding this mutex */ - volatile int trace; /* True to trace changes */ + volatile LONG trace; /* True to trace changes */ #endif }; @@ -52,10 +52,10 @@ struct sqlite3_mutex { #define SQLITE_W32_MUTEX_INITIALIZER { 0 } #ifdef SQLITE_DEBUG -#define SQLITE3_MUTEX_INITIALIZER { SQLITE_W32_MUTEX_INITIALIZER, 0, \ +#define SQLITE3_MUTEX_INITIALIZER(id) { SQLITE_W32_MUTEX_INITIALIZER, id, \ 0L, (DWORD)0, 0 } #else -#define SQLITE3_MUTEX_INITIALIZER { SQLITE_W32_MUTEX_INITIALIZER, 0 } +#define SQLITE3_MUTEX_INITIALIZER(id) { SQLITE_W32_MUTEX_INITIALIZER, id } #endif #ifdef SQLITE_DEBUG @@ -98,18 +98,18 @@ void sqlite3MemoryBarrier(void){ ** Initialize and deinitialize the mutex subsystem. */ static sqlite3_mutex winMutex_staticMutexes[] = { - SQLITE3_MUTEX_INITIALIZER, - SQLITE3_MUTEX_INITIALIZER, - SQLITE3_MUTEX_INITIALIZER, - SQLITE3_MUTEX_INITIALIZER, - SQLITE3_MUTEX_INITIALIZER, - SQLITE3_MUTEX_INITIALIZER, - SQLITE3_MUTEX_INITIALIZER, - SQLITE3_MUTEX_INITIALIZER, - SQLITE3_MUTEX_INITIALIZER, - SQLITE3_MUTEX_INITIALIZER, - SQLITE3_MUTEX_INITIALIZER, - SQLITE3_MUTEX_INITIALIZER + SQLITE3_MUTEX_INITIALIZER(2), + SQLITE3_MUTEX_INITIALIZER(3), + SQLITE3_MUTEX_INITIALIZER(4), + SQLITE3_MUTEX_INITIALIZER(5), + SQLITE3_MUTEX_INITIALIZER(6), + SQLITE3_MUTEX_INITIALIZER(7), + SQLITE3_MUTEX_INITIALIZER(8), + SQLITE3_MUTEX_INITIALIZER(9), + SQLITE3_MUTEX_INITIALIZER(10), + SQLITE3_MUTEX_INITIALIZER(11), + SQLITE3_MUTEX_INITIALIZER(12), + SQLITE3_MUTEX_INITIALIZER(13) }; static int winMutex_isInit = 0; @@ -239,15 +239,15 @@ static sqlite3_mutex *winMutexAlloc(int iType){ } #endif p = &winMutex_staticMutexes[iType-2]; - p->id = iType; #ifdef SQLITE_DEBUG #ifdef SQLITE_WIN32_MUTEX_TRACE_STATIC - p->trace = 1; + InterlockedCompareExchange(&p->trace, 1, 0); #endif #endif break; } } + assert( p==0 || p->id==iType ); return p; } diff --git a/src/os_unix.c b/src/os_unix.c index 94b1efd87d..5a1d396ce2 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -468,7 +468,11 @@ static struct unix_syscall { #endif #define osFchown ((int(*)(int,uid_t,gid_t))aSyscall[20].pCurrent) +#if defined(HAVE_FCHOWN) { "geteuid", (sqlite3_syscall_ptr)geteuid, 0 }, +#else + { "geteuid", (sqlite3_syscall_ptr)0, 0 }, +#endif #define osGeteuid ((uid_t(*)(void))aSyscall[21].pCurrent) #if !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 @@ -696,15 +700,16 @@ static int robust_open(const char *z, int f, mode_t m){ ** assert( unixMutexHeld() ); ** unixEnterLeave() */ +static sqlite3_mutex *unixBigLock = 0; static void unixEnterMutex(void){ - sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1)); + sqlite3_mutex_enter(unixBigLock); } static void unixLeaveMutex(void){ - sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1)); + sqlite3_mutex_leave(unixBigLock); } #ifdef SQLITE_DEBUG static int unixMutexHeld(void) { - return sqlite3_mutex_held(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1)); + return sqlite3_mutex_held(unixBigLock); } #endif @@ -4175,12 +4180,10 @@ static int unixShmSystemLock( if( pShmNode->h>=0 ){ /* Initialize the locking parameters */ - memset(&f, 0, sizeof(f)); f.l_type = lockType; f.l_whence = SEEK_SET; f.l_start = ofst; f.l_len = n; - rc = osFcntl(pShmNode->h, F_SETLK, &f); rc = (rc!=(-1)) ? SQLITE_OK : SQLITE_BUSY; } @@ -5846,7 +5849,6 @@ static int unixOpen( randomnessPid = osGetpid(0); sqlite3_randomness(0,0); } - memset(p, 0, sizeof(unixFile)); if( eType==SQLITE_OPEN_MAIN_DB ){ @@ -7721,6 +7723,7 @@ int sqlite3_os_init(void){ for(i=0; i<(sizeof(aVfs)/sizeof(sqlite3_vfs)); i++){ sqlite3_vfs_register(&aVfs[i], i==0); } + unixBigLock = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1); return SQLITE_OK; } @@ -7732,6 +7735,7 @@ int sqlite3_os_init(void){ ** This routine is a no-op for unix. */ int sqlite3_os_end(void){ + unixBigLock = 0; return SQLITE_OK; } diff --git a/src/os_win.c b/src/os_win.c index 2b2b8ebd56..534426977f 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -3631,15 +3631,16 @@ static SYSTEM_INFO winSysInfo; ** assert( winShmMutexHeld() ); ** winShmLeaveMutex() */ +static sqlite3_mutex *winBigLock = 0; static void winShmEnterMutex(void){ - sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1)); + sqlite3_mutex_enter(winBigLock); } static void winShmLeaveMutex(void){ - sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1)); + sqlite3_mutex_leave(winBigLock); } #ifndef NDEBUG static int winShmMutexHeld(void) { - return sqlite3_mutex_held(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1)); + return sqlite3_mutex_held(winBigLock); } #endif @@ -6062,6 +6063,10 @@ int sqlite3_os_init(void){ sqlite3_vfs_register(&winLongPathNolockVfs, 0); #endif +#ifndef SQLITE_OMIT_WAL + winBigLock = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1); +#endif + return SQLITE_OK; } @@ -6072,6 +6077,11 @@ int sqlite3_os_end(void){ sleepObj = NULL; } #endif + +#ifndef SQLITE_OMIT_WAL + winBigLock = 0; +#endif + return SQLITE_OK; } diff --git a/src/pager.c b/src/pager.c index 14237f6bed..175c94ddd8 100644 --- a/src/pager.c +++ b/src/pager.c @@ -4102,6 +4102,30 @@ static void pagerFreeMapHdrs(Pager *pPager){ } } +/* Verify that the database file has not be deleted or renamed out from +** under the pager. Return SQLITE_OK if the database is still where it ought +** to be on disk. Return non-zero (SQLITE_READONLY_DBMOVED or some other error +** code from sqlite3OsAccess()) if the database has gone missing. +*/ +static int databaseIsUnmoved(Pager *pPager){ + int bHasMoved = 0; + int rc; + + if( pPager->tempFile ) return SQLITE_OK; + if( pPager->dbSize==0 ) return SQLITE_OK; + assert( pPager->zFilename && pPager->zFilename[0] ); + rc = sqlite3OsFileControl(pPager->fd, SQLITE_FCNTL_HAS_MOVED, &bHasMoved); + if( rc==SQLITE_NOTFOUND ){ + /* If the HAS_MOVED file-control is unimplemented, assume that the file + ** has not been moved. That is the historical behavior of SQLite: prior to + ** version 3.8.3, it never checked */ + rc = SQLITE_OK; + }else if( rc==SQLITE_OK && bHasMoved ){ + rc = SQLITE_READONLY_DBMOVED; + } + return rc; +} + /* ** Shutdown the page cache. Free all memory and close all files. @@ -4118,8 +4142,7 @@ static void pagerFreeMapHdrs(Pager *pPager){ ** to the caller. */ int sqlite3PagerClose(Pager *pPager, sqlite3 *db){ - u8 *pTmp = (u8 *)pPager->pTmpSpace; - + u8 *pTmp = (u8*)pPager->pTmpSpace; assert( db || pagerUseWal(pPager)==0 ); assert( assert_pager_state(pPager) ); disable_simulated_io_errors(); @@ -4128,11 +4151,17 @@ int sqlite3PagerClose(Pager *pPager, sqlite3 *db){ /* pPager->errCode = 0; */ pPager->exclusiveMode = 0; #ifndef SQLITE_OMIT_WAL - assert( db || pPager->pWal==0 ); - sqlite3WalClose(pPager->pWal, db, pPager->walSyncFlags, pPager->pageSize, - (db && (db->flags & SQLITE_NoCkptOnClose) ? 0 : pTmp) - ); - pPager->pWal = 0; + { + u8 *a = 0; + assert( db || pPager->pWal==0 ); + if( db && 0==(db->flags & SQLITE_NoCkptOnClose) + && SQLITE_OK==databaseIsUnmoved(pPager) + ){ + a = pTmp; + } + sqlite3WalClose(pPager->pWal, db, pPager->walSyncFlags, pPager->pageSize,a); + pPager->pWal = 0; + } #endif pager_reset(pPager); if( MEMDB ){ @@ -4969,30 +4998,6 @@ act_like_temp_file: } -/* Verify that the database file has not be deleted or renamed out from -** under the pager. Return SQLITE_OK if the database is still were it ought -** to be on disk. Return non-zero (SQLITE_READONLY_DBMOVED or some other error -** code from sqlite3OsAccess()) if the database has gone missing. -*/ -static int databaseIsUnmoved(Pager *pPager){ - int bHasMoved = 0; - int rc; - - if( pPager->tempFile ) return SQLITE_OK; - if( pPager->dbSize==0 ) return SQLITE_OK; - assert( pPager->zFilename && pPager->zFilename[0] ); - rc = sqlite3OsFileControl(pPager->fd, SQLITE_FCNTL_HAS_MOVED, &bHasMoved); - if( rc==SQLITE_NOTFOUND ){ - /* If the HAS_MOVED file-control is unimplemented, assume that the file - ** has not been moved. That is the historical behavior of SQLite: prior to - ** version 3.8.3, it never checked */ - rc = SQLITE_OK; - }else if( rc==SQLITE_OK && bHasMoved ){ - rc = SQLITE_READONLY_DBMOVED; - } - return rc; -} - /* ** This function is called after transitioning from PAGER_UNLOCK to diff --git a/src/parse.y b/src/parse.y index d9cf1cb87d..9c41484ab2 100644 --- a/src/parse.y +++ b/src/parse.y @@ -313,6 +313,10 @@ ccons ::= DEFAULT MINUS(A) term(X) scanpt(Z). { } ccons ::= DEFAULT scanpt id(X). { Expr *p = tokenExpr(pParse, TK_STRING, X); + if( p ){ + sqlite3ExprIdToTrueFalse(p); + testcase( p->op==TK_TRUEFALSE && sqlite3ExprTruthValue(p) ); + } sqlite3AddDefaultValue(pParse,p,X.z,X.z+X.n); } diff --git a/src/printf.c b/src/printf.c index ca8d26e4f9..fcbd7fdbe0 100644 --- a/src/printf.c +++ b/src/printf.c @@ -206,6 +206,11 @@ void sqlite3VXPrintf( PrintfArguments *pArgList = 0; /* Arguments for SQLITE_PRINTF_SQLFUNC */ char buf[etBUFSIZE]; /* Conversion buffer */ + /* pAccum never starts out with an empty buffer that was obtained from + ** malloc(). This precondition is required by the mprintf("%z...") + ** optimization. */ + assert( pAccum->nChar>0 || (pAccum->printfFlags&SQLITE_PRINTF_MALLOCED)==0 ); + bufpt = 0; if( (pAccum->printfFlags & SQLITE_PRINTF_SQLFUNC)!=0 ){ pArgList = va_arg(ap, PrintfArguments*); @@ -624,9 +629,38 @@ void sqlite3VXPrintf( case etCHARX: if( bArgList ){ bufpt = getTextArg(pArgList); - c = bufpt ? bufpt[0] : 0; + length = 1; + if( bufpt ){ + buf[0] = c = *(bufpt++); + if( (c&0xc0)==0xc0 ){ + while( length<4 && (bufpt[0]&0xc0)==0x80 ){ + buf[length++] = *(bufpt++); + } + } + }else{ + buf[0] = 0; + } }else{ - c = va_arg(ap,int); + unsigned int ch = va_arg(ap,unsigned int); + if( ch<0x00080 ){ + buf[0] = ch & 0xff; + length = 1; + }else if( ch<0x00800 ){ + buf[0] = 0xc0 + (u8)((ch>>6)&0x1f); + buf[1] = 0x80 + (u8)(ch & 0x3f); + length = 2; + }else if( ch<0x10000 ){ + buf[0] = 0xe0 + (u8)((ch>>12)&0x0f); + buf[1] = 0x80 + (u8)((ch>>6) & 0x3f); + buf[2] = 0x80 + (u8)(ch & 0x3f); + length = 3; + }else{ + buf[0] = 0xf0 + (u8)((ch>>18) & 0x07); + buf[1] = 0x80 + (u8)((ch>>12) & 0x3f); + buf[2] = 0x80 + (u8)((ch>>6) & 0x3f); + buf[3] = 0x80 + (u8)(ch & 0x3f); + length = 4; + } } if( precision>1 ){ width -= precision-1; @@ -634,12 +668,13 @@ void sqlite3VXPrintf( sqlite3AppendChar(pAccum, width-1, ' '); width = 0; } - sqlite3AppendChar(pAccum, precision-1, c); + while( precision-- > 1 ){ + sqlite3StrAccumAppend(pAccum, buf, length); + } } - length = 1; - buf[0] = c; bufpt = buf; - break; + flag_altform2 = 1; + goto adjust_width_for_utf8; case etSTRING: case etDYNSTRING: if( bArgList ){ @@ -651,17 +686,45 @@ void sqlite3VXPrintf( if( bufpt==0 ){ bufpt = ""; }else if( xtype==etDYNSTRING ){ + if( pAccum->nChar==0 && pAccum->mxAlloc && width==0 && precision<0 ){ + /* Special optimization for sqlite3_mprintf("%z..."): + ** Extend an existing memory allocation rather than creating + ** a new one. */ + assert( (pAccum->printfFlags&SQLITE_PRINTF_MALLOCED)==0 ); + pAccum->zText = bufpt; + pAccum->nAlloc = sqlite3DbMallocSize(pAccum->db, bufpt); + pAccum->nChar = 0x7fffffff & (int)strlen(bufpt); + pAccum->printfFlags |= SQLITE_PRINTF_MALLOCED; + length = 0; + break; + } zExtra = bufpt; } if( precision>=0 ){ - for(length=0; length 0 && z[0] ){ + SQLITE_SKIP_UTF8(z); + } + length = (int)(z - (unsigned char*)bufpt); + }else{ + for(length=0; length0 ){ + /* Adjust width to account for extra bytes in UTF-8 characters */ + int ii = length - 1; + while( ii>=0 ) if( (bufpt[ii--] & 0xc0)==0x80 ) width++; + } break; - case etSQLESCAPE: /* Escape ' characters */ - case etSQLESCAPE2: /* Escape ' and enclose in '...' */ - case etSQLESCAPE3: { /* Escape " characters */ + case etSQLESCAPE: /* %q: Escape ' characters */ + case etSQLESCAPE2: /* %Q: Escape ' and enclose in '...' */ + case etSQLESCAPE3: { /* %w: Escape " characters */ int i, j, k, n, isnull; int needQuote; char ch; @@ -675,9 +738,17 @@ void sqlite3VXPrintf( } isnull = escarg==0; if( isnull ) escarg = (xtype==etSQLESCAPE2 ? "NULL" : "(NULL)"); + /* For %q, %Q, and %w, the precision is the number of byte (or + ** characters if the ! flags is present) to use from the input. + ** Because of the extra quoting characters inserted, the number + ** of output characters may be larger than the precision. + */ k = precision; for(i=n=0; k!=0 && (ch=escarg[i])!=0; i++, k--){ if( ch==q ) n++; + if( flag_altform2 && (ch&0xc0)==0xc0 ){ + while( (escarg[i+1]&0xc0)==0x80 ){ i++; } + } } needQuote = !isnull && xtype==etSQLESCAPE2; n += i + 3; @@ -700,10 +771,7 @@ void sqlite3VXPrintf( if( needQuote ) bufpt[j++] = q; bufpt[j] = 0; length = j; - /* The precision in %q and %Q means how many input characters to - ** consume, not the length of the output... - ** if( precision>=0 && precision0 ){ diff --git a/src/resolve.c b/src/resolve.c index f735fffa02..7ae49bde57 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -431,10 +431,16 @@ static int lookupName( ** Because no reference was made to outer contexts, the pNC->nRef ** fields are not changed in any context. */ - if( cnt==0 && zTab==0 && ExprHasProperty(pExpr,EP_DblQuoted) ){ - pExpr->op = TK_STRING; - pExpr->pTab = 0; - return WRC_Prune; + if( cnt==0 && zTab==0 ){ + assert( pExpr->op==TK_ID ); + if( ExprHasProperty(pExpr,EP_DblQuoted) ){ + pExpr->op = TK_STRING; + pExpr->pTab = 0; + return WRC_Prune; + } + if( sqlite3ExprIdToTrueFalse(pExpr) ){ + return WRC_Prune; + } } /* @@ -783,15 +789,30 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ notValid(pParse, pNC, "parameters", NC_IsCheck|NC_PartIdx|NC_IdxExpr); break; } + case TK_IS: + case TK_ISNOT: { + Expr *pRight; + assert( !ExprHasProperty(pExpr, EP_Reduced) ); + /* Handle special cases of "x IS TRUE", "x IS FALSE", "x IS NOT TRUE", + ** and "x IS NOT FALSE". */ + if( (pRight = pExpr->pRight)->op==TK_ID ){ + int rc = resolveExprStep(pWalker, pRight); + if( rc==WRC_Abort ) return WRC_Abort; + if( pRight->op==TK_TRUEFALSE ){ + pExpr->op2 = pExpr->op; + pExpr->op = TK_TRUTH; + return WRC_Continue; + } + } + /* Fall thru */ + } case TK_BETWEEN: case TK_EQ: case TK_NE: case TK_LT: case TK_LE: case TK_GT: - case TK_GE: - case TK_IS: - case TK_ISNOT: { + case TK_GE: { int nLeft, nRight; if( pParse->db->mallocFailed ) break; assert( pExpr->pLeft!=0 ); diff --git a/src/select.c b/src/select.c index c3cb4082fe..9ac4f9f198 100644 --- a/src/select.c +++ b/src/select.c @@ -1266,12 +1266,15 @@ static void generateSortTail( iSortTab = iTab; bSeq = 1; } - for(i=0, iCol=nKey+bSeq; i=0; i--){ int iRead; if( aOutEx[i].u.x.iOrderByCol ){ iRead = aOutEx[i].u.x.iOrderByCol-1; }else{ - iRead = iCol++; + iRead = iCol--; } sqlite3VdbeAddOp3(v, OP_Column, iSortTab, iRead, regRow+i); VdbeComment((v, "%s", aOutEx[i].zName ? aOutEx[i].zName : aOutEx[i].zSpan)); diff --git a/src/shell.c.in b/src/shell.c.in index a785517503..121a6c3506 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -1069,6 +1069,7 @@ struct ShellState { #define SHELL_OPEN_NORMAL 1 /* Normal database file */ #define SHELL_OPEN_APPENDVFS 2 /* Use appendvfs */ #define SHELL_OPEN_ZIPFILE 3 /* Use the zipfile virtual table */ +#define SHELL_OPEN_READONLY 4 /* Open a normal database read-only */ /* ** These are the allowed shellFlgs values @@ -1175,6 +1176,7 @@ static void shellPutsFunc( ** ** Also throw an error if the EDITOR program returns a non-zero exit code. */ +#ifndef SQLITE_NOHAVE_SYSTEM static void editFunc( sqlite3_context *context, int argc, @@ -1285,6 +1287,7 @@ edit_func_end: sqlite3_free(zTempFile); sqlite3_free(p); } +#endif /* SQLITE_NOHAVE_SYSTEM */ /* ** Save or restore the current output mode @@ -3256,6 +3259,7 @@ static char zHelp[] = " on the output.\n" ".open ?OPTIONS? ?FILE? Close existing database and reopen FILE\n" " The --new option starts with an empty file\n" + " Other options: --readonly --append --zip\n" ".output ?FILE? Send output to FILE or stdout\n" ".print STRING... Print literal STRING\n" ".prompt MAIN CONTINUE Replace the standard prompts\n" @@ -3273,10 +3277,14 @@ static char zHelp[] = ".session CMD ... Create or control sessions\n" #endif ".sha3sum ?OPTIONS...? Compute a SHA3 hash of database content\n" +#ifndef SQLITE_NOHAVE_SYSTEM ".shell CMD ARGS... Run CMD ARGS... in a system shell\n" +#endif ".show Show the current values for various settings\n" ".stats ?on|off? Show stats or turn stats on or off\n" +#ifndef SQLITE_NOHAVE_SYSTEM ".system CMD ARGS... Run CMD ARGS... in a system shell\n" +#endif ".tables ?TABLE? List names of tables\n" " If TABLE specified, only list tables matching\n" " LIKE pattern TABLE.\n" @@ -3448,6 +3456,10 @@ static void open_db(ShellState *p, int keepAlive){ sqlite3_open(":memory:", &p->db); break; } + case SHELL_OPEN_READONLY: { + sqlite3_open_v2(p->zDbFilename, &p->db, SQLITE_OPEN_READONLY, 0); + break; + } case SHELL_OPEN_UNSPEC: case SHELL_OPEN_NORMAL: { sqlite3_open(p->zDbFilename, &p->db); @@ -3477,10 +3489,12 @@ static void open_db(ShellState *p, int keepAlive){ shellModuleSchema, 0, 0); sqlite3_create_function(p->db, "shell_putsnl", 1, SQLITE_UTF8, p, shellPutsFunc, 0, 0); +#ifndef SQLITE_NOHAVE_SYSTEM sqlite3_create_function(p->db, "edit", 1, SQLITE_UTF8, 0, editFunc, 0, 0); sqlite3_create_function(p->db, "edit", 2, SQLITE_UTF8, 0, editFunc, 0, 0); +#endif if( p->openMode==SHELL_OPEN_ZIPFILE ){ char *zSql = sqlite3_mprintf( "CREATE VIRTUAL TABLE zip USING zipfile(%Q);", p->zDbFilename); @@ -4095,6 +4109,7 @@ static void output_reset(ShellState *p){ #endif }else{ output_file_close(p->out); +#ifndef SQLITE_NOHAVE_SYSTEM if( p->doXdgOpen ){ const char *zXdgOpenCmd = #if defined(_WIN32) @@ -4113,6 +4128,7 @@ static void output_reset(ShellState *p){ outputModePop(p); p->doXdgOpen = 0; } +#endif /* !defined(SQLITE_NOHAVE_SYSTEM) */ } p->outfile[0] = 0; p->out = stdout; @@ -6297,6 +6313,8 @@ static int do_meta_command(char *zLine, ShellState *p){ #endif }else if( optionMatch(z, "append") ){ p->openMode = SHELL_OPEN_APPENDVFS; + }else if( optionMatch(z, "readonly") ){ + p->openMode = SHELL_OPEN_READONLY; }else if( z[0]=='-' ){ utf8_printf(stderr, "unknown option: %s\n", z); rc = 1; @@ -6353,6 +6371,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } output_reset(p); if( zFile[0]=='-' && zFile[1]=='-' ) zFile++; +#ifndef SQLITE_NOHAVE_SYSTEM if( strcmp(zFile, "-e")==0 || strcmp(zFile, "-x")==0 ){ p->doXdgOpen = 1; outputModePush(p); @@ -6367,6 +6386,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } zFile = p->zTempFile; } +#endif /* SQLITE_NOHAVE_SYSTEM */ if( zFile[0]=='|' ){ #ifdef SQLITE_OMIT_POPEN raw_printf(stderr, "Error: pipes are not supported in this OS\n"); @@ -6486,7 +6506,6 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3_close(pSrc); }else - if( c=='s' && strncmp(azArg[0], "scanstats", n)==0 ){ if( nArg==2 ){ p->scanstatsOn = (u8)booleanValue(azArg[1]); @@ -7082,6 +7101,7 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3_free(zSql); }else +#ifndef SQLITE_NOHAVE_SYSTEM if( c=='s' && (strncmp(azArg[0], "shell", n)==0 || strncmp(azArg[0],"system",n)==0) ){ @@ -7101,6 +7121,7 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3_free(zCmd); if( x ) raw_printf(stderr, "System command returns %d\n", x); }else +#endif /* !defined(SQLITE_NOHAVE_SYSTEM) */ if( c=='s' && strncmp(azArg[0], "show", n)==0 ){ static const char *azBool[] = { "off", "on", "trigger", "full"}; @@ -7972,6 +7993,7 @@ static const char zOptions[] = " -nullvalue TEXT set text string for NULL values. Default ''\n" " -pagecache SIZE N use N slots of SZ bytes each for page cache memory\n" " -quote set output mode to 'quote'\n" + " -readonly open the database read-only\n" " -separator SEP set output column separator. Default: '|'\n" " -stats print memory stats before each finalize\n" " -version show SQLite version\n" @@ -8081,21 +8103,39 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ } #endif main_init(&data); + + /* On Windows, we must translate command-line arguments into UTF-8. + ** The SQLite memory allocator subsystem has to be enabled in order to + ** do this. But we want to run an sqlite3_shutdown() afterwards so that + ** subsequent sqlite3_config() calls will work. So copy all results into + ** memory that does not come from the SQLite memory allocator. + */ #if !SQLITE_SHELL_IS_UTF8 sqlite3_initialize(); - argv = sqlite3_malloc64(sizeof(argv[0])*argc); + argv = malloc(sizeof(argv[0])*argc); if( argv==0 ){ raw_printf(stderr, "out of memory\n"); exit(1); } for(i=0; i=1 && argv && argv[0] ); Argv0 = argv[0]; @@ -8221,6 +8261,8 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ #endif }else if( strcmp(z,"-append")==0 ){ data.openMode = SHELL_OPEN_APPENDVFS; + }else if( strcmp(z,"-readonly")==0 ){ + data.openMode = SHELL_OPEN_READONLY; } } if( data.zDbFilename==0 ){ @@ -8448,8 +8490,8 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ data.doXdgOpen = 0; clearTempFile(&data); #if !SQLITE_SHELL_IS_UTF8 - for(i=0; i
-**  char *zText = "It's a happy day!";
-**
-** -** One can use this text in an SQL statement as follows: -** -** 
-**  char *zSQL = sqlite3_mprintf("INSERT INTO table VALUES('%q')", zText);
-**  sqlite3_exec(db, zSQL, 0, 0, 0);
-**  sqlite3_free(zSQL);
-**
-** -** Because the %q format string is used, the '\'' character in zText -** is escaped and the SQL generated is as follows: -** -** 
-**  INSERT INTO table1 VALUES('It''s a happy day!')
-**
-** -** This is correct. Had we used %s instead of %q, the generated SQL -** would have looked like this: -** -** 
-**  INSERT INTO table1 VALUES('It's a happy day!');
-**
-** -** This second example is an SQL syntax error. As a general rule you should -** always use %q instead of %s when inserting text into a string literal. -** -** ^(The %Q option works like %q except it also adds single quotes around -** the outside of the total string. Additionally, if the parameter in the -** argument list is a NULL pointer, %Q substitutes the text "NULL" (without -** single quotes).)^ So, for example, one could say: -** -** 
-**  char *zSQL = sqlite3_mprintf("INSERT INTO table VALUES(%Q)", zText);
-**  sqlite3_exec(db, zSQL, 0, 0, 0);
-**  sqlite3_free(zSQL);
-**
-** -** The code above will render a correct SQL statement in the zSQL -** variable even if the zText variable is a NULL pointer. -** -** ^(The "%w" formatting option is like "%q" except that it expects to -** be contained within double-quotes instead of single quotes, and it -** escapes the double-quote character instead of the single-quote -** character.)^ The "%w" formatting option is intended for safely inserting -** table and column names into a constructed SQL statement. -** -** ^(The "%z" formatting option works like "%s" but with the -** addition that after the string has been read and copied into -** the result, [sqlite3_free()] is called on the input string.)^ +** See also: [built-in printf()], [printf() SQL function] */ char *sqlite3_mprintf(const char*,...); char *sqlite3_vmprintf(const char*, va_list); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index c2995b89ca..341cfe97a6 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1755,6 +1755,7 @@ struct Column { #define COLFLAG_PRIMKEY 0x0001 /* Column is part of the primary key */ #define COLFLAG_HIDDEN 0x0002 /* A hidden column in a virtual table */ #define COLFLAG_HASTYPE 0x0004 /* Type name follows column name */ +#define COLFLAG_UNIQUE 0x0008 /* Column def contains "UNIQUE" or "PK" */ /* ** A "Collating Sequence" is defined by an instance of the following @@ -3839,6 +3840,8 @@ void sqlite3EndTransaction(Parse*,int); void sqlite3Savepoint(Parse*, int, Token*); void sqlite3CloseSavepoints(sqlite3 *); void sqlite3LeaveMutexAndCloseZombie(sqlite3*); +int sqlite3ExprIdToTrueFalse(Expr*); +int sqlite3ExprTruthValue(const Expr*); int sqlite3ExprIsConstant(Expr*); int sqlite3ExprIsConstantNotJoin(Expr*); int sqlite3ExprIsConstantOrFunction(Expr*, u8); @@ -4073,6 +4076,9 @@ extern FuncDefHash sqlite3BuiltinFunctions; extern int sqlite3PendingByte; #endif #endif +#ifdef VDBE_PROFILE +extern sqlite3_uint64 sqlite3NProfileCnt; +#endif void sqlite3RootPageMoved(sqlite3*, int, int, int); void sqlite3Reindex(Parse*, Token*, Token*); void sqlite3AlterFunctions(void); diff --git a/src/test1.c b/src/test1.c index 55d92eb1fd..bc8f389dbd 100644 --- a/src/test1.c +++ b/src/test1.c @@ -4559,6 +4559,35 @@ static int SQLITE_TCLAPI test_complete16( return TCL_OK; } +/* +** Usage: sqlite3_normalize SQL +** +** Return the normalized value for an SQL statement. +*/ +static int SQLITE_TCLAPI test_normalize( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + char *zSql; + char *zNorm; + extern char *sqlite3_normalize(const char*); + + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "SQL"); + return TCL_ERROR; + } + + zSql = (char*)Tcl_GetString(objv[1]); + zNorm = sqlite3_normalize(zSql); + if( zNorm ){ + Tcl_SetObjResult(interp, Tcl_NewStringObj(zNorm, -1)); + sqlite3_free(zNorm); + } + return TCL_OK; +} + /* ** Usage: sqlite3_step STMT ** @@ -7547,6 +7576,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ { "sqlite3_open16", test_open16 ,0 }, { "sqlite3_open_v2", test_open_v2 ,0 }, { "sqlite3_complete16", test_complete16 ,0 }, + { "sqlite3_normalize", test_normalize ,0 }, { "sqlite3_prepare", test_prepare ,0 }, { "sqlite3_prepare16", test_prepare16 ,0 }, diff --git a/src/test_config.c b/src/test_config.c index aeb3393f0b..de693850d3 100644 --- a/src/test_config.c +++ b/src/test_config.c @@ -507,6 +507,12 @@ Tcl_SetVar2(interp, "sqlite_options", "long_double", Tcl_SetVar2(interp, "sqlite_options", "mergesort", "1", TCL_GLOBAL_ONLY); +#ifdef SQLITE_ENABLE_NULL_TRIM + Tcl_SetVar2(interp, "sqlite_options", "null_trim", "1", TCL_GLOBAL_ONLY); +#else + Tcl_SetVar2(interp, "sqlite_options", "null_trim", "0", TCL_GLOBAL_ONLY); +#endif + #ifdef SQLITE_OMIT_OR_OPTIMIZATION Tcl_SetVar2(interp, "sqlite_options", "or_opt", "0", TCL_GLOBAL_ONLY); #else diff --git a/src/treeview.c b/src/treeview.c index 6dd386ffe6..e33fe6e227 100644 --- a/src/treeview.c +++ b/src/treeview.c @@ -292,6 +292,11 @@ void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 moreToFollow){ sqlite3TreeViewLine(pView,"NULL"); break; } + case TK_TRUEFALSE: { + sqlite3TreeViewLine(pView, + sqlite3ExprTruthValue(pExpr) ? "TRUE" : "FALSE"); + break; + } #ifndef SQLITE_OMIT_BLOB_LITERAL case TK_BLOB: { sqlite3TreeViewLine(pView,"%s", pExpr->u.zToken); @@ -348,6 +353,19 @@ void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 moreToFollow){ case TK_ISNULL: zUniOp = "ISNULL"; break; case TK_NOTNULL: zUniOp = "NOTNULL"; break; + case TK_TRUTH: { + int x; + const char *azOp[] = { + "IS-FALSE", "IS-TRUE", "IS-NOT-FALSE", "IS-NOT-TRUE" + }; + assert( pExpr->op2==TK_IS || pExpr->op2==TK_ISNOT ); + assert( pExpr->pRight ); + assert( pExpr->pRight->op==TK_TRUEFALSE ); + x = (pExpr->op2==TK_ISNOT)*2 + sqlite3ExprTruthValue(pExpr->pRight); + zUniOp = azOp[x]; + break; + } + case TK_SPAN: { sqlite3TreeViewLine(pView, "SPAN %Q", pExpr->u.zToken); sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); diff --git a/src/util.c b/src/util.c index 456317cfa9..54f9b93887 100644 --- a/src/util.c +++ b/src/util.c @@ -595,7 +595,7 @@ static int compare2pow63(const char *zNum, int incr){ ** Returns: ** ** 0 Successful transformation. Fits in a 64-bit signed integer. -** 1 Excess text after the integer value +** 1 Excess non-space text after the integer value ** 2 Integer too large for a 64-bit signed integer or is malformed ** 3 Special case of 9223372036854775808 ** @@ -638,47 +638,57 @@ int sqlite3Atoi64(const char *zNum, i64 *pNum, int length, u8 enc){ for(i=0; &zNum[i]='0' && c<='9'; i+=incr){ u = u*10 + c - '0'; } + testcase( i==18*incr ); + testcase( i==19*incr ); + testcase( i==20*incr ); if( u>LARGEST_INT64 ){ + /* This test and assignment is needed only to suppress UB warnings + ** from clang and -fsanitize=undefined. This test and assignment make + ** the code a little larger and slower, and no harm comes from omitting + ** them, but we must appaise the undefined-behavior pharisees. */ *pNum = neg ? SMALLEST_INT64 : LARGEST_INT64; }else if( neg ){ *pNum = -(i64)u; }else{ *pNum = (i64)u; } - testcase( i==18 ); - testcase( i==19 ); - testcase( i==20 ); - if( &zNum[i]19*incr ){ /* Too many digits */ - /* zNum is empty or contains non-numeric text or is longer - ** than 19 digits (thus guaranteeing that it is too large) */ - return 2; - }else if( i<19*incr ){ + if( i<19*incr ){ /* Less than 19 digits, so we know that it fits in 64 bits */ assert( u<=LARGEST_INT64 ); return rc; }else{ /* zNum is a 19-digit numbers. Compare it against 9223372036854775808. */ - c = compare2pow63(zNum, incr); + c = i>19*incr ? 1 : compare2pow63(zNum, incr); if( c<0 ){ /* zNum is less than 9223372036854775808 so it fits */ assert( u<=LARGEST_INT64 ); return rc; - }else if( c>0 ){ - /* zNum is greater than 9223372036854775808 so it overflows */ - return 2; }else{ - /* zNum is exactly 9223372036854775808. Fits if negative. The - ** special case 2 overflow if positive */ - assert( u-1==LARGEST_INT64 ); - return neg ? rc : 3; + *pNum = neg ? SMALLEST_INT64 : LARGEST_INT64; + if( c>0 ){ + /* zNum is greater than 9223372036854775808 so it overflows */ + return 2; + }else{ + /* zNum is exactly 9223372036854775808. Fits if negative. The + ** special case 2 overflow if positive */ + assert( u-1==LARGEST_INT64 ); + return neg ? rc : 3; + } } } } diff --git a/src/vacuum.c b/src/vacuum.c index fde08ddc2a..c7a14a675d 100644 --- a/src/vacuum.c +++ b/src/vacuum.c @@ -39,8 +39,8 @@ static int execSql(sqlite3 *db, char **pzErrMsg, const char *zSql){ while( SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){ const char *zSubSql = (const char*)sqlite3_column_text(pStmt,0); assert( sqlite3_strnicmp(zSql,"SELECT",6)==0 ); - if( zSubSql ){ - assert( zSubSql[0]!='S' ); + assert( sqlite3_strnicmp(zSubSql,"SELECT",6)!=0 || CORRUPT_DB ); + if( zSubSql && zSubSql[0]!='S' ){ rc = execSql(db, pzErrMsg, zSubSql); if( rc!=SQLITE_OK ) break; } diff --git a/src/vdbe.c b/src/vdbe.c index 81a2361a55..10715a8232 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -264,6 +264,11 @@ static void applyNumericAffinity(Mem *pRec, int bTryForInt){ pRec->flags |= MEM_Real; if( bTryForInt ) sqlite3VdbeIntegerAffinity(pRec); } + /* TEXT->NUMERIC is many->one. Hence, it is important to invalidate the + ** string representation after computing a numeric equivalent, because the + ** string representation might not be the canonical representation for the + ** numeric value. Ticket [343634942dd54ab57b7024] 2018-01-31. */ + pRec->flags &= ~MEM_Str; } /* @@ -643,7 +648,7 @@ int sqlite3VdbeExec( assert( pOp>=aOp && pOp<&aOp[p->nOp]); #ifdef VDBE_PROFILE - start = sqlite3Hwtime(); + start = sqlite3NProfileCnt ? sqlite3NProfileCnt : sqlite3Hwtime(); #endif nVmStep++; #ifdef SQLITE_ENABLE_STMT_SCANSTATUS @@ -2167,18 +2172,8 @@ case OP_Or: { /* same as TK_OR, in1, in2, out3 */ int v1; /* Left operand: 0==FALSE, 1==TRUE, 2==UNKNOWN or NULL */ int v2; /* Right operand: 0==FALSE, 1==TRUE, 2==UNKNOWN or NULL */ - pIn1 = &aMem[pOp->p1]; - if( pIn1->flags & MEM_Null ){ - v1 = 2; - }else{ - v1 = sqlite3VdbeIntValue(pIn1)!=0; - } - pIn2 = &aMem[pOp->p2]; - if( pIn2->flags & MEM_Null ){ - v2 = 2; - }else{ - v2 = sqlite3VdbeIntValue(pIn2)!=0; - } + v1 = sqlite3VdbeBooleanValue(&aMem[pOp->p1], 2); + v2 = sqlite3VdbeBooleanValue(&aMem[pOp->p2], 2); if( pOp->opcode==OP_And ){ static const unsigned char and_logic[] = { 0, 0, 0, 0, 1, 2, 0, 2, 2 }; v1 = and_logic[v1*3+v2]; @@ -2196,6 +2191,35 @@ case OP_Or: { /* same as TK_OR, in1, in2, out3 */ break; } +/* Opcode: IsTrue P1 P2 P3 P4 * +** Synopsis: r[P2] = coalesce(r[P1]==TRUE,P3) ^ P4 +** +** This opcode implements the IS TRUE, IS FALSE, IS NOT TRUE, and +** IS NOT FALSE operators. +** +** Interpret the value in register P1 as a boolean value. Store that +** boolean (a 0 or 1) in register P2. Or if the value in register P1 is +** NULL, then the P3 is stored in register P2. Invert the answer if P4 +** is 1. +** +** The logic is summarized like this: +** +**
    +**
  • If P3==0 and P4==0 then r[P2] := r[P1] IS TRUE +**
  • If P3==1 and P4==1 then r[P2] := r[P1] IS FALSE +**
  • If P3==0 and P4==1 then r[P2] := r[P1] IS NOT TRUE +**
  • If P3==1 and P4==0 then r[P2] := r[P1] IS NOT FALSE +**
+*/ +case OP_IsTrue: { /* in1, out2 */ + assert( pOp->p4type==P4_INT32 ); + assert( pOp->p4.i==0 || pOp->p4.i==1 ); + assert( pOp->p3==0 || pOp->p3==1 ); + sqlite3VdbeMemSetInt64(&aMem[pOp->p2], + sqlite3VdbeBooleanValue(&aMem[pOp->p1], pOp->p3) ^ pOp->p4.i); + break; +} + /* Opcode: Not P1 P2 * * * ** Synopsis: r[P2]= !r[P1] ** @@ -2206,10 +2230,10 @@ case OP_Or: { /* same as TK_OR, in1, in2, out3 */ case OP_Not: { /* same as TK_NOT, in1, out2 */ pIn1 = &aMem[pOp->p1]; pOut = &aMem[pOp->p2]; - sqlite3VdbeMemSetNull(pOut); if( (pIn1->flags & MEM_Null)==0 ){ - pOut->flags = MEM_Int; - pOut->u.i = !sqlite3VdbeIntValue(pIn1); + sqlite3VdbeMemSetInt64(pOut, !sqlite3VdbeBooleanValue(pIn1,0)); + }else{ + sqlite3VdbeMemSetNull(pOut); } break; } @@ -2276,30 +2300,25 @@ case OP_Once: { /* jump */ ** is considered true if it is numeric and non-zero. If the value ** in P1 is NULL then take the jump if and only if P3 is non-zero. */ +case OP_If: { /* jump, in1 */ + int c; + c = sqlite3VdbeBooleanValue(&aMem[pOp->p1], pOp->p3); + VdbeBranchTaken(c!=0, 2); + if( c ) goto jump_to_p2; + break; +} + /* Opcode: IfNot P1 P2 P3 * * ** ** Jump to P2 if the value in register P1 is False. The value ** is considered false if it has a numeric value of zero. If the value ** in P1 is NULL then take the jump if and only if P3 is non-zero. */ -case OP_If: /* jump, in1 */ case OP_IfNot: { /* jump, in1 */ int c; - pIn1 = &aMem[pOp->p1]; - if( pIn1->flags & MEM_Null ){ - c = pOp->p3; - }else{ -#ifdef SQLITE_OMIT_FLOATING_POINT - c = sqlite3VdbeIntValue(pIn1)!=0; -#else - c = sqlite3VdbeRealValue(pIn1)!=0.0; -#endif - if( pOp->opcode==OP_IfNot ) c = !c; - } + c = !sqlite3VdbeBooleanValue(&aMem[pOp->p1], !pOp->p3); VdbeBranchTaken(c!=0, 2); - if( c ){ - goto jump_to_p2; - } + if( c ) goto jump_to_p2; break; } @@ -6756,6 +6775,7 @@ case OP_VColumn: { rc = pModule->xColumn(pCur->uc.pVCur, &sContext, pOp->p2); sqlite3VtabImportErrmsg(p, pVtab); if( sContext.isError>0 ){ + sqlite3VdbeError(p, "%s", sqlite3_value_text(pDest)); rc = sContext.isError; } sqlite3VdbeChangeEncoding(pDest, encoding); @@ -7227,7 +7247,7 @@ default: { /* This is really OP_Noop and OP_Explain */ #ifdef VDBE_PROFILE { - u64 endTime = sqlite3Hwtime(); + u64 endTime = sqlite3NProfileCnt ? sqlite3NProfileCnt : sqlite3Hwtime(); if( endTime>start ) pOrigOp->cycles += endTime - start; pOrigOp->cnt++; } diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 1e4f615ba2..44f901abf7 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -486,6 +486,7 @@ int sqlite3VdbeMemStringify(Mem*, u8, u8); i64 sqlite3VdbeIntValue(Mem*); int sqlite3VdbeMemIntegerify(Mem*); double sqlite3VdbeRealValue(Mem*); +int sqlite3VdbeBooleanValue(Mem*, int ifNull); void sqlite3VdbeIntegerAffinity(Mem*); int sqlite3VdbeMemRealify(Mem*); int sqlite3VdbeMemNumerify(Mem*); diff --git a/src/vdbemem.c b/src/vdbemem.c index e2912b3a6f..8df3c0d53b 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -93,6 +93,51 @@ int sqlite3VdbeCheckMemInvariants(Mem *p){ } #endif +#ifdef SQLITE_DEBUG +/* +** Check that string value of pMem agrees with its integer or real value. +** +** A single int or real value always converts to the same strings. But +** many different strings can be converted into the same int or real. +** If a table contains a numeric value and an index is based on the +** corresponding string value, then it is important that the string be +** derived from the numeric value, not the other way around, to ensure +** that the index and table are consistent. See ticket +** https://www.sqlite.org/src/info/343634942dd54ab (2018-01-31) for +** an example. +** +** This routine looks at pMem to verify that if it has both a numeric +** representation and a string representation then the string rep has +** been derived from the numeric and not the other way around. It returns +** true if everything is ok and false if there is a problem. +** +** This routine is for use inside of assert() statements only. +*/ +int sqlite3VdbeMemConsistentDualRep(Mem *p){ + char zBuf[100]; + char *z; + int i, j, incr; + if( (p->flags & MEM_Str)==0 ) return 1; + if( (p->flags & (MEM_Int|MEM_Real))==0 ) return 1; + if( p->flags & MEM_Int ){ + sqlite3_snprintf(sizeof(zBuf),zBuf,"%lld",p->u.i); + }else{ + sqlite3_snprintf(sizeof(zBuf),zBuf,"%!.15g",p->u.r); + } + z = p->z; + i = j = 0; + incr = 1; + if( p->enc!=SQLITE_UTF8 ){ + incr = 2; + if( p->enc==SQLITE_UTF16BE ) z++; + } + while( zBuf[j] ){ + if( zBuf[j++]!=z[i] ) return 0; + i += incr; + } + return 1; +} +#endif /* SQLITE_DEBUG */ /* ** If pMem is an object with a valid string representation, this routine @@ -527,6 +572,16 @@ double sqlite3VdbeRealValue(Mem *pMem){ } } +/* +** Return 1 if pMem represents true, and return 0 if pMem represents false. +** Return the value ifNull if pMem is NULL. +*/ +int sqlite3VdbeBooleanValue(Mem *pMem, int ifNull){ + if( pMem->flags & MEM_Int ) return pMem->u.i!=0; + if( pMem->flags & MEM_Null ) return ifNull; + return sqlite3VdbeRealValue(pMem)!=0.0; +} + /* ** The MEM structure is already a MEM_Real. Try to also make it a ** MEM_Int if we can. @@ -1096,6 +1151,7 @@ static SQLITE_NOINLINE const void *valueToText(sqlite3_value* pVal, u8 enc){ assert(pVal->enc==(enc & ~SQLITE_UTF16_ALIGNED) || pVal->db==0 || pVal->db->mallocFailed ); if( pVal->enc==(enc & ~SQLITE_UTF16_ALIGNED) ){ + assert( sqlite3VdbeMemConsistentDualRep(pVal) ); return pVal->z; }else{ return 0; @@ -1118,6 +1174,7 @@ const void *sqlite3ValueText(sqlite3_value* pVal, u8 enc){ assert( (enc&3)==(enc&~SQLITE_UTF16_ALIGNED) ); assert( (pVal->flags & MEM_RowSet)==0 ); if( (pVal->flags&(MEM_Str|MEM_Term))==(MEM_Str|MEM_Term) && pVal->enc==enc ){ + assert( sqlite3VdbeMemConsistentDualRep(pVal) ); return pVal->z; } if( pVal->flags&MEM_Null ){ diff --git a/src/wal.c b/src/wal.c index 7b49f3dab4..d56946e309 100644 --- a/src/wal.c +++ b/src/wal.c @@ -554,7 +554,11 @@ struct WalIterator { ** page and SQLITE_OK is returned. If an error (an OOM or VFS error) occurs, ** then an SQLite error code is returned and *ppPage is set to 0. */ -static int walIndexPage(Wal *pWal, int iPage, volatile u32 **ppPage){ +static SQLITE_NOINLINE int walIndexPageRealloc( + Wal *pWal, /* The WAL context */ + int iPage, /* The page we seek */ + volatile u32 **ppPage /* Write the page pointer here */ +){ int rc = SQLITE_OK; /* Enlarge the pWal->apWiData[] array if required */ @@ -573,21 +577,20 @@ static int walIndexPage(Wal *pWal, int iPage, volatile u32 **ppPage){ } /* Request a pointer to the required page from the VFS */ - if( pWal->apWiData[iPage]==0 ){ - if( pWal->exclusiveMode==WAL_HEAPMEMORY_MODE ){ - pWal->apWiData[iPage] = (u32 volatile *)sqlite3MallocZero(WALINDEX_PGSZ); - if( !pWal->apWiData[iPage] ) rc = SQLITE_NOMEM_BKPT; - }else{ - rc = sqlite3OsShmMap(pWal->pDbFd, iPage, WALINDEX_PGSZ, - pWal->writeLock, (void volatile **)&pWal->apWiData[iPage] - ); - assert( pWal->apWiData[iPage]!=0 || rc!=SQLITE_OK || pWal->writeLock==0 ); - testcase( pWal->apWiData[iPage]==0 && rc==SQLITE_OK ); - if( (rc&0xff)==SQLITE_READONLY ){ - pWal->readOnly |= WAL_SHM_RDONLY; - if( rc==SQLITE_READONLY ){ - rc = SQLITE_OK; - } + assert( pWal->apWiData[iPage]==0 ); + if( pWal->exclusiveMode==WAL_HEAPMEMORY_MODE ){ + pWal->apWiData[iPage] = (u32 volatile *)sqlite3MallocZero(WALINDEX_PGSZ); + if( !pWal->apWiData[iPage] ) rc = SQLITE_NOMEM_BKPT; + }else{ + rc = sqlite3OsShmMap(pWal->pDbFd, iPage, WALINDEX_PGSZ, + pWal->writeLock, (void volatile **)&pWal->apWiData[iPage] + ); + assert( pWal->apWiData[iPage]!=0 || rc!=SQLITE_OK || pWal->writeLock==0 ); + testcase( pWal->apWiData[iPage]==0 && rc==SQLITE_OK ); + if( (rc&0xff)==SQLITE_READONLY ){ + pWal->readOnly |= WAL_SHM_RDONLY; + if( rc==SQLITE_READONLY ){ + rc = SQLITE_OK; } } } @@ -596,6 +599,16 @@ static int walIndexPage(Wal *pWal, int iPage, volatile u32 **ppPage){ assert( iPage==0 || *ppPage || rc!=SQLITE_OK ); return rc; } +static int walIndexPage( + Wal *pWal, /* The WAL context */ + int iPage, /* The page we seek */ + volatile u32 **ppPage /* Write the page pointer here */ +){ + if( pWal->nWiData<=iPage || (*ppPage = pWal->apWiData[iPage])==0 ){ + return walIndexPageRealloc(pWal, iPage, ppPage); + } + return SQLITE_OK; +} /* ** Return a pointer to the WalCkptInfo structure in the wal-index. @@ -2868,7 +2881,7 @@ int sqlite3WalFindFrame( ** table after the current read-transaction had started. */ iMinHash = walFramePage(pWal->minFrame); - for(iHash=walFramePage(iLast); iHash>=iMinHash && iRead==0; iHash--){ + for(iHash=walFramePage(iLast); iHash>=iMinHash; iHash--){ volatile ht_slot *aHash; /* Pointer to hash table */ volatile u32 *aPgno; /* Pointer to array of page numbers */ u32 iZero; /* Frame number corresponding to aPgno[0] */ @@ -2891,6 +2904,7 @@ int sqlite3WalFindFrame( return SQLITE_CORRUPT_BKPT; } } + if( iRead ) break; } #ifdef SQLITE_ENABLE_EXPENSIVE_ASSERT diff --git a/src/where.c b/src/where.c index 3152c8e9aa..8c1e3cd01f 100644 --- a/src/where.c +++ b/src/where.c @@ -2481,10 +2481,12 @@ static int whereLoopAddBtreeIndex( if( iCol==XN_ROWID || (iCol>=0 && nInMul==0 && saved_nEq==pProbe->nKeyCol-1) ){ - if( iCol>=0 && pProbe->uniqNotNull==0 ){ - pNew->wsFlags |= WHERE_UNQ_WANTED; - }else{ + if( iCol==XN_ROWID || pProbe->uniqNotNull + || (pProbe->nKeyCol==1 && pProbe->onError && eOp==WO_EQ) + ){ pNew->wsFlags |= WHERE_ONEROW; + }else{ + pNew->wsFlags |= WHERE_UNQ_WANTED; } } }else if( eOp & WO_ISNULL ){ @@ -4631,6 +4633,7 @@ WhereInfo *sqlite3WhereBegin( */ for(ii=0; iinTerm; ii++){ WhereTerm *pT = &sWLB.pWC->a[ii]; + if( pT->wtFlags & TERM_VIRTUAL ) continue; if( pT->prereqAll==0 && (nTabList==0 || exprIsDeterministic(pT->pExpr)) ){ sqlite3ExprIfFalse(pParse, pT->pExpr, pWInfo->iBreak, SQLITE_JUMPIFNULL); pT->wtFlags |= TERM_CODED; diff --git a/src/wherecode.c b/src/wherecode.c index 8e8672ff9f..e40a940ac1 100644 --- a/src/wherecode.c +++ b/src/wherecode.c @@ -1395,7 +1395,15 @@ Bitmask sqlite3WhereCodeOneLoopStart( if( sqlite3ExprIsVector(pX->pRight) ){ r1 = rTemp = sqlite3GetTempReg(pParse); codeExprOrVector(pParse, pX->pRight, r1, 1); - op = aMoveOp[(pX->op - TK_GT) | 0x0001]; + testcase( pX->op==TK_GT ); + testcase( pX->op==TK_GE ); + testcase( pX->op==TK_LT ); + testcase( pX->op==TK_LE ); + op = aMoveOp[((pX->op - TK_GT - 1) & 0x3) | 0x1]; + assert( pX->op!=TK_GT || op==OP_SeekGE ); + assert( pX->op!=TK_GE || op==OP_SeekGE ); + assert( pX->op!=TK_LT || op==OP_SeekLE ); + assert( pX->op!=TK_LE || op==OP_SeekLE ); }else{ r1 = sqlite3ExprCodeTemp(pParse, pX->pRight, &rTemp); disableTerm(pLevel, pStart); diff --git a/src/whereexpr.c b/src/whereexpr.c index 58f1908cf8..313c5ee9bc 100644 --- a/src/whereexpr.c +++ b/src/whereexpr.c @@ -876,6 +876,9 @@ static Bitmask exprSelectUsage(WhereMaskSet *pMaskSet, Select *pS){ for(i=0; inSrc; i++){ mask |= exprSelectUsage(pMaskSet, pSrc->a[i].pSelect); mask |= sqlite3WhereExprUsage(pMaskSet, pSrc->a[i].pOn); + if( pSrc->a[i].fg.isTabFunc ){ + mask |= sqlite3WhereExprListUsage(pMaskSet, pSrc->a[i].u1.pFuncArg); + } } } pS = pS->pPrior; @@ -1288,7 +1291,7 @@ static void exprAnalyze( exprAnalyze(pSrc, pWC, idxNew); } pTerm = &pWC->a[idxTerm]; - pTerm->wtFlags = TERM_CODED|TERM_VIRTUAL; /* Disable the original */ + pTerm->wtFlags |= TERM_CODED|TERM_VIRTUAL; /* Disable the original */ pTerm->eOperator = 0; } diff --git a/test/cast.test b/test/cast.test index f47f4bb2bf..f43aa48560 100644 --- a/test/cast.test +++ b/test/cast.test @@ -343,4 +343,49 @@ do_test cast-4.4 { } } {0 abc 0.0 abc} +# Added 2018-01-26 +# +# EVIDENCE-OF: R-48741-32454 If the prefix integer is greater than +# +9223372036854775807 then the result of the cast is exactly +# +9223372036854775807. +do_execsql_test cast-5.1 { + SELECT CAST('9223372036854775808' AS integer); + SELECT CAST(' +000009223372036854775808' AS integer); + SELECT CAST('12345678901234567890123' AS INTEGER); +} {9223372036854775807 9223372036854775807 9223372036854775807} + +# EVIDENCE-OF: R-06028-16857 Similarly, if the prefix integer is less +# than -9223372036854775808 then the result of the cast is exactly +# -9223372036854775808. +do_execsql_test cast-5.2 { + SELECT CAST('-9223372036854775808' AS integer); + SELECT CAST('-9223372036854775809' AS integer); + SELECT CAST('-12345678901234567890123' AS INTEGER); +} {-9223372036854775808 -9223372036854775808 -9223372036854775808} + +# EVIDENCE-OF: R-33990-33527 When casting to INTEGER, if the text looks +# like a floating point value with an exponent, the exponent will be +# ignored because it is no part of the integer prefix. +# EVIDENCE-OF: R-24225-46995 For example, "(CAST '123e+5' AS INTEGER)" +# results in 123, not in 12300000. +do_execsql_test case-5.3 { + SELECT CAST('123e+5' AS INTEGER); + SELECT CAST('123e+5' AS NUMERIC); +} {123 12300000.0} + + +# The following does not have anything to do with the CAST operator, +# but it does deal with affinity transformations. +# +do_execsql_test case-6.1 { + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(a NUMERIC); + INSERT INTO t1 VALUES + ('9000000000000000001'), + ('9000000000000000001 '), + (' 9000000000000000001'), + (' 9000000000000000001 '); + SELECT * FROM t1; +} {9000000000000000001 9000000000000000001 9000000000000000001 9000000000000000001} + finish_test diff --git a/test/expr.test b/test/expr.test index 7a6d477259..3cdc9180e8 100644 --- a/test/expr.test +++ b/test/expr.test @@ -977,6 +977,63 @@ do_execsql_test expr-13.9 { SELECT '' <= ""; } {1} +# 2018-02-26. Ticket https://www.sqlite.org/src/tktview/36fae083b450e3af85 +# +do_execsql_test expr-14.1 { + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(x); + INSERT INTO t1 VALUES(0),(1),(NULL),(0.5),('1x'),('0x'); + SELECT count(*) FROM t1 + WHERE (x OR (8==9)) != (CASE WHEN x THEN 1 ELSE 0 END); +} {0} +do_execsql_test expr-14.2 { + SELECT count(*) FROM t1 + WHERE (x OR (8==9)) != (NOT NOT x); +} {0} +do_execsql_test expr-14.3 { + SELECT sum(NOT x) FROM t1 + WHERE x +} {0} +do_execsql_test expr-14.4 { + SELECT sum(CASE WHEN x THEN 0 ELSE 1 END) FROM t1 + WHERE x +} {0} +foreach {tn val} [list 1 NaN 2 -NaN 3 NaN0 4 -NaN0 5 Inf 6 -Inf] { + do_execsql_test expr-15.$tn.1 { + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(x); + INSERT INTO t1 VALUES(0),(1),(NULL),(0.5),('1x'),('0x'); + } + + do_test expr-15.$tn.2 { + set ::STMT [sqlite3_prepare db "INSERT INTO t1 VALUES(?)" -1 TAIL] + sqlite3_bind_double $::STMT 1 $val + sqlite3_step $::STMT + sqlite3_reset $::STMT + sqlite3_finalize $::STMT + } {SQLITE_OK} + + do_execsql_test expr-15.$tn.3 { + SELECT count(*) FROM t1 + WHERE (x OR (8==9)) != (CASE WHEN x THEN 1 ELSE 0 END); + } {0} + + do_execsql_test expr-15.$tn.4 { + SELECT count(*) FROM t1 + WHERE (x OR (8==9)) != (NOT NOT x); + } {0} + + do_execsql_test expr-15.$tn.5 { + SELECT sum(NOT x) FROM t1 + WHERE x + } {0} + + do_execsql_test expr-15.$tn.6 { + SELECT sum(CASE WHEN x THEN 0 ELSE 1 END) FROM t1 + WHERE x + } {0} +} + finish_test diff --git a/test/fts3aa.test b/test/fts3aa.test index 10ec273cbf..d5f96d81a7 100644 --- a/test/fts3aa.test +++ b/test/fts3aa.test @@ -250,4 +250,5 @@ do_execsql_test 9.2 { CREATE VIRTUAL TABLE t10 USING fts3(<, b, c); } +expand_all_sql db finish_test diff --git a/test/fts4onepass.test b/test/fts4onepass.test index 46cb4b794b..344be4b1df 100644 --- a/test/fts4onepass.test +++ b/test/fts4onepass.test @@ -143,4 +143,18 @@ foreach {tn tcl1 tcl2} { eval $tcl2 } +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE zt USING fts4(a, b); + INSERT INTO zt(rowid, a, b) VALUES(1, 'unus duo', NULL); + INSERT INTO zt(rowid, a, b) VALUES(2, NULL, NULL); + + BEGIN; + UPDATE zt SET b='septum' WHERE rowid = 1; + UPDATE zt SET b='octo' WHERE rowid = 1; + COMMIT; + + SELECT count(*) FROM zt_segdir; +} {3} + + finish_test diff --git a/test/func.test b/test/func.test index 98ae8ddeb5..23a3ae4392 100644 --- a/test/func.test +++ b/test/func.test @@ -507,6 +507,17 @@ if {$encoding=="UTF-16le"} { execsql {SELECT hex(replace('aabcdefg','a','aaa'))} } {616161616161626364656667} } +do_execsql_test func-9.14 { + WITH RECURSIVE c(x) AS ( + VALUES(1) + UNION ALL + SELECT x+1 FROM c WHERE x<1040 + ) + SELECT + count(*), + sum(length(replace(printf('abc%.*cxyz',x,'m'),'m','nnnn'))-(6+x*4)) + FROM c; +} {1040 0} # Use the "sqlite_register_test_function" TCL command which is part of # the text fixture in order to verify correct operation of some of diff --git a/test/func6.test b/test/func6.test index 02e1998cd0..1e16a7ca38 100644 --- a/test/func6.test +++ b/test/func6.test @@ -22,6 +22,11 @@ ifcapable !offset_sql_func { return } +set bNullTrim 0 +ifcapable null_trim { + set bNullTrim 1 +} + do_execsql_test func6-100 { PRAGMA page_size=4096; PRAGMA auto_vacuum=NONE; @@ -33,54 +38,137 @@ do_execsql_test func6-100 { CREATE TABLE t2(x TEXT PRIMARY KEY, y) WITHOUT ROWID; INSERT INTO t2(x,y) SELECT a, b FROM t1; } + +# Load the contents of $file from disk and return it encoded as a hex +# string. +proc loadhex {file} { + set fd [open $file] + fconfigure $fd -translation binary -encoding binary + set data [read $fd] + close $fd + binary encode hex $data +} + +# Each argument is either an integer between 0 and 65535, a text value, or +# an empty string representing an SQL NULL. This command builds an SQLite +# record containing the values passed as arguments and returns it encoded +# as a hex string. +proc hexrecord {args} { + set hdr "" + set body "" + + if {$::bNullTrim} { + while {[llength $args] && [lindex $args end]=={}} { + set args [lrange $args 0 end-1] + } + } + + foreach x $args { + if {$x==""} { + append hdr 00 + } elseif {[string is integer $x]==0} { + set n [string length $x] + append hdr [format %02x [expr $n*2 + 13]] + append body [binary encode hex $x] + } elseif {$x == 0} { + append hdr 08 + } elseif {$x == 1} { + append hdr 09 + } elseif {$x <= 127} { + append hdr 01 + append body [format %02x $x] + } else { + append hdr 02 + append body [format %04x $x] + } + } + set res [format %02x [expr 1 + [string length $hdr]/2]] + append res $hdr + append res $body +} + +# Argument $off is an offset into the database image encoded as a hex string +# in argument $hexdb. This command returns 0 if the offset contains the hex +# $hexrec, or throws an exception otherwise. +# +proc offset_contains_record {off hexdb hexrec} { + set n [string length $hexrec] + set off [expr $off*2] + if { [string compare $hexrec [string range $hexdb $off [expr $off+$n-1]]] } { + error "record not found!" + } + return 0 +} + +# This command is the implementation of SQL function "offrec()". The first +# argument to this is an offset value. The remaining values are used to +# formulate an SQLite record. If database file test.db does not contain +# an equivalent record at the specified offset, an exception is thrown. +# Otherwise, 0 is returned. +# +proc offrec {args} { + set offset [lindex $args 0] + set rec [hexrecord {*}[lrange $args 1 end]] + offset_contains_record $offset $::F $rec +} +set F [loadhex test.db] +db func offrec offrec + +# Test the sanity of the tests. +if {$bNullTrim} { + set offset 8180 +} else { + set offset 8179 +} +do_execsql_test func6-105 { + SELECT sqlite_offset(d) FROM t1 ORDER BY rowid LIMIT 1; +} $offset +do_test func6-106 { + set r [hexrecord abc001 1 999 {}] + offset_contains_record $offset $F $r +} 0 + +set z100 [string trim [string repeat "0 " 100]] + +# Test offsets within table b-tree t1. do_execsql_test func6-110 { - SELECT a, sqlite_offset(d)/4096 + 1, - sqlite_offset(d)%4096 FROM t1 - ORDER BY rowid LIMIT 2; -} {abc001 2 4084 abc002 2 4069} + SELECT offrec(sqlite_offset(d), a, b, c, d) FROM t1 ORDER BY rowid +} $z100 + do_execsql_test func6-120 { SELECT a, typeof(sqlite_offset(+a)) FROM t1 ORDER BY rowid LIMIT 2; } {abc001 null abc002 null} + +# Test offsets within index b-tree t1a. do_execsql_test func6-130 { - SELECT a, sqlite_offset(a)/4096+1, - sqlite_offset(a)%4096 - FROM t1 - ORDER BY a LIMIT 2; -} {abc001 3 4087 abc002 3 4076} + SELECT offrec(sqlite_offset(a), a, rowid) FROM t1 ORDER BY a +} $z100 + +# Test offsets within table b-tree t1 with a temp b-tree ORDER BY. do_execsql_test func6-140 { - SELECT a, sqlite_offset(d)/4096+1, - sqlite_offset(d)%4096 - FROM t1 - ORDER BY a LIMIT 2; -} {abc001 2 4084 abc002 2 4069} + SELECT offrec(sqlite_offset(d), a, b, c, d) FROM t1 ORDER BY a +} $z100 + +# Test offsets from both index t1a and table t1 in the same query. do_execsql_test func6-150 { - SELECT a, - sqlite_offset(a)/4096+1, - sqlite_offset(a)%4096, - sqlite_offset(d)/4096+1, - sqlite_offset(d)%4096 - FROM t1 - ORDER BY a LIMIT 2; -} {abc001 3 4087 2 4084 abc002 3 4076 2 4069} + SELECT offrec(sqlite_offset(a), a, rowid), + offrec(sqlite_offset(d), a, b, c, d) + FROM t1 ORDER BY a +} [concat $z100 $z100] + +# Test offsets from both index t1bc and table t1 in the same query. do_execsql_test func6-160 { - SELECT b, - sqlite_offset(b)/4096+1, - sqlite_offset(b)%4096, - sqlite_offset(c)/4096+1, - sqlite_offset(c)%4096, - sqlite_offset(d)/4096+1, - sqlite_offset(d)%4096 - FROM t1 - ORDER BY b LIMIT 2; -} {1 4 4090 4 4090 2 4084 2 4 4081 4 4081 2 4069} - + SELECT offrec(sqlite_offset(b), b, c, rowid), + offrec(sqlite_offset(c), b, c, rowid), + offrec(sqlite_offset(d), a, b, c, d) + FROM t1 + ORDER BY b +} [concat $z100 $z100 $z100] +# Test offsets in WITHOUT ROWID table t2. do_execsql_test func6-200 { - SELECT y, sqlite_offset(y)/4096+1, - sqlite_offset(y)%4096 - FROM t2 - ORDER BY x LIMIT 2; -} {1 5 4087 2 5 4076} + SELECT offrec( sqlite_offset(y), x, y ) FROM t2 ORDER BY x +} $z100 finish_test diff --git a/test/indexexpr1.test b/test/indexexpr1.test index 28c23b9089..e93dcc0cd1 100644 --- a/test/indexexpr1.test +++ b/test/indexexpr1.test @@ -425,4 +425,25 @@ do_execsql_test indexexpr-1510 { REPLACE INTO t1 SELECT a, randomblob(a) FROM t1 } {} +# 2018-01-31 https://www.sqlite.org/src/tktview/343634942dd54ab57b702411 +# When an index on an expression depends on the string representation of +# a numeric table column, trouble can arise since there are multiple +# string that can map to the same numeric value. (Ex: 123, 0123, 000123). +# +do_execsql_test indexexpr-1600 { + DROP TABLE IF EXISTS t1; + CREATE TABLE t1 (a INTEGER, b); + CREATE INDEX idx1 ON t1 (lower(a)); + INSERT INTO t1 VALUES('0001234',3); + PRAGMA integrity_check; +} {ok} +do_execsql_test indexexpr-1610 { + INSERT INTO t1 VALUES('1234',0),('001234',2),('01234',1); + SELECT b FROM t1 WHERE lower(a)='1234' ORDER BY +b; +} {0 1 2 3} +do_execsql_test indexexpr-1620 { + SELECT b FROM t1 WHERE lower(a)='01234' ORDER BY +b; +} {} + + finish_test diff --git a/test/istrue.test b/test/istrue.test new file mode 100644 index 0000000000..250f1f9d5d --- /dev/null +++ b/test/istrue.test @@ -0,0 +1,126 @@ +# 2018-02-26 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing expressions of the form +# +# x IS TRUE +# x IS FALSE +# x IS NOT TRUE +# x IS NOT FALSE +# +# Tests are also included for the use of TRUE and FALSE as +# literal values. + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +do_execsql_test istrue-100 { + CREATE TABLE t1(x INTEGER PRIMARY KEY, y BOOLEAN); + INSERT INTO t1 VALUES(1, true),(2, false),(3, null); + SELECT x FROM t1 WHERE y IS TRUE; +} {1} +do_execsql_test istrue-110 { + SELECT x FROM t1 WHERE y IS FALSE; +} {2} +do_execsql_test istrue-120 { + SELECT x FROM t1 WHERE y IS NULL; +} {3} +do_execsql_test istrue-130 { + SELECT x FROM t1 WHERE y IS NOT TRUE; +} {2 3} +do_execsql_test istrue-140 { + SELECT x FROM t1 WHERE y IS NOT FALSE; +} {1 3} +do_execsql_test istrue-150 { + SELECT x FROM t1 WHERE y IS NOT NULL; +} {1 2} +unset -nocomplain X +set X 9 +do_execsql_test istrue-160 { + SELECT x FROM t1 WHERE y IS TRUE OR (8==$X) +} {1} +do_execsql_test istrue-170 { + SELECT x FROM t1 WHERE y IS FALSE OR (8==$X) +} {2} +do_execsql_test istrue-180 { + SELECT x FROM t1 WHERE y IS NULL OR (8==$X); +} {3} +do_execsql_test istrue-190 { + SELECT x FROM t1 WHERE y IS NOT TRUE OR (8==$X); +} {2 3} +do_execsql_test istrue-200 { + SELECT x FROM t1 WHERE y IS NOT FALSE OR (8==$X); +} {1 3} +do_execsql_test istrue-210 { + SELECT x FROM t1 WHERE y IS NOT NULL OR (8==$X); +} {1 2} + +do_execsql_test istrue-300 { + SELECT x, + y IS TRUE, y IS FALSE, y is NULL, + y IS NOT TRUE, y IS NOT FALSE, y IS NOT NULL, '|' + FROM t1 ORDER BY x; +} {1 1 0 0 0 1 1 | 2 0 1 0 1 0 1 | 3 0 0 1 1 1 0 |} + +do_execsql_test istrue-400 { + SELECT x FROM t1 WHERE true; +} {1 2 3} +do_execsql_test istrue-410 { + SELECT x FROM t1 WHERE false; +} {} + +do_execsql_test istrue-500 { + CREATE TABLE t2( + a INTEGER PRIMARY KEY, + b BOOLEAN DEFAULT true, + c BOOLEAN DEFAULT(true), + d BOOLEAN DEFAULT false, + e BOOLEAN DEFAULT(false) + ); + INSERT INTO t2 DEFAULT VALUES; + SELECT * FROM t2; +} {1 1 1 0 0} +do_execsql_test istrue-510 { + DROP TABLE t2; + CREATE TABLE t2( + a INTEGER PRIMARY KEY, + b BOOLEAN DEFAULT(not true), + c BOOLEAN DEFAULT(not false) + ); + INSERT INTO t2(a) VALUES(99); + SELECT * FROM t2; +} {99 0 1} +do_execsql_test istrue-520 { + DROP TABLE t2; + CREATE TABLE t2( + a INTEGER PRIMARY KEY, + b BOOLEAN CHECK(b IS TRUE), + c BOOLEAN CHECK(c IS FALSE), + d BOOLEAN CHECK(d IS NOT TRUE), + e BOOLEAN CHECK(e IS NOT FALSE) + ); + INSERT INTO t2 VALUES(1,true,false,null,null); + SELECT * FROM t2; +} {1 1 0 {} {}} +do_catchsql_test istrue-521 { + INSERT INTO t2 VALUES(2,false,false,null,null); +} {1 {CHECK constraint failed: t2}} +do_catchsql_test istrue-522 { + INSERT INTO t2 VALUES(2,true,true,null,null); +} {1 {CHECK constraint failed: t2}} +do_catchsql_test istrue-523 { + INSERT INTO t2 VALUES(2,true,false,true,null); +} {1 {CHECK constraint failed: t2}} +do_catchsql_test istrue-524 { + INSERT INTO t2 VALUES(2,true,false,null,false); +} {1 {CHECK constraint failed: t2}} + +finish_test diff --git a/test/join2.test b/test/join2.test index 76006f1ecd..256846ac91 100644 --- a/test/join2.test +++ b/test/join2.test @@ -155,20 +155,65 @@ do_execsql_test 4.1.4 { SELECT v1, v3 FROM c1 LEFT JOIN c2 LEFT JOIN c3 ON (c3.k=v1+1); } {2 v3 2 v3 1112 {} 1112 {}} -do_eqp_test 4.2.1 { +do_eqp_test 4.1.5 { SELECT v1, v3 FROM c1 LEFT JOIN c2 ON (c2.k=v1) LEFT JOIN c3 ON (c3.k=v2); } { 0 0 0 {SCAN TABLE c1} 0 1 1 {SEARCH TABLE c2 USING INTEGER PRIMARY KEY (rowid=?)} 0 2 2 {SEARCH TABLE c3 USING INTEGER PRIMARY KEY (rowid=?)} } -do_eqp_test 4.2.2 { +do_eqp_test 4.1.6 { SELECT v1, v3 FROM c1 LEFT JOIN c2 ON (c2.k=v1) LEFT JOIN c3 ON (c3.k=v1+1); } { 0 0 0 {SCAN TABLE c1} 0 1 2 {SEARCH TABLE c3 USING INTEGER PRIMARY KEY (rowid=?)} } +do_execsql_test 4.2.0 { + DROP TABLE c1; + DROP TABLE c2; + DROP TABLE c3; + CREATE TABLE c1(k UNIQUE, v1); + CREATE TABLE c2(k UNIQUE, v2); + CREATE TABLE c3(k UNIQUE, v3); + + INSERT INTO c1 VALUES(1, 2); + INSERT INTO c2 VALUES(2, 3); + INSERT INTO c3 VALUES(3, 'v3'); + + INSERT INTO c1 VALUES(111, 1112); + INSERT INTO c2 VALUES(112, 1113); + INSERT INTO c3 VALUES(113, 'v1113'); +} +do_execsql_test 4.2.1 { + SELECT v1, v3 FROM c1 LEFT JOIN c2 ON (c2.k=v1) LEFT JOIN c3 ON (c3.k=v2); +} {2 v3 1112 {}} +do_execsql_test 4.2.2 { + SELECT v1, v3 FROM c1 LEFT JOIN c2 ON (c2.k=v1) LEFT JOIN c3 ON (c3.k=v1+1); +} {2 v3 1112 {}} + +do_execsql_test 4.2.3 { + SELECT DISTINCT v1, v3 FROM c1 LEFT JOIN c2 LEFT JOIN c3 ON (c3.k=v1+1); +} {2 v3 1112 {}} + +do_execsql_test 4.2.4 { + SELECT v1, v3 FROM c1 LEFT JOIN c2 LEFT JOIN c3 ON (c3.k=v1+1); +} {2 v3 2 v3 1112 {} 1112 {}} + +do_eqp_test 4.2.5 { + SELECT v1, v3 FROM c1 LEFT JOIN c2 ON (c2.k=v1) LEFT JOIN c3 ON (c3.k=v2); +} { + 0 0 0 {SCAN TABLE c1} + 0 1 1 {SEARCH TABLE c2 USING INDEX sqlite_autoindex_c2_1 (k=?)} + 0 2 2 {SEARCH TABLE c3 USING INDEX sqlite_autoindex_c3_1 (k=?)} +} +do_eqp_test 4.2.6 { + SELECT v1, v3 FROM c1 LEFT JOIN c2 ON (c2.k=v1) LEFT JOIN c3 ON (c3.k=v1+1); +} { + 0 0 0 {SCAN TABLE c1} + 0 1 2 {SEARCH TABLE c3 USING INDEX sqlite_autoindex_c3_1 (k=?)} +} + # 2017-11-23 (Thanksgiving day) # OSSFuzz found an assertion fault in the new LEFT JOIN eliminator code. # @@ -192,4 +237,32 @@ do_execsql_test 4.3.1 { LEFT JOIN t2 AS c ON (a.x=c.x); } {1 {} 2 {} 3 {} 4 {} 5 {} 6 {} 7 {} 8 {} 9 {} 10 10} +do_execsql_test 5.0 { + CREATE TABLE s1 (a INTEGER PRIMARY KEY); + CREATE TABLE s2 (a INTEGER PRIMARY KEY); + CREATE TABLE s3 (a INTEGER); + CREATE UNIQUE INDEX ndx on s3(a); +} +do_eqp_test 5.1 { + SELECT s1.a FROM s1 left join s2 using (a); +} { + 0 0 0 {SCAN TABLE s1} +} +do_eqp_test 5.2 { + SELECT s1.a FROM s1 left join s3 using (a); +} { + 0 0 0 {SCAN TABLE s1} +} + +do_execsql_test 6.0 { + CREATE TABLE u1(a INTEGER PRIMARY KEY, b, c); + CREATE TABLE u2(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX u1ab ON u1(b, c); +} +do_eqp_test 6.1 { + SELECT u2.* FROM u2 LEFT JOIN u1 ON( u1.a=u2.a AND u1.b=u2.b AND u1.c=u2.c ); +} { + 0 0 0 {SCAN TABLE u2} +} + finish_test diff --git a/test/json101.test b/test/json101.test index 4bfcc2d80d..ac4e11ebc2 100644 --- a/test/json101.test +++ b/test/json101.test @@ -759,7 +759,30 @@ do_execsql_test json-12.120 { FROM t12; } {0} - +# 2018-01-26 +# ticket https://www.sqlite.org/src/tktview/80177f0c226ff54f6ddd41 +# Make sure the query planner knows about the arguments to table-valued functions. +# +do_execsql_test json-13.100 { + DROP TABLE IF EXISTS t1; + DROP TABLE IF EXISTS t2; + CREATE TABLE t1(id, json); + INSERT INTO t1(id,json) VALUES(1,'{"items":[3,5]}'); + CREATE TABLE t2(id, json); + INSERT INTO t2(id,json) VALUES(2,'{"value":2}'); + INSERT INTO t2(id,json) VALUES(3,'{"value":3}'); + INSERT INTO t2(id,json) VALUES(4,'{"value":4}'); + INSERT INTO t2(id,json) VALUES(5,'{"value":5}'); + INSERT INTO t2(id,json) VALUES(6,'{"value":6}'); + SELECT * FROM t1 CROSS JOIN t2 + WHERE EXISTS(SELECT 1 FROM json_each(t1.json,'$.items') AS Z + WHERE Z.value==t2.id); +} {1 {{"items":[3,5]}} 3 {{"value":3}} 1 {{"items":[3,5]}} 5 {{"value":5}}} +do_execsql_test json-13.110 { + SELECT * FROM t2 CROSS JOIN t1 + WHERE EXISTS(SELECT 1 FROM json_each(t1.json,'$.items') AS Z + WHERE Z.value==t2.id); +} {3 {{"value":3}} 1 {{"items":[3,5]}} 5 {{"value":5}} 1 {{"items":[3,5]}}} finish_test diff --git a/test/nockpt.test b/test/nockpt.test index bd3953f1ee..6da3313ad8 100644 --- a/test/nockpt.test +++ b/test/nockpt.test @@ -61,6 +61,86 @@ do_test 1.14 { sqlite3_db_config db NO_CKPT_ON_CLOSE 1 } {1} do_execsql_test 1.14 { PRAGMA main.journal_mode = delete } {delete} do_test 1.15 { file exists test.db-wal } {0} +if {$::tcl_platform(platform)!="windows"} { +#------------------------------------------------------------------------- +# Test an unusual scenario: +# +# 1. A wal mode db is opened and written. Then sqlite3_close_v2() used +# to close the db handle while there is still an unfinalized +# statement (so the db handle stays open). +# +# 2. The db, wal and *-shm files are deleted from the file system. +# +# 3. Another connection creates a new wal mode db at the same file-system +# location as the previous one. +# +# 4. The statement left unfinalized in (1) is finalized. +# +# The test is to ensure that the connection left open in step (1) does +# not try to delete the wal file from the file-system as part of step +# 4. +# +reset_db +db close + +# Open a connection on a wal database. Write to it a bit. Then prepare +# a statement and call sqlite3_close_v2() (so that the statement handle +# holds the db connection open). +# +set ::db1 [sqlite3_open_v2 test.db SQLITE_OPEN_READWRITE ""] +do_test 2.0 { + lindex [ + sqlite3_exec $::db1 { + PRAGMA journal_mode = wal; + CREATE TABLE t1(x PRIMARY KEY, y UNIQUE, z); + INSERT INTO t1 VALUES(1, 2, 3); + PRAGMA wal_checkpoint; + }] 0 +} {0} +set ::stmt [sqlite3_prepare $::db1 "SELECT * FROM t1" -1 dummy] +sqlite3_close_v2 $::db1 + +# Delete the database, wal and shm files. +# +forcedelete test.db test.db-wal test.db-shm + +# Open and populate a new database file at the same file-system location +# as the one just deleted. Contrive a partial checkpoint on it. +# +sqlite3 db test.db +sqlite3 db2 test.db +do_execsql_test 2.1 { + PRAGMA journal_mode = wal; + CREATE TABLE y1(a PRIMARY KEY, b UNIQUE, c); + INSERT INTO y1 VALUES('a', 'b', 'c'); + INSERT INTO y1 VALUES('d', 'e', 'f'); +} {wal} +do_execsql_test -db db2 2.2 { + BEGIN; + SELECT * FROM y1; +} {a b c d e f} +do_execsql_test 2.3 { + UPDATE y1 SET c='g' WHERE a='d'; + PRAGMA wal_checkpoint; +} {0 11 10} +do_execsql_test -db db2 2.4 { + COMMIT +} + +# Finalize the statement handle, causing the first connection to be +# closed. Test that this has not corrupted the database file by +# deleting the new wal file from the file-system. If it has, this +# test should fail with an IO or corruption error. +# +do_test 2.5 { + sqlite3_finalize $::stmt + sqlite3 db3 test.db + execsql { + PRAGMA integrity_check; + SELECT * FROM y1; + } db3 +} {ok a b c d e g} +} finish_test diff --git a/test/normalize.test b/test/normalize.test new file mode 100644 index 0000000000..8932650c83 --- /dev/null +++ b/test/normalize.test @@ -0,0 +1,72 @@ +# 2018-01-08 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests for the sqlite3_normalize() extension function. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix normalize + +foreach {tnum sql norm} { + 100 + {SELECT * FROM t1 WHERE a IN (1) AND b=51.42} + {select*from t1 where a in(?,?,?)and b=?;} + + 110 + {SELECT a, b+15, c FROM t1 WHERE d NOT IN (SELECT x FROM t2);} + {select a,b+?,c from t1 where d not in(select x from t2);} + + 120 + { SELECT NULL, b FROM t1 -- comment text + WHERE d IN (WITH t(a) AS (VALUES(5)) /* CTE */ + SELECT a FROM t) + OR e='hello'; + } + {select?,b from t1 where d in(with t(a)as(values(?))select a from t)or e=?;} + + 121 + {/*Initial comment*/ + -- another comment line + SELECT NULL /* comment */ , b FROM t1 -- comment text + WHERE d IN (WITH t(a) AS (VALUES(5)) /* CTE */ + SELECT a FROM t) + OR e='hello'; + } + {select?,b from t1 where d in(with t(a)as(values(?))select a from t)or e=?;} + + 130 + {/* Query containing parameters */ + SELECT x,$::abc(15),y,@abc,z,?99,w FROM t1 /* Trailing comment */} + {select x,?,y,?,z,?,w from t1;} + + 140 + {/* Long list on the RHS of IN */ + SELECT 15 IN (1,2,3,(SELECT * FROM t1),'xyz',x'abcd',22*(x+5),null);} + {select?in(?,?,?);} + + 150 + {SELECT x'abc'; -- illegal token} + {} + + 160 + {SELECT a,NULL,b FROM t1 WHERE c IS NOT NULL or D is null or e=5} + {select a,?,b from t1 where c is not null or d is null or e=?;} + + 170 + {/* IN list exactly 5 bytes long */ + SELECT * FROM t1 WHERE x IN (1,2,3);} + {select*from t1 where x in(?,?,?);} +} { + do_test $tnum [list sqlite3_normalize $sql] $norm +} + +finish_test diff --git a/test/notnull.test b/test/notnull.test index 23fd33d4ba..32d95eaf24 100644 --- a/test/notnull.test +++ b/test/notnull.test @@ -561,4 +561,49 @@ do_test notnull-5.5 { execsql { SELECT * FROM t1 } } {1 2} +#------------------------------------------------------------------------- +# Check that UNIQUE NOT NULL indexes are always recognized as such. +# +proc uses_op_next {sql} { + db eval "EXPLAIN $sql" a { + if {$a(opcode)=="Next"} { return 1 } + } + return 0 +} + +proc do_uses_op_next_test {tn sql res} { + uplevel [list do_test $tn [list uses_op_next $sql] $res] +} + +reset_db +do_execsql_test notnull-6.0 { + CREATE TABLE t1(a UNIQUE); + CREATE TABLE t2(a NOT NULL UNIQUE); + CREATE TABLE t3(a UNIQUE NOT NULL); + CREATE TABLE t4(a NOT NULL); + CREATE UNIQUE INDEX t4a ON t4(a); + + CREATE TABLE t5(a PRIMARY KEY); + CREATE TABLE t6(a PRIMARY KEY NOT NULL); + CREATE TABLE t7(a NOT NULL PRIMARY KEY); + CREATE TABLE t8(a PRIMARY KEY) WITHOUT ROWID; + + CREATE TABLE t9(a PRIMARY KEY UNIQUE NOT NULL); + CREATE TABLE t10(a UNIQUE PRIMARY KEY NOT NULL); +} + +do_uses_op_next_test notnull-6.1 "SELECT * FROM t1 WHERE a IS ?" 1 +do_uses_op_next_test notnull-6.2 "SELECT * FROM t2 WHERE a IS ?" 0 +do_uses_op_next_test notnull-6.3 "SELECT * FROM t3 WHERE a IS ?" 0 +do_uses_op_next_test notnull-6.4 "SELECT * FROM t4 WHERE a IS ?" 0 + +do_uses_op_next_test notnull-6.5 "SELECT * FROM t5 WHERE a IS ?" 1 +do_uses_op_next_test notnull-6.6 "SELECT * FROM t6 WHERE a IS ?" 0 +do_uses_op_next_test notnull-6.7 "SELECT * FROM t7 WHERE a IS ?" 0 +do_uses_op_next_test notnull-6.8 "SELECT * FROM t8 WHERE a IS ?" 0 + +do_uses_op_next_test notnull-6.9 "SELECT * FROM t8 WHERE a IS ?" 0 +do_uses_op_next_test notnull-6.10 "SELECT * FROM t8 WHERE a IS ?" 0 + finish_test + diff --git a/test/printf2.test b/test/printf2.test index d30966d167..998038f88e 100644 --- a/test/printf2.test +++ b/test/printf2.test @@ -148,6 +148,63 @@ do_execsql_test printf2-4.10 { SELECT printf('|%,d|%,d|',1234567890,-1234567890); } {|1,234,567,890|-1,234,567,890|} +# 2018-02-19. Unicode characters with %c +do_execsql_test printf2-5.100 { + SELECT printf('(%8c)',char(11106)); +} {{( ⭢)}} +do_execsql_test printf2-5.101 { + SELECT printf('(%-8c)',char(11106)); +} {{(⭢ )}} +do_execsql_test printf2-5.102 { + SELECT printf('(%5.3c)',char(1492)); +} {{( ההה)}} +do_execsql_test printf2-5.103 { + SELECT printf('(%-5.3c)',char(1492)); +} {{(ההה )}} +do_execsql_test printf2-5.104 { + SELECT printf('(%3.3c)',char(1492)); +} {{(ההה)}} +do_execsql_test printf2-5.105 { + SELECT printf('(%-3.3c)',char(1492)); +} {{(ההה)}} +do_execsql_test printf2-5.104 { + SELECT printf('(%2c)',char(1513)); +} {{( ש)}} +do_execsql_test printf2-5.106 { + SELECT printf('(%-2c)',char(1513)); +} {{(ש )}} + +# 2018-02-19. Unicode characters with the "!" flag in %s and friends. +do_execsql_test printf2-6.100 { + SELECT printf('(%!.3s)','הנה מה־טוב ומה־נעים שבת אחים גם־יחד'); +} {(הנה)} +do_execsql_test printf2-6.101 { + SELECT printf('(%.6s)','הנה מה־טוב ומה־נעים שבת אחים גם־יחד'); +} {(הנה)} +do_execsql_test printf2-6.102 { + SELECT printf('(%!5.3s)','הנה מה־טוב ומה־נעים שבת אחים גם־יחד'); +} {{( הנה)}} +do_execsql_test printf2-6.103 { + SELECT printf('(%8.6s)','הנה מה־טוב ומה־נעים שבת אחים גם־יחד'); +} {{( הנה)}} +do_execsql_test printf2-6.104 { + SELECT printf('(%!-5.3s)','הנה מה־טוב ומה־נעים שבת אחים גם־יחד'); +} {{(הנה )}} +do_execsql_test printf2-6.105 { + SELECT printf('(%-8.6s)','הנה מה־טוב ומה־נעים שבת אחים גם־יחד'); +} {{(הנה )}} +do_execsql_test printf2-6.106 { + SELECT printf('(%!.3Q)','הנה מה־טוב ומה־נעים שבת אחים גם־יחד'); +} {('הנה')} +do_execsql_test printf2-6.107 { + SELECT printf('(%.6Q)','הנה מה־טוב ומה־נעים שבת אחים גם־יחד'); +} {('הנה')} +do_execsql_test printf2-6.108 { + SELECT printf('(%!7.3Q)','הנה מה־טוב ומה־נעים שבת אחים גם־יחד'); +} {{( 'הנה')}} +do_execsql_test printf2-6.109 { + SELECT printf('(%10.6Q)','הנה מה־טוב ומה־נעים שבת אחים גם־יחד'); +} {{( 'הנה')}} finish_test diff --git a/test/rowvalue.test b/test/rowvalue.test index d900e0daa3..00d9395331 100644 --- a/test/rowvalue.test +++ b/test/rowvalue.test @@ -437,6 +437,113 @@ do_execsql_test 18.6 { } {1 1 1 1 2 1} -finish_test +# 2018-02-13 Ticket https://www.sqlite.org/src/tktview/f484b65f3d6230593c3 +# Incorrect result from a row-value comparison in the WHERE clause. +# +do_execsql_test 19.1 { + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(a INTEGER PRIMARY KEY,b); + INSERT INTO t1(a,b) VALUES(1,11),(2,22),(3,33),(4,44); + SELECT * FROM t1 WHERE (a,b)>(0,0) ORDER BY a; +} {1 11 2 22 3 33 4 44} +do_execsql_test 19.2 { + SELECT * FROM t1 WHERE (a,b)>=(0,0) ORDER BY a; +} {1 11 2 22 3 33 4 44} +do_execsql_test 19.3 { + SELECT * FROM t1 WHERE (a,b)<(5,0) ORDER BY a DESC; +} {4 44 3 33 2 22 1 11} +do_execsql_test 19.4 { + SELECT * FROM t1 WHERE (a,b)<=(5,0) ORDER BY a DESC; +} {4 44 3 33 2 22 1 11} +do_execsql_test 19.5 { + SELECT * FROM t1 WHERE (a,b)>(3,0) ORDER BY a; +} {3 33 4 44} +do_execsql_test 19.6 { + SELECT * FROM t1 WHERE (a,b)>=(3,0) ORDER BY a; +} {3 33 4 44} +do_execsql_test 19.7 { + SELECT * FROM t1 WHERE (a,b)<(3,0) ORDER BY a DESC; +} {2 22 1 11} +do_execsql_test 19.8 { + SELECT * FROM t1 WHERE (a,b)<=(3,0) ORDER BY a DESC; +} {2 22 1 11} +do_execsql_test 19.9 { + SELECT * FROM t1 WHERE (a,b)>(3,32) ORDER BY a; +} {3 33 4 44} +do_execsql_test 19.10 { + SELECT * FROM t1 WHERE (a,b)>(3,33) ORDER BY a; +} {4 44} +do_execsql_test 19.11 { + SELECT * FROM t1 WHERE (a,b)>=(3,33) ORDER BY a; +} {3 33 4 44} +do_execsql_test 19.12 { + SELECT * FROM t1 WHERE (a,b)>=(3,34) ORDER BY a; +} {4 44} +do_execsql_test 19.13 { + SELECT * FROM t1 WHERE (a,b)<(3,34) ORDER BY a DESC; +} {3 33 2 22 1 11} +do_execsql_test 19.14 { + SELECT * FROM t1 WHERE (a,b)<(3,33) ORDER BY a DESC; +} {2 22 1 11} +do_execsql_test 19.15 { + SELECT * FROM t1 WHERE (a,b)<=(3,33) ORDER BY a DESC; +} {3 33 2 22 1 11} +do_execsql_test 19.16 { + SELECT * FROM t1 WHERE (a,b)<=(3,32) ORDER BY a DESC; +} {2 22 1 11} +do_execsql_test 19.21 { + SELECT * FROM t1 WHERE (0,0)<(a,b) ORDER BY a; +} {1 11 2 22 3 33 4 44} +do_execsql_test 19.22 { + SELECT * FROM t1 WHERE (0,0)<=(a,b) ORDER BY a; +} {1 11 2 22 3 33 4 44} +do_execsql_test 19.23 { + SELECT * FROM t1 WHERE (5,0)>(a,b) ORDER BY a DESC; +} {4 44 3 33 2 22 1 11} +do_execsql_test 19.24 { + SELECT * FROM t1 WHERE (5,0)>=(a,b) ORDER BY a DESC; +} {4 44 3 33 2 22 1 11} +do_execsql_test 19.25 { + SELECT * FROM t1 WHERE (3,0)<(a,b) ORDER BY a; +} {3 33 4 44} +do_execsql_test 19.26 { + SELECT * FROM t1 WHERE (3,0)<=(a,b) ORDER BY a; +} {3 33 4 44} +do_execsql_test 19.27 { + SELECT * FROM t1 WHERE (3,0)>(a,b) ORDER BY a DESC; +} {2 22 1 11} +do_execsql_test 19.28 { + SELECT * FROM t1 WHERE (3,0)>=(a,b) ORDER BY a DESC; +} {2 22 1 11} +do_execsql_test 19.29 { + SELECT * FROM t1 WHERE (3,32)<(a,b) ORDER BY a; +} {3 33 4 44} +do_execsql_test 19.30 { + SELECT * FROM t1 WHERE (3,33)<(a,b) ORDER BY a; +} {4 44} +do_execsql_test 19.31 { + SELECT * FROM t1 WHERE (3,33)<=(a,b) ORDER BY a; +} {3 33 4 44} +do_execsql_test 19.32 { + SELECT * FROM t1 WHERE (3,34)<=(a,b) ORDER BY a; +} {4 44} +do_execsql_test 19.33 { + SELECT * FROM t1 WHERE (3,34)>(a,b) ORDER BY a DESC; +} {3 33 2 22 1 11} +do_execsql_test 19.34 { + SELECT * FROM t1 WHERE (3,33)>(a,b) ORDER BY a DESC; +} {2 22 1 11} +do_execsql_test 19.35 { + SELECT * FROM t1 WHERE (3,33)>=(a,b) ORDER BY a DESC; +} {3 33 2 22 1 11} +do_execsql_test 19.36 { + SELECT * FROM t1 WHERE (3,32)>=(a,b) ORDER BY a DESC; +} {2 22 1 11} + +# 2018-02-18: Memory leak nexted row-value. Detected by OSSFuzz. +# +do_catchsql_test 20.1 { + SELECT 1 WHERE (2,(2,0)) IS (2,(2,0)); +} {0 1} finish_test diff --git a/test/spellfix.test b/test/spellfix.test index 8128bb59d2..68bcfd5adb 100644 --- a/test/spellfix.test +++ b/test/spellfix.test @@ -279,7 +279,7 @@ ifcapable trace { do_tracesql_test 6.2.3 { SELECT word, distance FROM t3 WHERE rowid = 10 AND word MATCH 'kiiner'; } {keener 300 - {SELECT id, word, rank, k1 FROM "main"."t3_vocab" WHERE langid=0 AND k2>=?1 AND k2=?1 AND k2 ( + SELECT COUNT( * ) + FROM cd rownum__emulation + WHERE + ( + me.genreid IS NOT NULL + AND + rownum__emulation.genreid IS NULL + ) + OR + ( + me.genreid IS NOT NULL + AND + rownum__emulation.genreid IS NOT NULL + AND + rownum__emulation.genreid < me.genreid + ) + OR + ( + ( me.genreid = rownum__emulation.genreid OR ( me.genreid IS NULL + AND rownum__emulation.genreid IS NULL ) ) + AND + rownum__emulation.cdid > me.cdid + ) + ); +} {4 5} + +# Simplified test cases from the ticket +# +do_execsql_test 7.2 { + DROP TABLE IF EXISTS t1; + DROP TABLE IF EXISTS t2; + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + INSERT INTO t1(a,b) VALUES(1,1); + CREATE TABLE t2(aa INTEGER PRIMARY KEY, bb); + INSERT INTO t2(aa,bb) VALUES(1,1),(2,NULL),(3,NULL); + SELECT ( + SELECT COUNT(*) FROM t2 + WHERE ( t1.b IS NOT NULL AND t2.bb IS NULL ) + OR ( t2.bb < t1.b ) + OR ( t1.b IS t2.bb AND t2.aa > t1.a ) + ) + FROM t1; +} {2} + +# The fix for ticket ec32177c99ccac2b180fd3ea2083 only makes a difference +# in the output when there is a TERM_VNULL entry in the WhereClause array. +# And TERM_VNULL entries are only generated when compiling with +# SQLITE_ENABLE_STAT4. Nevertheless, it is correct that TERM_VIRTUAL terms +# should not participate in the factoring optimization. In all cases other +# than TERM_VNULL, participation is harmless, but it does consume a few +# extra CPU cycles. +# +# The following test verifies that the TERM_VIRTUAL terms resulting from +# a GLOB operator do not appear anywhere in the generated code. This +# confirms that the problem is fixed, even on builds that omit STAT4. +# +do_execsql_test 7.3 { + DROP TABLE IF EXISTS t1; + DROP TABLE IF EXISTS t2; + CREATE TABLE t1(a INTEGER PRIMARY KEY, b TEXT); + INSERT INTO t1(a,b) VALUES(1,'abcxyz'); + CREATE TABLE t2(aa INTEGER PRIMARY KEY, bb TEXT); + INSERT INTO t2(aa,bb) VALUES(1,'abc'),(2,'wxyz'),(3,'xyz'); + CREATE INDEX t2bb ON t2(bb); + EXPLAIN SELECT ( + SELECT COUNT(*) FROM t2 + WHERE ( t1.b GLOB 'a*z' AND t2.bb='xyz' ) + OR ( t2.bb = t1.b ) + OR ( t2.aa = t1.a ) + ) + FROM t1; +} {~/ (Lt|Ge) /} + finish_test diff --git a/test/with4.test b/test/with4.test new file mode 100644 index 0000000000..b0eeba6d14 --- /dev/null +++ b/test/with4.test @@ -0,0 +1,52 @@ +# 2018-02-15 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing the WITH clause in TRIGGERs and VIEWs. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set ::testprefix with4 + +ifcapable {!cte} { + finish_test + return +} + +do_execsql_test 100 { + ATTACH ':memory:' AS aux; + CREATE TABLE main.t1(a,b); + CREATE TABLE aux.t2(x,y); + INSERT INTO t1 VALUES(1,2); + INSERT INTO t2 VALUES(3,4); +} {} +do_catchsql_test 110 { + CREATE VIEW v1 AS SELECT * FROM t1, aux.t2; +} {1 {view v1 cannot reference objects in database aux}} +do_catchsql_test 120 { + CREATE VIEW v2 AS WITH v(m,n) AS (SELECT x,y FROM aux.t2) SELECT * FROM t1, v; +} {1 {view v2 cannot reference objects in database aux}} +do_catchsql_test 130 { + CREATE VIEW v2 AS WITH v(m,n) AS (SELECT 5,?2) SELECT * FROM t1, v; +} {1 {parameters are not allowed in views}} + +do_catchsql_test 200 { + CREATE TRIGGER r1 AFTER INSERT ON t1 BEGIN + WITH v(m,n) AS (SELECT x,y FROM aux.t2) SELECT * FROM t1, v; + END; +} {1 {trigger r1 cannot reference objects in database aux}} +do_catchsql_test 210 { + CREATE TRIGGER r1 AFTER INSERT ON t1 BEGIN + WITH v(m,n) AS (SELECT 5,?2) SELECT * FROM t1, v; + END; +} {1 {trigger cannot use variables}} + +finish_test diff --git a/test/zipfile.test b/test/zipfile.test index 07fc80f602..2bb3f07892 100644 --- a/test/zipfile.test +++ b/test/zipfile.test @@ -22,6 +22,117 @@ if {[catch {load_static_extension db zipfile} error]} { finish_test; return } +proc readfile {f} { + set fd [open $f] + fconfigure $fd -translation binary -encoding binary + set data [read $fd] + close $fd + set data +} + +if {$::tcl_platform(platform)=="unix" && [catch {exec unzip}]==0} { + set ::UNZIP 1 + load_static_extension db fileio + proc do_unzip {file} { + forcedelete test_unzip + file mkdir test_unzip + exec unzip -d test_unzip $file + + set res [db eval { + SELECT replace(name,'test_unzip/',''),mode,mtime,data + FROM fsdir('test_unzip') + WHERE name!='test_unzip' + ORDER BY name + }] + set res + } +} + + +# The argument is a blob (not a hex string) containing a zip archive. +# This proc removes the extended timestamp fields from the archive +# and returns the result. +# +proc remove_timestamps {blob} { + set hex [binary encode hex $blob] + set hex [string map {55540500 00000500} $hex] + binary decode hex $hex +} + + +# Argument $file is the name of a zip archive on disk. This function +# executes test cases to check that the results of each of the following +# are the same: +# +# SELECT * FROM zipfile($file) +# SELECT * FROM zipfile( readfile($file) ) +# SELECT * FROM zipfile( +# (SELECT zipfile(name,mode,mtime,data,method) FROM zipfile($file)) +# ) +# +proc do_zipfile_blob_test {tn file} { + + db func r readfile + set q1 {SELECT name,mode,mtime,method,quote(data) FROM zipfile($file)} + set q2 {SELECT name,mode,mtime,method,quote(data) FROM zipfile( r($file) )} + set q3 {SELECT name,mode,mtime,method,quote(data) FROM zipfile( + ( SELECT zipfile(name,mode,mtime,data,method) FROM zipfile($file) ) + )} + + + set r1 [db eval $q1] + set r2 [db eval $q2] + set r3 [db eval $q3] + #puts $r1 + #puts $r2 + #puts $r3 + + uplevel [list do_test $tn.1 [list set {} $r2] $r1] + uplevel [list do_test $tn.2 [list set {} $r3] $r1] +} + +# Argument $file is a zip file on disk. This command runs tests to: +# +# 1. Unpack the archive with unix command [unzip] and compare the +# results to reading the same archive using the zipfile() table +# valued function. +# +# 2. Creates a new archive with the same contents using the zipfile() +# aggregate function as follows: +# +# SELECT writefile('test_unzip.zip', +# ( SELECT zipfile(name,mode,mtime,data,method) FROM zipfile($file) ) +# ); +# +# Then tests that unpacking the new archive using [unzip] produces +# the same results as in (1). +# +proc do_unzip_test {tn file} { + if {[info vars ::UNZIP]==""} { return } + db func sss strip_slash + + db eval { + SELECT writefile('test_unzip.zip', + ( SELECT zipfile(name,mode,mtime,data,method) FROM zipfile($file) ) + ); + } + + set r1 [db eval { + SELECT sss(name),mode,mtime,data FROM zipfile($file) ORDER BY name + }] + set r2 [do_unzip $file] + set r3 [do_unzip test_unzip.zip] + + uplevel [list do_test $tn.1 [list set {} $r2] $r1] + uplevel [list do_test $tn.2 [list set {} $r3] $r1] +} +proc strip_slash {in} { regsub {/$} $in {} } + +proc do_zip_tests {tn file} { + uplevel do_zipfile_blob_test $tn.1 $file + uplevel do_unzip_test $tn.2 $file +} + forcedelete test.zip do_execsql_test 1.0 { CREATE VIRTUAL TABLE temp.zz USING zipfile('test.zip'); @@ -40,10 +151,18 @@ do_catchsql_test 1.1.0.1 { INSERT INTO zz(name, mode, mtime, sz, rawdata, method) VALUES('f.txt', '-rw-r--r--', 1000000000, 5, 'abcde', 0); } {1 {constraint failed}} -do_catchsql_test 1.1.0.1 { - INSERT INTO zz(name, mtime, sz, rawdata, method) +do_catchsql_test 1.1.0.2 { + INSERT INTO zz(name, mtime, sz, data, method) VALUES('g.txt', 1000000002, 5, '12345', 0); } {1 {constraint failed}} +do_catchsql_test 1.1.0.3 { + INSERT INTO zz(name, mtime, rawdata, method) + VALUES('g.txt', 1000000002, '12345', 0); +} {1 {constraint failed}} +do_catchsql_test 1.1.0.4 { + INSERT INTO zz(name, data, method) + VALUES('g.txt', '12345', 7); +} {1 {constraint failed}} do_execsql_test 1.1.1 { INSERT INTO zz(name, mode, mtime, data, method) @@ -60,12 +179,14 @@ do_execsql_test 1.2 { f.txt 1000000000 abcde g.txt 1000000002 12345 } +do_zip_tests 1.2a test.zip do_execsql_test 1.3 { INSERT INTO zz(name, mode, mtime, data) VALUES('h.txt', '-rw-r--r--', 1000000004, 'aaaaaaaaaabbbbbbbbbb' ); } +do_zip_tests 1.3a test.zip do_execsql_test 1.4 { SELECT name, mtime, data, method FROM zipfile('test.zip'); @@ -85,6 +206,9 @@ ifcapable json1 { h.txt 1 } } +do_catchsql_test 1.4.2 { + SELECT zipfile_cds(mode) FROM zipfile('test.zip'); +} {0 {{} {} {}}} do_execsql_test 1.5.1 { BEGIN; @@ -112,6 +236,7 @@ do_execsql_test 1.6.1 { h.txt 33188 1000000004 aaaaaaaaaabbbbbbbbbb 8 i.txt 33188 1000000006 zxcvb 0 } +do_zip_tests 1.6.1a test.zip do_execsql_test 1.6.2 { UPDATE zz SET mtime=4 WHERE name='i.txt'; @@ -130,6 +255,7 @@ do_execsql_test 1.6.3 { h.txt 33189 1000000004 aaaaaaaaaabbbbbbbbbb 8 i.txt 33188 4 zxcvb 0 } +do_zip_tests 1.6.3a test.zip do_execsql_test 1.6.4 { UPDATE zz SET name = 'blue.txt' WHERE name='f.txt'; @@ -139,6 +265,7 @@ do_execsql_test 1.6.4 { h.txt 33189 1000000004 aaaaaaaaaabbbbbbbbbb 8 i.txt 33188 4 zxcvb 0 } +do_zip_tests 1.6.4a test.zip do_execsql_test 1.6.5 { UPDATE zz SET data = 'edcba' WHERE name='blue.txt'; @@ -169,10 +296,42 @@ do_execsql_test 1.6.8 { i.txt 33188 4 zxcvb 0 } +do_execsql_test 1.6.8 { + UPDATE zz SET data = '' WHERE name='i.txt'; + SELECT name,mode,mtime,data,method from zipfile('test.zip'); +} { + blue.txt/ 16877 1000000000 {} 0 + h.txt 33189 1000000004 aaaaaaaaaabbbbbbbbbb 8 + i.txt 33188 4 {} 0 +} + +do_execsql_test 1.6.9 { + SELECT a.name, a.data + FROM zz AS a, zz AS b + WHERE a.name=+b.name AND +a.mode=b.mode +} { + blue.txt/ {} + h.txt aaaaaaaaaabbbbbbbbbb + i.txt {} +} + +do_execsql_test 1.6.10 { + SELECT name, data FROM zz WHERE name LIKE '%txt' +} { + h.txt aaaaaaaaaabbbbbbbbbb + i.txt {} +} + +do_execsql_test 1.7 { + DELETE FROM zz; + SELECT * FROM zz; +} {} + #------------------------------------------------------------------------- db close forcedelete test.zip reset_db +load_static_extension db fileio load_static_extension db zipfile do_execsql_test 2.1 { CREATE VIRTUAL TABLE zzz USING zipfile('test.zip'); @@ -199,6 +358,7 @@ do_execsql_test 2.4 { dirname2/ 16877 {} dirname2/file1.txt 33188 abcdefghijklmnop } +do_zip_tests 2.4a test.zip # If on unix, check that the [unzip] utility can unpack our archive. # @@ -251,6 +411,230 @@ do_catchsql_test 3.2 { SELECT rowid FROM x1 } {1 {no such column: rowid}} +#------------------------------------------------------------------------- +# Test some error conditions. +# +do_catchsql_test 4.1 { + CREATE VIRTUAL TABLE yyy USING zipfile(); +} {1 {zipfile constructor requires one argument}} +do_catchsql_test 4.2 { + CREATE VIRTUAL TABLE yyy USING zipfile('test.zip', 'test.zip'); +} {1 {zipfile constructor requires one argument}} +do_catchsql_test 4.3 { + SELECT * FROM zipfile() +} {1 {zipfile() function requires an argument}} + +do_catchsql_test 4.4 { + SELECT * FROM zipfile('/path/that/does/not/exist') +} {1 {cannot open file: /path/that/does/not/exist}} + +foreach {tn mode} { + 1 abcd + 2 brwxrwxrwx + 3 lrwxrrxrwx +} { + do_catchsql_test 4.5.$tn { + WITH m(m) AS ( SELECT $mode) + SELECT zipfile('a.txt', m, 1000, 'xyz') FROM m + } [list 1 "zipfile: parse error in mode: $mode"] +} + +do_catchsql_test 4.6 { + WITH c(name,data) AS ( SELECT 'a.txt', 'abc') + SELECT zipfile(name) FROM c +} {1 {wrong number of arguments to function zipfile()}} + +do_catchsql_test 4.7 { + WITH c(name,data) AS ( + SELECT 'a.txt', 'abc' UNION ALL + SELECT NULL, 'def' + ) + SELECT zipfile(name,data) FROM c +} {1 {first argument to zipfile() must be non-NULL}} + +do_catchsql_test 4.7 { + WITH c(name,data,method) AS ( + SELECT 'a.txt', 'abc', 0 + UNION SELECT 'b.txt', 'def', 8 + UNION SELECT 'c.txt', 'ghi', 16 + ) + SELECT zipfile(name,NULL,NULL,data,method) FROM c +} {1 {illegal method value: 16}} + +do_catchsql_test 4.8 { + WITH c(name,data) AS ( + SELECT 'a.txt', 'abc' + UNION SELECT 'b.txt', 'def' + UNION SELECT 'c.txt/', 'ghi' + ) + SELECT zipfile(name,NULL,NULL,data) FROM c +} {1 {non-directory name must not end with /}} + +#-------------------------------------------------------------------------- + +db func rt remove_timestamps +do_execsql_test 5.0 { + WITH c(name,mtime,data) AS ( + SELECT 'a.txt', 946684800, 'abc' + ) + SELECT name,mtime,data FROM zipfile( + ( SELECT rt( zipfile(name,NULL,mtime,data,NULL) ) FROM c ) + ) +} { + a.txt 946684800 abc +} + +if {[info vars ::UNZIP]!=""} { +ifcapable datetime { + load_static_extension db fileio + forcedelete test1.zip test2.zip + do_test 6.0 { + execsql { + WITH c(name,mtime,data) AS ( + SELECT 'a.txt', 946684800, 'abc' UNION ALL + SELECT 'b.txt', 1000000000, 'abc' UNION ALL + SELECT 'c.txt', 1111111000, 'abc' + ) + SELECT writefile('test1.zip', rt( zipfile(name, NULL, mtime, data) ) ), + writefile('test2.zip', ( zipfile(name, NULL, mtime, data) ) ) + FROM c; + } + forcedelete test_unzip + file mkdir test_unzip + exec unzip -d test_unzip test1.zip + + db eval { + SELECT name, strftime('%s', mtime, 'unixepoch', 'localtime') + FROM fsdir('test_unzip') WHERE name!='test_unzip' + ORDER BY name + } + } [list {*}{ + test_unzip/a.txt 946684800 + test_unzip/b.txt 1000000000 + test_unzip/c.txt 1111111000 + }] + + do_execsql_test 6.1 { + SELECT name, mtime, data FROM zipfile('test1.zip') + } { + a.txt 946684800 abc + b.txt 1000000000 abc + c.txt 1111111000 abc + } + + do_test 6.2 { + forcedelete test_unzip + file mkdir test_unzip + exec unzip -d test_unzip test2.zip + + db eval { + SELECT name, mtime + FROM fsdir('test_unzip') WHERE name!='test_unzip' + ORDER BY name + } + } [list {*}{ + test_unzip/a.txt 946684800 + test_unzip/b.txt 1000000000 + test_unzip/c.txt 1111111000 + }] + + do_execsql_test 6.3 { + SELECT name, mtime, sz, rawdata, data FROM zipfile('test2.zip') + } { + a.txt 946684800 3 abc abc + b.txt 1000000000 3 abc abc + c.txt 1111111000 3 abc abc + } +} +} + +#------------------------------------------------------------------------- +# Force an IO error by truncating the zip archive to zero bytes in size +# while it is being read. +forcedelete test.zip +do_test 7.0 { + execsql { + WITH c(name,data) AS ( + SELECT '1', randomblob(1000000) UNION ALL + SELECT '2', randomblob(1000000) UNION ALL + SELECT '3', randomblob(1000000) + ) + SELECT writefile('test.zip', zipfile(name, data) ) FROM c; + } + + list [catch { + db eval { SELECT name, data FROM zipfile('test.zip') } { + if {$name==2} { close [open test.zip w+] } + } + } msg] $msg +} {1 {error in fread()}} + +forcedelete test.zip +do_execsql_test 8.0.1 { + CREATE VIRTUAL TABLE zz USING zipfile('test.zip'); + BEGIN; + INSERT INTO zz(name, data) VALUES('a.txt', '1'); + INSERT INTO zz(name, data) VALUES('b.txt', '2'); + INSERT INTO zz(name, data) VALUES('c.txt', '1'); + INSERT INTO zz(name, data) VALUES('d.txt', '2'); + SELECT name, data FROM zz; +} { + a.txt 1 b.txt 2 c.txt 1 d.txt 2 +} +do_test 8.0.2 { + db eval { SELECT name, data FROM zz } { + if { $data=="2" } { db eval { DELETE FROM zz WHERE name=$name } } + } + execsql { SELECT name, data FROM zz } +} {a.txt 1 c.txt 1} +do_test 8.0.3 { + db eval { SELECT name, data FROM zz } { + db eval { DELETE FROM zz WHERE name=$name } + } + execsql { SELECT name, data FROM zz } +} {} +execsql COMMIT + +do_execsql_test 8.1.1 { + CREATE VIRTUAL TABLE nogood USING zipfile('test_unzip'); +} +do_catchsql_test 8.1.2 { + INSERT INTO nogood(name, data) VALUES('abc', 'def'); +} {1 {zipfile: failed to open file test_unzip for writing}} + +do_execsql_test 8.2.1 { + DROP TABLE nogood; + BEGIN; + CREATE VIRTUAL TABLE nogood USING zipfile('test_unzip'); +} +do_catchsql_test 8.2.2 { + INSERT INTO nogood(name, data) VALUES('abc', 'def'); +} {1 {zipfile: failed to open file test_unzip for writing}} +do_execsql_test 8.2.3 { + COMMIT; +} + +forcedelete test.zip +do_execsql_test 8.3.1 { + BEGIN; + CREATE VIRTUAL TABLE ok USING zipfile('test.zip'); + INSERT INTO ok(name, data) VALUES ('sqlite3', 'elf'); + COMMIT; +} + +#------------------------------------------------------------------------- +# Test that the zipfile aggregate correctly adds and removes "/" from +# the ends of directory file names. +do_execsql_test 9.0 { + WITH src(nm) AS ( + VALUES('dir1') UNION ALL + VALUES('dir2/') UNION ALL + VALUES('dir3//') UNION ALL + VALUES('dir4///') UNION ALL + VALUES('/') + ) + SELECT name FROM zipfile((SELECT zipfile(nm, NULL) FROM src)) +} {dir1/ dir2/ dir3/ dir4/ /} finish_test diff --git a/test/zipfile2.test b/test/zipfile2.test new file mode 100644 index 0000000000..f3509d0a54 --- /dev/null +++ b/test/zipfile2.test @@ -0,0 +1,208 @@ +# 2018 January 30 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix zipfile2 + +ifcapable !vtab { + finish_test; return +} +if {[catch {load_static_extension db zipfile} error]} { + puts "Skipping zipfile2 tests, hit load error: $error" + finish_test; return +} + +proc blobliteral {str} { + set concat [string map {" " "" "\n" ""} $str] + return "X'$concat'" +} + +proc blob {str} { + binary decode hex $str +} + +proc findall {needle haystack} { + set L [list] + set start 0 + while { [set idx [string first $needle $haystack $start]]>=0 } { + lappend L $idx + set start [expr $idx+1] + } + set L +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE aaa USING zipfile('testzip'); + CREATE VIRTUAL TABLE bbb USING zipfile("testzip"); + CREATE VIRTUAL TABLE ccc USING zipfile(`testzip`); + CREATE VIRTUAL TABLE ddd USING zipfile([testzip]); + CREATE VIRTUAL TABLE eee USING zipfile(testzip); + CREATE VIRTUAL TABLE fff USING zipfile('test''zip'); +} + +do_test 2.0 { + forcedelete testdir + file mkdir testdir + execsql { CREATE VIRTUAL TABLE hhh USING zipfile('testdir') } + catchsql { SELECT * FROM hhh } +} {1 {error in fread()}} + + +set archive { + 504B0304140000080000D4A52BEC09F3B6E0110000001100000005000900612E + 747874555405000140420F00636F6E74656E7473206F6620612E747874504B03 + 04140000080000D4A52BECD98916A7110000001100000005000900622E747874 + 555405000140420F00636F6E74656E7473206F6620622E747874504B01021E03 + 140000080000D4A52BEC09F3B6E0110000001100000005000900000000000000 + 0000A48100000000612E747874555405000140420F00504B01021E0314000008 + 0000D4A52BECD98916A71100000011000000050009000000000000000000A481 + 3D000000622E747874555405000140420F00504B050600000000020002007800 + 00007A0000000000 +} + +if 0 { + # This test is broken - the archive generated is slightly different + # depending on the zlib version used. + do_execsql_test 3.1 { + WITH contents(name,mtime,data) AS ( + VALUES('a.txt', 1000000, 'contents of a.txt') UNION ALL + VALUES('b.txt', 1000000, 'contents of b.txt') + ) SELECT quote( zipfile(name,NULL,mtime,data) ) FROM contents; + } [blobliteral $archive] +} + + +set blob [blob $archive] +do_execsql_test 3.2 { + SELECT name,mtime,data FROM zipfile($blob) +} { + a.txt 1000000 {contents of a.txt} + b.txt 1000000 {contents of b.txt} +} + +# Corrupt each of the 0x50 0x4B (ascii "PK") headers in the file +# Test that in each case this causes an error. +# +set L [findall 504B $archive] +for {set i 0} {$i < [llength $L]} {incr i} { + set idx [lindex $L $i] + set a [string replace $archive $idx [expr $idx+3] 0000] + set blob [blob $a] + do_catchsql_test 3.3.$i { + SELECT name,mtime,data FROM zipfile($blob) + } {/1 .*/} +} + +# Change the "extra info id" for all extended-timestamp fields. +set L [findall 5554 $archive] +for {set i 0} {$i < [llength $L]} {incr i} { + set idx [lindex $L $i] + set a [string replace $archive $idx [expr $idx+3] 1234] + set blob [blob $a] + do_execsql_test 3.4.$i { + SELECT name,data FROM zipfile($blob) + } { + a.txt {contents of a.txt} + b.txt {contents of b.txt} + } +} + +for {set i 0} {$i < [llength $L]} {incr i} { + set idx [lindex $L $i] + set a [string replace $archive [expr $idx+8] [expr $idx+9] 00] + set blob [blob $a] + do_execsql_test 3.5.$i { + SELECT name,data FROM zipfile($blob) + } { + a.txt {contents of a.txt} + b.txt {contents of b.txt} + } +} + +# set blob [db one { +# WITH contents(name,mtime,data) AS ( +# VALUES('a.txt', 1000000, 'aaaaaaaaaaaaaaaaaaaaaaa') +# ) SELECT quote( zipfile(name,NULL,mtime,data) ) FROM contents; +# }] +# set blob [string range $blob 2 end] +# set blob [string range $blob 0 end-1] +# while {[string length $blob]>0} { +# puts [string range $blob 0 63] +# set blob [string range $blob 64 end] +# } +# exit + +set archive2 { + 504B0304140000080800D4A52BEC08F54C6E050000001700000005000900612E + 747874555405000140420F004B4CC40A00504B01021E03140000080800D4A52B + EC08F54C6E0500000017000000050009000000000000000000A4810000000061 + 2E747874555405000140420F00504B050600000000010001003C000000310000 + 000000 +} +set blob [blob $archive2] +do_execsql_test 4.0 { + SELECT name,mtime,data,method FROM zipfile($blob) +} { + a.txt 1000000 aaaaaaaaaaaaaaaaaaaaaaa 8 +} + +set L [findall 17000000 $archive2] +set a $archive2 +foreach i $L { set a [string replace $a $i [expr $i+7] 16000000] } +set blob [blob $a] +do_catchsql_test 4.1 { + SELECT name,mtime,data,method FROM zipfile($blob) +} {1 {inflate() failed (0)}} + +# Check the response to an unknown compression method (set data to NULL). +set blob [blob [string map {0800 0900} $archive2]] +do_execsql_test 4.2 { + SELECT name,mtime,data IS NULL,method FROM zipfile($blob) +} {a.txt 1000000 1 9} + +# Corrupt the EOCDS signature bytes in various ways. +foreach {tn sub} { + 1 {504B0500} + 2 {504B0006} + 3 {50000506} + 4 {004B0506} +} { + set blob [blob [string map [list 504B0506 $sub] $archive2]] + do_catchsql_test 4.3.$tn { + SELECT * FROM zipfile($blob) + } {1 {cannot find end of central directory record}} +} + +#------------------------------------------------------------------------- +# Test that a zero-length file with a '/' at the end is treated as +# a directory (data IS NULL). Even if the mode doesn't indicate +# that it is a directory. + +do_test 5.0 { + set blob [db one { + WITH c(n, d) AS ( + SELECT 'notadir', '' + ) + SELECT zipfile(n, d) FROM c + }] + + set hex [binary encode hex $blob] + set hex [string map {6e6f7461646972 6e6f746164692f} $hex] + set blob2 [binary decode hex $hex] + + execsql { SELECT name, data IS NULL FROM zipfile($blob2) } +} {notadi/ 1} + + +finish_test + diff --git a/test/zipfilefault.test b/test/zipfilefault.test new file mode 100644 index 0000000000..158370695e --- /dev/null +++ b/test/zipfilefault.test @@ -0,0 +1,165 @@ +# 2018 January 30 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/malloc_common.tcl +set testprefix zipfilefault + +ifcapable !vtab { + finish_test; return +} +if {[catch {load_static_extension db zipfile} error]} { + puts "Skipping zipfile2 tests, hit load error: $error" + finish_test; return +} + +faultsim_save_and_close +do_faultsim_test 1 -prep { + faultsim_restore_and_reopen + load_static_extension db zipfile + execsql { DROP TABLE IF EXISTS aaa } +} -body { + execsql { CREATE VIRTUAL TABLE aaa USING zipfile('test.zip') } +} -test { + faultsim_test_result {0 {}} +} + +forcedelete test.zip +sqlite3 db test.db +load_static_extension db zipfile +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE setup USING zipfile('test.zip'); + INSERT INTO setup(name, data) VALUES('a.txt', '1234567890'); +} + +do_faultsim_test 2.1 -faults oom* -body { + execsql { SELECT name,data FROM zipfile('test.zip') } +} -test { + faultsim_test_result {0 {a.txt 1234567890}} +} +do_faultsim_test 2.2 -faults oom* -body { + execsql { + SELECT json_extract( zipfile_cds(z), '$.version-made-by' ) + FROM zipfile('test.zip') + } +} -test { + faultsim_test_result {0 798} +} + +forcedelete test.zip +reset_db +load_static_extension db zipfile +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE setup USING zipfile('test.zip'); + INSERT INTO setup(name, data) VALUES('a.txt', 'aaaaaaaaaaaaaaaaaaaaaaaaaaaa'); +} + +do_faultsim_test 3 -faults oom* -body { + execsql { SELECT name,data FROM zipfile('test.zip') } +} -test { + faultsim_test_result {0 {a.txt aaaaaaaaaaaaaaaaaaaaaaaaaaaa}} +} + +do_faultsim_test 4 -faults oom* -body { + execsql { + WITH c(n, d) AS ( + SELECT 1, 'aaaaaaaaaaabbbbbbbbbbaaaaaaaaaabbbbbbbbbb' + ) + SELECT name, data FROM zipfile( + (SELECT zipfile(n, d) FROM c) + ); + } +} -test { + faultsim_test_result {0 {1 aaaaaaaaaaabbbbbbbbbbaaaaaaaaaabbbbbbbbbb}} +} + +reset_db +sqlite3_db_config_lookaside db 0 0 0 +load_static_extension db zipfile + +do_execsql_test 5.0 { + CREATE VIRTUAL TABLE setup USING zipfile('test.zip') +} + +do_faultsim_test 5.1 -faults oom* -prep { + forcedelete test.zip +} -body { + execsql { + INSERT INTO setup(name, data) + VALUES('a.txt', 'aaaaaaaaaaaaaaaaaaaaaaaaaaaa'); + } +} -test { + faultsim_test_result {0 {}} +} + +do_faultsim_test 5.2 -faults oom* -prep { + forcedelete test.zip +} -body { + execsql { + INSERT INTO setup(name, data) VALUES('dir', NULL) + } +} -test { + faultsim_test_result {0 {}} +} + +do_faultsim_test 5.3 -faults oom* -prep { + forcedelete test.zip + execsql { + DROP TABLE IF EXISTS setup; + BEGIN; + CREATE VIRTUAL TABLE setup USING zipfile('test.zip') + } +} -body { + execsql { + INSERT INTO setup(name, data) VALUES('dir', NULL) + } +} -test { + catchsql { COMMIT } + faultsim_test_result {0 {}} +} + +do_faultsim_test 6.1 -faults oom* -body { + execsql { + WITH c(n, d) AS ( + VALUES('a.txt', '1234567890') UNION ALL + VALUES('dir', NULL) + ) + SELECT zipfile(n, d) IS NULL FROM c; + } +} -test { + faultsim_test_result {0 0} +} + +set big [string repeat 0123456789 1000] +do_faultsim_test 6.2 -faults oom* -body { + execsql { + WITH c(n, d) AS ( + VALUES('a.txt', $big) + ) + SELECT zipfile(n, NULL, NULL, d, 0) IS NULL FROM c; + } +} -test { + faultsim_test_result {0 0} +} + +do_faultsim_test 7.0 -faults oom* -prep { + catch { db close } + sqlite3 db "" +} -body { + load_static_extension db zipfile +} -test { +} + + +finish_test + diff --git a/tool/addopcodes.tcl b/tool/addopcodes.tcl index dfd7fac88c..070779029b 100644 --- a/tool/addopcodes.tcl +++ b/tool/addopcodes.tcl @@ -22,6 +22,7 @@ close $in # ILLEGAL *must* be the last two token codes and they must be in that order. # set extras { + TRUEFALSE ISNOT FUNCTION COLUMN @@ -29,6 +30,7 @@ set extras { AGG_COLUMN UMINUS UPLUS + TRUTH REGISTER VECTOR SELECT_COLUMN diff --git a/tool/lempar.c b/tool/lempar.c index ecc0e6389b..1ade666916 100644 --- a/tool/lempar.c +++ b/tool/lempar.c @@ -511,7 +511,8 @@ static unsigned int yy_find_shift_action( #endif do{ i = yy_shift_ofst[stateno]; - assert( i>=0 && i+YYNTOKEN<=sizeof(yy_lookahead)/sizeof(yy_lookahead[0]) ); + assert( i>=0 ); + assert( i+YYNTOKEN<=(int)sizeof(yy_lookahead)/sizeof(yy_lookahead[0]) ); assert( iLookAhead!=YYNOCODE ); assert( iLookAhead < YYNTOKEN ); i += iLookAhead; diff --git a/tool/speed-check.sh b/tool/speed-check.sh index 6cc3018981..fc05ac98ef 100644 --- a/tool/speed-check.sh +++ b/tool/speed-check.sh @@ -39,6 +39,8 @@ LEAN_OPTS="$LEAN_OPTS -DSQLITE_USE_ALLOCA" BASELINE="trunk" doExplain=0 doCachegrind=1 +doVdbeProfile=0 +doWal=1 while test "$1" != ""; do case $1 in --reprepare) @@ -62,8 +64,11 @@ while test "$1" != ""; do --temp) SPEEDTEST_OPTS="$SPEEDTEST_OPTS --temp 6" ;; + --legacy) + doWal=0 + ;; --wal) - SPEEDTEST_OPTS="$SPEEDTEST_OPTS --journal wal" + doWal=1 ;; --size) shift; SIZE=$1 @@ -78,6 +83,7 @@ while test "$1" != ""; do rm -f vdbe_profile.out CC_OPTS="$CC_OPTS -DVDBE_PROFILE" doCachegrind=0 + doVdbeProfile=1 ;; --lean) CC_OPTS="$CC_OPTS $LEAN_OPTS" @@ -132,11 +138,17 @@ while test "$1" != ""; do esac shift done +if test $doWal -eq 1; then + SPEEDTEST_OPTS="$SPEEDTEST_OPTS --journal wal" +fi SPEEDTEST_OPTS="$SPEEDTEST_OPTS --size $SIZE" echo "NAME = $NAME" | tee summary-$NAME.txt echo "SPEEDTEST_OPTS = $SPEEDTEST_OPTS" | tee -a summary-$NAME.txt echo "CC_OPTS = $CC_OPTS" | tee -a summary-$NAME.txt rm -f cachegrind.out.* speedtest1 speedtest1.db sqlite3.o +if test $doVdbeProfile -eq 1; then + rm -f vdbe_profile.out +fi $CC -g -Os -Wall -I. $CC_OPTS -c sqlite3.c size sqlite3.o | tee -a summary-$NAME.txt if test $doExplain -eq 1; then @@ -163,6 +175,10 @@ fi if test $doExplain -eq 1; then ./speedtest1 --explain $SPEEDTEST_OPTS | ./sqlite3 >explain-$NAME.txt fi -if test "$NAME" != "$BASELINE"; then +if test $doVdbeProfile -eq 1; then + tclsh ../sqlite/tool/vdbe_profile.tcl >vdbeprofile-$NAME.txt + open vdbeprofile-$NAME.txt +fi +if test "$NAME" != "$BASELINE" -a $doVdbeProfile -ne 1; then fossil test-diff --tk -c 20 cout-$BASELINE.txt cout-$NAME.txt fi