From a41f2855e9310602d0680fe6056c3653504705a4 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 8 Jan 2018 16:54:26 +0000 Subject: [PATCH 001/107] First code for an auxiliary function that will normalize an SQL statement. FossilOrigin-Name: 84814aac81e54f03430f180926156ab0fc01e22bbce2bb228b698ea0d4a588ea --- ext/misc/normalize.c | 639 +++++++++++++++++++++++++++++++++++++++++++ manifest | 14 +- manifest.uuid | 2 +- 3 files changed, 649 insertions(+), 6 deletions(-) create mode 100644 ext/misc/normalize.c diff --git a/ext/misc/normalize.c b/ext/misc/normalize.c new file mode 100644 index 0000000000..914a154d9a --- /dev/null +++ b/ext/misc/normalize.c @@ -0,0 +1,639 @@ +/* +** 2018-01-08 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains code to implement the sqlite3_normalize() function. +** +** char *sqlite3_normalize(const char *zSql); +** +** This function takes an SQL string as input and returns a "normalized" +** version of that string in memory obtained from sqlite3_malloc64(). The +** caller is responsible for ensuring that the returned memory is freed. +** +** If a memory allocation error occurs, this routine returns NULL. +** +** The normalization consists of the following transformations: +** +** (1) Convert every literal (string, blob literal, numeric constant, +** or "NULL" constant) into a ? +** +** (2) Remove all superfluous whitespace, including comments. Change +** all required whitespace to a single space character. +** +** (3) Lowercase all ASCII characters. +** +** (4) If an IN or NOT IN operator is followed by a list of 1 or more +** values, convert that list into "(?,?,?)". +** +** The purpose of normalization is two-fold: +** +** (1) Sanitize queries by removing possibly sensitive information contained +** in literals. +** +** (2) Identify structurally identical queries by comparing their +** normalized forms. +*/ +#include +#include + +/* +** Implementation note: +** +** Much of the tokenizer logic is copied out of the tokenize.c source file +** of SQLite. This logic could be simplified for this particular application, +** but that would impose a risk of introducing subtle errors. It is best to +** keep the code as close to the original as possible. +*/ + + +/* Character classes for tokenizing +** +** In the sqlite3GetToken() function, a switch() on aiClass[c] is implemented +** using a lookup table, whereas a switch() directly on c uses a binary search. +** The lookup table is much faster. To maximize speed, and to ensure that +** a lookup table is used, all of the classes need to be small integers and +** all of them need to be used within the switch. +*/ +#define CC_X 0 /* The letter 'x', or start of BLOB literal */ +#define CC_KYWD 1 /* Alphabetics or '_'. Usable in a keyword */ +#define CC_ID 2 /* unicode characters usable in IDs */ +#define CC_DIGIT 3 /* Digits */ +#define CC_DOLLAR 4 /* '$' */ +#define CC_VARALPHA 5 /* '@', '#', ':'. Alphabetic SQL variables */ +#define CC_VARNUM 6 /* '?'. Numeric SQL variables */ +#define CC_SPACE 7 /* Space characters */ +#define CC_QUOTE 8 /* '"', '\'', or '`'. String literals, quoted ids */ +#define CC_QUOTE2 9 /* '['. [...] style quoted ids */ +#define CC_PIPE 10 /* '|'. Bitwise OR or concatenate */ +#define CC_MINUS 11 /* '-'. Minus or SQL-style comment */ +#define CC_LT 12 /* '<'. Part of < or <= or <> */ +#define CC_GT 13 /* '>'. Part of > or >= */ +#define CC_EQ 14 /* '='. Part of = or == */ +#define CC_BANG 15 /* '!'. Part of != */ +#define CC_SLASH 16 /* '/'. / or c-style comment */ +#define CC_LP 17 /* '(' */ +#define CC_RP 18 /* ')' */ +#define CC_SEMI 19 /* ';' */ +#define CC_PLUS 20 /* '+' */ +#define CC_STAR 21 /* '*' */ +#define CC_PERCENT 22 /* '%' */ +#define CC_COMMA 23 /* ',' */ +#define CC_AND 24 /* '&' */ +#define CC_TILDA 25 /* '~' */ +#define CC_DOT 26 /* '.' */ +#define CC_ILLEGAL 27 /* Illegal character */ + +static const unsigned char aiClass[] = { +/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xa xb xc xd xe xf */ +/* 0x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 7, 7, 27, 7, 7, 27, 27, +/* 1x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, +/* 2x */ 7, 15, 8, 5, 4, 22, 24, 8, 17, 18, 21, 20, 23, 11, 26, 16, +/* 3x */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 5, 19, 12, 14, 13, 6, +/* 4x */ 5, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, +/* 5x */ 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 9, 27, 27, 27, 1, +/* 6x */ 8, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, +/* 7x */ 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 27, 10, 27, 25, 27, +/* 8x */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* 9x */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* Ax */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* Bx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* Cx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* Dx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* Ex */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* Fx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 +}; + +/* An array to map all upper-case characters into their corresponding +** lower-case character. +** +** SQLite only considers US-ASCII (or EBCDIC) characters. We do not +** handle case conversions for the UTF character set since the tables +** involved are nearly as big or bigger than SQLite itself. +*/ +const unsigned char sqlite3UpperToLower[] = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, + 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, + 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, + 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 97, 98, 99,100,101,102,103, + 104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121, + 122, 91, 92, 93, 94, 95, 96, 97, 98, 99,100,101,102,103,104,105,106,107, + 108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125, + 126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143, + 144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161, + 162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179, + 180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197, + 198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215, + 216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233, + 234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251, + 252,253,254,255 +}; + +/* +** The following 256 byte lookup table is used to support SQLites built-in +** equivalents to the following standard library functions: +** +** isspace() 0x01 +** isalpha() 0x02 +** isdigit() 0x04 +** isalnum() 0x06 +** isxdigit() 0x08 +** toupper() 0x20 +** SQLite identifier character 0x40 +** Quote character 0x80 +** +** Bit 0x20 is set if the mapped character requires translation to upper +** case. i.e. if the character is a lower-case ASCII character. +** If x is a lower-case ASCII character, then its upper-case equivalent +** is (x - 0x20). Therefore toupper() can be implemented as: +** +** (x & ~(map[x]&0x20)) +** +** The equivalent of tolower() is implemented using the sqlite3UpperToLower[] +** array. tolower() is used more often than toupper() by SQLite. +** +** Bit 0x40 is set if the character is non-alphanumeric and can be used in an +** SQLite identifier. Identifiers are alphanumerics, "_", "$", and any +** non-ASCII UTF character. Hence the test for whether or not a character is +** part of an identifier is 0x46. +*/ +const unsigned char sqlite3CtypeMap[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 00..07 ........ */ + 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00, /* 08..0f ........ */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 10..17 ........ */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 18..1f ........ */ + 0x01, 0x00, 0x80, 0x00, 0x40, 0x00, 0x00, 0x80, /* 20..27 !"#$%&' */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 28..2f ()*+,-./ */ + 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, /* 30..37 01234567 */ + 0x0c, 0x0c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 38..3f 89:;<=>? */ + + 0x00, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x02, /* 40..47 @ABCDEFG */ + 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 48..4f HIJKLMNO */ + 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 50..57 PQRSTUVW */ + 0x02, 0x02, 0x02, 0x80, 0x00, 0x00, 0x00, 0x40, /* 58..5f XYZ[\]^_ */ + 0x80, 0x2a, 0x2a, 0x2a, 0x2a, 0x2a, 0x2a, 0x22, /* 60..67 `abcdefg */ + 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, /* 68..6f hijklmno */ + 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, /* 70..77 pqrstuvw */ + 0x22, 0x22, 0x22, 0x00, 0x00, 0x00, 0x00, 0x00, /* 78..7f xyz{|}~. */ + + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* 80..87 ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* 88..8f ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* 90..97 ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* 98..9f ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* a0..a7 ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* a8..af ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* b0..b7 ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* b8..bf ........ */ + + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* c0..c7 ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* c8..cf ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* d0..d7 ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* d8..df ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* e0..e7 ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* e8..ef ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* f0..f7 ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40 /* f8..ff ........ */ +}; +#define sqlite3Toupper(x) ((x)&~(sqlite3CtypeMap[(unsigned char)(x)]&0x20)) +#define sqlite3Isspace(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x01) +#define sqlite3Isalnum(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x06) +#define sqlite3Isalpha(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x02) +#define sqlite3Isdigit(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x04) +#define sqlite3Isxdigit(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x08) +#define sqlite3Tolower(x) (sqlite3UpperToLower[(unsigned char)(x)]) +#define sqlite3Isquote(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x80) + + +/* +** If X is a character that can be used in an identifier then +** IdChar(X) will be true. Otherwise it is false. +** +** For ASCII, any character with the high-order bit set is +** allowed in an identifier. For 7-bit characters, +** sqlite3IsIdChar[X] must be 1. +** +** For EBCDIC, the rules are more complex but have the same +** end result. +** +** Ticket #1066. the SQL standard does not allow '$' in the +** middle of identifiers. But many SQL implementations do. +** SQLite will allow '$' in identifiers for compatibility. +** But the feature is undocumented. +*/ +#define IdChar(C) ((sqlite3CtypeMap[(unsigned char)C]&0x46)!=0) + +/* +** Ignore testcase() macros +*/ +#define testcase(X) + +/* +** Token values +*/ +#define TK_SPACE 0 +#define TK_NAME 1 +#define TK_LITERAL 2 +#define TK_PUNCT 3 +#define TK_ERROR 4 + +#define TK_MINUS TK_PUNCT +#define TK_LP TK_PUNCT +#define TK_RP TK_PUNCT +#define TK_SEMI TK_PUNCT +#define TK_PLUS TK_PUNCT +#define TK_STAR TK_PUNCT +#define TK_SLASH TK_PUNCT +#define TK_REM TK_PUNCT +#define TK_EQ TK_PUNCT +#define TK_LE TK_PUNCT +#define TK_NE TK_PUNCT +#define TK_LSHIFT TK_PUNCT +#define TK_LT TK_PUNCT +#define TK_GE TK_PUNCT +#define TK_RSHIFT TK_PUNCT +#define TK_GT TK_PUNCT +#define TK_GE TK_PUNCT +#define TK_BITOR TK_PUNCT +#define TK_CONCAT TK_PUNCT +#define TK_COMMA TK_PUNCT +#define TK_BITAND TK_PUNCT +#define TK_BITNOT TK_PUNCT +#define TK_STRING TK_LITERAL +#define TK_ID TK_NAME +#define TK_ILLEGAL TK_ERROR +#define TK_DOT TK_PUNCT +#define TK_INTEGER TK_LITERAL +#define TK_FLOAT TK_LITERAL +#define TK_VARIABLE TK_LITERAL +#define TK_BLOB TK_LITERAL + +/* +** Return the length (in bytes) of the token that begins at z[0]. +** Store the token type in *tokenType before returning. +*/ +static int sqlite3GetToken(const unsigned char *z, int *tokenType){ + int i, c; + switch( aiClass[*z] ){ /* Switch on the character-class of the first byte + ** of the token. See the comment on the CC_ defines + ** above. */ + case CC_SPACE: { + for(i=1; sqlite3Isspace(z[i]); i++){} + *tokenType = TK_SPACE; + return i; + } + case CC_MINUS: { + if( z[1]=='-' ){ + for(i=2; (c=z[i])!=0 && c!='\n'; i++){} + *tokenType = TK_SPACE; + return i; + } + *tokenType = TK_MINUS; + return 1; + } + case CC_LP: { + *tokenType = TK_LP; + return 1; + } + case CC_RP: { + *tokenType = TK_RP; + return 1; + } + case CC_SEMI: { + *tokenType = TK_SEMI; + return 1; + } + case CC_PLUS: { + *tokenType = TK_PLUS; + return 1; + } + case CC_STAR: { + *tokenType = TK_STAR; + return 1; + } + case CC_SLASH: { + if( z[1]!='*' || z[2]==0 ){ + *tokenType = TK_SLASH; + return 1; + } + for(i=3, c=z[2]; (c!='*' || z[i]!='/') && (c=z[i])!=0; i++){} + if( c ) i++; + *tokenType = TK_SPACE; + return i; + } + case CC_PERCENT: { + *tokenType = TK_REM; + return 1; + } + case CC_EQ: { + *tokenType = TK_EQ; + return 1 + (z[1]=='='); + } + case CC_LT: { + if( (c=z[1])=='=' ){ + *tokenType = TK_LE; + return 2; + }else if( c=='>' ){ + *tokenType = TK_NE; + return 2; + }else if( c=='<' ){ + *tokenType = TK_LSHIFT; + return 2; + }else{ + *tokenType = TK_LT; + return 1; + } + } + case CC_GT: { + if( (c=z[1])=='=' ){ + *tokenType = TK_GE; + return 2; + }else if( c=='>' ){ + *tokenType = TK_RSHIFT; + return 2; + }else{ + *tokenType = TK_GT; + return 1; + } + } + case CC_BANG: { + if( z[1]!='=' ){ + *tokenType = TK_ILLEGAL; + return 1; + }else{ + *tokenType = TK_NE; + return 2; + } + } + case CC_PIPE: { + if( z[1]!='|' ){ + *tokenType = TK_BITOR; + return 1; + }else{ + *tokenType = TK_CONCAT; + return 2; + } + } + case CC_COMMA: { + *tokenType = TK_COMMA; + return 1; + } + case CC_AND: { + *tokenType = TK_BITAND; + return 1; + } + case CC_TILDA: { + *tokenType = TK_BITNOT; + return 1; + } + case CC_QUOTE: { + int delim = z[0]; + testcase( delim=='`' ); + testcase( delim=='\'' ); + testcase( delim=='"' ); + for(i=1; (c=z[i])!=0; i++){ + if( c==delim ){ + if( z[i+1]==delim ){ + i++; + }else{ + break; + } + } + } + if( c=='\'' ){ + *tokenType = TK_STRING; + return i+1; + }else if( c!=0 ){ + *tokenType = TK_ID; + return i+1; + }else{ + *tokenType = TK_ILLEGAL; + return i; + } + } + case CC_DOT: { + if( !sqlite3Isdigit(z[1]) ){ + *tokenType = TK_DOT; + return 1; + } + /* If the next character is a digit, this is a floating point + ** number that begins with ".". Fall thru into the next case */ + } + case CC_DIGIT: { + *tokenType = TK_INTEGER; + if( z[0]=='0' && (z[1]=='x' || z[1]=='X') && sqlite3Isxdigit(z[2]) ){ + for(i=3; sqlite3Isxdigit(z[i]); i++){} + return i; + } + for(i=0; sqlite3Isdigit(z[i]); i++){} + if( z[i]=='.' ){ + i++; + while( sqlite3Isdigit(z[i]) ){ i++; } + *tokenType = TK_FLOAT; + } + if( (z[i]=='e' || z[i]=='E') && + ( sqlite3Isdigit(z[i+1]) + || ((z[i+1]=='+' || z[i+1]=='-') && sqlite3Isdigit(z[i+2])) + ) + ){ + i += 2; + while( sqlite3Isdigit(z[i]) ){ i++; } + *tokenType = TK_FLOAT; + } + while( IdChar(z[i]) ){ + *tokenType = TK_ILLEGAL; + i++; + } + return i; + } + case CC_QUOTE2: { + for(i=1, c=z[0]; c!=']' && (c=z[i])!=0; i++){} + *tokenType = c==']' ? TK_ID : TK_ILLEGAL; + return i; + } + case CC_VARNUM: { + *tokenType = TK_VARIABLE; + for(i=1; sqlite3Isdigit(z[i]); i++){} + return i; + } + case CC_DOLLAR: + case CC_VARALPHA: { + int n = 0; + testcase( z[0]=='$' ); testcase( z[0]=='@' ); + testcase( z[0]==':' ); testcase( z[0]=='#' ); + *tokenType = TK_VARIABLE; + for(i=1; (c=z[i])!=0; i++){ + if( IdChar(c) ){ + n++; + }else if( c=='(' && n>0 ){ + do{ + i++; + }while( (c=z[i])!=0 && !sqlite3Isspace(c) && c!=')' ); + if( c==')' ){ + i++; + }else{ + *tokenType = TK_ILLEGAL; + } + break; + }else if( c==':' && z[i+1]==':' ){ + i++; + }else{ + break; + } + } + if( n==0 ) *tokenType = TK_ILLEGAL; + return i; + } + case CC_KYWD: { + for(i=1; aiClass[z[i]]<=CC_KYWD; i++){} + if( IdChar(z[i]) ){ + /* This token started out using characters that can appear in keywords, + ** but z[i] is a character not allowed within keywords, so this must + ** be an identifier instead */ + i++; + break; + } + *tokenType = TK_ID; + return i; + } + case CC_X: { + testcase( z[0]=='x' ); testcase( z[0]=='X' ); + if( z[1]=='\'' ){ + *tokenType = TK_BLOB; + for(i=2; sqlite3Isxdigit(z[i]); i++){} + if( z[i]!='\'' || i%2 ){ + *tokenType = TK_ILLEGAL; + while( z[i] && z[i]!='\'' ){ i++; } + } + if( z[i] ) i++; + return i; + } + /* If it is not a BLOB literal, then it must be an ID, since no + ** SQL keywords start with the letter 'x'. Fall through */ + } + case CC_ID: { + i = 1; + break; + } + default: { + *tokenType = TK_ILLEGAL; + return 1; + } + } + while( IdChar(z[i]) ){ i++; } + *tokenType = TK_ID; + return i; +} + +char *sqlite3_normalize(const char *zSql){ + char *z; /* The output string */ + sqlite3_int64 nZ; /* Size of the output string in bytes */ + sqlite3_int64 nSql; /* Size of the input string in bytes */ + int i; /* Next character to read from zSql[] */ + int j; /* Next slot to fill in on z[] */ + int tokenType; /* Type of the next token */ + int n; /* Size of the next token */ + int k; /* Loop counter */ + + nSql = strlen(zSql); + nZ = nSql; + z = sqlite3_malloc64( nZ+2 ); + if( z==0 ) return 0; + for(i=j=0; zSql[i]; i += n){ + n = sqlite3GetToken((unsigned char*)zSql+i, &tokenType); + switch( tokenType ){ + case TK_SPACE: { + break; + } + case TK_ERROR: { + sqlite3_free(z); + return 0; + } + case TK_LITERAL: { + z[j++] = '?'; + break; + } + case TK_PUNCT: + case TK_NAME: { + if( j>0 && IdChar(z[j-1]) && IdChar(zSql[i]) ) z[j++] = ' '; + for(k=0; k0 && z[j-1]==' ' ){ j--; } + if( i>0 && z[j-1]!=';' ){ z[j++] = ';'; } + z[j] = 0; + return z; +} + +#ifdef NORMALIZE_TEST +#include +#include + +static void normalizeFile(char *zIn){ + int i; + if( zIn==0 ) return; + for(i=0; zIn[i]; i++){ + char cSaved; + if( zIn[i]!=';' ) continue; + cSaved = zIn[i+1]; + zIn[i+1] = 0; + if( sqlite3_complete(zIn) ){ + char *zOut = sqlite3_normalize(zIn); + if( zOut ){ + printf("%s\n", zOut); + sqlite3_free(zOut); + }else{ + fprintf(stderr, "ERROR: %s\n", zIn); + } + zIn[i+1] = cSaved; + zIn += i+1; + i = -1; + }else{ + zIn[i+1] = cSaved; + } + } +} + +int main(int argc, char **argv){ + int i; + FILE *in; + char *zBuf = 0; + sqlite3_int64 sz, got; + + for(i=1; i Date: Mon, 8 Jan 2018 19:18:27 +0000 Subject: [PATCH 002/107] Add IN-operator normalizating and the SQLITE_NORMALIZE_CLI compile-time option for generating a stand-alone program. FossilOrigin-Name: d77dbb398afa80c1b3373c55f278491e83d5c80ddc22dbc303876bdcbc127df9 --- ext/misc/normalize.c | 67 ++++++++++++++++++++++++++++++++++++++++---- manifest | 15 ++++------ manifest.uuid | 2 +- 3 files changed, 69 insertions(+), 15 deletions(-) diff --git a/ext/misc/normalize.c b/ext/misc/normalize.c index 914a154d9a..fe5fb01b4d 100644 --- a/ext/misc/normalize.c +++ b/ext/misc/normalize.c @@ -35,11 +35,19 @@ ** ** The purpose of normalization is two-fold: ** -** (1) Sanitize queries by removing possibly sensitive information contained -** in literals. +** (1) Sanitize queries by removing potentially private or sensitive +** information contained in literals. ** ** (2) Identify structurally identical queries by comparing their ** normalized forms. +** +** Command-Line Utility +** -------------------- +** +** This file also contains code for a command-line utility that converts +** SQL queries in text files into their normalized forms. To build the +** command-line program, compile this file with -DSQLITE_NORMALIZE_CLI +** and link it against the SQLite library. */ #include #include @@ -48,9 +56,13 @@ ** Implementation note: ** ** Much of the tokenizer logic is copied out of the tokenize.c source file -** of SQLite. This logic could be simplified for this particular application, +** of SQLite. That logic could be simplified for this particular application, ** but that would impose a risk of introducing subtle errors. It is best to ** keep the code as close to the original as possible. +** +** The tokenize code is in sync with the SQLite core as of 2018-01-08. +** Any future changes to the core tokenizer might require corresponding +** adjustments to the tokenizer logic in this module. */ @@ -572,13 +584,54 @@ char *sqlite3_normalize(const char *zSql){ while( j>0 && z[j-1]==' ' ){ j--; } if( i>0 && z[j-1]!=';' ){ z[j++] = ';'; } z[j] = 0; + + /* Make a second pass converting "in(...)" where the "..." is not a + ** SELECT statement into "in(?,?,?)" */ + for(i=0; i5 ){ + memmove(z+n+5, z+n+k, j-(n+k)); + } + j = j-k+5; + z[j] = 0; + memcpy(z+n, "?,?,?", 5); + } return z; } -#ifdef NORMALIZE_TEST +/* +** For testing purposes, or to build a stand-alone SQL normalizer program, +** compile this one source file with the -DSQLITE_NORMALIZE_CLI and link +** it against any SQLite library. The resulting command-line program will +** run sqlite3_normalize() over the text of all files named on the command- +** line and show the result on standard output. +*/ +#ifdef SQLITE_NORMALIZE_CLI #include #include +/* +** Break zIn up into separate SQL statements and run sqlite3_normalize() +** on each one. Print the result of each run. +*/ static void normalizeFile(char *zIn){ int i; if( zIn==0 ) return; @@ -604,6 +657,10 @@ static void normalizeFile(char *zIn){ } } +/* +** The main routine for "sql_normalize". Read files named on the +** command-line and run the text of each through sqlite3_normalize(). +*/ int main(int argc, char **argv){ int i; FILE *in; @@ -636,4 +693,4 @@ int main(int argc, char **argv){ } sqlite3_free(zBuf); } -#endif /* NORMALIZE_TEST */ +#endif /* SQLITE_NORMALIZE_CLI */ diff --git a/manifest b/manifest index 7cd42ecfd6..4cc5e208a8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C First\scode\sfor\san\sauxiliary\sfunction\sthat\swill\snormalize\san\sSQL\sstatement. -D 2018-01-08T16:54:26.258 +C Add\sIN-operator\snormalizating\sand\sthe\sSQLITE_NORMALIZE_CLI\scompile-time\noption\sfor\sgenerating\sa\sstand-alone\sprogram. +D 2018-01-08T19:18:27.144 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 12b6daa4bdb03fa87da27cbc205ff88ace645475b5be79414a3038b68ade14cb @@ -284,7 +284,7 @@ F ext/misc/json1.c dbe086615b9546c156bf32b9378fc09383b58bd17513b866cfd24c1e15281 F ext/misc/memvfs.c ab36f49e02ebcdf85a1e08dc4d8599ea8f343e073ac9e0bca18a98b7e1ec9567 F ext/misc/mmapwarm.c 70b618f2d0bde43fae288ad0b7498a629f2b6f61b50a27e06fae3cd23c83af29 F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 -F ext/misc/normalize.c 5ca65b1332e41d679d7c07374398b49d020281d4ea30f267c6a6fb500e71f7c7 +F ext/misc/normalize.c c5f9b3968bf0bfac6ca1c1f802a30ca08e96a9ac7ab3297190b2745046d7d838 F ext/misc/percentile.c 92699c8cd7d517ff610e6037e56506f8904dae2e F ext/misc/regexp.c a68d25c659bd2d893cd1215667bbf75ecb9dc7d4 F ext/misc/remember.c add730f0f7e7436cd15ea3fd6a90fd83c3f706ab44169f7f048438b7d6baa69c @@ -1698,10 +1698,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 90cb01d8d6ac12d0b88f2952a75aeefa81ba66f5e4a5377fdd8b9f86aec8e927 -R 9bd921419b4f2c3b5e55f041d45b29ce -T *branch * normalize -T *sym-normalize * -T -sym-trunk * +P 84814aac81e54f03430f180926156ab0fc01e22bbce2bb228b698ea0d4a588ea +R 52aee785605536a5becd57b22eca4f38 U drh -Z c8d4afe60d21d90fa2f1cb97ff587c09 +Z be9e0b00c5811d466485564c364b04ac diff --git a/manifest.uuid b/manifest.uuid index 68c796f48f..6aaff97c39 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -84814aac81e54f03430f180926156ab0fc01e22bbce2bb228b698ea0d4a588ea \ No newline at end of file +d77dbb398afa80c1b3373c55f278491e83d5c80ddc22dbc303876bdcbc127df9 \ No newline at end of file From d19866bb98c548049757d4e54efda009badd655c Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 8 Jan 2018 19:29:28 +0000 Subject: [PATCH 003/107] Special handling of the NULL keyword. Sometimes it is a literal, and sometimes it is a keyword. FossilOrigin-Name: db5d138e97f22ad4d4d11dbef96df93696ba0e557809066bc263ca3c3898f349 --- ext/misc/normalize.c | 11 +++++++++++ manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/ext/misc/normalize.c b/ext/misc/normalize.c index fe5fb01b4d..a3c00210f6 100644 --- a/ext/misc/normalize.c +++ b/ext/misc/normalize.c @@ -573,6 +573,17 @@ char *sqlite3_normalize(const char *zSql){ } case TK_PUNCT: case TK_NAME: { + if( n==4 && sqlite3_strnicmp(zSql+i,"NULL",4)==0 ){ + if( (j>=3 && strncmp(z+j-2,"is",2)==0 && !IdChar(z[j-3])) + || (j>=4 && strncmp(z+j-3,"not",3)==0 && !IdChar(z[j-4])) + ){ + /* NULL is a keyword in this case, not a literal value */ + }else{ + /* Here the NULL is a literal value */ + z[j++] = '?'; + break; + } + } if( j>0 && IdChar(z[j-1]) && IdChar(zSql[i]) ) z[j++] = ' '; for(k=0; k Date: Mon, 8 Jan 2018 20:04:57 +0000 Subject: [PATCH 004/107] Test cases for sqlite3_normalize(). FossilOrigin-Name: 658f42257d56a3562dfa8e55023c6e497f55c565cd751d942de3a9e0cb2fa708 --- Makefile.in | 1 + Makefile.msc | 1 + ext/misc/normalize.c | 4 +-- main.mk | 1 + manifest | 21 +++++++------ manifest.uuid | 2 +- src/test1.c | 30 ++++++++++++++++++ test/normalize.test | 72 ++++++++++++++++++++++++++++++++++++++++++++ 8 files changed, 119 insertions(+), 13 deletions(-) create mode 100644 test/normalize.test diff --git a/Makefile.in b/Makefile.in index af1bd61548..98655bb820 100644 --- a/Makefile.in +++ b/Makefile.in @@ -440,6 +440,7 @@ TESTSRC += \ $(TOP)/ext/misc/ieee754.c \ $(TOP)/ext/misc/mmapwarm.c \ $(TOP)/ext/misc/nextchar.c \ + $(TOP)/ext/misc/normalize.c \ $(TOP)/ext/misc/percentile.c \ $(TOP)/ext/misc/regexp.c \ $(TOP)/ext/misc/remember.c \ diff --git a/Makefile.msc b/Makefile.msc index 4d6c5c0cc2..640f95deaa 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1500,6 +1500,7 @@ TESTEXT = \ $(TOP)\ext\misc\ieee754.c \ $(TOP)\ext\misc\mmapwarm.c \ $(TOP)\ext\misc\nextchar.c \ + $(TOP)\ext\misc\normalize.c \ $(TOP)\ext\misc\percentile.c \ $(TOP)\ext\misc\regexp.c \ $(TOP)\ext\misc\remember.c \ diff --git a/ext/misc/normalize.c b/ext/misc/normalize.c index a3c00210f6..fd656f1303 100644 --- a/ext/misc/normalize.c +++ b/ext/misc/normalize.c @@ -130,7 +130,7 @@ static const unsigned char aiClass[] = { ** handle case conversions for the UTF character set since the tables ** involved are nearly as big or bigger than SQLite itself. */ -const unsigned char sqlite3UpperToLower[] = { +static const unsigned char sqlite3UpperToLower[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, @@ -176,7 +176,7 @@ const unsigned char sqlite3UpperToLower[] = { ** non-ASCII UTF character. Hence the test for whether or not a character is ** part of an identifier is 0x46. */ -const unsigned char sqlite3CtypeMap[256] = { +static const unsigned char sqlite3CtypeMap[256] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 00..07 ........ */ 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00, /* 08..0f ........ */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 10..17 ........ */ diff --git a/main.mk b/main.mk index fc68d9d307..6271e4b682 100644 --- a/main.mk +++ b/main.mk @@ -361,6 +361,7 @@ TESTSRC += \ $(TOP)/ext/misc/ieee754.c \ $(TOP)/ext/misc/mmapwarm.c \ $(TOP)/ext/misc/nextchar.c \ + $(TOP)/ext/misc/normalize.c \ $(TOP)/ext/misc/percentile.c \ $(TOP)/ext/misc/regexp.c \ $(TOP)/ext/misc/remember.c \ diff --git a/manifest b/manifest index 2c752a40db..14067d0999 100644 --- a/manifest +++ b/manifest @@ -1,10 +1,10 @@ -C Special\shandling\sof\sthe\sNULL\skeyword.\s\sSometimes\sit\sis\sa\sliteral,\sand\ssometimes\nit\sis\sa\skeyword. -D 2018-01-08T19:29:28.438 +C Test\scases\sfor\ssqlite3_normalize(). +D 2018-01-08T20:04:57.992 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea -F Makefile.in 12b6daa4bdb03fa87da27cbc205ff88ace645475b5be79414a3038b68ade14cb +F Makefile.in 822052988bbf6204ca2c3e2665618b5a48acec24105d5afa8c4ebb59cb941d3e F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 -F Makefile.msc b33f630258ce72fb3150c1a6819a484eba3950dc9397571d48672087e12abf4a +F Makefile.msc 0f2875c7d4cb93ae4dab661f58f9d68699d323cfb87ebd7d8dccbb9949979b83 F README.md eeae1e552f93ef72ef7c5b8f6647b368a001c28820ad1df179d3dae602bef681 F VERSION 0c10cdfed866fdd2d80434f64f042c3330f1daaed12e54287beb104f04b3faaf F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -284,7 +284,7 @@ F ext/misc/json1.c dbe086615b9546c156bf32b9378fc09383b58bd17513b866cfd24c1e15281 F ext/misc/memvfs.c ab36f49e02ebcdf85a1e08dc4d8599ea8f343e073ac9e0bca18a98b7e1ec9567 F ext/misc/mmapwarm.c 70b618f2d0bde43fae288ad0b7498a629f2b6f61b50a27e06fae3cd23c83af29 F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 -F ext/misc/normalize.c f26e69c7539f74dbc0a00468df117bb9085f229ecb64d65c10ebf59a010511bb +F ext/misc/normalize.c 19262ef3ef29d4de2f281b423326865c8916c63d0cb09f1dc98d24d5c1e8ba64 F ext/misc/percentile.c 92699c8cd7d517ff610e6037e56506f8904dae2e F ext/misc/regexp.c a68d25c659bd2d893cd1215667bbf75ecb9dc7d4 F ext/misc/remember.c add730f0f7e7436cd15ea3fd6a90fd83c3f706ab44169f7f048438b7d6baa69c @@ -407,7 +407,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk 7965d01485f7bec7945407c5238985ea8c7cb2cb686d2bfdbe3d5f79d6fd4eb2 +F main.mk c6b8ee2fa871e01ad6a460d750cd074010292464a60a37a2207930ae0df629dc F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 @@ -494,7 +494,7 @@ F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6 F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 F src/tclsqlite.c 1833388c01e3b77f4c712185ee7250b9423ee0981ce6ae7e401e47db0319a696 -F src/test1.c b52f9e7fe62016d357c3266fcfa0793cc1883d3cb2b11dfa39fcba2e70b0305c +F src/test1.c 1ab7cbbb6693e08364c1a9241e2aee17f8c4925e4cc52396be77ae6845a05828 F src/test2.c 3efb99ab7f1fc8d154933e02ae1378bac9637da5 F src/test3.c b8434949dfb8aff8dfa082c8b592109e77844c2135ed3c492113839b6956255b F src/test4.c 18ec393bb4d0ad1de729f0b94da7267270f3d8e6 @@ -1093,6 +1093,7 @@ F test/mutex2.test bfeaeac2e73095b2ac32285d2756e3a65e681660 F test/nan.test 437d40e6d0778b050d7750726c0cbd2c9936b81962926e8f8c48ca698f00f4d1 F test/nockpt.test 9a436a7213ba5ef7a32304998d386d3ea3f76c9d F test/nolock.test f196cf8b8fbea4e2ca345140a2b3f3b0da45c76e +F test/normalize.test 501630ab49b0b26b65c74124bf03e3374c1b57fa97aae750f84803609141d167 F test/notify1.test 669b2b743618efdc18ca4b02f45423d5d2304abf F test/notify2.test 2ecabaa1305083856b7c39cf32816b612740c161 F test/notify3.test 10ff25cde502e72a92053a2f215d64bece4ef934 @@ -1698,7 +1699,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d77dbb398afa80c1b3373c55f278491e83d5c80ddc22dbc303876bdcbc127df9 -R 52c05b49206cf2984d560c493371a137 +P db5d138e97f22ad4d4d11dbef96df93696ba0e557809066bc263ca3c3898f349 +R 5585a6831c91176202d5e0f81c59c944 U drh -Z 26ab19c5b91f60ca62a12106abc7db8e +Z ca698384a23dd84f9bb875fa9da10ba1 diff --git a/manifest.uuid b/manifest.uuid index c338d86fc5..ac2ec7cffb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -db5d138e97f22ad4d4d11dbef96df93696ba0e557809066bc263ca3c3898f349 \ No newline at end of file +658f42257d56a3562dfa8e55023c6e497f55c565cd751d942de3a9e0cb2fa708 \ No newline at end of file diff --git a/src/test1.c b/src/test1.c index 55d92eb1fd..bc8f389dbd 100644 --- a/src/test1.c +++ b/src/test1.c @@ -4559,6 +4559,35 @@ static int SQLITE_TCLAPI test_complete16( return TCL_OK; } +/* +** Usage: sqlite3_normalize SQL +** +** Return the normalized value for an SQL statement. +*/ +static int SQLITE_TCLAPI test_normalize( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + char *zSql; + char *zNorm; + extern char *sqlite3_normalize(const char*); + + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "SQL"); + return TCL_ERROR; + } + + zSql = (char*)Tcl_GetString(objv[1]); + zNorm = sqlite3_normalize(zSql); + if( zNorm ){ + Tcl_SetObjResult(interp, Tcl_NewStringObj(zNorm, -1)); + sqlite3_free(zNorm); + } + return TCL_OK; +} + /* ** Usage: sqlite3_step STMT ** @@ -7547,6 +7576,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ { "sqlite3_open16", test_open16 ,0 }, { "sqlite3_open_v2", test_open_v2 ,0 }, { "sqlite3_complete16", test_complete16 ,0 }, + { "sqlite3_normalize", test_normalize ,0 }, { "sqlite3_prepare", test_prepare ,0 }, { "sqlite3_prepare16", test_prepare16 ,0 }, diff --git a/test/normalize.test b/test/normalize.test new file mode 100644 index 0000000000..8932650c83 --- /dev/null +++ b/test/normalize.test @@ -0,0 +1,72 @@ +# 2018-01-08 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests for the sqlite3_normalize() extension function. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix normalize + +foreach {tnum sql norm} { + 100 + {SELECT * FROM t1 WHERE a IN (1) AND b=51.42} + {select*from t1 where a in(?,?,?)and b=?;} + + 110 + {SELECT a, b+15, c FROM t1 WHERE d NOT IN (SELECT x FROM t2);} + {select a,b+?,c from t1 where d not in(select x from t2);} + + 120 + { SELECT NULL, b FROM t1 -- comment text + WHERE d IN (WITH t(a) AS (VALUES(5)) /* CTE */ + SELECT a FROM t) + OR e='hello'; + } + {select?,b from t1 where d in(with t(a)as(values(?))select a from t)or e=?;} + + 121 + {/*Initial comment*/ + -- another comment line + SELECT NULL /* comment */ , b FROM t1 -- comment text + WHERE d IN (WITH t(a) AS (VALUES(5)) /* CTE */ + SELECT a FROM t) + OR e='hello'; + } + {select?,b from t1 where d in(with t(a)as(values(?))select a from t)or e=?;} + + 130 + {/* Query containing parameters */ + SELECT x,$::abc(15),y,@abc,z,?99,w FROM t1 /* Trailing comment */} + {select x,?,y,?,z,?,w from t1;} + + 140 + {/* Long list on the RHS of IN */ + SELECT 15 IN (1,2,3,(SELECT * FROM t1),'xyz',x'abcd',22*(x+5),null);} + {select?in(?,?,?);} + + 150 + {SELECT x'abc'; -- illegal token} + {} + + 160 + {SELECT a,NULL,b FROM t1 WHERE c IS NOT NULL or D is null or e=5} + {select a,?,b from t1 where c is not null or d is null or e=?;} + + 170 + {/* IN list exactly 5 bytes long */ + SELECT * FROM t1 WHERE x IN (1,2,3);} + {select*from t1 where x in(?,?,?);} +} { + do_test $tnum [list sqlite3_normalize $sql] $norm +} + +finish_test From da6bc6792fa0a647f11c73b1ee5a57f0355e39b9 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 24 Jan 2018 16:04:21 +0000 Subject: [PATCH 005/107] Rearrange fields of the BtCursor object so that it is smaller and requires less initialization, for a small performance improvement. FossilOrigin-Name: 0ddf5292cc0411ec6fcb7399ecf2904c899e0488404d3f65490fbe1db15efdf4 --- manifest | 15 +++++++-------- manifest.uuid | 2 +- src/btree.c | 7 ++++--- src/btreeInt.h | 18 +++++++++--------- 4 files changed, 21 insertions(+), 21 deletions(-) diff --git a/manifest b/manifest index 57e34466ef..e20e592681 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\snormalize.c\sextension. -D 2018-01-24T15:07:37.708 +C Rearrange\sfields\sof\sthe\sBtCursor\sobject\sso\sthat\sit\sis\ssmaller\sand\srequires\sless\ninitialization,\sfor\sa\ssmall\sperformance\simprovement. +D 2018-01-24T16:04:21.587 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -429,9 +429,9 @@ F src/auth.c 6277d63837357549fe14e723490d6dc1a38768d71c795c5eb5c0f8a99f918f73 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca -F src/btree.c 1beceb1c5f9563271241fd0c294484668e4ad28cf6aa970eab70e50ff6f75e25 +F src/btree.c d711228cac336fb35fff21f3f4a0efe2ad58aa9a800dd02929cdf184be1e78a3 F src/btree.h 0866c0a08255142ea0e754aabd211c843cab32045c978a592a43152405ed0c84 -F src/btreeInt.h 6e70d15435b60a29386ae9864875618ca225c31bf815038bbd8f510a66bb9cb1 +F src/btreeInt.h 620ab4c7235f43572cf3ac2ac8723cbdf68073be4d29da24897c7b77dda5fd96 F src/build.c 9f9647454f236cab097f266ae970f899b53c71cadab6756c47e2b2e81392c2a1 F src/callback.c fe677cb5f5abb02f7a772a62a98c2f516426081df68856e8f2d5f950929b966a F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e @@ -1702,8 +1702,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e0f192ea6dda4fa0b243d58c8ce41932519141bcae0689a90318b4f866f54edd 658f42257d56a3562dfa8e55023c6e497f55c565cd751d942de3a9e0cb2fa708 -R 01bd832eb21dc9b5a19a52a2b0f11c48 -T +closed 658f42257d56a3562dfa8e55023c6e497f55c565cd751d942de3a9e0cb2fa708 +P 16ebe558d2f0982f630c39b6c1b00d14d27011d454cced213f17c26491a2c06e +R fc9e02eb09b9540ef2f7fa9944ac0566 U drh -Z 043a35201b0c771ecddffe95c53bb65f +Z c7e32730b85a19ed6c94c7a445674e7d diff --git a/manifest.uuid b/manifest.uuid index 498f994dc6..1dd418e847 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -16ebe558d2f0982f630c39b6c1b00d14d27011d454cced213f17c26491a2c06e \ No newline at end of file +0ddf5292cc0411ec6fcb7399ecf2904c899e0488404d3f65490fbe1db15efdf4 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 168f3105fb..33f5842282 100644 --- a/src/btree.c +++ b/src/btree.c @@ -4345,7 +4345,7 @@ int sqlite3BtreeCursorSize(void){ ** of run-time by skipping the initialization of those elements. */ void sqlite3BtreeCursorZero(BtCursor *p){ - memset(p, 0, offsetof(BtCursor, iPage)); + memset(p, 0, offsetof(BtCursor, BTCURSOR_FIRST_UNINIT)); } /* @@ -4668,14 +4668,15 @@ static int accessPayload( */ if( (pCur->curFlags & BTCF_ValidOvfl)==0 ){ int nOvfl = (pCur->info.nPayload-pCur->info.nLocal+ovflSize-1)/ovflSize; - if( nOvfl>pCur->nOvflAlloc ){ + if( pCur->aOverflow==0 + || nOvfl*sizeof(Pgno) > sqlite3MallocSize(pCur->aOverflow) + ){ Pgno *aNew = (Pgno*)sqlite3Realloc( pCur->aOverflow, nOvfl*2*sizeof(Pgno) ); if( aNew==0 ){ return SQLITE_NOMEM_BKPT; }else{ - pCur->nOvflAlloc = nOvfl*2; pCur->aOverflow = aNew; } } diff --git a/src/btreeInt.h b/src/btreeInt.h index 87d3247e13..d1e2c08d18 100644 --- a/src/btreeInt.h +++ b/src/btreeInt.h @@ -503,20 +503,20 @@ struct BtCursor { u8 curFlags; /* zero or more BTCF_* flags defined below */ u8 curPagerFlags; /* Flags to send to sqlite3PagerGet() */ u8 hints; /* As configured by CursorSetHints() */ - int nOvflAlloc; /* Allocated size of aOverflow[] array */ - Btree *pBtree; /* The Btree to which this cursor belongs */ - BtShared *pBt; /* The BtShared this cursor points to */ - BtCursor *pNext; /* Forms a linked list of all cursors */ - Pgno *aOverflow; /* Cache of overflow page locations */ - CellInfo info; /* A parse of the cell we are pointing at */ - i64 nKey; /* Size of pKey, or last integer key */ - void *pKey; /* Saved key that was cursor last known position */ - Pgno pgnoRoot; /* The root page of this tree */ int skipNext; /* Prev() is noop if negative. Next() is noop if positive. ** Error code if eState==CURSOR_FAULT */ + Btree *pBtree; /* The Btree to which this cursor belongs */ + Pgno *aOverflow; /* Cache of overflow page locations */ + void *pKey; /* Saved key that was cursor last known position */ /* All fields above are zeroed when the cursor is allocated. See ** sqlite3BtreeCursorZero(). Fields that follow must be manually ** initialized. */ +#define BTCURSOR_FIRST_UNINIT pBt /* Name of first uninitialized field */ + BtShared *pBt; /* The BtShared this cursor points to */ + BtCursor *pNext; /* Forms a linked list of all cursors */ + CellInfo info; /* A parse of the cell we are pointing at */ + i64 nKey; /* Size of pKey, or last integer key */ + Pgno pgnoRoot; /* The root page of this tree */ i8 iPage; /* Index of current page in apPage */ u8 curIntKey; /* Value of apPage[0]->intKey */ u16 ix; /* Current index for apPage[iPage] */ From 1a0e5b37b242b640c00d876a085ed657fe7b8012 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 24 Jan 2018 18:28:39 +0000 Subject: [PATCH 006/107] Reorganize spellfix.c to make it easier to edit automatically (e.g. using a script). No changes to functionality. FossilOrigin-Name: 090a64faaac579c6ed1cddb02d14e7089468a4aeaeec11e78d461790bcb6eb0a --- ext/misc/spellfix.c | 832 ++++++++++++++++++++++---------------------- manifest | 14 +- manifest.uuid | 2 +- 3 files changed, 432 insertions(+), 416 deletions(-) diff --git a/ext/misc/spellfix.c b/ext/misc/spellfix.c index 4f17b88e1d..2d7ef1b02b 100644 --- a/ext/misc/spellfix.c +++ b/ext/misc/spellfix.c @@ -1200,404 +1200,412 @@ static int utf8Charlen(const char *zIn, int nIn){ return nChar; } +typedef struct Transliteration Transliteration; +struct Transliteration { + unsigned short int cFrom; + unsigned char cTo0, cTo1, cTo2, cTo3; +}; + /* ** Table of translations from unicode characters into ASCII. */ -static const struct { - unsigned short int cFrom; - unsigned char cTo0, cTo1; -} translit[] = { - { 0x00A0, 0x20, 0x00 }, /*   to */ - { 0x00B5, 0x75, 0x00 }, /* µ to u */ - { 0x00C0, 0x41, 0x00 }, /* À to A */ - { 0x00C1, 0x41, 0x00 }, /* Á to A */ - { 0x00C2, 0x41, 0x00 }, /*  to A */ - { 0x00C3, 0x41, 0x00 }, /* à to A */ - { 0x00C4, 0x41, 0x65 }, /* Ä to Ae */ - { 0x00C5, 0x41, 0x61 }, /* Å to Aa */ - { 0x00C6, 0x41, 0x45 }, /* Æ to AE */ - { 0x00C7, 0x43, 0x00 }, /* Ç to C */ - { 0x00C8, 0x45, 0x00 }, /* È to E */ - { 0x00C9, 0x45, 0x00 }, /* É to E */ - { 0x00CA, 0x45, 0x00 }, /* Ê to E */ - { 0x00CB, 0x45, 0x00 }, /* Ë to E */ - { 0x00CC, 0x49, 0x00 }, /* Ì to I */ - { 0x00CD, 0x49, 0x00 }, /* Í to I */ - { 0x00CE, 0x49, 0x00 }, /* Î to I */ - { 0x00CF, 0x49, 0x00 }, /* Ï to I */ - { 0x00D0, 0x44, 0x00 }, /* Ð to D */ - { 0x00D1, 0x4E, 0x00 }, /* Ñ to N */ - { 0x00D2, 0x4F, 0x00 }, /* Ò to O */ - { 0x00D3, 0x4F, 0x00 }, /* Ó to O */ - { 0x00D4, 0x4F, 0x00 }, /* Ô to O */ - { 0x00D5, 0x4F, 0x00 }, /* Õ to O */ - { 0x00D6, 0x4F, 0x65 }, /* Ö to Oe */ - { 0x00D7, 0x78, 0x00 }, /* × to x */ - { 0x00D8, 0x4F, 0x00 }, /* Ø to O */ - { 0x00D9, 0x55, 0x00 }, /* Ù to U */ - { 0x00DA, 0x55, 0x00 }, /* Ú to U */ - { 0x00DB, 0x55, 0x00 }, /* Û to U */ - { 0x00DC, 0x55, 0x65 }, /* Ü to Ue */ - { 0x00DD, 0x59, 0x00 }, /* Ý to Y */ - { 0x00DE, 0x54, 0x68 }, /* Þ to Th */ - { 0x00DF, 0x73, 0x73 }, /* ß to ss */ - { 0x00E0, 0x61, 0x00 }, /* à to a */ - { 0x00E1, 0x61, 0x00 }, /* á to a */ - { 0x00E2, 0x61, 0x00 }, /* â to a */ - { 0x00E3, 0x61, 0x00 }, /* ã to a */ - { 0x00E4, 0x61, 0x65 }, /* ä to ae */ - { 0x00E5, 0x61, 0x61 }, /* å to aa */ - { 0x00E6, 0x61, 0x65 }, /* æ to ae */ - { 0x00E7, 0x63, 0x00 }, /* ç to c */ - { 0x00E8, 0x65, 0x00 }, /* è to e */ - { 0x00E9, 0x65, 0x00 }, /* é to e */ - { 0x00EA, 0x65, 0x00 }, /* ê to e */ - { 0x00EB, 0x65, 0x00 }, /* ë to e */ - { 0x00EC, 0x69, 0x00 }, /* ì to i */ - { 0x00ED, 0x69, 0x00 }, /* í to i */ - { 0x00EE, 0x69, 0x00 }, /* î to i */ - { 0x00EF, 0x69, 0x00 }, /* ï to i */ - { 0x00F0, 0x64, 0x00 }, /* ð to d */ - { 0x00F1, 0x6E, 0x00 }, /* ñ to n */ - { 0x00F2, 0x6F, 0x00 }, /* ò to o */ - { 0x00F3, 0x6F, 0x00 }, /* ó to o */ - { 0x00F4, 0x6F, 0x00 }, /* ô to o */ - { 0x00F5, 0x6F, 0x00 }, /* õ to o */ - { 0x00F6, 0x6F, 0x65 }, /* ö to oe */ - { 0x00F7, 0x3A, 0x00 }, /* ÷ to : */ - { 0x00F8, 0x6F, 0x00 }, /* ø to o */ - { 0x00F9, 0x75, 0x00 }, /* ù to u */ - { 0x00FA, 0x75, 0x00 }, /* ú to u */ - { 0x00FB, 0x75, 0x00 }, /* û to u */ - { 0x00FC, 0x75, 0x65 }, /* ü to ue */ - { 0x00FD, 0x79, 0x00 }, /* ý to y */ - { 0x00FE, 0x74, 0x68 }, /* þ to th */ - { 0x00FF, 0x79, 0x00 }, /* ÿ to y */ - { 0x0100, 0x41, 0x00 }, /* Ā to A */ - { 0x0101, 0x61, 0x00 }, /* ā to a */ - { 0x0102, 0x41, 0x00 }, /* Ă to A */ - { 0x0103, 0x61, 0x00 }, /* ă to a */ - { 0x0104, 0x41, 0x00 }, /* Ą to A */ - { 0x0105, 0x61, 0x00 }, /* ą to a */ - { 0x0106, 0x43, 0x00 }, /* Ć to C */ - { 0x0107, 0x63, 0x00 }, /* ć to c */ - { 0x0108, 0x43, 0x68 }, /* Ĉ to Ch */ - { 0x0109, 0x63, 0x68 }, /* ĉ to ch */ - { 0x010A, 0x43, 0x00 }, /* Ċ to C */ - { 0x010B, 0x63, 0x00 }, /* ċ to c */ - { 0x010C, 0x43, 0x00 }, /* Č to C */ - { 0x010D, 0x63, 0x00 }, /* č to c */ - { 0x010E, 0x44, 0x00 }, /* Ď to D */ - { 0x010F, 0x64, 0x00 }, /* ď to d */ - { 0x0110, 0x44, 0x00 }, /* Đ to D */ - { 0x0111, 0x64, 0x00 }, /* đ to d */ - { 0x0112, 0x45, 0x00 }, /* Ē to E */ - { 0x0113, 0x65, 0x00 }, /* ē to e */ - { 0x0114, 0x45, 0x00 }, /* Ĕ to E */ - { 0x0115, 0x65, 0x00 }, /* ĕ to e */ - { 0x0116, 0x45, 0x00 }, /* Ė to E */ - { 0x0117, 0x65, 0x00 }, /* ė to e */ - { 0x0118, 0x45, 0x00 }, /* Ę to E */ - { 0x0119, 0x65, 0x00 }, /* ę to e */ - { 0x011A, 0x45, 0x00 }, /* Ě to E */ - { 0x011B, 0x65, 0x00 }, /* ě to e */ - { 0x011C, 0x47, 0x68 }, /* Ĝ to Gh */ - { 0x011D, 0x67, 0x68 }, /* ĝ to gh */ - { 0x011E, 0x47, 0x00 }, /* Ğ to G */ - { 0x011F, 0x67, 0x00 }, /* ğ to g */ - { 0x0120, 0x47, 0x00 }, /* Ġ to G */ - { 0x0121, 0x67, 0x00 }, /* ġ to g */ - { 0x0122, 0x47, 0x00 }, /* Ģ to G */ - { 0x0123, 0x67, 0x00 }, /* ģ to g */ - { 0x0124, 0x48, 0x68 }, /* Ĥ to Hh */ - { 0x0125, 0x68, 0x68 }, /* ĥ to hh */ - { 0x0126, 0x48, 0x00 }, /* Ħ to H */ - { 0x0127, 0x68, 0x00 }, /* ħ to h */ - { 0x0128, 0x49, 0x00 }, /* Ĩ to I */ - { 0x0129, 0x69, 0x00 }, /* ĩ to i */ - { 0x012A, 0x49, 0x00 }, /* Ī to I */ - { 0x012B, 0x69, 0x00 }, /* ī to i */ - { 0x012C, 0x49, 0x00 }, /* Ĭ to I */ - { 0x012D, 0x69, 0x00 }, /* ĭ to i */ - { 0x012E, 0x49, 0x00 }, /* Į to I */ - { 0x012F, 0x69, 0x00 }, /* į to i */ - { 0x0130, 0x49, 0x00 }, /* İ to I */ - { 0x0131, 0x69, 0x00 }, /* ı to i */ - { 0x0132, 0x49, 0x4A }, /* IJ to IJ */ - { 0x0133, 0x69, 0x6A }, /* ij to ij */ - { 0x0134, 0x4A, 0x68 }, /* Ĵ to Jh */ - { 0x0135, 0x6A, 0x68 }, /* ĵ to jh */ - { 0x0136, 0x4B, 0x00 }, /* Ķ to K */ - { 0x0137, 0x6B, 0x00 }, /* ķ to k */ - { 0x0138, 0x6B, 0x00 }, /* ĸ to k */ - { 0x0139, 0x4C, 0x00 }, /* Ĺ to L */ - { 0x013A, 0x6C, 0x00 }, /* ĺ to l */ - { 0x013B, 0x4C, 0x00 }, /* Ļ to L */ - { 0x013C, 0x6C, 0x00 }, /* ļ to l */ - { 0x013D, 0x4C, 0x00 }, /* Ľ to L */ - { 0x013E, 0x6C, 0x00 }, /* ľ to l */ - { 0x013F, 0x4C, 0x2E }, /* Ŀ to L. */ - { 0x0140, 0x6C, 0x2E }, /* ŀ to l. */ - { 0x0141, 0x4C, 0x00 }, /* Ł to L */ - { 0x0142, 0x6C, 0x00 }, /* ł to l */ - { 0x0143, 0x4E, 0x00 }, /* Ń to N */ - { 0x0144, 0x6E, 0x00 }, /* ń to n */ - { 0x0145, 0x4E, 0x00 }, /* Ņ to N */ - { 0x0146, 0x6E, 0x00 }, /* ņ to n */ - { 0x0147, 0x4E, 0x00 }, /* Ň to N */ - { 0x0148, 0x6E, 0x00 }, /* ň to n */ - { 0x0149, 0x27, 0x6E }, /* ʼn to 'n */ - { 0x014A, 0x4E, 0x47 }, /* Ŋ to NG */ - { 0x014B, 0x6E, 0x67 }, /* ŋ to ng */ - { 0x014C, 0x4F, 0x00 }, /* Ō to O */ - { 0x014D, 0x6F, 0x00 }, /* ō to o */ - { 0x014E, 0x4F, 0x00 }, /* Ŏ to O */ - { 0x014F, 0x6F, 0x00 }, /* ŏ to o */ - { 0x0150, 0x4F, 0x00 }, /* Ő to O */ - { 0x0151, 0x6F, 0x00 }, /* ő to o */ - { 0x0152, 0x4F, 0x45 }, /* Œ to OE */ - { 0x0153, 0x6F, 0x65 }, /* œ to oe */ - { 0x0154, 0x52, 0x00 }, /* Ŕ to R */ - { 0x0155, 0x72, 0x00 }, /* ŕ to r */ - { 0x0156, 0x52, 0x00 }, /* Ŗ to R */ - { 0x0157, 0x72, 0x00 }, /* ŗ to r */ - { 0x0158, 0x52, 0x00 }, /* Ř to R */ - { 0x0159, 0x72, 0x00 }, /* ř to r */ - { 0x015A, 0x53, 0x00 }, /* Ś to S */ - { 0x015B, 0x73, 0x00 }, /* ś to s */ - { 0x015C, 0x53, 0x68 }, /* Ŝ to Sh */ - { 0x015D, 0x73, 0x68 }, /* ŝ to sh */ - { 0x015E, 0x53, 0x00 }, /* Ş to S */ - { 0x015F, 0x73, 0x00 }, /* ş to s */ - { 0x0160, 0x53, 0x00 }, /* Š to S */ - { 0x0161, 0x73, 0x00 }, /* š to s */ - { 0x0162, 0x54, 0x00 }, /* Ţ to T */ - { 0x0163, 0x74, 0x00 }, /* ţ to t */ - { 0x0164, 0x54, 0x00 }, /* Ť to T */ - { 0x0165, 0x74, 0x00 }, /* ť to t */ - { 0x0166, 0x54, 0x00 }, /* Ŧ to T */ - { 0x0167, 0x74, 0x00 }, /* ŧ to t */ - { 0x0168, 0x55, 0x00 }, /* Ũ to U */ - { 0x0169, 0x75, 0x00 }, /* ũ to u */ - { 0x016A, 0x55, 0x00 }, /* Ū to U */ - { 0x016B, 0x75, 0x00 }, /* ū to u */ - { 0x016C, 0x55, 0x00 }, /* Ŭ to U */ - { 0x016D, 0x75, 0x00 }, /* ŭ to u */ - { 0x016E, 0x55, 0x00 }, /* Ů to U */ - { 0x016F, 0x75, 0x00 }, /* ů to u */ - { 0x0170, 0x55, 0x00 }, /* Ű to U */ - { 0x0171, 0x75, 0x00 }, /* ű to u */ - { 0x0172, 0x55, 0x00 }, /* Ų to U */ - { 0x0173, 0x75, 0x00 }, /* ų to u */ - { 0x0174, 0x57, 0x00 }, /* Ŵ to W */ - { 0x0175, 0x77, 0x00 }, /* ŵ to w */ - { 0x0176, 0x59, 0x00 }, /* Ŷ to Y */ - { 0x0177, 0x79, 0x00 }, /* ŷ to y */ - { 0x0178, 0x59, 0x00 }, /* Ÿ to Y */ - { 0x0179, 0x5A, 0x00 }, /* Ź to Z */ - { 0x017A, 0x7A, 0x00 }, /* ź to z */ - { 0x017B, 0x5A, 0x00 }, /* Ż to Z */ - { 0x017C, 0x7A, 0x00 }, /* ż to z */ - { 0x017D, 0x5A, 0x00 }, /* Ž to Z */ - { 0x017E, 0x7A, 0x00 }, /* ž to z */ - { 0x017F, 0x73, 0x00 }, /* ſ to s */ - { 0x0192, 0x66, 0x00 }, /* ƒ to f */ - { 0x0218, 0x53, 0x00 }, /* Ș to S */ - { 0x0219, 0x73, 0x00 }, /* ș to s */ - { 0x021A, 0x54, 0x00 }, /* Ț to T */ - { 0x021B, 0x74, 0x00 }, /* ț to t */ - { 0x0386, 0x41, 0x00 }, /* Ά to A */ - { 0x0388, 0x45, 0x00 }, /* Έ to E */ - { 0x0389, 0x49, 0x00 }, /* Ή to I */ - { 0x038A, 0x49, 0x00 }, /* Ί to I */ - { 0x038C, 0x4f, 0x00 }, /* Ό to O */ - { 0x038E, 0x59, 0x00 }, /* Ύ to Y */ - { 0x038F, 0x4f, 0x00 }, /* Ώ to O */ - { 0x0390, 0x69, 0x00 }, /* ΐ to i */ - { 0x0391, 0x41, 0x00 }, /* Α to A */ - { 0x0392, 0x42, 0x00 }, /* Β to B */ - { 0x0393, 0x47, 0x00 }, /* Γ to G */ - { 0x0394, 0x44, 0x00 }, /* Δ to D */ - { 0x0395, 0x45, 0x00 }, /* Ε to E */ - { 0x0396, 0x5a, 0x00 }, /* Ζ to Z */ - { 0x0397, 0x49, 0x00 }, /* Η to I */ - { 0x0398, 0x54, 0x68 }, /* Θ to Th */ - { 0x0399, 0x49, 0x00 }, /* Ι to I */ - { 0x039A, 0x4b, 0x00 }, /* Κ to K */ - { 0x039B, 0x4c, 0x00 }, /* Λ to L */ - { 0x039C, 0x4d, 0x00 }, /* Μ to M */ - { 0x039D, 0x4e, 0x00 }, /* Ν to N */ - { 0x039E, 0x58, 0x00 }, /* Ξ to X */ - { 0x039F, 0x4f, 0x00 }, /* Ο to O */ - { 0x03A0, 0x50, 0x00 }, /* Π to P */ - { 0x03A1, 0x52, 0x00 }, /* Ρ to R */ - { 0x03A3, 0x53, 0x00 }, /* Σ to S */ - { 0x03A4, 0x54, 0x00 }, /* Τ to T */ - { 0x03A5, 0x59, 0x00 }, /* Υ to Y */ - { 0x03A6, 0x46, 0x00 }, /* Φ to F */ - { 0x03A7, 0x43, 0x68 }, /* Χ to Ch */ - { 0x03A8, 0x50, 0x73 }, /* Ψ to Ps */ - { 0x03A9, 0x4f, 0x00 }, /* Ω to O */ - { 0x03AA, 0x49, 0x00 }, /* Ϊ to I */ - { 0x03AB, 0x59, 0x00 }, /* Ϋ to Y */ - { 0x03AC, 0x61, 0x00 }, /* ά to a */ - { 0x03AD, 0x65, 0x00 }, /* έ to e */ - { 0x03AE, 0x69, 0x00 }, /* ή to i */ - { 0x03AF, 0x69, 0x00 }, /* ί to i */ - { 0x03B1, 0x61, 0x00 }, /* α to a */ - { 0x03B2, 0x62, 0x00 }, /* β to b */ - { 0x03B3, 0x67, 0x00 }, /* γ to g */ - { 0x03B4, 0x64, 0x00 }, /* δ to d */ - { 0x03B5, 0x65, 0x00 }, /* ε to e */ - { 0x03B6, 0x7a, 0x00 }, /* ζ to z */ - { 0x03B7, 0x69, 0x00 }, /* η to i */ - { 0x03B8, 0x74, 0x68 }, /* θ to th */ - { 0x03B9, 0x69, 0x00 }, /* ι to i */ - { 0x03BA, 0x6b, 0x00 }, /* κ to k */ - { 0x03BB, 0x6c, 0x00 }, /* λ to l */ - { 0x03BC, 0x6d, 0x00 }, /* μ to m */ - { 0x03BD, 0x6e, 0x00 }, /* ν to n */ - { 0x03BE, 0x78, 0x00 }, /* ξ to x */ - { 0x03BF, 0x6f, 0x00 }, /* ο to o */ - { 0x03C0, 0x70, 0x00 }, /* π to p */ - { 0x03C1, 0x72, 0x00 }, /* ρ to r */ - { 0x03C3, 0x73, 0x00 }, /* σ to s */ - { 0x03C4, 0x74, 0x00 }, /* τ to t */ - { 0x03C5, 0x79, 0x00 }, /* υ to y */ - { 0x03C6, 0x66, 0x00 }, /* φ to f */ - { 0x03C7, 0x63, 0x68 }, /* χ to ch */ - { 0x03C8, 0x70, 0x73 }, /* ψ to ps */ - { 0x03C9, 0x6f, 0x00 }, /* ω to o */ - { 0x03CA, 0x69, 0x00 }, /* ϊ to i */ - { 0x03CB, 0x79, 0x00 }, /* ϋ to y */ - { 0x03CC, 0x6f, 0x00 }, /* ό to o */ - { 0x03CD, 0x79, 0x00 }, /* ύ to y */ - { 0x03CE, 0x69, 0x00 }, /* ώ to i */ - { 0x0400, 0x45, 0x00 }, /* Ѐ to E */ - { 0x0401, 0x45, 0x00 }, /* Ё to E */ - { 0x0402, 0x44, 0x00 }, /* Ђ to D */ - { 0x0403, 0x47, 0x00 }, /* Ѓ to G */ - { 0x0404, 0x45, 0x00 }, /* Є to E */ - { 0x0405, 0x5a, 0x00 }, /* Ѕ to Z */ - { 0x0406, 0x49, 0x00 }, /* І to I */ - { 0x0407, 0x49, 0x00 }, /* Ї to I */ - { 0x0408, 0x4a, 0x00 }, /* Ј to J */ - { 0x0409, 0x49, 0x00 }, /* Љ to I */ - { 0x040A, 0x4e, 0x00 }, /* Њ to N */ - { 0x040B, 0x44, 0x00 }, /* Ћ to D */ - { 0x040C, 0x4b, 0x00 }, /* Ќ to K */ - { 0x040D, 0x49, 0x00 }, /* Ѝ to I */ - { 0x040E, 0x55, 0x00 }, /* Ў to U */ - { 0x040F, 0x44, 0x00 }, /* Џ to D */ - { 0x0410, 0x41, 0x00 }, /* А to A */ - { 0x0411, 0x42, 0x00 }, /* Б to B */ - { 0x0412, 0x56, 0x00 }, /* В to V */ - { 0x0413, 0x47, 0x00 }, /* Г to G */ - { 0x0414, 0x44, 0x00 }, /* Д to D */ - { 0x0415, 0x45, 0x00 }, /* Е to E */ - { 0x0416, 0x5a, 0x68 }, /* Ж to Zh */ - { 0x0417, 0x5a, 0x00 }, /* З to Z */ - { 0x0418, 0x49, 0x00 }, /* И to I */ - { 0x0419, 0x49, 0x00 }, /* Й to I */ - { 0x041A, 0x4b, 0x00 }, /* К to K */ - { 0x041B, 0x4c, 0x00 }, /* Л to L */ - { 0x041C, 0x4d, 0x00 }, /* М to M */ - { 0x041D, 0x4e, 0x00 }, /* Н to N */ - { 0x041E, 0x4f, 0x00 }, /* О to O */ - { 0x041F, 0x50, 0x00 }, /* П to P */ - { 0x0420, 0x52, 0x00 }, /* Р to R */ - { 0x0421, 0x53, 0x00 }, /* С to S */ - { 0x0422, 0x54, 0x00 }, /* Т to T */ - { 0x0423, 0x55, 0x00 }, /* У to U */ - { 0x0424, 0x46, 0x00 }, /* Ф to F */ - { 0x0425, 0x4b, 0x68 }, /* Х to Kh */ - { 0x0426, 0x54, 0x63 }, /* Ц to Tc */ - { 0x0427, 0x43, 0x68 }, /* Ч to Ch */ - { 0x0428, 0x53, 0x68 }, /* Ш to Sh */ - { 0x0429, 0x53, 0x68 }, /* Щ to Shch */ - { 0x042A, 0x61, 0x00 }, /* to A */ - { 0x042B, 0x59, 0x00 }, /* Ы to Y */ - { 0x042C, 0x59, 0x00 }, /* to Y */ - { 0x042D, 0x45, 0x00 }, /* Э to E */ - { 0x042E, 0x49, 0x75 }, /* Ю to Iu */ - { 0x042F, 0x49, 0x61 }, /* Я to Ia */ - { 0x0430, 0x61, 0x00 }, /* а to a */ - { 0x0431, 0x62, 0x00 }, /* б to b */ - { 0x0432, 0x76, 0x00 }, /* в to v */ - { 0x0433, 0x67, 0x00 }, /* г to g */ - { 0x0434, 0x64, 0x00 }, /* д to d */ - { 0x0435, 0x65, 0x00 }, /* е to e */ - { 0x0436, 0x7a, 0x68 }, /* ж to zh */ - { 0x0437, 0x7a, 0x00 }, /* з to z */ - { 0x0438, 0x69, 0x00 }, /* и to i */ - { 0x0439, 0x69, 0x00 }, /* й to i */ - { 0x043A, 0x6b, 0x00 }, /* к to k */ - { 0x043B, 0x6c, 0x00 }, /* л to l */ - { 0x043C, 0x6d, 0x00 }, /* м to m */ - { 0x043D, 0x6e, 0x00 }, /* н to n */ - { 0x043E, 0x6f, 0x00 }, /* о to o */ - { 0x043F, 0x70, 0x00 }, /* п to p */ - { 0x0440, 0x72, 0x00 }, /* р to r */ - { 0x0441, 0x73, 0x00 }, /* с to s */ - { 0x0442, 0x74, 0x00 }, /* т to t */ - { 0x0443, 0x75, 0x00 }, /* у to u */ - { 0x0444, 0x66, 0x00 }, /* ф to f */ - { 0x0445, 0x6b, 0x68 }, /* х to kh */ - { 0x0446, 0x74, 0x63 }, /* ц to tc */ - { 0x0447, 0x63, 0x68 }, /* ч to ch */ - { 0x0448, 0x73, 0x68 }, /* ш to sh */ - { 0x0449, 0x73, 0x68 }, /* щ to shch */ - { 0x044A, 0x61, 0x00 }, /* to a */ - { 0x044B, 0x79, 0x00 }, /* ы to y */ - { 0x044C, 0x79, 0x00 }, /* to y */ - { 0x044D, 0x65, 0x00 }, /* э to e */ - { 0x044E, 0x69, 0x75 }, /* ю to iu */ - { 0x044F, 0x69, 0x61 }, /* я to ia */ - { 0x0450, 0x65, 0x00 }, /* ѐ to e */ - { 0x0451, 0x65, 0x00 }, /* ё to e */ - { 0x0452, 0x64, 0x00 }, /* ђ to d */ - { 0x0453, 0x67, 0x00 }, /* ѓ to g */ - { 0x0454, 0x65, 0x00 }, /* є to e */ - { 0x0455, 0x7a, 0x00 }, /* ѕ to z */ - { 0x0456, 0x69, 0x00 }, /* і to i */ - { 0x0457, 0x69, 0x00 }, /* ї to i */ - { 0x0458, 0x6a, 0x00 }, /* ј to j */ - { 0x0459, 0x69, 0x00 }, /* љ to i */ - { 0x045A, 0x6e, 0x00 }, /* њ to n */ - { 0x045B, 0x64, 0x00 }, /* ћ to d */ - { 0x045C, 0x6b, 0x00 }, /* ќ to k */ - { 0x045D, 0x69, 0x00 }, /* ѝ to i */ - { 0x045E, 0x75, 0x00 }, /* ў to u */ - { 0x045F, 0x64, 0x00 }, /* џ to d */ - { 0x1E02, 0x42, 0x00 }, /* Ḃ to B */ - { 0x1E03, 0x62, 0x00 }, /* ḃ to b */ - { 0x1E0A, 0x44, 0x00 }, /* Ḋ to D */ - { 0x1E0B, 0x64, 0x00 }, /* ḋ to d */ - { 0x1E1E, 0x46, 0x00 }, /* Ḟ to F */ - { 0x1E1F, 0x66, 0x00 }, /* ḟ to f */ - { 0x1E40, 0x4D, 0x00 }, /* Ṁ to M */ - { 0x1E41, 0x6D, 0x00 }, /* ṁ to m */ - { 0x1E56, 0x50, 0x00 }, /* Ṗ to P */ - { 0x1E57, 0x70, 0x00 }, /* ṗ to p */ - { 0x1E60, 0x53, 0x00 }, /* Ṡ to S */ - { 0x1E61, 0x73, 0x00 }, /* ṡ to s */ - { 0x1E6A, 0x54, 0x00 }, /* Ṫ to T */ - { 0x1E6B, 0x74, 0x00 }, /* ṫ to t */ - { 0x1E80, 0x57, 0x00 }, /* Ẁ to W */ - { 0x1E81, 0x77, 0x00 }, /* ẁ to w */ - { 0x1E82, 0x57, 0x00 }, /* Ẃ to W */ - { 0x1E83, 0x77, 0x00 }, /* ẃ to w */ - { 0x1E84, 0x57, 0x00 }, /* Ẅ to W */ - { 0x1E85, 0x77, 0x00 }, /* ẅ to w */ - { 0x1EF2, 0x59, 0x00 }, /* Ỳ to Y */ - { 0x1EF3, 0x79, 0x00 }, /* ỳ to y */ - { 0xFB00, 0x66, 0x66 }, /* ff to ff */ - { 0xFB01, 0x66, 0x69 }, /* fi to fi */ - { 0xFB02, 0x66, 0x6C }, /* fl to fl */ - { 0xFB05, 0x73, 0x74 }, /* ſt to st */ - { 0xFB06, 0x73, 0x74 }, /* st to st */ +static const Transliteration translit[] = { + { 0x00A0, 0x20, 0x00, 0x00, 0x00 }, /*   to */ + { 0x00B5, 0x75, 0x00, 0x00, 0x00 }, /* µ to u */ + { 0x00C0, 0x41, 0x00, 0x00, 0x00 }, /* À to A */ + { 0x00C1, 0x41, 0x00, 0x00, 0x00 }, /* Á to A */ + { 0x00C2, 0x41, 0x00, 0x00, 0x00 }, /*  to A */ + { 0x00C3, 0x41, 0x00, 0x00, 0x00 }, /* à to A */ + { 0x00C4, 0x41, 0x65, 0x00, 0x00 }, /* Ä to Ae */ + { 0x00C5, 0x41, 0x61, 0x00, 0x00 }, /* Å to Aa */ + { 0x00C6, 0x41, 0x45, 0x00, 0x00 }, /* Æ to AE */ + { 0x00C7, 0x43, 0x00, 0x00, 0x00 }, /* Ç to C */ + { 0x00C8, 0x45, 0x00, 0x00, 0x00 }, /* È to E */ + { 0x00C9, 0x45, 0x00, 0x00, 0x00 }, /* É to E */ + { 0x00CA, 0x45, 0x00, 0x00, 0x00 }, /* Ê to E */ + { 0x00CB, 0x45, 0x00, 0x00, 0x00 }, /* Ë to E */ + { 0x00CC, 0x49, 0x00, 0x00, 0x00 }, /* Ì to I */ + { 0x00CD, 0x49, 0x00, 0x00, 0x00 }, /* Í to I */ + { 0x00CE, 0x49, 0x00, 0x00, 0x00 }, /* Î to I */ + { 0x00CF, 0x49, 0x00, 0x00, 0x00 }, /* Ï to I */ + { 0x00D0, 0x44, 0x00, 0x00, 0x00 }, /* Ð to D */ + { 0x00D1, 0x4E, 0x00, 0x00, 0x00 }, /* Ñ to N */ + { 0x00D2, 0x4F, 0x00, 0x00, 0x00 }, /* Ò to O */ + { 0x00D3, 0x4F, 0x00, 0x00, 0x00 }, /* Ó to O */ + { 0x00D4, 0x4F, 0x00, 0x00, 0x00 }, /* Ô to O */ + { 0x00D5, 0x4F, 0x00, 0x00, 0x00 }, /* Õ to O */ + { 0x00D6, 0x4F, 0x65, 0x00, 0x00 }, /* Ö to Oe */ + { 0x00D7, 0x78, 0x00, 0x00, 0x00 }, /* × to x */ + { 0x00D8, 0x4F, 0x00, 0x00, 0x00 }, /* Ø to O */ + { 0x00D9, 0x55, 0x00, 0x00, 0x00 }, /* Ù to U */ + { 0x00DA, 0x55, 0x00, 0x00, 0x00 }, /* Ú to U */ + { 0x00DB, 0x55, 0x00, 0x00, 0x00 }, /* Û to U */ + { 0x00DC, 0x55, 0x65, 0x00, 0x00 }, /* Ü to Ue */ + { 0x00DD, 0x59, 0x00, 0x00, 0x00 }, /* Ý to Y */ + { 0x00DE, 0x54, 0x68, 0x00, 0x00 }, /* Þ to Th */ + { 0x00DF, 0x73, 0x73, 0x00, 0x00 }, /* ß to ss */ + { 0x00E0, 0x61, 0x00, 0x00, 0x00 }, /* à to a */ + { 0x00E1, 0x61, 0x00, 0x00, 0x00 }, /* á to a */ + { 0x00E2, 0x61, 0x00, 0x00, 0x00 }, /* â to a */ + { 0x00E3, 0x61, 0x00, 0x00, 0x00 }, /* ã to a */ + { 0x00E4, 0x61, 0x65, 0x00, 0x00 }, /* ä to ae */ + { 0x00E5, 0x61, 0x61, 0x00, 0x00 }, /* å to aa */ + { 0x00E6, 0x61, 0x65, 0x00, 0x00 }, /* æ to ae */ + { 0x00E7, 0x63, 0x00, 0x00, 0x00 }, /* ç to c */ + { 0x00E8, 0x65, 0x00, 0x00, 0x00 }, /* è to e */ + { 0x00E9, 0x65, 0x00, 0x00, 0x00 }, /* é to e */ + { 0x00EA, 0x65, 0x00, 0x00, 0x00 }, /* ê to e */ + { 0x00EB, 0x65, 0x00, 0x00, 0x00 }, /* ë to e */ + { 0x00EC, 0x69, 0x00, 0x00, 0x00 }, /* ì to i */ + { 0x00ED, 0x69, 0x00, 0x00, 0x00 }, /* í to i */ + { 0x00EE, 0x69, 0x00, 0x00, 0x00 }, /* î to i */ + { 0x00EF, 0x69, 0x00, 0x00, 0x00 }, /* ï to i */ + { 0x00F0, 0x64, 0x00, 0x00, 0x00 }, /* ð to d */ + { 0x00F1, 0x6E, 0x00, 0x00, 0x00 }, /* ñ to n */ + { 0x00F2, 0x6F, 0x00, 0x00, 0x00 }, /* ò to o */ + { 0x00F3, 0x6F, 0x00, 0x00, 0x00 }, /* ó to o */ + { 0x00F4, 0x6F, 0x00, 0x00, 0x00 }, /* ô to o */ + { 0x00F5, 0x6F, 0x00, 0x00, 0x00 }, /* õ to o */ + { 0x00F6, 0x6F, 0x65, 0x00, 0x00 }, /* ö to oe */ + { 0x00F7, 0x3A, 0x00, 0x00, 0x00 }, /* ÷ to : */ + { 0x00F8, 0x6F, 0x00, 0x00, 0x00 }, /* ø to o */ + { 0x00F9, 0x75, 0x00, 0x00, 0x00 }, /* ù to u */ + { 0x00FA, 0x75, 0x00, 0x00, 0x00 }, /* ú to u */ + { 0x00FB, 0x75, 0x00, 0x00, 0x00 }, /* û to u */ + { 0x00FC, 0x75, 0x65, 0x00, 0x00 }, /* ü to ue */ + { 0x00FD, 0x79, 0x00, 0x00, 0x00 }, /* ý to y */ + { 0x00FE, 0x74, 0x68, 0x00, 0x00 }, /* þ to th */ + { 0x00FF, 0x79, 0x00, 0x00, 0x00 }, /* ÿ to y */ + { 0x0100, 0x41, 0x00, 0x00, 0x00 }, /* Ā to A */ + { 0x0101, 0x61, 0x00, 0x00, 0x00 }, /* ā to a */ + { 0x0102, 0x41, 0x00, 0x00, 0x00 }, /* Ă to A */ + { 0x0103, 0x61, 0x00, 0x00, 0x00 }, /* ă to a */ + { 0x0104, 0x41, 0x00, 0x00, 0x00 }, /* Ą to A */ + { 0x0105, 0x61, 0x00, 0x00, 0x00 }, /* ą to a */ + { 0x0106, 0x43, 0x00, 0x00, 0x00 }, /* Ć to C */ + { 0x0107, 0x63, 0x00, 0x00, 0x00 }, /* ć to c */ + { 0x0108, 0x43, 0x68, 0x00, 0x00 }, /* Ĉ to Ch */ + { 0x0109, 0x63, 0x68, 0x00, 0x00 }, /* ĉ to ch */ + { 0x010A, 0x43, 0x00, 0x00, 0x00 }, /* Ċ to C */ + { 0x010B, 0x63, 0x00, 0x00, 0x00 }, /* ċ to c */ + { 0x010C, 0x43, 0x00, 0x00, 0x00 }, /* Č to C */ + { 0x010D, 0x63, 0x00, 0x00, 0x00 }, /* č to c */ + { 0x010E, 0x44, 0x00, 0x00, 0x00 }, /* Ď to D */ + { 0x010F, 0x64, 0x00, 0x00, 0x00 }, /* ď to d */ + { 0x0110, 0x44, 0x00, 0x00, 0x00 }, /* Đ to D */ + { 0x0111, 0x64, 0x00, 0x00, 0x00 }, /* đ to d */ + { 0x0112, 0x45, 0x00, 0x00, 0x00 }, /* Ē to E */ + { 0x0113, 0x65, 0x00, 0x00, 0x00 }, /* ē to e */ + { 0x0114, 0x45, 0x00, 0x00, 0x00 }, /* Ĕ to E */ + { 0x0115, 0x65, 0x00, 0x00, 0x00 }, /* ĕ to e */ + { 0x0116, 0x45, 0x00, 0x00, 0x00 }, /* Ė to E */ + { 0x0117, 0x65, 0x00, 0x00, 0x00 }, /* ė to e */ + { 0x0118, 0x45, 0x00, 0x00, 0x00 }, /* Ę to E */ + { 0x0119, 0x65, 0x00, 0x00, 0x00 }, /* ę to e */ + { 0x011A, 0x45, 0x00, 0x00, 0x00 }, /* Ě to E */ + { 0x011B, 0x65, 0x00, 0x00, 0x00 }, /* ě to e */ + { 0x011C, 0x47, 0x68, 0x00, 0x00 }, /* Ĝ to Gh */ + { 0x011D, 0x67, 0x68, 0x00, 0x00 }, /* ĝ to gh */ + { 0x011E, 0x47, 0x00, 0x00, 0x00 }, /* Ğ to G */ + { 0x011F, 0x67, 0x00, 0x00, 0x00 }, /* ğ to g */ + { 0x0120, 0x47, 0x00, 0x00, 0x00 }, /* Ġ to G */ + { 0x0121, 0x67, 0x00, 0x00, 0x00 }, /* ġ to g */ + { 0x0122, 0x47, 0x00, 0x00, 0x00 }, /* Ģ to G */ + { 0x0123, 0x67, 0x00, 0x00, 0x00 }, /* ģ to g */ + { 0x0124, 0x48, 0x68, 0x00, 0x00 }, /* Ĥ to Hh */ + { 0x0125, 0x68, 0x68, 0x00, 0x00 }, /* ĥ to hh */ + { 0x0126, 0x48, 0x00, 0x00, 0x00 }, /* Ħ to H */ + { 0x0127, 0x68, 0x00, 0x00, 0x00 }, /* ħ to h */ + { 0x0128, 0x49, 0x00, 0x00, 0x00 }, /* Ĩ to I */ + { 0x0129, 0x69, 0x00, 0x00, 0x00 }, /* ĩ to i */ + { 0x012A, 0x49, 0x00, 0x00, 0x00 }, /* Ī to I */ + { 0x012B, 0x69, 0x00, 0x00, 0x00 }, /* ī to i */ + { 0x012C, 0x49, 0x00, 0x00, 0x00 }, /* Ĭ to I */ + { 0x012D, 0x69, 0x00, 0x00, 0x00 }, /* ĭ to i */ + { 0x012E, 0x49, 0x00, 0x00, 0x00 }, /* Į to I */ + { 0x012F, 0x69, 0x00, 0x00, 0x00 }, /* į to i */ + { 0x0130, 0x49, 0x00, 0x00, 0x00 }, /* İ to I */ + { 0x0131, 0x69, 0x00, 0x00, 0x00 }, /* ı to i */ + { 0x0132, 0x49, 0x4A, 0x00, 0x00 }, /* IJ to IJ */ + { 0x0133, 0x69, 0x6A, 0x00, 0x00 }, /* ij to ij */ + { 0x0134, 0x4A, 0x68, 0x00, 0x00 }, /* Ĵ to Jh */ + { 0x0135, 0x6A, 0x68, 0x00, 0x00 }, /* ĵ to jh */ + { 0x0136, 0x4B, 0x00, 0x00, 0x00 }, /* Ķ to K */ + { 0x0137, 0x6B, 0x00, 0x00, 0x00 }, /* ķ to k */ + { 0x0138, 0x6B, 0x00, 0x00, 0x00 }, /* ĸ to k */ + { 0x0139, 0x4C, 0x00, 0x00, 0x00 }, /* Ĺ to L */ + { 0x013A, 0x6C, 0x00, 0x00, 0x00 }, /* ĺ to l */ + { 0x013B, 0x4C, 0x00, 0x00, 0x00 }, /* Ļ to L */ + { 0x013C, 0x6C, 0x00, 0x00, 0x00 }, /* ļ to l */ + { 0x013D, 0x4C, 0x00, 0x00, 0x00 }, /* Ľ to L */ + { 0x013E, 0x6C, 0x00, 0x00, 0x00 }, /* ľ to l */ + { 0x013F, 0x4C, 0x2E, 0x00, 0x00 }, /* Ŀ to L. */ + { 0x0140, 0x6C, 0x2E, 0x00, 0x00 }, /* ŀ to l. */ + { 0x0141, 0x4C, 0x00, 0x00, 0x00 }, /* Ł to L */ + { 0x0142, 0x6C, 0x00, 0x00, 0x00 }, /* ł to l */ + { 0x0143, 0x4E, 0x00, 0x00, 0x00 }, /* Ń to N */ + { 0x0144, 0x6E, 0x00, 0x00, 0x00 }, /* ń to n */ + { 0x0145, 0x4E, 0x00, 0x00, 0x00 }, /* Ņ to N */ + { 0x0146, 0x6E, 0x00, 0x00, 0x00 }, /* ņ to n */ + { 0x0147, 0x4E, 0x00, 0x00, 0x00 }, /* Ň to N */ + { 0x0148, 0x6E, 0x00, 0x00, 0x00 }, /* ň to n */ + { 0x0149, 0x27, 0x6E, 0x00, 0x00 }, /* ʼn to 'n */ + { 0x014A, 0x4E, 0x47, 0x00, 0x00 }, /* Ŋ to NG */ + { 0x014B, 0x6E, 0x67, 0x00, 0x00 }, /* ŋ to ng */ + { 0x014C, 0x4F, 0x00, 0x00, 0x00 }, /* Ō to O */ + { 0x014D, 0x6F, 0x00, 0x00, 0x00 }, /* ō to o */ + { 0x014E, 0x4F, 0x00, 0x00, 0x00 }, /* Ŏ to O */ + { 0x014F, 0x6F, 0x00, 0x00, 0x00 }, /* ŏ to o */ + { 0x0150, 0x4F, 0x00, 0x00, 0x00 }, /* Ő to O */ + { 0x0151, 0x6F, 0x00, 0x00, 0x00 }, /* ő to o */ + { 0x0152, 0x4F, 0x45, 0x00, 0x00 }, /* Œ to OE */ + { 0x0153, 0x6F, 0x65, 0x00, 0x00 }, /* œ to oe */ + { 0x0154, 0x52, 0x00, 0x00, 0x00 }, /* Ŕ to R */ + { 0x0155, 0x72, 0x00, 0x00, 0x00 }, /* ŕ to r */ + { 0x0156, 0x52, 0x00, 0x00, 0x00 }, /* Ŗ to R */ + { 0x0157, 0x72, 0x00, 0x00, 0x00 }, /* ŗ to r */ + { 0x0158, 0x52, 0x00, 0x00, 0x00 }, /* Ř to R */ + { 0x0159, 0x72, 0x00, 0x00, 0x00 }, /* ř to r */ + { 0x015A, 0x53, 0x00, 0x00, 0x00 }, /* Ś to S */ + { 0x015B, 0x73, 0x00, 0x00, 0x00 }, /* ś to s */ + { 0x015C, 0x53, 0x68, 0x00, 0x00 }, /* Ŝ to Sh */ + { 0x015D, 0x73, 0x68, 0x00, 0x00 }, /* ŝ to sh */ + { 0x015E, 0x53, 0x00, 0x00, 0x00 }, /* Ş to S */ + { 0x015F, 0x73, 0x00, 0x00, 0x00 }, /* ş to s */ + { 0x0160, 0x53, 0x00, 0x00, 0x00 }, /* Š to S */ + { 0x0161, 0x73, 0x00, 0x00, 0x00 }, /* š to s */ + { 0x0162, 0x54, 0x00, 0x00, 0x00 }, /* Ţ to T */ + { 0x0163, 0x74, 0x00, 0x00, 0x00 }, /* ţ to t */ + { 0x0164, 0x54, 0x00, 0x00, 0x00 }, /* Ť to T */ + { 0x0165, 0x74, 0x00, 0x00, 0x00 }, /* ť to t */ + { 0x0166, 0x54, 0x00, 0x00, 0x00 }, /* Ŧ to T */ + { 0x0167, 0x74, 0x00, 0x00, 0x00 }, /* ŧ to t */ + { 0x0168, 0x55, 0x00, 0x00, 0x00 }, /* Ũ to U */ + { 0x0169, 0x75, 0x00, 0x00, 0x00 }, /* ũ to u */ + { 0x016A, 0x55, 0x00, 0x00, 0x00 }, /* Ū to U */ + { 0x016B, 0x75, 0x00, 0x00, 0x00 }, /* ū to u */ + { 0x016C, 0x55, 0x00, 0x00, 0x00 }, /* Ŭ to U */ + { 0x016D, 0x75, 0x00, 0x00, 0x00 }, /* ŭ to u */ + { 0x016E, 0x55, 0x00, 0x00, 0x00 }, /* Ů to U */ + { 0x016F, 0x75, 0x00, 0x00, 0x00 }, /* ů to u */ + { 0x0170, 0x55, 0x00, 0x00, 0x00 }, /* Ű to U */ + { 0x0171, 0x75, 0x00, 0x00, 0x00 }, /* ű to u */ + { 0x0172, 0x55, 0x00, 0x00, 0x00 }, /* Ų to U */ + { 0x0173, 0x75, 0x00, 0x00, 0x00 }, /* ų to u */ + { 0x0174, 0x57, 0x00, 0x00, 0x00 }, /* Ŵ to W */ + { 0x0175, 0x77, 0x00, 0x00, 0x00 }, /* ŵ to w */ + { 0x0176, 0x59, 0x00, 0x00, 0x00 }, /* Ŷ to Y */ + { 0x0177, 0x79, 0x00, 0x00, 0x00 }, /* ŷ to y */ + { 0x0178, 0x59, 0x00, 0x00, 0x00 }, /* Ÿ to Y */ + { 0x0179, 0x5A, 0x00, 0x00, 0x00 }, /* Ź to Z */ + { 0x017A, 0x7A, 0x00, 0x00, 0x00 }, /* ź to z */ + { 0x017B, 0x5A, 0x00, 0x00, 0x00 }, /* Ż to Z */ + { 0x017C, 0x7A, 0x00, 0x00, 0x00 }, /* ż to z */ + { 0x017D, 0x5A, 0x00, 0x00, 0x00 }, /* Ž to Z */ + { 0x017E, 0x7A, 0x00, 0x00, 0x00 }, /* ž to z */ + { 0x017F, 0x73, 0x00, 0x00, 0x00 }, /* ſ to s */ + { 0x0192, 0x66, 0x00, 0x00, 0x00 }, /* ƒ to f */ + { 0x0218, 0x53, 0x00, 0x00, 0x00 }, /* Ș to S */ + { 0x0219, 0x73, 0x00, 0x00, 0x00 }, /* ș to s */ + { 0x021A, 0x54, 0x00, 0x00, 0x00 }, /* Ț to T */ + { 0x021B, 0x74, 0x00, 0x00, 0x00 }, /* ț to t */ + { 0x0386, 0x41, 0x00, 0x00, 0x00 }, /* Ά to A */ + { 0x0388, 0x45, 0x00, 0x00, 0x00 }, /* Έ to E */ + { 0x0389, 0x49, 0x00, 0x00, 0x00 }, /* Ή to I */ + { 0x038A, 0x49, 0x00, 0x00, 0x00 }, /* Ί to I */ + { 0x038C, 0x4f, 0x00, 0x00, 0x00 }, /* Ό to O */ + { 0x038E, 0x59, 0x00, 0x00, 0x00 }, /* Ύ to Y */ + { 0x038F, 0x4f, 0x00, 0x00, 0x00 }, /* Ώ to O */ + { 0x0390, 0x69, 0x00, 0x00, 0x00 }, /* ΐ to i */ + { 0x0391, 0x41, 0x00, 0x00, 0x00 }, /* Α to A */ + { 0x0392, 0x42, 0x00, 0x00, 0x00 }, /* Β to B */ + { 0x0393, 0x47, 0x00, 0x00, 0x00 }, /* Γ to G */ + { 0x0394, 0x44, 0x00, 0x00, 0x00 }, /* Δ to D */ + { 0x0395, 0x45, 0x00, 0x00, 0x00 }, /* Ε to E */ + { 0x0396, 0x5a, 0x00, 0x00, 0x00 }, /* Ζ to Z */ + { 0x0397, 0x49, 0x00, 0x00, 0x00 }, /* Η to I */ + { 0x0398, 0x54, 0x68, 0x00, 0x00 }, /* Θ to Th */ + { 0x0399, 0x49, 0x00, 0x00, 0x00 }, /* Ι to I */ + { 0x039A, 0x4b, 0x00, 0x00, 0x00 }, /* Κ to K */ + { 0x039B, 0x4c, 0x00, 0x00, 0x00 }, /* Λ to L */ + { 0x039C, 0x4d, 0x00, 0x00, 0x00 }, /* Μ to M */ + { 0x039D, 0x4e, 0x00, 0x00, 0x00 }, /* Ν to N */ + { 0x039E, 0x58, 0x00, 0x00, 0x00 }, /* Ξ to X */ + { 0x039F, 0x4f, 0x00, 0x00, 0x00 }, /* Ο to O */ + { 0x03A0, 0x50, 0x00, 0x00, 0x00 }, /* Π to P */ + { 0x03A1, 0x52, 0x00, 0x00, 0x00 }, /* Ρ to R */ + { 0x03A3, 0x53, 0x00, 0x00, 0x00 }, /* Σ to S */ + { 0x03A4, 0x54, 0x00, 0x00, 0x00 }, /* Τ to T */ + { 0x03A5, 0x59, 0x00, 0x00, 0x00 }, /* Υ to Y */ + { 0x03A6, 0x46, 0x00, 0x00, 0x00 }, /* Φ to F */ + { 0x03A7, 0x43, 0x68, 0x00, 0x00 }, /* Χ to Ch */ + { 0x03A8, 0x50, 0x73, 0x00, 0x00 }, /* Ψ to Ps */ + { 0x03A9, 0x4f, 0x00, 0x00, 0x00 }, /* Ω to O */ + { 0x03AA, 0x49, 0x00, 0x00, 0x00 }, /* Ϊ to I */ + { 0x03AB, 0x59, 0x00, 0x00, 0x00 }, /* Ϋ to Y */ + { 0x03AC, 0x61, 0x00, 0x00, 0x00 }, /* ά to a */ + { 0x03AD, 0x65, 0x00, 0x00, 0x00 }, /* έ to e */ + { 0x03AE, 0x69, 0x00, 0x00, 0x00 }, /* ή to i */ + { 0x03AF, 0x69, 0x00, 0x00, 0x00 }, /* ί to i */ + { 0x03B1, 0x61, 0x00, 0x00, 0x00 }, /* α to a */ + { 0x03B2, 0x62, 0x00, 0x00, 0x00 }, /* β to b */ + { 0x03B3, 0x67, 0x00, 0x00, 0x00 }, /* γ to g */ + { 0x03B4, 0x64, 0x00, 0x00, 0x00 }, /* δ to d */ + { 0x03B5, 0x65, 0x00, 0x00, 0x00 }, /* ε to e */ + { 0x03B6, 0x7a, 0x00, 0x00, 0x00 }, /* ζ to z */ + { 0x03B7, 0x69, 0x00, 0x00, 0x00 }, /* η to i */ + { 0x03B8, 0x74, 0x68, 0x00, 0x00 }, /* θ to th */ + { 0x03B9, 0x69, 0x00, 0x00, 0x00 }, /* ι to i */ + { 0x03BA, 0x6b, 0x00, 0x00, 0x00 }, /* κ to k */ + { 0x03BB, 0x6c, 0x00, 0x00, 0x00 }, /* λ to l */ + { 0x03BC, 0x6d, 0x00, 0x00, 0x00 }, /* μ to m */ + { 0x03BD, 0x6e, 0x00, 0x00, 0x00 }, /* ν to n */ + { 0x03BE, 0x78, 0x00, 0x00, 0x00 }, /* ξ to x */ + { 0x03BF, 0x6f, 0x00, 0x00, 0x00 }, /* ο to o */ + { 0x03C0, 0x70, 0x00, 0x00, 0x00 }, /* π to p */ + { 0x03C1, 0x72, 0x00, 0x00, 0x00 }, /* ρ to r */ + { 0x03C3, 0x73, 0x00, 0x00, 0x00 }, /* σ to s */ + { 0x03C4, 0x74, 0x00, 0x00, 0x00 }, /* τ to t */ + { 0x03C5, 0x79, 0x00, 0x00, 0x00 }, /* υ to y */ + { 0x03C6, 0x66, 0x00, 0x00, 0x00 }, /* φ to f */ + { 0x03C7, 0x63, 0x68, 0x00, 0x00 }, /* χ to ch */ + { 0x03C8, 0x70, 0x73, 0x00, 0x00 }, /* ψ to ps */ + { 0x03C9, 0x6f, 0x00, 0x00, 0x00 }, /* ω to o */ + { 0x03CA, 0x69, 0x00, 0x00, 0x00 }, /* ϊ to i */ + { 0x03CB, 0x79, 0x00, 0x00, 0x00 }, /* ϋ to y */ + { 0x03CC, 0x6f, 0x00, 0x00, 0x00 }, /* ό to o */ + { 0x03CD, 0x79, 0x00, 0x00, 0x00 }, /* ύ to y */ + { 0x03CE, 0x69, 0x00, 0x00, 0x00 }, /* ώ to i */ + { 0x0400, 0x45, 0x00, 0x00, 0x00 }, /* Ѐ to E */ + { 0x0401, 0x45, 0x00, 0x00, 0x00 }, /* Ё to E */ + { 0x0402, 0x44, 0x00, 0x00, 0x00 }, /* Ђ to D */ + { 0x0403, 0x47, 0x00, 0x00, 0x00 }, /* Ѓ to G */ + { 0x0404, 0x45, 0x00, 0x00, 0x00 }, /* Є to E */ + { 0x0405, 0x5a, 0x00, 0x00, 0x00 }, /* Ѕ to Z */ + { 0x0406, 0x49, 0x00, 0x00, 0x00 }, /* І to I */ + { 0x0407, 0x49, 0x00, 0x00, 0x00 }, /* Ї to I */ + { 0x0408, 0x4a, 0x00, 0x00, 0x00 }, /* Ј to J */ + { 0x0409, 0x49, 0x00, 0x00, 0x00 }, /* Љ to I */ + { 0x040A, 0x4e, 0x00, 0x00, 0x00 }, /* Њ to N */ + { 0x040B, 0x44, 0x00, 0x00, 0x00 }, /* Ћ to D */ + { 0x040C, 0x4b, 0x00, 0x00, 0x00 }, /* Ќ to K */ + { 0x040D, 0x49, 0x00, 0x00, 0x00 }, /* Ѝ to I */ + { 0x040E, 0x55, 0x00, 0x00, 0x00 }, /* Ў to U */ + { 0x040F, 0x44, 0x00, 0x00, 0x00 }, /* Џ to D */ + { 0x0410, 0x41, 0x00, 0x00, 0x00 }, /* А to A */ + { 0x0411, 0x42, 0x00, 0x00, 0x00 }, /* Б to B */ + { 0x0412, 0x56, 0x00, 0x00, 0x00 }, /* В to V */ + { 0x0413, 0x47, 0x00, 0x00, 0x00 }, /* Г to G */ + { 0x0414, 0x44, 0x00, 0x00, 0x00 }, /* Д to D */ + { 0x0415, 0x45, 0x00, 0x00, 0x00 }, /* Е to E */ + { 0x0416, 0x5a, 0x68, 0x00, 0x00 }, /* Ж to Zh */ + { 0x0417, 0x5a, 0x00, 0x00, 0x00 }, /* З to Z */ + { 0x0418, 0x49, 0x00, 0x00, 0x00 }, /* И to I */ + { 0x0419, 0x49, 0x00, 0x00, 0x00 }, /* Й to I */ + { 0x041A, 0x4b, 0x00, 0x00, 0x00 }, /* К to K */ + { 0x041B, 0x4c, 0x00, 0x00, 0x00 }, /* Л to L */ + { 0x041C, 0x4d, 0x00, 0x00, 0x00 }, /* М to M */ + { 0x041D, 0x4e, 0x00, 0x00, 0x00 }, /* Н to N */ + { 0x041E, 0x4f, 0x00, 0x00, 0x00 }, /* О to O */ + { 0x041F, 0x50, 0x00, 0x00, 0x00 }, /* П to P */ + { 0x0420, 0x52, 0x00, 0x00, 0x00 }, /* Р to R */ + { 0x0421, 0x53, 0x00, 0x00, 0x00 }, /* С to S */ + { 0x0422, 0x54, 0x00, 0x00, 0x00 }, /* Т to T */ + { 0x0423, 0x55, 0x00, 0x00, 0x00 }, /* У to U */ + { 0x0424, 0x46, 0x00, 0x00, 0x00 }, /* Ф to F */ + { 0x0425, 0x4b, 0x68, 0x00, 0x00 }, /* Х to Kh */ + { 0x0426, 0x54, 0x63, 0x00, 0x00 }, /* Ц to Tc */ + { 0x0427, 0x43, 0x68, 0x00, 0x00 }, /* Ч to Ch */ + { 0x0428, 0x53, 0x68, 0x00, 0x00 }, /* Ш to Sh */ + { 0x0429, 0x53, 0x68, 0x63, 0x68 }, /* Щ to Shch */ + { 0x042A, 0x61, 0x00, 0x00, 0x00 }, /* to A */ + { 0x042B, 0x59, 0x00, 0x00, 0x00 }, /* Ы to Y */ + { 0x042C, 0x59, 0x00, 0x00, 0x00 }, /* to Y */ + { 0x042D, 0x45, 0x00, 0x00, 0x00 }, /* Э to E */ + { 0x042E, 0x49, 0x75, 0x00, 0x00 }, /* Ю to Iu */ + { 0x042F, 0x49, 0x61, 0x00, 0x00 }, /* Я to Ia */ + { 0x0430, 0x61, 0x00, 0x00, 0x00 }, /* а to a */ + { 0x0431, 0x62, 0x00, 0x00, 0x00 }, /* б to b */ + { 0x0432, 0x76, 0x00, 0x00, 0x00 }, /* в to v */ + { 0x0433, 0x67, 0x00, 0x00, 0x00 }, /* г to g */ + { 0x0434, 0x64, 0x00, 0x00, 0x00 }, /* д to d */ + { 0x0435, 0x65, 0x00, 0x00, 0x00 }, /* е to e */ + { 0x0436, 0x7a, 0x68, 0x00, 0x00 }, /* ж to zh */ + { 0x0437, 0x7a, 0x00, 0x00, 0x00 }, /* з to z */ + { 0x0438, 0x69, 0x00, 0x00, 0x00 }, /* и to i */ + { 0x0439, 0x69, 0x00, 0x00, 0x00 }, /* й to i */ + { 0x043A, 0x6b, 0x00, 0x00, 0x00 }, /* к to k */ + { 0x043B, 0x6c, 0x00, 0x00, 0x00 }, /* л to l */ + { 0x043C, 0x6d, 0x00, 0x00, 0x00 }, /* м to m */ + { 0x043D, 0x6e, 0x00, 0x00, 0x00 }, /* н to n */ + { 0x043E, 0x6f, 0x00, 0x00, 0x00 }, /* о to o */ + { 0x043F, 0x70, 0x00, 0x00, 0x00 }, /* п to p */ + { 0x0440, 0x72, 0x00, 0x00, 0x00 }, /* р to r */ + { 0x0441, 0x73, 0x00, 0x00, 0x00 }, /* с to s */ + { 0x0442, 0x74, 0x00, 0x00, 0x00 }, /* т to t */ + { 0x0443, 0x75, 0x00, 0x00, 0x00 }, /* у to u */ + { 0x0444, 0x66, 0x00, 0x00, 0x00 }, /* ф to f */ + { 0x0445, 0x6b, 0x68, 0x00, 0x00 }, /* х to kh */ + { 0x0446, 0x74, 0x63, 0x00, 0x00 }, /* ц to tc */ + { 0x0447, 0x63, 0x68, 0x00, 0x00 }, /* ч to ch */ + { 0x0448, 0x73, 0x68, 0x00, 0x00 }, /* ш to sh */ + { 0x0449, 0x73, 0x68, 0x63, 0x68 }, /* щ to shch */ + { 0x044A, 0x61, 0x00, 0x00, 0x00 }, /* to a */ + { 0x044B, 0x79, 0x00, 0x00, 0x00 }, /* ы to y */ + { 0x044C, 0x79, 0x00, 0x00, 0x00 }, /* to y */ + { 0x044D, 0x65, 0x00, 0x00, 0x00 }, /* э to e */ + { 0x044E, 0x69, 0x75, 0x00, 0x00 }, /* ю to iu */ + { 0x044F, 0x69, 0x61, 0x00, 0x00 }, /* я to ia */ + { 0x0450, 0x65, 0x00, 0x00, 0x00 }, /* ѐ to e */ + { 0x0451, 0x65, 0x00, 0x00, 0x00 }, /* ё to e */ + { 0x0452, 0x64, 0x00, 0x00, 0x00 }, /* ђ to d */ + { 0x0453, 0x67, 0x00, 0x00, 0x00 }, /* ѓ to g */ + { 0x0454, 0x65, 0x00, 0x00, 0x00 }, /* є to e */ + { 0x0455, 0x7a, 0x00, 0x00, 0x00 }, /* ѕ to z */ + { 0x0456, 0x69, 0x00, 0x00, 0x00 }, /* і to i */ + { 0x0457, 0x69, 0x00, 0x00, 0x00 }, /* ї to i */ + { 0x0458, 0x6a, 0x00, 0x00, 0x00 }, /* ј to j */ + { 0x0459, 0x69, 0x00, 0x00, 0x00 }, /* љ to i */ + { 0x045A, 0x6e, 0x00, 0x00, 0x00 }, /* њ to n */ + { 0x045B, 0x64, 0x00, 0x00, 0x00 }, /* ћ to d */ + { 0x045C, 0x6b, 0x00, 0x00, 0x00 }, /* ќ to k */ + { 0x045D, 0x69, 0x00, 0x00, 0x00 }, /* ѝ to i */ + { 0x045E, 0x75, 0x00, 0x00, 0x00 }, /* ў to u */ + { 0x045F, 0x64, 0x00, 0x00, 0x00 }, /* џ to d */ + { 0x1E02, 0x42, 0x00, 0x00, 0x00 }, /* Ḃ to B */ + { 0x1E03, 0x62, 0x00, 0x00, 0x00 }, /* ḃ to b */ + { 0x1E0A, 0x44, 0x00, 0x00, 0x00 }, /* Ḋ to D */ + { 0x1E0B, 0x64, 0x00, 0x00, 0x00 }, /* ḋ to d */ + { 0x1E1E, 0x46, 0x00, 0x00, 0x00 }, /* Ḟ to F */ + { 0x1E1F, 0x66, 0x00, 0x00, 0x00 }, /* ḟ to f */ + { 0x1E40, 0x4D, 0x00, 0x00, 0x00 }, /* Ṁ to M */ + { 0x1E41, 0x6D, 0x00, 0x00, 0x00 }, /* ṁ to m */ + { 0x1E56, 0x50, 0x00, 0x00, 0x00 }, /* Ṗ to P */ + { 0x1E57, 0x70, 0x00, 0x00, 0x00 }, /* ṗ to p */ + { 0x1E60, 0x53, 0x00, 0x00, 0x00 }, /* Ṡ to S */ + { 0x1E61, 0x73, 0x00, 0x00, 0x00 }, /* ṡ to s */ + { 0x1E6A, 0x54, 0x00, 0x00, 0x00 }, /* Ṫ to T */ + { 0x1E6B, 0x74, 0x00, 0x00, 0x00 }, /* ṫ to t */ + { 0x1E80, 0x57, 0x00, 0x00, 0x00 }, /* Ẁ to W */ + { 0x1E81, 0x77, 0x00, 0x00, 0x00 }, /* ẁ to w */ + { 0x1E82, 0x57, 0x00, 0x00, 0x00 }, /* Ẃ to W */ + { 0x1E83, 0x77, 0x00, 0x00, 0x00 }, /* ẃ to w */ + { 0x1E84, 0x57, 0x00, 0x00, 0x00 }, /* Ẅ to W */ + { 0x1E85, 0x77, 0x00, 0x00, 0x00 }, /* ẅ to w */ + { 0x1EF2, 0x59, 0x00, 0x00, 0x00 }, /* Ỳ to Y */ + { 0x1EF3, 0x79, 0x00, 0x00, 0x00 }, /* ỳ to y */ + { 0xFB00, 0x66, 0x66, 0x00, 0x00 }, /* ff to ff */ + { 0xFB01, 0x66, 0x69, 0x00, 0x00 }, /* fi to fi */ + { 0xFB02, 0x66, 0x6C, 0x00, 0x00 }, /* fl to fl */ + { 0xFB05, 0x73, 0x74, 0x00, 0x00 }, /* ſt to st */ + { 0xFB06, 0x73, 0x74, 0x00, 0x00 }, /* st to st */ }; +static const Transliteration *spellfixFindTranslit(int c, int *pxTop){ + *pxTop = (sizeof(translit)/sizeof(translit[0])) - 1; + return translit; +} + /* ** Convert the input string from UTF-8 into pure ASCII by converting ** all non-ASCII characters to some combination of characters in the @@ -1621,23 +1629,24 @@ static unsigned char *transliterate(const unsigned char *zIn, int nIn){ zOut[nOut++] = (unsigned char)c; }else{ int xTop, xBtm, x; - xTop = sizeof(translit)/sizeof(translit[0]) - 1; + const Transliteration *tbl = spellfixFindTranslit(c, &xTop); xBtm = 0; while( xTop>=xBtm ){ x = (xTop + xBtm)/2; - if( translit[x].cFrom==c ){ - zOut[nOut++] = translit[x].cTo0; - if( translit[x].cTo1 ){ - zOut[nOut++] = translit[x].cTo1; - /* Add an extra "ch" after the "sh" for Щ and щ */ - if( c==0x0429 || c== 0x0449 ){ - zOut[nOut++] = 'c'; - zOut[nOut++] = 'h'; + if( tbl[x].cFrom==c ){ + zOut[nOut++] = tbl[x].cTo0; + if( tbl[x].cTo1 ){ + zOut[nOut++] = tbl[x].cTo1; + if( tbl[x].cTo2 ){ + zOut[nOut++] = tbl[x].cTo2; + if( tbl[x].cTo3 ){ + zOut[nOut++] = tbl[x].cTo3; + } } } c = 0; break; - }else if( translit[x].cFrom>c ){ + }else if( tbl[x].cFrom>c ){ xTop = x-1; }else{ xBtm = x+1; @@ -1668,15 +1677,22 @@ static int translen_to_charlen(const char *zIn, int nIn, int nTrans){ nOut++; if( c>=128 ){ int xTop, xBtm, x; - xTop = sizeof(translit)/sizeof(translit[0]) - 1; + const Transliteration *tbl = spellfixFindTranslit(c, &xTop); xBtm = 0; while( xTop>=xBtm ){ x = (xTop + xBtm)/2; - if( translit[x].cFrom==c ){ - if( translit[x].cTo1 ) nOut++; - if( c==0x0429 || c== 0x0449 ) nOut += 2; + if( tbl[x].cFrom==c ){ + if( tbl[x].cTo1 ){ + nOut++; + if( tbl[x].cTo2 ){ + nOut++; + if( tbl[x].cTo3 ){ + nOut++; + } + } + } break; - }else if( translit[x].cFrom>c ){ + }else if( tbl[x].cFrom>c ){ xTop = x-1; }else{ xBtm = x+1; diff --git a/manifest b/manifest index e20e592681..db881f438c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Rearrange\sfields\sof\sthe\sBtCursor\sobject\sso\sthat\sit\sis\ssmaller\sand\srequires\sless\ninitialization,\sfor\sa\ssmall\sperformance\simprovement. -D 2018-01-24T16:04:21.587 +C Reorganize\sspellfix.c\sto\smake\sit\seasier\sto\sedit\sautomatically\s(e.g.\susing\sa\s\nscript).\sNo\schanges\sto\sfunctionality. +D 2018-01-24T18:28:39.575 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -294,7 +294,7 @@ F ext/misc/series.c f3c0dba5c5c749ce1782b53076108f87cf0b71041eb6023f727a9c50681d F ext/misc/sha1.c 0b9e9b855354910d3ca467bf39099d570e73db56 F ext/misc/shathree.c 9e960ba50483214c6a7a4b1517f8d8cef799e9db381195178c3fd3ad207e10c0 F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52 -F ext/misc/spellfix.c 41cf26c6b89fcaa8798ae10ae64d39c1f1d9d6995152e545bd491c13058b8fac +F ext/misc/spellfix.c 705ce1ed701b38f43c8ee7dd848877ff51c09eda322e39ccac6873adb4b74735 F ext/misc/sqlar.c 57d5bc45cd5492208e451f697404be88f8612527d64c9d42f96b325b64983d74 F ext/misc/stmt.c 6f16443abb3551e3f5813bb13ba19a30e7032830015b0f92fe0c0453045c0a11 F ext/misc/totype.c 4a167594e791abeed95e0a8db028822b5e8fe512 @@ -1702,7 +1702,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 16ebe558d2f0982f630c39b6c1b00d14d27011d454cced213f17c26491a2c06e -R fc9e02eb09b9540ef2f7fa9944ac0566 -U drh -Z c7e32730b85a19ed6c94c7a445674e7d +P 0ddf5292cc0411ec6fcb7399ecf2904c899e0488404d3f65490fbe1db15efdf4 +R 9f30c4a7d442a62a313819b7586e7fb7 +U dan +Z 5b6a5ad739adf30d436b457769ddc2cb diff --git a/manifest.uuid b/manifest.uuid index 1dd418e847..795b17508a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0ddf5292cc0411ec6fcb7399ecf2904c899e0488404d3f65490fbe1db15efdf4 \ No newline at end of file +090a64faaac579c6ed1cddb02d14e7089468a4aeaeec11e78d461790bcb6eb0a \ No newline at end of file From 9f8952390a97140d8dffa9be53fd61121726629f Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 24 Jan 2018 20:42:42 +0000 Subject: [PATCH 007/107] When unpacking a sorter record, do so from right to left instead of left to right, since by starting with the right-most column, the work done by OP_Column opcodes is reduced. FossilOrigin-Name: 8055e4f42446ceb5bcf752bbf41a73289c3ca759c56c9f779edc3d7f202b7881 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/select.c | 7 +++++-- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index db881f438c..36be0b5904 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Reorganize\sspellfix.c\sto\smake\sit\seasier\sto\sedit\sautomatically\s(e.g.\susing\sa\s\nscript).\sNo\schanges\sto\sfunctionality. -D 2018-01-24T18:28:39.575 +C When\sunpacking\sa\ssorter\srecord,\sdo\sso\sfrom\sright\sto\sleft\sinstead\sof\sleft\sto\nright,\ssince\sby\sstarting\swith\sthe\sright-most\scolumn,\sthe\swork\sdone\sby\nOP_Column\sopcodes\sis\sreduced. +D 2018-01-24T20:42:42.319 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -487,7 +487,7 @@ F src/printf.c 9506b4b96e59c0467047155f09015750cb2878aeda3d39e5610c1192ddc3c41c F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac -F src/select.c bebe7cce45d899d2237c76bce059d525abf5b861f2fce92f6b53914a961c01ba +F src/select.c f02352ac5cbb6fad3804add825743b924cfb6c742ba2e8666d726828a9985d73 F src/shell.c.in 7cea439c3f7f2e4ed6eb4b3a633cd93dccb1349241400de4da0c1291285ed514 F src/sqlite.h.in 51f9acf52c80113d793ddd38b3940ad6895d97b4752503b19291fb8fcbf54c5e F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -1702,7 +1702,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0ddf5292cc0411ec6fcb7399ecf2904c899e0488404d3f65490fbe1db15efdf4 -R 9f30c4a7d442a62a313819b7586e7fb7 -U dan -Z 5b6a5ad739adf30d436b457769ddc2cb +P 090a64faaac579c6ed1cddb02d14e7089468a4aeaeec11e78d461790bcb6eb0a +R 36fadb5748a2d2919f267681ce3c3a75 +U drh +Z 4006c7b9f32358fa0bed65eb183dba8e diff --git a/manifest.uuid b/manifest.uuid index 795b17508a..4ff4675403 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -090a64faaac579c6ed1cddb02d14e7089468a4aeaeec11e78d461790bcb6eb0a \ No newline at end of file +8055e4f42446ceb5bcf752bbf41a73289c3ca759c56c9f779edc3d7f202b7881 \ No newline at end of file diff --git a/src/select.c b/src/select.c index c3cb4082fe..9ac4f9f198 100644 --- a/src/select.c +++ b/src/select.c @@ -1266,12 +1266,15 @@ static void generateSortTail( iSortTab = iTab; bSeq = 1; } - for(i=0, iCol=nKey+bSeq; i=0; i--){ int iRead; if( aOutEx[i].u.x.iOrderByCol ){ iRead = aOutEx[i].u.x.iOrderByCol-1; }else{ - iRead = iCol++; + iRead = iCol--; } sqlite3VdbeAddOp3(v, OP_Column, iSortTab, iRead, regRow+i); VdbeComment((v, "%s", aOutEx[i].zName ? aOutEx[i].zName : aOutEx[i].zSpan)); From cad42838d718fdabb18d0ea4fc72ced389489cb0 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 25 Jan 2018 01:20:29 +0000 Subject: [PATCH 008/107] Slightly smaller and faster implementation of OP_If and OP_IfNot. FossilOrigin-Name: 6ab42934e2c4957b5d8927bf4434a9db07ab6078987a6a2d25f35cc468d21203 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbe.c | 10 +++++----- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 36be0b5904..7d40e3172b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\sunpacking\sa\ssorter\srecord,\sdo\sso\sfrom\sright\sto\sleft\sinstead\sof\sleft\sto\nright,\ssince\sby\sstarting\swith\sthe\sright-most\scolumn,\sthe\swork\sdone\sby\nOP_Column\sopcodes\sis\sreduced. -D 2018-01-24T20:42:42.319 +C Slightly\ssmaller\sand\sfaster\simplementation\sof\sOP_If\sand\sOP_IfNot. +D 2018-01-25T01:20:29.953 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -556,7 +556,7 @@ F src/update.c a90a32ffc0100265b0693dbbdbe490756447af181f5ea2c138cce515b08c8795 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5 F src/util.c ef4a5f904d942e660abade7fbf3e6bdb402dabe9e7c27f3361ecf40b945538b5 F src/vacuum.c 90839322fd5f00df9617eb21b68beda9b6e2a2937576b0d65985e4aeb1c53739 -F src/vdbe.c d4684bd5efad095d9e4405e5ca2e1168dc7aff643cae04c424e4bade70641d70 +F src/vdbe.c 0e13b8c33a9ecf236b3045f190fb58933049582307b621cbd924b6582a2b705f F src/vdbe.h 134beb7a12a6213c00eba58febaede33447cc4441bc568a0d9c144b33fc3720a F src/vdbeInt.h 8d7d07f13cb3c4cbca91e22ba4a1920e542dda7c5d9299920432a0b3d5b009f5 F src/vdbeapi.c fea41171884a4de119f8b10ab514c788674eeeb7f27218bb6d008e1310bfd07f @@ -1702,7 +1702,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 090a64faaac579c6ed1cddb02d14e7089468a4aeaeec11e78d461790bcb6eb0a -R 36fadb5748a2d2919f267681ce3c3a75 +P 8055e4f42446ceb5bcf752bbf41a73289c3ca759c56c9f779edc3d7f202b7881 +R 3a57ffa9e8084df96fa05ea7edc4d068 U drh -Z 4006c7b9f32358fa0bed65eb183dba8e +Z 1f11fb78272f5afcdf3b192bdf3cca0d diff --git a/manifest.uuid b/manifest.uuid index 4ff4675403..8b78bd0022 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8055e4f42446ceb5bcf752bbf41a73289c3ca759c56c9f779edc3d7f202b7881 \ No newline at end of file +6ab42934e2c4957b5d8927bf4434a9db07ab6078987a6a2d25f35cc468d21203 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 81a2361a55..495b56aa1d 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2289,11 +2289,11 @@ case OP_IfNot: { /* jump, in1 */ if( pIn1->flags & MEM_Null ){ c = pOp->p3; }else{ -#ifdef SQLITE_OMIT_FLOATING_POINT - c = sqlite3VdbeIntValue(pIn1)!=0; -#else - c = sqlite3VdbeRealValue(pIn1)!=0.0; -#endif + if( pIn1->flags & MEM_Int ){ + c = pIn1->u.i!=0; + }else{ + c = sqlite3VdbeRealValue(pIn1)!=0.0; + } if( pOp->opcode==OP_IfNot ) c = !c; } VdbeBranchTaken(c!=0, 2); From 8558ef2e199c63455305ea3ffb7f31a56193814d Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 25 Jan 2018 20:50:46 +0000 Subject: [PATCH 009/107] Reorganize zipfile.c code to make it easier to add support for in-memory zip archive processing. FossilOrigin-Name: 30b9258294e3028ef4ea467e332e013995509544e9a23c8bbf5168772a7e895d --- ext/misc/zipfile.c | 733 +++++++++++++++++++++++---------------------- manifest | 15 +- manifest.uuid | 2 +- 3 files changed, 382 insertions(+), 368 deletions(-) diff --git a/ext/misc/zipfile.c b/ext/misc/zipfile.c index fa31d62d11..d378606941 100644 --- a/ext/misc/zipfile.c +++ b/ext/misc/zipfile.c @@ -222,10 +222,9 @@ struct ZipfileLFH { typedef struct ZipfileEntry ZipfileEntry; struct ZipfileEntry { - char *zPath; /* Path of zipfile entry */ - u8 *aCdsEntry; /* Buffer containing entire CDS entry */ - int nCdsEntry; /* Size of buffer aCdsEntry[] in bytes */ - int bDeleted; /* True if entry has been deleted */ + ZipfileCDS cds; /* Parsed CDS record */ + u32 mUnixTime; /* Modification time, in UNIX format */ + u8 *aExtra; /* cds.nExtra+cds.nComment bytes of extra data */ ZipfileEntry *pNext; /* Next element in in-memory CDS */ }; @@ -236,28 +235,30 @@ typedef struct ZipfileCsr ZipfileCsr; struct ZipfileCsr { sqlite3_vtab_cursor base; /* Base class - must be first */ i64 iId; /* Cursor ID */ - int bEof; /* True when at EOF */ + u8 bEof; /* True when at EOF */ + u8 bNoop; /* If next xNext() call is no-op */ /* Used outside of write transactions */ FILE *pFile; /* Zip file */ i64 iNextOff; /* Offset of next record in central directory */ ZipfileEOCD eocd; /* Parse of central directory record */ - /* Used inside write transactions */ - ZipfileEntry *pCurrent; - - ZipfileCDS cds; /* Central Directory Structure */ + ZipfileEntry *pCurrent; /* Current entry */ ZipfileLFH lfh; /* Local File Header for current entry */ i64 iDataOff; /* Offset in zipfile to data */ - u32 mTime; /* Extended mtime value */ - int flags; /* Flags byte (see below for bits) */ ZipfileCsr *pCsrNext; /* Next cursor on same virtual table */ }; +/* +** Possible values for ZipfileCsr.eType. Set in zipfileFilter(). +*/ +#define ZIPFILE_CSR_LIST 1 /* Cursor reads from ZipfileTab.pFirstEntry */ +#define ZIPFILE_CSR_FILE 2 /* Cursor reads from file on disk */ + /* ** Values for ZipfileCsr.flags. */ -#define ZIPFILE_MTIME_VALID 0x0001 +#define ZIPFILE_NOOP_NEXT 0x0002 /* Next xNext() is a no-op */ typedef struct ZipfileTab ZipfileTab; struct ZipfileTab { @@ -335,10 +336,36 @@ static int zipfileConnect( return rc; } +static void zipfileEntryFree(ZipfileEntry *p){ + if( p ){ + sqlite3_free(p->cds.zFile); + sqlite3_free(p); + } +} + +static void zipfileCleanupTransaction(ZipfileTab *pTab){ + ZipfileEntry *pEntry; + ZipfileEntry *pNext; + + if( pTab->pWriteFd ){ + fclose(pTab->pWriteFd); + pTab->pWriteFd = 0; + } + for(pEntry=pTab->pFirstEntry; pEntry; pEntry=pNext){ + pNext = pEntry->pNext; + zipfileEntryFree(pEntry); + } + pTab->pFirstEntry = 0; + pTab->pLastEntry = 0; + pTab->szCurrent = 0; + pTab->szOrig = 0; +} + /* ** This method is the destructor for zipfile vtab objects. */ static int zipfileDisconnect(sqlite3_vtab *pVtab){ + zipfileCleanupTransaction((ZipfileTab*)pVtab); sqlite3_free(pVtab); return SQLITE_OK; } @@ -366,12 +393,12 @@ static int zipfileOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCsr){ ** by zipfileOpen(). */ static void zipfileResetCursor(ZipfileCsr *pCsr){ - sqlite3_free(pCsr->cds.zFile); - pCsr->cds.zFile = 0; pCsr->bEof = 0; if( pCsr->pFile ){ fclose(pCsr->pFile); pCsr->pFile = 0; + zipfileEntryFree(pCsr->pCurrent); + pCsr->pCurrent = 0; } } @@ -512,77 +539,117 @@ static int zipfileReadCDS(u8 *aBuf, ZipfileCDS *pCDS){ } /* -** Read the CDS record for the current entry from disk into pCsr->cds. +** The general format of an extra field is: +** +** Header ID 2 bytes +** Data Size 2 bytes +** Data N bytes */ -static int zipfileCsrReadCDS(ZipfileCsr *pCsr){ - char **pzErr = &pCsr->base.pVtab->zErrMsg; - u8 *aRead; - int rc = SQLITE_OK; +static int zipfileScanExtra(u8 *aExtra, int nExtra, u32 *pmTime){ + int ret = 0; + u8 *p = aExtra; + u8 *pEnd = &aExtra[nExtra]; - sqlite3_free(pCsr->cds.zFile); - pCsr->cds.zFile = 0; + while( ppCurrent==0 ){ - aRead = zipfileCsrBuffer(pCsr); - rc = zipfileReadData( - pCsr->pFile, aRead, ZIPFILE_CDS_FIXED_SZ, pCsr->iNextOff, pzErr - ); - }else{ - aRead = pCsr->pCurrent->aCdsEntry; - } - - if( rc==SQLITE_OK ){ - rc = zipfileReadCDS(aRead, &pCsr->cds); - if( rc!=SQLITE_OK ){ - assert( pCsr->pCurrent==0 ); - zipfileSetErrmsg(pCsr,"failed to read CDS at offset %lld",pCsr->iNextOff); - }else{ - int nRead; - if( pCsr->pCurrent==0 ){ - nRead = pCsr->cds.nFile + pCsr->cds.nExtra; - aRead = zipfileCsrBuffer(pCsr); - pCsr->iNextOff += ZIPFILE_CDS_FIXED_SZ; - rc = zipfileReadData(pCsr->pFile, aRead, nRead, pCsr->iNextOff, pzErr); - }else{ - aRead = &aRead[ZIPFILE_CDS_FIXED_SZ]; - } - - if( rc==SQLITE_OK ){ - pCsr->cds.zFile = sqlite3_mprintf("%.*s", (int)pCsr->cds.nFile, aRead); - pCsr->iNextOff += pCsr->cds.nFile; - pCsr->iNextOff += pCsr->cds.nExtra; - pCsr->iNextOff += pCsr->cds.nComment; - } - - /* Scan the cds.nExtra bytes of "extra" fields for any that can - ** be interpreted. The general format of an extra field is: - ** - ** Header ID 2 bytes - ** Data Size 2 bytes - ** Data N bytes - ** - */ - if( rc==SQLITE_OK ){ - u8 *p = &aRead[pCsr->cds.nFile]; - u8 *pEnd = &p[pCsr->cds.nExtra]; - - while( p modtime is present */ - pCsr->mTime = zipfileGetU32(&p[1]); - pCsr->flags |= ZIPFILE_MTIME_VALID; - } - break; - } - } - - p += nByte; + switch( id ){ + case ZIPFILE_EXTRA_TIMESTAMP: { + u8 b = p[0]; + if( b & 0x01 ){ /* 0x01 -> modtime is present */ + *pmTime = zipfileGetU32(&p[1]); + ret = 1; } + break; + } + } + + p += nByte; + } + return ret; +} + +/* +** "Standard" MS-DOS time format: +** +** File modification time: +** Bits 00-04: seconds divided by 2 +** Bits 05-10: minute +** Bits 11-15: hour +** File modification date: +** Bits 00-04: day +** Bits 05-08: month (1-12) +** Bits 09-15: years from 1980 +*/ +static time_t zipfileMtime(ZipfileCDS *pCDS){ + struct tm t; + memset(&t, 0, sizeof(t)); + t.tm_sec = (pCDS->mTime & 0x1F)*2; + t.tm_min = (pCDS->mTime >> 5) & 0x2F; + t.tm_hour = (pCDS->mTime >> 11) & 0x1F; + + t.tm_mday = (pCDS->mDate & 0x1F); + t.tm_mon = ((pCDS->mDate >> 5) & 0x0F) - 1; + t.tm_year = 80 + ((pCDS->mDate >> 9) & 0x7F); + + return mktime(&t); +} + +/* +** Read a Zip archive CDS header from offset iOff of file pFile. Return +** SQLITE_OK if successful, or an SQLite error code otherwise. +*/ +static int zipfileGetEntry( + ZipfileTab *pTab, /* Store any error message here */ + FILE *pFile, + i64 iOff, + ZipfileEntry **ppEntry +){ + u8 *aRead = pTab->aBuffer; + char **pzErr = &pTab->base.zErrMsg; + int rc; + + rc = zipfileReadData(pFile, aRead, ZIPFILE_CDS_FIXED_SZ, iOff, pzErr); + if( rc==SQLITE_OK ){ + if( rc!=SQLITE_OK ){ + *pzErr = sqlite3_mprintf("failed to read CDS at offset %lld", iOff); + }else{ + ZipfileEntry *pNew; + + int nFile = zipfileGetU16(&aRead[ZIPFILE_CDS_NFILE_OFF]); + int nExtra = zipfileGetU16(&aRead[ZIPFILE_CDS_NFILE_OFF+2]); + nExtra += zipfileGetU16(&aRead[ZIPFILE_CDS_NFILE_OFF+4]); + + pNew = (ZipfileEntry*)sqlite3_malloc(sizeof(ZipfileEntry) + nExtra); + if( pNew==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(pNew, 0, sizeof(ZipfileEntry)); + rc = zipfileReadCDS(aRead, &pNew->cds); + if( rc!=SQLITE_OK ){ + *pzErr = sqlite3_mprintf("failed to read CDS at offset %lld", iOff); + }else{ + rc = zipfileReadData( + pFile, aRead, nExtra+nFile, iOff+ZIPFILE_CDS_FIXED_SZ, pzErr + ); + } + } + + if( rc==SQLITE_OK ){ + u32 *pt = &pNew->mUnixTime; + pNew->cds.zFile = sqlite3_mprintf("%.*s", nFile, aRead); + if( pNew->cds.zFile==0 ){ + rc = SQLITE_NOMEM; + }else if( 0==zipfileScanExtra(&aRead[nFile], pNew->cds.nExtra, pt) ){ + pNew->mUnixTime = zipfileMtime(&pNew->cds); + } + } + + if( rc!=SQLITE_OK ){ + zipfileEntryFree(pNew); + }else{ + *ppEntry = pNew; } } } @@ -633,8 +700,9 @@ static int zipfileCsrReadLFH(ZipfileCsr *pCsr){ FILE *pFile = zipfileGetFd(pCsr); char **pzErr = &pCsr->base.pVtab->zErrMsg; u8 *aRead = zipfileCsrBuffer(pCsr); - int rc = zipfileReadLFH(pFile, pCsr->cds.iOffset, aRead, &pCsr->lfh, pzErr); - pCsr->iDataOff = pCsr->cds.iOffset + ZIPFILE_LFH_FIXED_SZ; + ZipfileCDS *pCDS = &pCsr->pCurrent->cds; + int rc = zipfileReadLFH(pFile, pCDS->iOffset, aRead, &pCsr->lfh, pzErr); + pCsr->iDataOff = pCDS->iOffset + ZIPFILE_LFH_FIXED_SZ; pCsr->iDataOff += pCsr->lfh.nFile+pCsr->lfh.nExtra; return rc; } @@ -646,59 +714,40 @@ static int zipfileCsrReadLFH(ZipfileCsr *pCsr){ static int zipfileNext(sqlite3_vtab_cursor *cur){ ZipfileCsr *pCsr = (ZipfileCsr*)cur; int rc = SQLITE_OK; - pCsr->flags = 0; - if( pCsr->pCurrent==0 ){ + if( pCsr->pFile ){ i64 iEof = pCsr->eocd.iOffset + pCsr->eocd.nSize; + zipfileEntryFree(pCsr->pCurrent); + pCsr->pCurrent = 0; if( pCsr->iNextOff>=iEof ){ pCsr->bEof = 1; + }else{ + ZipfileEntry *p = 0; + ZipfileTab *pTab = (ZipfileTab*)(cur->pVtab); + rc = zipfileGetEntry(pTab, pCsr->pFile, pCsr->iNextOff, &p); + if( rc==SQLITE_OK ){ + pCsr->iNextOff += ZIPFILE_CDS_FIXED_SZ; + pCsr->iNextOff += (int)p->cds.nExtra + p->cds.nFile + p->cds.nComment; + } + pCsr->pCurrent = p; } }else{ - assert( pCsr->pFile==0 ); - do { + if( !pCsr->bNoop ){ pCsr->pCurrent = pCsr->pCurrent->pNext; - }while( pCsr->pCurrent && pCsr->pCurrent->bDeleted ); + } if( pCsr->pCurrent==0 ){ pCsr->bEof = 1; } } - if( pCsr->bEof==0 ){ - rc = zipfileCsrReadCDS(pCsr); - if( rc==SQLITE_OK ){ - rc = zipfileCsrReadLFH(pCsr); - } + pCsr->bNoop = 0; + if( rc==SQLITE_OK && pCsr->bEof==0 ){ + rc = zipfileCsrReadLFH(pCsr); } return rc; } -/* -** "Standard" MS-DOS time format: -** -** File modification time: -** Bits 00-04: seconds divided by 2 -** Bits 05-10: minute -** Bits 11-15: hour -** File modification date: -** Bits 00-04: day -** Bits 05-08: month (1-12) -** Bits 09-15: years from 1980 -*/ -static time_t zipfileMtime(ZipfileCsr *pCsr){ - struct tm t; - memset(&t, 0, sizeof(t)); - t.tm_sec = (pCsr->cds.mTime & 0x1F)*2; - t.tm_min = (pCsr->cds.mTime >> 5) & 0x2F; - t.tm_hour = (pCsr->cds.mTime >> 11) & 0x1F; - - t.tm_mday = (pCsr->cds.mDate & 0x1F); - t.tm_mon = ((pCsr->cds.mDate >> 5) & 0x0F) - 1; - t.tm_year = 80 + ((pCsr->cds.mDate >> 9) & 0x7F); - - return mktime(&t); -} - static void zipfileMtimeToDos(ZipfileCDS *pCds, u32 mTime){ time_t t = (time_t)mTime; struct tm res; @@ -804,36 +853,33 @@ static int zipfileColumn( int i /* Which column to return */ ){ ZipfileCsr *pCsr = (ZipfileCsr*)cur; + ZipfileCDS *pCDS = &pCsr->pCurrent->cds; int rc = SQLITE_OK; switch( i ){ case 0: /* name */ - sqlite3_result_text(ctx, pCsr->cds.zFile, -1, SQLITE_TRANSIENT); + sqlite3_result_text(ctx, pCDS->zFile, -1, SQLITE_TRANSIENT); break; case 1: /* mode */ /* TODO: Whether or not the following is correct surely depends on ** the platform on which the archive was created. */ - sqlite3_result_int(ctx, pCsr->cds.iExternalAttr >> 16); + sqlite3_result_int(ctx, pCDS->iExternalAttr >> 16); break; case 2: { /* mtime */ - if( pCsr->flags & ZIPFILE_MTIME_VALID ){ - sqlite3_result_int64(ctx, pCsr->mTime); - }else{ - sqlite3_result_int64(ctx, zipfileMtime(pCsr)); - } + sqlite3_result_int64(ctx, pCsr->pCurrent->mUnixTime); break; } case 3: { /* sz */ if( sqlite3_vtab_nochange(ctx)==0 ){ - sqlite3_result_int64(ctx, pCsr->cds.szUncompressed); + sqlite3_result_int64(ctx, pCDS->szUncompressed); } break; } case 4: /* rawdata */ if( sqlite3_vtab_nochange(ctx) ) break; case 5: { /* data */ - if( i==4 || pCsr->cds.iCompression==0 || pCsr->cds.iCompression==8 ){ - int sz = pCsr->cds.szCompressed; - int szFinal = pCsr->cds.szUncompressed; + if( i==4 || pCDS->iCompression==0 || pCDS->iCompression==8 ){ + int sz = pCDS->szCompressed; + int szFinal = pCDS->szUncompressed; if( szFinal>0 ){ u8 *aBuf = sqlite3_malloc(sz); if( aBuf==0 ){ @@ -845,7 +891,7 @@ static int zipfileColumn( ); } if( rc==SQLITE_OK ){ - if( i==5 && pCsr->cds.iCompression ){ + if( i==5 && pCDS->iCompression ){ zipfileInflate(ctx, aBuf, sz, szFinal); }else{ sqlite3_result_blob(ctx, aBuf, sz, SQLITE_TRANSIENT); @@ -856,8 +902,8 @@ static int zipfileColumn( /* Figure out if this is a directory or a zero-sized file. Consider ** it to be a directory either if the mode suggests so, or if ** the final character in the name is '/'. */ - u32 mode = pCsr->cds.iExternalAttr >> 16; - if( !(mode & S_IFDIR) && pCsr->cds.zFile[pCsr->cds.nFile-1]!='/' ){ + u32 mode = pCDS->iExternalAttr >> 16; + if( !(mode & S_IFDIR) && pCDS->zFile[pCDS->nFile-1]!='/' ){ sqlite3_result_blob(ctx, "", 0, SQLITE_STATIC); } } @@ -865,7 +911,7 @@ static int zipfileColumn( break; } case 6: /* method */ - sqlite3_result_int(ctx, pCsr->cds.iCompression); + sqlite3_result_int(ctx, pCDS->iCompression); break; case 7: /* z */ sqlite3_result_int64(ctx, pCsr->iId); @@ -965,6 +1011,7 @@ static int zipfileFilter( ZipfileCsr *pCsr = (ZipfileCsr*)cur; const char *zFile; /* Zip file to scan */ int rc = SQLITE_OK; /* Return Code */ + int bInMemory = 0; /* True for an in-memory zipfile */ zipfileResetCursor(pCsr); @@ -975,11 +1022,12 @@ static int zipfileFilter( ** supplied a file name. */ zipfileSetErrmsg(pCsr, "table function zipfile() requires an argument"); return SQLITE_ERROR; + }else if( sqlite3_value_type(argv[0])==SQLITE_BLOB ){ }else{ zFile = (const char*)sqlite3_value_text(argv[0]); } - if( pTab->pWriteFd==0 ){ + if( 0==pTab->pWriteFd && 0==bInMemory ){ pCsr->pFile = fopen(zFile, "rb"); if( pCsr->pFile==0 ){ zipfileSetErrmsg(pCsr, "cannot open file: %s", zFile); @@ -996,12 +1044,9 @@ static int zipfileFilter( } } }else{ - ZipfileEntry e; - memset(&e, 0, sizeof(e)); - e.pNext = pTab->pFirstEntry; - pCsr->pCurrent = &e; + pCsr->bNoop = 1; + pCsr->pCurrent = pTab->pFirstEntry; rc = zipfileNext(cur); - assert( pCsr->pCurrent!=&e ); } return rc; @@ -1067,110 +1112,35 @@ static void zipfileAddEntry( static int zipfileLoadDirectory(ZipfileTab *pTab){ ZipfileEOCD eocd; int rc; + int i; + i64 iOff; rc = zipfileReadEOCD(pTab, pTab->pWriteFd, &eocd); - if( rc==SQLITE_OK && eocd.nEntry>0 ){ - int i; - int iOff = 0; - u8 *aBuf = sqlite3_malloc(eocd.nSize); - if( aBuf==0 ){ - rc = SQLITE_NOMEM; - }else{ - rc = zipfileReadData( - pTab->pWriteFd, aBuf, eocd.nSize, eocd.iOffset, &pTab->base.zErrMsg - ); + iOff = eocd.iOffset; + for(i=0; rc==SQLITE_OK && ipWriteFd, iOff, &pNew); + + if( rc==SQLITE_OK ){ + zipfileAddEntry(pTab, 0, pNew); + iOff += ZIPFILE_CDS_FIXED_SZ; + iOff += (int)pNew->cds.nExtra + pNew->cds.nFile + pNew->cds.nComment; } - - for(i=0; rc==SQLITE_OK && izPath = (char*)&pNew[1]; - memcpy(pNew->zPath, &aRec[ZIPFILE_CDS_FIXED_SZ], nFile); - pNew->zPath[nFile] = '\0'; - pNew->aCdsEntry = (u8*)&pNew->zPath[nFile+1]; - pNew->nCdsEntry = ZIPFILE_CDS_FIXED_SZ+nFile+nExtra+nComment; - memcpy(pNew->aCdsEntry, aRec, pNew->nCdsEntry); - zipfileAddEntry(pTab, 0, pNew); - } - - iOff += ZIPFILE_CDS_FIXED_SZ+nFile+nExtra+nComment; - } - - sqlite3_free(aBuf); } - return rc; } -static ZipfileEntry *zipfileNewEntry( - ZipfileCDS *pCds, /* Values for fixed size part of CDS */ - const char *zPath, /* Path for new entry */ - int nPath, /* strlen(zPath) */ - u32 mTime /* Modification time (or 0) */ -){ - u8 *aWrite; +static ZipfileEntry *zipfileNewEntry(const char *zPath){ ZipfileEntry *pNew; - pCds->nFile = (u16)nPath; - pCds->nExtra = mTime ? 9 : 0; - pNew = (ZipfileEntry*)sqlite3_malloc( - sizeof(ZipfileEntry) + - nPath+1 + - ZIPFILE_CDS_FIXED_SZ + nPath + pCds->nExtra - ); - + pNew = sqlite3_malloc(sizeof(ZipfileEntry)); if( pNew ){ memset(pNew, 0, sizeof(ZipfileEntry)); - pNew->zPath = (char*)&pNew[1]; - pNew->aCdsEntry = (u8*)&pNew->zPath[nPath+1]; - pNew->nCdsEntry = ZIPFILE_CDS_FIXED_SZ + nPath + pCds->nExtra; - memcpy(pNew->zPath, zPath, nPath+1); - - aWrite = pNew->aCdsEntry; - zipfileWrite32(aWrite, ZIPFILE_SIGNATURE_CDS); - zipfileWrite16(aWrite, pCds->iVersionMadeBy); - zipfileWrite16(aWrite, pCds->iVersionExtract); - zipfileWrite16(aWrite, pCds->flags); - zipfileWrite16(aWrite, pCds->iCompression); - zipfileWrite16(aWrite, pCds->mTime); - zipfileWrite16(aWrite, pCds->mDate); - zipfileWrite32(aWrite, pCds->crc32); - zipfileWrite32(aWrite, pCds->szCompressed); - zipfileWrite32(aWrite, pCds->szUncompressed); - zipfileWrite16(aWrite, pCds->nFile); - zipfileWrite16(aWrite, pCds->nExtra); - zipfileWrite16(aWrite, pCds->nComment); assert( pCds->nComment==0 ); - zipfileWrite16(aWrite, pCds->iDiskStart); - zipfileWrite16(aWrite, pCds->iInternalAttr); - zipfileWrite32(aWrite, pCds->iExternalAttr); - zipfileWrite32(aWrite, pCds->iOffset); - assert( aWrite==&pNew->aCdsEntry[ZIPFILE_CDS_FIXED_SZ] ); - memcpy(aWrite, zPath, nPath); - if( pCds->nExtra ){ - aWrite += nPath; - zipfileWrite16(aWrite, ZIPFILE_EXTRA_TIMESTAMP); - zipfileWrite16(aWrite, 5); - *aWrite++ = 0x01; - zipfileWrite32(aWrite, mTime); + pNew->cds.zFile = sqlite3_mprintf("%s", zPath); + if( pNew->cds.zFile==0 ){ + sqlite3_free(pNew); + pNew = 0; } } - return pNew; } @@ -1186,6 +1156,8 @@ static int zipfileAppendEntry( u8 *aBuf = pTab->aBuffer; int rc; + pCds->nExtra = 9; + zipfileWrite32(aBuf, ZIPFILE_SIGNATURE_LFH); zipfileWrite16(aBuf, pCds->iVersionExtract); zipfileWrite16(aBuf, pCds->flags); @@ -1291,7 +1263,6 @@ static int zipfileUpdate( int iMethod = 0; /* Compression method for new entry */ u8 *pFree = 0; /* Free this */ char *zFree = 0; /* Also free this */ - ZipfileCDS cds; /* New Central Directory Structure entry */ ZipfileEntry *pOld = 0; int bIsDir = 0; u32 iCrc32 = 0; @@ -1299,136 +1270,145 @@ static int zipfileUpdate( assert( pTab->zFile ); assert( pTab->pWriteFd ); + /* If this is a DELETE or UPDATE, find the archive entry to delete. */ if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){ const char *zDelete = (const char*)sqlite3_value_text(apVal[0]); int nDelete = (int)strlen(zDelete); for(pOld=pTab->pFirstEntry; 1; pOld=pOld->pNext){ - if( pOld->bDeleted ) continue; - if( zipfileComparePath(pOld->zPath, zDelete, nDelete)==0 ){ - pOld->bDeleted = 1; + if( zipfileComparePath(pOld->cds.zFile, zDelete, nDelete)==0 ){ break; } assert( pOld->pNext ); } - if( nVal==1 ) return SQLITE_OK; } - /* Check that "sz" and "rawdata" are both NULL: */ - if( sqlite3_value_type(apVal[5])!=SQLITE_NULL - || sqlite3_value_type(apVal[6])!=SQLITE_NULL - ){ - rc = SQLITE_CONSTRAINT; - } - - if( rc==SQLITE_OK ){ - if( sqlite3_value_type(apVal[7])==SQLITE_NULL ){ - /* data=NULL. A directory */ - bIsDir = 1; - }else{ - /* Value specified for "data", and possibly "method". This must be - ** a regular file or a symlink. */ - const u8 *aIn = sqlite3_value_blob(apVal[7]); - int nIn = sqlite3_value_bytes(apVal[7]); - int bAuto = sqlite3_value_type(apVal[8])==SQLITE_NULL; - - iMethod = sqlite3_value_int(apVal[8]); - sz = nIn; - pData = aIn; - nData = nIn; - if( iMethod!=0 && iMethod!=8 ){ - rc = SQLITE_CONSTRAINT; - }else{ - if( bAuto || iMethod ){ - int nCmp; - rc = zipfileDeflate(pTab, aIn, nIn, &pFree, &nCmp); - if( rc==SQLITE_OK ){ - if( iMethod || nCmp1 ){ + /* Check that "sz" and "rawdata" are both NULL: */ + if( sqlite3_value_type(apVal[5])!=SQLITE_NULL + || sqlite3_value_type(apVal[6])!=SQLITE_NULL + ){ rc = SQLITE_CONSTRAINT; } - } - if( rc==SQLITE_OK ){ - zPath = (const char*)sqlite3_value_text(apVal[2]); - nPath = (int)strlen(zPath); - if( sqlite3_value_type(apVal[4])==SQLITE_NULL ){ - mTime = (sqlite3_int64)time(0); - }else{ - mTime = sqlite3_value_int64(apVal[4]); + if( rc==SQLITE_OK ){ + if( sqlite3_value_type(apVal[7])==SQLITE_NULL ){ + /* data=NULL. A directory */ + bIsDir = 1; + }else{ + /* Value specified for "data", and possibly "method". This must be + ** a regular file or a symlink. */ + const u8 *aIn = sqlite3_value_blob(apVal[7]); + int nIn = sqlite3_value_bytes(apVal[7]); + int bAuto = sqlite3_value_type(apVal[8])==SQLITE_NULL; + + iMethod = sqlite3_value_int(apVal[8]); + sz = nIn; + pData = aIn; + nData = nIn; + if( iMethod!=0 && iMethod!=8 ){ + rc = SQLITE_CONSTRAINT; + }else{ + if( bAuto || iMethod ){ + int nCmp; + rc = zipfileDeflate(pTab, aIn, nIn, &pFree, &nCmp); + if( rc==SQLITE_OK ){ + if( iMethod || nCmppFirstEntry; p; p=p->pNext){ - if( p->bDeleted ) continue; - if( zipfileComparePath(p->zPath, zPath, nPath)==0 ){ + if( rc==SQLITE_OK ){ + rc = zipfileGetMode(pTab, apVal[3], + (bIsDir ? (S_IFDIR + 0755) : (S_IFREG + 0644)), &mode + ); + if( rc==SQLITE_OK && (bIsDir == ((mode & S_IFDIR)==0)) ){ + /* The "mode" attribute is a directory, but data has been specified. + ** Or vice-versa - no data but "mode" is a file or symlink. */ rc = SQLITE_CONSTRAINT; - break; + } + } + + if( rc==SQLITE_OK ){ + zPath = (const char*)sqlite3_value_text(apVal[2]); + nPath = (int)strlen(zPath); + if( sqlite3_value_type(apVal[4])==SQLITE_NULL ){ + mTime = (sqlite3_int64)time(0); + }else{ + mTime = sqlite3_value_int64(apVal[4]); + } + } + + if( rc==SQLITE_OK && bIsDir ){ + /* For a directory, check that the last character in the path is a + ** '/'. This appears to be required for compatibility with info-zip + ** (the unzip command on unix). It does not create directories + ** otherwise. */ + if( zPath[nPath-1]!='/' ){ + zFree = sqlite3_mprintf("%s/", zPath); + if( zFree==0 ){ rc = SQLITE_NOMEM; } + zPath = (const char*)zFree; + nPath++; + } + } + + /* Check that we're not inserting a duplicate entry */ + if( pOld==0 && rc==SQLITE_OK ){ + ZipfileEntry *p; + for(p=pTab->pFirstEntry; p; p=p->pNext){ + if( zipfileComparePath(p->cds.zFile, zPath, nPath)==0 ){ + rc = SQLITE_CONSTRAINT; + break; + } + } + } + + if( rc==SQLITE_OK ){ + /* Create the new CDS record. */ + pNew = zipfileNewEntry(zPath); + if( pNew==0 ){ + rc = SQLITE_NOMEM; + }else{ + pNew->cds.iVersionMadeBy = ZIPFILE_NEWENTRY_MADEBY; + pNew->cds.iVersionExtract = ZIPFILE_NEWENTRY_REQUIRED; + pNew->cds.flags = ZIPFILE_NEWENTRY_FLAGS; + pNew->cds.iCompression = (u16)iMethod; + zipfileMtimeToDos(&pNew->cds, (u32)mTime); + pNew->cds.crc32 = iCrc32; + pNew->cds.szCompressed = nData; + pNew->cds.szUncompressed = (u32)sz; + pNew->cds.iExternalAttr = (mode<<16); + pNew->cds.iOffset = (u32)pTab->szCurrent; + pNew->cds.nFile = nPath; + pNew->mUnixTime = (u32)mTime; + rc = zipfileAppendEntry( + pTab, &pNew->cds, zPath, nPath, pData, nData, pNew->mUnixTime + ); + zipfileAddEntry(pTab, pOld, pNew); } } } - if( rc==SQLITE_OK ){ - /* Create the new CDS record. */ - memset(&cds, 0, sizeof(cds)); - cds.iVersionMadeBy = ZIPFILE_NEWENTRY_MADEBY; - cds.iVersionExtract = ZIPFILE_NEWENTRY_REQUIRED; - cds.flags = ZIPFILE_NEWENTRY_FLAGS; - cds.iCompression = (u16)iMethod; - zipfileMtimeToDos(&cds, (u32)mTime); - cds.crc32 = iCrc32; - cds.szCompressed = nData; - cds.szUncompressed = (u32)sz; - cds.iExternalAttr = (mode<<16); - cds.iOffset = (u32)pTab->szCurrent; - pNew = zipfileNewEntry(&cds, zPath, nPath, (u32)mTime); - if( pNew==0 ){ - rc = SQLITE_NOMEM; - }else{ - zipfileAddEntry(pTab, pOld, pNew); + if( rc==SQLITE_OK && pOld ){ + ZipfileEntry **pp; + ZipfileCsr *pCsr; + for(pCsr=pTab->pCsrList; pCsr; pCsr=pCsr->pCsrNext){ + if( pCsr->pCurrent==pOld ){ + pCsr->pCurrent = pOld->pNext; + pCsr->bNoop = 1; + } } + for(pp=&pTab->pFirstEntry; (*pp)!=pOld; pp=&((*pp)->pNext)); + *pp = (*pp)->pNext; + zipfileEntryFree(pOld); } - /* Append the new header+file to the archive */ - if( rc==SQLITE_OK ){ - rc = zipfileAppendEntry(pTab, &cds, zPath, nPath, pData, nData, (u32)mTime); - } - - if( rc!=SQLITE_OK && pOld ){ - pOld->bDeleted = 0; - } sqlite3_free(pFree); sqlite3_free(zFree); return rc; @@ -1450,22 +1430,6 @@ static int zipfileAppendEOCD(ZipfileTab *pTab, ZipfileEOCD *p){ return zipfileAppendData(pTab, pTab->aBuffer, (int)(aBuf - pTab->aBuffer)); } -static void zipfileCleanupTransaction(ZipfileTab *pTab){ - ZipfileEntry *pEntry; - ZipfileEntry *pNext; - - for(pEntry=pTab->pFirstEntry; pEntry; pEntry=pNext){ - pNext = pEntry->pNext; - sqlite3_free(pEntry); - } - pTab->pFirstEntry = 0; - pTab->pLastEntry = 0; - fclose(pTab->pWriteFd); - pTab->pWriteFd = 0; - pTab->szCurrent = 0; - pTab->szOrig = 0; -} - static int zipfileBegin(sqlite3_vtab *pVtab){ ZipfileTab *pTab = (ZipfileTab*)pVtab; int rc = SQLITE_OK; @@ -1504,6 +1468,55 @@ static int zipfileBegin(sqlite3_vtab *pVtab){ return rc; } +/* +** Serialize the CDS structure into buffer aBuf[]. Return the number +** of bytes written. +*/ +static int zipfileSerializeCDS(ZipfileEntry *pEntry, u8 *aBuf){ + u8 *a = aBuf; + ZipfileCDS *pCDS = &pEntry->cds; + + if( pEntry->aExtra==0 ){ + pCDS->nExtra = 9; + } + + zipfileWrite32(a, ZIPFILE_SIGNATURE_CDS); + zipfileWrite16(a, pCDS->iVersionMadeBy); + zipfileWrite16(a, pCDS->iVersionExtract); + zipfileWrite16(a, pCDS->flags); + zipfileWrite16(a, pCDS->iCompression); + zipfileWrite16(a, pCDS->mTime); + zipfileWrite16(a, pCDS->mDate); + zipfileWrite32(a, pCDS->crc32); + zipfileWrite32(a, pCDS->szCompressed); + zipfileWrite32(a, pCDS->szUncompressed); + assert( a==&aBuf[ZIPFILE_CDS_NFILE_OFF] ); + zipfileWrite16(a, pCDS->nFile); + zipfileWrite16(a, pCDS->nExtra); + zipfileWrite16(a, pCDS->nComment); + zipfileWrite16(a, pCDS->iDiskStart); + zipfileWrite16(a, pCDS->iInternalAttr); + zipfileWrite32(a, pCDS->iExternalAttr); + zipfileWrite32(a, pCDS->iOffset); + + memcpy(a, pCDS->zFile, pCDS->nFile); + a += pCDS->nFile; + + if( pEntry->aExtra ){ + int n = (int)pCDS->nExtra + (int)pCDS->nComment; + memcpy(a, pEntry->aExtra, n); + a += n; + }else{ + assert( pCDS->nExtra==9 ); + zipfileWrite16(a, ZIPFILE_EXTRA_TIMESTAMP); + zipfileWrite16(a, 5); + *a++ = 0x01; + zipfileWrite32(a, pEntry->mUnixTime); + } + + return a-aBuf; +} + static int zipfileCommit(sqlite3_vtab *pVtab){ ZipfileTab *pTab = (ZipfileTab*)pVtab; int rc = SQLITE_OK; @@ -1513,10 +1526,10 @@ static int zipfileCommit(sqlite3_vtab *pVtab){ ZipfileEOCD eocd; int nEntry = 0; - /* Write out all undeleted entries */ + /* Write out all entries */ for(p=pTab->pFirstEntry; rc==SQLITE_OK && p; p=p->pNext){ - if( p->bDeleted ) continue; - rc = zipfileAppendData(pTab, p->aCdsEntry, p->nCdsEntry); + int n = zipfileSerializeCDS(p, pTab->aBuffer); + rc = zipfileAppendData(pTab, pTab->aBuffer, n); nEntry++; } @@ -1557,7 +1570,7 @@ static void zipfileFunctionCds( pCsr = zipfileFindCursor(pTab, sqlite3_value_int64(argv[0])); if( pCsr ){ - ZipfileCDS *p = &pCsr->cds; + ZipfileCDS *p = &pCsr->pCurrent->cds; char *zRes = sqlite3_mprintf("{" "\"version-made-by\" : %u, " "\"version-to-extract\" : %u, " diff --git a/manifest b/manifest index 7d40e3172b..9b84aa5a52 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Slightly\ssmaller\sand\sfaster\simplementation\sof\sOP_If\sand\sOP_IfNot. -D 2018-01-25T01:20:29.953 +C Reorganize\szipfile.c\scode\sto\smake\sit\seasier\sto\sadd\ssupport\sfor\sin-memory\szip\narchive\sprocessing. +D 2018-01-25T20:50:46.467 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -304,7 +304,7 @@ F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178 F ext/misc/vtablog.c 31d0d8f4406795679dcd3a67917c213d3a2a5fb3ea5de35f6e773491ed7e13c9 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 -F ext/misc/zipfile.c d99efb67ecdfcae7e1855984c218c8c33d0d46a833eaa4b5a5c3d7a4f6690ce4 +F ext/misc/zipfile.c 45cc42164444fa49c9a04ba8361e38e4727862a3f2f64a35e4f6f661f8208c5a F ext/rbu/rbu.c ea7d1b7eb44c123a2a619332e19fe5313500705c4a58aaa1887905c0d83ffc2e F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842 F ext/rbu/rbu10.test 1846519a438697f45e9dcb246908af81b551c29e1078d0304fae83f1fed7e9ee @@ -1702,7 +1702,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8055e4f42446ceb5bcf752bbf41a73289c3ca759c56c9f779edc3d7f202b7881 -R 3a57ffa9e8084df96fa05ea7edc4d068 -U drh -Z 1f11fb78272f5afcdf3b192bdf3cca0d +P 6ab42934e2c4957b5d8927bf4434a9db07ab6078987a6a2d25f35cc468d21203 +R 7248e9fa39da1851c7faf3b8da443f98 +T +closed 469694857a5301e1da8cf067017d400403f3b12882d4265cefb3fd185c536305 +U dan +Z 84871d5400876eca8ff4f985699fe3ef diff --git a/manifest.uuid b/manifest.uuid index 8b78bd0022..ae4e344061 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6ab42934e2c4957b5d8927bf4434a9db07ab6078987a6a2d25f35cc468d21203 \ No newline at end of file +30b9258294e3028ef4ea467e332e013995509544e9a23c8bbf5168772a7e895d \ No newline at end of file From 4eb57ceffc2735a078fd66e58650f10f44e56a63 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 26 Jan 2018 18:37:34 +0000 Subject: [PATCH 010/107] Improve text-to-integer conversion in boundary cases. The sqlite3Atoi64() function always returns the minimum or maximum integer if the magnitude of the text value is too large. Trailing whitespace is now ignored. FossilOrigin-Name: ace0644a1a2a42a3ea42d44f00a31915b8a7e56c9ba90f90a6c02001f89f9c86 --- manifest | 17 ++++++++--------- manifest.uuid | 2 +- src/util.c | 52 +++++++++++++++++++++++++++----------------------- test/cast.test | 45 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 82 insertions(+), 34 deletions(-) diff --git a/manifest b/manifest index 9b84aa5a52..c964e14825 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Reorganize\szipfile.c\scode\sto\smake\sit\seasier\sto\sadd\ssupport\sfor\sin-memory\szip\narchive\sprocessing. -D 2018-01-25T20:50:46.467 +C Improve\stext-to-integer\sconversion\sin\sboundary\scases.\s\sThe\nsqlite3Atoi64()\sfunction\salways\sreturns\sthe\sminimum\sor\smaximum\sinteger\nif\sthe\smagnitude\sof\sthe\stext\svalue\sis\stoo\slarge.\s\sTrailing\swhitespace\sis\nnow\signored. +D 2018-01-26T18:37:34.351 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -554,7 +554,7 @@ F src/treeview.c eae35972ff44f67064de2eaf35f04afe94e7aea3271a8b3bcebb3f954880fec F src/trigger.c a34539c69433276d37b0da9a89c117726ff2d292c0902895af1f393a983cd3a1 F src/update.c a90a32ffc0100265b0693dbbdbe490756447af181f5ea2c138cce515b08c8795 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5 -F src/util.c ef4a5f904d942e660abade7fbf3e6bdb402dabe9e7c27f3361ecf40b945538b5 +F src/util.c 8628fe1daa9f5db9fa3a70d6fcfd828b73e839feafe7d7faecd7073be56eaec2 F src/vacuum.c 90839322fd5f00df9617eb21b68beda9b6e2a2937576b0d65985e4aeb1c53739 F src/vdbe.c 0e13b8c33a9ecf236b3045f190fb58933049582307b621cbd924b6582a2b705f F src/vdbe.h 134beb7a12a6213c00eba58febaede33447cc4441bc568a0d9c144b33fc3720a @@ -673,7 +673,7 @@ F test/capi3b.test efb2b9cfd127efa84433cd7a2d72ce0454ae0dc4 F test/capi3c.test 7ebed1d8fa2f3190149d556fe8cff5a006be62af437c5c4640db614470126098 F test/capi3d.test 485048dc5cd07bc68011e4917ad035ad6047ab82 F test/capi3e.test 3d49c01ef2a1a55f41d73cba2b23b5059ec460fe -F test/cast.test 4c275cbdc8202d6f9c54a3596701719868ac7dc3 +F test/cast.test 5ceb920718d280b61163500a7d29e0e0a86458b1cbd92d96f962c9d970aa3857 F test/cffault.test 9d6b20606afe712374952eec4f8fd74b1a8097ef F test/check.test 33a698e8c63613449d85d624a38ef669bf20331daabebe3891c9405dd6df463a F test/close.test 799ea4599d2f5704b0a30f477d17c2c760d8523fa5d0c8be4a7df2a8cad787d8 @@ -1702,8 +1702,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6ab42934e2c4957b5d8927bf4434a9db07ab6078987a6a2d25f35cc468d21203 -R 7248e9fa39da1851c7faf3b8da443f98 -T +closed 469694857a5301e1da8cf067017d400403f3b12882d4265cefb3fd185c536305 -U dan -Z 84871d5400876eca8ff4f985699fe3ef +P 30b9258294e3028ef4ea467e332e013995509544e9a23c8bbf5168772a7e895d +R a3a69d646870f53631d68482ffc811f2 +U drh +Z ebcc0484dfae31c55e47d329daf31900 diff --git a/manifest.uuid b/manifest.uuid index ae4e344061..5fee1327fd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -30b9258294e3028ef4ea467e332e013995509544e9a23c8bbf5168772a7e895d \ No newline at end of file +ace0644a1a2a42a3ea42d44f00a31915b8a7e56c9ba90f90a6c02001f89f9c86 \ No newline at end of file diff --git a/src/util.c b/src/util.c index 456317cfa9..9954d709df 100644 --- a/src/util.c +++ b/src/util.c @@ -595,7 +595,7 @@ static int compare2pow63(const char *zNum, int incr){ ** Returns: ** ** 0 Successful transformation. Fits in a 64-bit signed integer. -** 1 Excess text after the integer value +** 1 Excess non-space text after the integer value ** 2 Integer too large for a 64-bit signed integer or is malformed ** 3 Special case of 9223372036854775808 ** @@ -638,47 +638,51 @@ int sqlite3Atoi64(const char *zNum, i64 *pNum, int length, u8 enc){ for(i=0; &zNum[i]='0' && c<='9'; i+=incr){ u = u*10 + c - '0'; } - if( u>LARGEST_INT64 ){ - *pNum = neg ? SMALLEST_INT64 : LARGEST_INT64; - }else if( neg ){ + testcase( i==18*incr ); + testcase( i==19*incr ); + testcase( i==20*incr ); + if( neg ){ *pNum = -(i64)u; }else{ *pNum = (i64)u; } - testcase( i==18 ); - testcase( i==19 ); - testcase( i==20 ); - if( &zNum[i]19*incr ){ /* Too many digits */ - /* zNum is empty or contains non-numeric text or is longer - ** than 19 digits (thus guaranteeing that it is too large) */ - return 2; - }else if( i<19*incr ){ + if( i<19*incr ){ /* Less than 19 digits, so we know that it fits in 64 bits */ assert( u<=LARGEST_INT64 ); return rc; }else{ /* zNum is a 19-digit numbers. Compare it against 9223372036854775808. */ - c = compare2pow63(zNum, incr); + c = i>19*incr ? 1 : compare2pow63(zNum, incr); if( c<0 ){ /* zNum is less than 9223372036854775808 so it fits */ assert( u<=LARGEST_INT64 ); return rc; - }else if( c>0 ){ - /* zNum is greater than 9223372036854775808 so it overflows */ - return 2; }else{ - /* zNum is exactly 9223372036854775808. Fits if negative. The - ** special case 2 overflow if positive */ - assert( u-1==LARGEST_INT64 ); - return neg ? rc : 3; + *pNum = neg ? SMALLEST_INT64 : LARGEST_INT64; + if( c>0 ){ + /* zNum is greater than 9223372036854775808 so it overflows */ + return 2; + }else{ + /* zNum is exactly 9223372036854775808. Fits if negative. The + ** special case 2 overflow if positive */ + assert( u-1==LARGEST_INT64 ); + return neg ? rc : 3; + } } } } diff --git a/test/cast.test b/test/cast.test index f47f4bb2bf..f43aa48560 100644 --- a/test/cast.test +++ b/test/cast.test @@ -343,4 +343,49 @@ do_test cast-4.4 { } } {0 abc 0.0 abc} +# Added 2018-01-26 +# +# EVIDENCE-OF: R-48741-32454 If the prefix integer is greater than +# +9223372036854775807 then the result of the cast is exactly +# +9223372036854775807. +do_execsql_test cast-5.1 { + SELECT CAST('9223372036854775808' AS integer); + SELECT CAST(' +000009223372036854775808' AS integer); + SELECT CAST('12345678901234567890123' AS INTEGER); +} {9223372036854775807 9223372036854775807 9223372036854775807} + +# EVIDENCE-OF: R-06028-16857 Similarly, if the prefix integer is less +# than -9223372036854775808 then the result of the cast is exactly +# -9223372036854775808. +do_execsql_test cast-5.2 { + SELECT CAST('-9223372036854775808' AS integer); + SELECT CAST('-9223372036854775809' AS integer); + SELECT CAST('-12345678901234567890123' AS INTEGER); +} {-9223372036854775808 -9223372036854775808 -9223372036854775808} + +# EVIDENCE-OF: R-33990-33527 When casting to INTEGER, if the text looks +# like a floating point value with an exponent, the exponent will be +# ignored because it is no part of the integer prefix. +# EVIDENCE-OF: R-24225-46995 For example, "(CAST '123e+5' AS INTEGER)" +# results in 123, not in 12300000. +do_execsql_test case-5.3 { + SELECT CAST('123e+5' AS INTEGER); + SELECT CAST('123e+5' AS NUMERIC); +} {123 12300000.0} + + +# The following does not have anything to do with the CAST operator, +# but it does deal with affinity transformations. +# +do_execsql_test case-6.1 { + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(a NUMERIC); + INSERT INTO t1 VALUES + ('9000000000000000001'), + ('9000000000000000001 '), + (' 9000000000000000001'), + (' 9000000000000000001 '); + SELECT * FROM t1; +} {9000000000000000001 9000000000000000001 9000000000000000001 9000000000000000001} + finish_test From 8005d60587d572b226dab4d0b7c3ecf441ab595a Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 26 Jan 2018 18:59:25 +0000 Subject: [PATCH 011/107] If the argument to table function zipfile() is a blob (not text), assume that it contains a zip file image to interpret, not the name of a file on disk. FossilOrigin-Name: 029ebcd30cb261d949f7587ac54c23d7479796b6716fd4ca7512361b8f32de3e --- ext/misc/zipfile.c | 380 +++++++++++++++++++++++++-------------------- manifest | 16 +- manifest.uuid | 2 +- test/zipfile.test | 20 +++ 4 files changed, 239 insertions(+), 179 deletions(-) diff --git a/ext/misc/zipfile.c b/ext/misc/zipfile.c index d378606941..c199f8ec1b 100644 --- a/ext/misc/zipfile.c +++ b/ext/misc/zipfile.c @@ -225,6 +225,8 @@ struct ZipfileEntry { ZipfileCDS cds; /* Parsed CDS record */ u32 mUnixTime; /* Modification time, in UNIX format */ u8 *aExtra; /* cds.nExtra+cds.nComment bytes of extra data */ + i64 iDataOff; + u8 *aData; /* cds.szCompressed bytes of compressed data */ ZipfileEntry *pNext; /* Next element in in-memory CDS */ }; @@ -243,9 +245,9 @@ struct ZipfileCsr { i64 iNextOff; /* Offset of next record in central directory */ ZipfileEOCD eocd; /* Parse of central directory record */ + ZipfileEntry *pFreeEntry; + ZipfileEntry *pCurrent; /* Current entry */ - ZipfileLFH lfh; /* Local File Header for current entry */ - i64 iDataOff; /* Offset in zipfile to data */ ZipfileCsr *pCsrNext; /* Next cursor on same virtual table */ }; @@ -393,6 +395,9 @@ static int zipfileOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCsr){ ** by zipfileOpen(). */ static void zipfileResetCursor(ZipfileCsr *pCsr){ + ZipfileEntry *p; + ZipfileEntry *pNext; + pCsr->bEof = 0; if( pCsr->pFile ){ fclose(pCsr->pFile); @@ -400,6 +405,11 @@ static void zipfileResetCursor(ZipfileCsr *pCsr){ zipfileEntryFree(pCsr->pCurrent); pCsr->pCurrent = 0; } + + for(p=pCsr->pFreeEntry; p; p=pNext){ + pNext = p->pNext; + zipfileEntryFree(p); + } } /* @@ -503,6 +513,7 @@ static u8* zipfileCsrBuffer(ZipfileCsr *pCsr){ */ #define ZIPFILE_CDS_FIXED_SZ 46 #define ZIPFILE_CDS_NFILE_OFF 28 +#define ZIPFILE_CDS_SZCOMPRESSED_OFF 20 /* ** Decode the CDS record in buffer aBuf into (*pCDS). Return SQLITE_ERROR @@ -596,62 +607,125 @@ static time_t zipfileMtime(ZipfileCDS *pCDS){ return mktime(&t); } +static int zipfileReadLFH( + u8 *aBuffer, + ZipfileLFH *pLFH +){ + u8 *aRead = aBuffer; + int rc = SQLITE_OK; + + u32 sig = zipfileRead32(aRead); + if( sig!=ZIPFILE_SIGNATURE_LFH ){ + rc = SQLITE_ERROR; + }else{ + pLFH->iVersionExtract = zipfileRead16(aRead); + pLFH->flags = zipfileRead16(aRead); + pLFH->iCompression = zipfileRead16(aRead); + pLFH->mTime = zipfileRead16(aRead); + pLFH->mDate = zipfileRead16(aRead); + pLFH->crc32 = zipfileRead32(aRead); + pLFH->szCompressed = zipfileRead32(aRead); + pLFH->szUncompressed = zipfileRead32(aRead); + pLFH->nFile = zipfileRead16(aRead); + pLFH->nExtra = zipfileRead16(aRead); + } + return rc; +} + /* ** Read a Zip archive CDS header from offset iOff of file pFile. Return ** SQLITE_OK if successful, or an SQLite error code otherwise. */ static int zipfileGetEntry( ZipfileTab *pTab, /* Store any error message here */ - FILE *pFile, + const u8 *aBlob, /* Pointer to in-memory file image */ + int nBlob, /* Size of aBlob[] in bytes */ + FILE *pFile, /* If aBlob==0, read from this file */ i64 iOff, ZipfileEntry **ppEntry ){ - u8 *aRead = pTab->aBuffer; + u8 *aRead; char **pzErr = &pTab->base.zErrMsg; - int rc; + int rc = SQLITE_OK; + + if( aBlob==0 ){ + aRead = pTab->aBuffer; + rc = zipfileReadData(pFile, aRead, ZIPFILE_CDS_FIXED_SZ, iOff, pzErr); + }else{ + aRead = (u8*)&aBlob[iOff]; + } - rc = zipfileReadData(pFile, aRead, ZIPFILE_CDS_FIXED_SZ, iOff, pzErr); if( rc==SQLITE_OK ){ - if( rc!=SQLITE_OK ){ - *pzErr = sqlite3_mprintf("failed to read CDS at offset %lld", iOff); + int nAlloc; + ZipfileEntry *pNew; + + int nFile = zipfileGetU16(&aRead[ZIPFILE_CDS_NFILE_OFF]); + int nExtra = zipfileGetU16(&aRead[ZIPFILE_CDS_NFILE_OFF+2]); + nExtra += zipfileGetU16(&aRead[ZIPFILE_CDS_NFILE_OFF+4]); + + nAlloc = sizeof(ZipfileEntry) + nExtra; + if( aBlob ){ + nAlloc += zipfileGetU32(&aRead[ZIPFILE_CDS_SZCOMPRESSED_OFF]); + } + + pNew = (ZipfileEntry*)sqlite3_malloc(nAlloc); + if( pNew==0 ){ + rc = SQLITE_NOMEM; }else{ - ZipfileEntry *pNew; - - int nFile = zipfileGetU16(&aRead[ZIPFILE_CDS_NFILE_OFF]); - int nExtra = zipfileGetU16(&aRead[ZIPFILE_CDS_NFILE_OFF+2]); - nExtra += zipfileGetU16(&aRead[ZIPFILE_CDS_NFILE_OFF+4]); - - pNew = (ZipfileEntry*)sqlite3_malloc(sizeof(ZipfileEntry) + nExtra); - if( pNew==0 ){ - rc = SQLITE_NOMEM; - }else{ - memset(pNew, 0, sizeof(ZipfileEntry)); - rc = zipfileReadCDS(aRead, &pNew->cds); - if( rc!=SQLITE_OK ){ - *pzErr = sqlite3_mprintf("failed to read CDS at offset %lld", iOff); - }else{ - rc = zipfileReadData( - pFile, aRead, nExtra+nFile, iOff+ZIPFILE_CDS_FIXED_SZ, pzErr - ); - } - } - - if( rc==SQLITE_OK ){ - u32 *pt = &pNew->mUnixTime; - pNew->cds.zFile = sqlite3_mprintf("%.*s", nFile, aRead); - if( pNew->cds.zFile==0 ){ - rc = SQLITE_NOMEM; - }else if( 0==zipfileScanExtra(&aRead[nFile], pNew->cds.nExtra, pt) ){ - pNew->mUnixTime = zipfileMtime(&pNew->cds); - } - } - + memset(pNew, 0, sizeof(ZipfileEntry)); + rc = zipfileReadCDS(aRead, &pNew->cds); if( rc!=SQLITE_OK ){ - zipfileEntryFree(pNew); + *pzErr = sqlite3_mprintf("failed to read CDS at offset %lld", iOff); + }else if( aBlob==0 ){ + rc = zipfileReadData( + pFile, aRead, nExtra+nFile, iOff+ZIPFILE_CDS_FIXED_SZ, pzErr + ); }else{ - *ppEntry = pNew; + aRead = (u8*)&aBlob[iOff + ZIPFILE_CDS_FIXED_SZ]; } } + + if( rc==SQLITE_OK ){ + u32 *pt = &pNew->mUnixTime; + pNew->cds.zFile = sqlite3_mprintf("%.*s", nFile, aRead); + pNew->aExtra = (u8*)&pNew[1]; + memcpy(pNew->aExtra, &aRead[nFile], nExtra); + if( pNew->cds.zFile==0 ){ + rc = SQLITE_NOMEM; + }else if( 0==zipfileScanExtra(&aRead[nFile], pNew->cds.nExtra, pt) ){ + pNew->mUnixTime = zipfileMtime(&pNew->cds); + } + } + + if( rc==SQLITE_OK ){ + static const int szFix = ZIPFILE_LFH_FIXED_SZ; + ZipfileLFH lfh; + if( pFile ){ + rc = zipfileReadData(pFile, aRead, szFix, pNew->cds.iOffset, pzErr); + }else{ + aRead = &aBlob[pNew->cds.iOffset]; + } + + rc = zipfileReadLFH(aRead, &lfh); + if( rc==SQLITE_OK ){ + pNew->iDataOff = pNew->cds.iOffset + ZIPFILE_LFH_FIXED_SZ; + pNew->iDataOff += lfh.nFile + lfh.nExtra; + if( aBlob && pNew->cds.szCompressed ){ + pNew->aData = &pNew->aExtra[nExtra]; + memcpy(pNew->aData, &aBlob[pNew->iDataOff], pNew->cds.szCompressed); + } + }else{ + *pzErr = sqlite3_mprintf("failed to read LFH at offset %d", + (int)pNew->cds.iOffset + ); + } + } + + if( rc!=SQLITE_OK ){ + zipfileEntryFree(pNew); + }else{ + *ppEntry = pNew; + } } return rc; @@ -662,51 +736,6 @@ static FILE *zipfileGetFd(ZipfileCsr *pCsr){ return ((ZipfileTab*)(pCsr->base.pVtab))->pWriteFd; } -static int zipfileReadLFH( - FILE *pFd, - i64 iOffset, - u8 *aTmp, - ZipfileLFH *pLFH, - char **pzErr -){ - u8 *aRead = aTmp; - static const int szFix = ZIPFILE_LFH_FIXED_SZ; - int rc; - - rc = zipfileReadData(pFd, aRead, szFix, iOffset, pzErr); - if( rc==SQLITE_OK ){ - u32 sig = zipfileRead32(aRead); - if( sig!=ZIPFILE_SIGNATURE_LFH ){ - *pzErr = sqlite3_mprintf("failed to read LFH at offset %d", (int)iOffset); - rc = SQLITE_ERROR; - }else{ - pLFH->iVersionExtract = zipfileRead16(aRead); - pLFH->flags = zipfileRead16(aRead); - pLFH->iCompression = zipfileRead16(aRead); - pLFH->mTime = zipfileRead16(aRead); - pLFH->mDate = zipfileRead16(aRead); - pLFH->crc32 = zipfileRead32(aRead); - pLFH->szCompressed = zipfileRead32(aRead); - pLFH->szUncompressed = zipfileRead32(aRead); - pLFH->nFile = zipfileRead16(aRead); - pLFH->nExtra = zipfileRead16(aRead); - assert( aRead==&aTmp[szFix] ); - } - } - return rc; -} - -static int zipfileCsrReadLFH(ZipfileCsr *pCsr){ - FILE *pFile = zipfileGetFd(pCsr); - char **pzErr = &pCsr->base.pVtab->zErrMsg; - u8 *aRead = zipfileCsrBuffer(pCsr); - ZipfileCDS *pCDS = &pCsr->pCurrent->cds; - int rc = zipfileReadLFH(pFile, pCDS->iOffset, aRead, &pCsr->lfh, pzErr); - pCsr->iDataOff = pCDS->iOffset + ZIPFILE_LFH_FIXED_SZ; - pCsr->iDataOff += pCsr->lfh.nFile+pCsr->lfh.nExtra; - return rc; -} - /* ** Advance an ZipfileCsr to its next row of output. @@ -724,7 +753,7 @@ static int zipfileNext(sqlite3_vtab_cursor *cur){ }else{ ZipfileEntry *p = 0; ZipfileTab *pTab = (ZipfileTab*)(cur->pVtab); - rc = zipfileGetEntry(pTab, pCsr->pFile, pCsr->iNextOff, &p); + rc = zipfileGetEntry(pTab, 0, 0, pCsr->pFile, pCsr->iNextOff, &p); if( rc==SQLITE_OK ){ pCsr->iNextOff += ZIPFILE_CDS_FIXED_SZ; pCsr->iNextOff += (int)p->cds.nExtra + p->cds.nFile + p->cds.nComment; @@ -741,10 +770,6 @@ static int zipfileNext(sqlite3_vtab_cursor *cur){ } pCsr->bNoop = 0; - if( rc==SQLITE_OK && pCsr->bEof==0 ){ - rc = zipfileCsrReadLFH(pCsr); - } - return rc; } @@ -881,14 +906,22 @@ static int zipfileColumn( int sz = pCDS->szCompressed; int szFinal = pCDS->szUncompressed; if( szFinal>0 ){ - u8 *aBuf = sqlite3_malloc(sz); - if( aBuf==0 ){ - rc = SQLITE_NOMEM; + u8 *aBuf; + u8 *aFree = 0; + if( pCsr->pCurrent->aData ){ + aBuf = pCsr->pCurrent->aData; }else{ - FILE *pFile = zipfileGetFd(pCsr); - rc = zipfileReadData(pFile, aBuf, sz, pCsr->iDataOff, - &pCsr->base.pVtab->zErrMsg - ); + aBuf = aFree = sqlite3_malloc(sz); + if( aBuf==0 ){ + rc = SQLITE_NOMEM; + }else{ + FILE *pFile = zipfileGetFd(pCsr); + if( rc==SQLITE_OK ){ + rc = zipfileReadData(pFile, aBuf, sz, pCsr->pCurrent->iDataOff, + &pCsr->base.pVtab->zErrMsg + ); + } + } } if( rc==SQLITE_OK ){ if( i==5 && pCDS->iCompression ){ @@ -896,8 +929,8 @@ static int zipfileColumn( }else{ sqlite3_result_blob(ctx, aBuf, sz, SQLITE_TRANSIENT); } - sqlite3_free(aBuf); } + sqlite3_free(aFree); }else{ /* Figure out if this is a directory or a zero-sized file. Consider ** it to be a directory either if the mode suggests so, or if @@ -942,25 +975,32 @@ static int zipfileEof(sqlite3_vtab_cursor *cur){ */ static int zipfileReadEOCD( ZipfileTab *pTab, /* Return errors here */ - FILE *pFile, /* Read from this file */ + const u8 *aBlob, /* Pointer to in-memory file image */ + int nBlob, /* Size of aBlob[] in bytes */ + FILE *pFile, /* Read from this file if aBlob==0 */ ZipfileEOCD *pEOCD /* Object to populate */ ){ u8 *aRead = pTab->aBuffer; /* Temporary buffer */ - i64 szFile; /* Total size of file in bytes */ int nRead; /* Bytes to read from file */ - i64 iOff; /* Offset to read from */ - int rc; + int rc = SQLITE_OK; - fseek(pFile, 0, SEEK_END); - szFile = (i64)ftell(pFile); - if( szFile==0 ){ - memset(pEOCD, 0, sizeof(ZipfileEOCD)); - return SQLITE_OK; + if( aBlob==0 ){ + i64 iOff; /* Offset to read from */ + i64 szFile; /* Total size of file in bytes */ + fseek(pFile, 0, SEEK_END); + szFile = (i64)ftell(pFile); + if( szFile==0 ){ + memset(pEOCD, 0, sizeof(ZipfileEOCD)); + return SQLITE_OK; + } + nRead = (int)(MIN(szFile, ZIPFILE_BUFFER_SIZE)); + iOff = szFile - nRead; + rc = zipfileReadData(pFile, aRead, nRead, iOff, &pTab->base.zErrMsg); + }else{ + nRead = (int)(MIN(nBlob, ZIPFILE_BUFFER_SIZE)); + aRead = (u8*)&aBlob[nBlob-nRead]; } - nRead = (int)(MIN(szFile, ZIPFILE_BUFFER_SIZE)); - iOff = szFile - nRead; - rc = zipfileReadData(pFile, aRead, nRead, iOff, &pTab->base.zErrMsg); if( rc==SQLITE_OK ){ int i; @@ -986,19 +1026,59 @@ static int zipfileReadEOCD( pEOCD->nEntryTotal = zipfileRead16(aRead); pEOCD->nSize = zipfileRead32(aRead); pEOCD->iOffset = zipfileRead32(aRead); - -#if 0 - printf("iDisk=%d iFirstDisk=%d nEntry=%d " - "nEntryTotal=%d nSize=%d iOffset=%d", - (int)pEOCD->iDisk, (int)pEOCD->iFirstDisk, (int)pEOCD->nEntry, - (int)pEOCD->nEntryTotal, (int)pEOCD->nSize, (int)pEOCD->iOffset - ); -#endif } return SQLITE_OK; } +/* +** Add object pNew to the end of the linked list that begins at +** ZipfileTab.pFirstEntry and ends with pLastEntry. +*/ +static void zipfileAddEntry( + ZipfileTab *pTab, + ZipfileEntry *pBefore, + ZipfileEntry *pNew +){ + assert( (pTab->pFirstEntry==0)==(pTab->pLastEntry==0) ); + assert( pNew->pNext==0 ); + if( pBefore==0 ){ + if( pTab->pFirstEntry==0 ){ + pTab->pFirstEntry = pTab->pLastEntry = pNew; + }else{ + assert( pTab->pLastEntry->pNext==0 ); + pTab->pLastEntry->pNext = pNew; + pTab->pLastEntry = pNew; + } + }else{ + ZipfileEntry **pp; + for(pp=&pTab->pFirstEntry; *pp!=pBefore; pp=&((*pp)->pNext)); + pNew->pNext = pBefore; + *pp = pNew; + } +} + +static int zipfileLoadDirectory(ZipfileTab *pTab, const u8 *aBlob, int nBlob){ + ZipfileEOCD eocd; + int rc; + int i; + i64 iOff; + + rc = zipfileReadEOCD(pTab, aBlob, nBlob, pTab->pWriteFd, &eocd); + iOff = eocd.iOffset; + for(i=0; rc==SQLITE_OK && ipWriteFd, iOff, &pNew); + + if( rc==SQLITE_OK ){ + zipfileAddEntry(pTab, 0, pNew); + iOff += ZIPFILE_CDS_FIXED_SZ; + iOff += (int)pNew->cds.nExtra + pNew->cds.nFile + pNew->cds.nComment; + } + } + return rc; +} + /* ** xFilter callback. */ @@ -1023,6 +1103,14 @@ static int zipfileFilter( zipfileSetErrmsg(pCsr, "table function zipfile() requires an argument"); return SQLITE_ERROR; }else if( sqlite3_value_type(argv[0])==SQLITE_BLOB ){ + const u8 *aBlob = (const u8*)sqlite3_value_blob(argv[0]); + int nBlob = sqlite3_value_bytes(argv[0]); + assert( pTab->pFirstEntry==0 ); + rc = zipfileLoadDirectory(pTab, aBlob, nBlob); + pCsr->pFreeEntry = pTab->pFirstEntry; + pTab->pFirstEntry = pTab->pLastEntry = 0; + if( rc!=SQLITE_OK ) return rc; + bInMemory = 1; }else{ zFile = (const char*)sqlite3_value_text(argv[0]); } @@ -1033,7 +1121,7 @@ static int zipfileFilter( zipfileSetErrmsg(pCsr, "cannot open file: %s", zFile); rc = SQLITE_ERROR; }else{ - rc = zipfileReadEOCD(pTab, pCsr->pFile, &pCsr->eocd); + rc = zipfileReadEOCD(pTab, 0, 0, pCsr->pFile, &pCsr->eocd); if( rc==SQLITE_OK ){ if( pCsr->eocd.nEntry==0 ){ pCsr->bEof = 1; @@ -1045,7 +1133,7 @@ static int zipfileFilter( } }else{ pCsr->bNoop = 1; - pCsr->pCurrent = pTab->pFirstEntry; + pCsr->pCurrent = pCsr->pFreeEntry ? pCsr->pFreeEntry : pTab->pFirstEntry; rc = zipfileNext(cur); } @@ -1082,54 +1170,6 @@ static int zipfileBestIndex( return SQLITE_OK; } -/* -** Add object pNew to the end of the linked list that begins at -** ZipfileTab.pFirstEntry and ends with pLastEntry. -*/ -static void zipfileAddEntry( - ZipfileTab *pTab, - ZipfileEntry *pBefore, - ZipfileEntry *pNew -){ - assert( (pTab->pFirstEntry==0)==(pTab->pLastEntry==0) ); - assert( pNew->pNext==0 ); - if( pBefore==0 ){ - if( pTab->pFirstEntry==0 ){ - pTab->pFirstEntry = pTab->pLastEntry = pNew; - }else{ - assert( pTab->pLastEntry->pNext==0 ); - pTab->pLastEntry->pNext = pNew; - pTab->pLastEntry = pNew; - } - }else{ - ZipfileEntry **pp; - for(pp=&pTab->pFirstEntry; *pp!=pBefore; pp=&((*pp)->pNext)); - pNew->pNext = pBefore; - *pp = pNew; - } -} - -static int zipfileLoadDirectory(ZipfileTab *pTab){ - ZipfileEOCD eocd; - int rc; - int i; - i64 iOff; - - rc = zipfileReadEOCD(pTab, pTab->pWriteFd, &eocd); - iOff = eocd.iOffset; - for(i=0; rc==SQLITE_OK && ipWriteFd, iOff, &pNew); - - if( rc==SQLITE_OK ){ - zipfileAddEntry(pTab, 0, pNew); - iOff += ZIPFILE_CDS_FIXED_SZ; - iOff += (int)pNew->cds.nExtra + pNew->cds.nFile + pNew->cds.nComment; - } - } - return rc; -} - static ZipfileEntry *zipfileNewEntry(const char *zPath){ ZipfileEntry *pNew; pNew = sqlite3_malloc(sizeof(ZipfileEntry)); @@ -1458,7 +1498,7 @@ static int zipfileBegin(sqlite3_vtab *pVtab){ }else{ fseek(pTab->pWriteFd, 0, SEEK_END); pTab->szCurrent = pTab->szOrig = (i64)ftell(pTab->pWriteFd); - rc = zipfileLoadDirectory(pTab); + rc = zipfileLoadDirectory(pTab, 0, 0); } if( rc!=SQLITE_OK ){ diff --git a/manifest b/manifest index c964e14825..7ae6ef7c87 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improve\stext-to-integer\sconversion\sin\sboundary\scases.\s\sThe\nsqlite3Atoi64()\sfunction\salways\sreturns\sthe\sminimum\sor\smaximum\sinteger\nif\sthe\smagnitude\sof\sthe\stext\svalue\sis\stoo\slarge.\s\sTrailing\swhitespace\sis\nnow\signored. -D 2018-01-26T18:37:34.351 +C If\sthe\sargument\sto\stable\sfunction\szipfile()\sis\sa\sblob\s(not\stext),\sassume\sthat\nit\scontains\sa\szip\sfile\simage\sto\sinterpret,\snot\sthe\sname\sof\sa\sfile\son\sdisk. +D 2018-01-26T18:59:25.439 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -304,7 +304,7 @@ F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178 F ext/misc/vtablog.c 31d0d8f4406795679dcd3a67917c213d3a2a5fb3ea5de35f6e773491ed7e13c9 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 -F ext/misc/zipfile.c 45cc42164444fa49c9a04ba8361e38e4727862a3f2f64a35e4f6f661f8208c5a +F ext/misc/zipfile.c 071fb5af4d0e6403f01cfac755d78ed2008b04793ec6f87a1078755a6359a26f F ext/rbu/rbu.c ea7d1b7eb44c123a2a619332e19fe5313500705c4a58aaa1887905c0d83ffc2e F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842 F ext/rbu/rbu10.test 1846519a438697f45e9dcb246908af81b551c29e1078d0304fae83f1fed7e9ee @@ -1603,7 +1603,7 @@ F test/wordcount.c cb589cec469a1d90add05b1f8cee75c7210338d87a5afd65260ed5c0f4bbf F test/writecrash.test f1da7f7adfe8d7f09ea79b42e5ca6dcc41102f27f8e334ad71539501ddd910cc F test/zeroblob.test 3857870fe681b8185654414a9bccfde80b62a0fa F test/zerodamage.test 9c41628db7e8d9e8a0181e59ea5f189df311a9f6ce99cc376dc461f66db6f8dc -F test/zipfile.test cb42e8fa6ba5db4a03ce6baa4401fc6236baf6eb5e62b44f3e463bf6aafd631d +F test/zipfile.test 0834b33e000991a80d94167e84af346162d212526d8efee0b9d4ba84589fe292 F tool/GetFile.cs a15e08acb5dd7539b75ba23501581d7c2b462cb5 F tool/GetTclKit.bat 8995df40c4209808b31f24de0b58f90930239a234f7591e3675d45bfbb990c5d F tool/Replace.cs 02c67258801c2fb5f63231e0ac0f220b4b36ba91 @@ -1702,7 +1702,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 30b9258294e3028ef4ea467e332e013995509544e9a23c8bbf5168772a7e895d -R a3a69d646870f53631d68482ffc811f2 -U drh -Z ebcc0484dfae31c55e47d329daf31900 +P ace0644a1a2a42a3ea42d44f00a31915b8a7e56c9ba90f90a6c02001f89f9c86 +R 6f618424b3fb9707a3fb679f6f287314 +U dan +Z ab7fcdd492fa54ea891500a7623bc431 diff --git a/manifest.uuid b/manifest.uuid index 5fee1327fd..e4da9d78da 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ace0644a1a2a42a3ea42d44f00a31915b8a7e56c9ba90f90a6c02001f89f9c86 \ No newline at end of file +029ebcd30cb261d949f7587ac54c23d7479796b6716fd4ca7512361b8f32de3e \ No newline at end of file diff --git a/test/zipfile.test b/test/zipfile.test index 07fc80f602..60fde2ee15 100644 --- a/test/zipfile.test +++ b/test/zipfile.test @@ -22,6 +22,20 @@ if {[catch {load_static_extension db zipfile} error]} { finish_test; return } +proc do_zipfile_blob_test {tn file} { + set res1 [ + db eval { SELECT name,mode,mtime,method,quote(data) FROM zipfile($file) } + ] + + set fd [open $file] + fconfigure $fd -translation binary -encoding binary + set data [read $fd] + close $fd + + set res2 [db eval { SELECT name,mode,mtime,method,quote(data) FROM zipfile($data) }] + uplevel [list do_test $tn [list set {} $res2] $res1] +} + forcedelete test.zip do_execsql_test 1.0 { CREATE VIRTUAL TABLE temp.zz USING zipfile('test.zip'); @@ -60,12 +74,14 @@ do_execsql_test 1.2 { f.txt 1000000000 abcde g.txt 1000000002 12345 } +do_zipfile_blob_test 1.2.1 test.zip do_execsql_test 1.3 { INSERT INTO zz(name, mode, mtime, data) VALUES('h.txt', '-rw-r--r--', 1000000004, 'aaaaaaaaaabbbbbbbbbb' ); } +do_zipfile_blob_test 1.3.1 test.zip do_execsql_test 1.4 { SELECT name, mtime, data, method FROM zipfile('test.zip'); @@ -112,6 +128,7 @@ do_execsql_test 1.6.1 { h.txt 33188 1000000004 aaaaaaaaaabbbbbbbbbb 8 i.txt 33188 1000000006 zxcvb 0 } +do_zipfile_blob_test 1.6.1a test.zip do_execsql_test 1.6.2 { UPDATE zz SET mtime=4 WHERE name='i.txt'; @@ -130,6 +147,7 @@ do_execsql_test 1.6.3 { h.txt 33189 1000000004 aaaaaaaaaabbbbbbbbbb 8 i.txt 33188 4 zxcvb 0 } +do_zipfile_blob_test 1.6.3a test.zip do_execsql_test 1.6.4 { UPDATE zz SET name = 'blue.txt' WHERE name='f.txt'; @@ -139,6 +157,7 @@ do_execsql_test 1.6.4 { h.txt 33189 1000000004 aaaaaaaaaabbbbbbbbbb 8 i.txt 33188 4 zxcvb 0 } +do_zipfile_blob_test 1.6.4a test.zip do_execsql_test 1.6.5 { UPDATE zz SET data = 'edcba' WHERE name='blue.txt'; @@ -199,6 +218,7 @@ do_execsql_test 2.4 { dirname2/ 16877 {} dirname2/file1.txt 33188 abcdefghijklmnop } +do_zipfile_blob_test 2.4.1 test.zip # If on unix, check that the [unzip] utility can unpack our archive. # From 33f763d1540b07b9e8088b193991d8a436f45f0a Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 26 Jan 2018 22:41:59 +0000 Subject: [PATCH 012/107] Fix the query planner so that it takes into account dependencies in the arguments to table-valued functions in subexpressions in the WHERE clause. Fix for ticket [80177f0c226ff54f6dd]. FossilOrigin-Name: 7daa687340e475972d6849f84fe1419a1f5700934f53fd80656849adc73d6d04 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/whereexpr.c | 3 +++ test/json101.test | 25 ++++++++++++++++++++++++- 4 files changed, 36 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 7ae6ef7c87..d9b3091b28 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C If\sthe\sargument\sto\stable\sfunction\szipfile()\sis\sa\sblob\s(not\stext),\sassume\sthat\nit\scontains\sa\szip\sfile\simage\sto\sinterpret,\snot\sthe\sname\sof\sa\sfile\son\sdisk. -D 2018-01-26T18:59:25.439 +C Fix\sthe\squery\splanner\sso\sthat\sit\stakes\sinto\saccount\sdependencies\sin\sthe\narguments\sto\stable-valued\sfunctions\sin\ssubexpressions\sin\sthe\sWHERE\sclause.\nFix\sfor\sticket\s[80177f0c226ff54f6dd]. +D 2018-01-26T22:41:59.788 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -573,7 +573,7 @@ F src/walker.c da987a20d40145c0a03c07d8fefcb2ed363becc7680d0500d9c79915591f5b1f F src/where.c caf0b6c9d31f22f0b2c91aba723858de52b5d665aaa89034099015aaf9bb8219 F src/whereInt.h 82c04c5075308abbac59180c8bad5ecb45b07453981f60a53f3c7dee21e1e971 F src/wherecode.c cb360c511f69294ddf00340d7b390e6b1d601a1cfb77b42c4d316fe2f4cd01c3 -F src/whereexpr.c 427ea8e96ec24f2a7814c67b8024ad664a9c7656264c4566c34743cb23186e46 +F src/whereexpr.c 22dbfd3bf5f6051a61523dd0ebef7a944fb29ee4aa7d2a62feb8aac6ffbbc0eb F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test a6d901b436328bd67a79b41bb0ac2663918fe3bd F test/affinity3.test 6a101af2fc945ce2912f6fe54dd646018551710d @@ -1012,7 +1012,7 @@ F test/journal3.test c9c29883f5bf535ae82ae21c472df6263806a22e467b6db7cd0d6d54530 F test/jrnlmode.test a6693f2bed4541a21e703aaa37bb3e10de154130645952933b82b2dec0a8b539 F test/jrnlmode2.test 8759a1d4657c064637f8b079592651530db738419e1d649c6df7048cd724363d F test/jrnlmode3.test 556b447a05be0e0963f4311e95ab1632b11c9eaa -F test/json101.test d7cdf3e6731d41e0c4bde1c88806abd17f1f478486a1409933c1d8eac9120095 +F test/json101.test 24e97954e3bd6404f3715888c7f8f835e36e19c7ae6513b5d9ab2d381498962d F test/json102.test eeb54efa221e50b74a2d6fb9259963b48d7414dca3ce2fdfdeed45cb28487bc1 F test/json103.test c5f6b85e69de05f6b3195f9f9d5ce9cd179099a0 F test/json104.test 877d5845f6303899b7889ea5dd1bea99076e3100574d5c536082245c5805dcaa @@ -1702,7 +1702,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ace0644a1a2a42a3ea42d44f00a31915b8a7e56c9ba90f90a6c02001f89f9c86 -R 6f618424b3fb9707a3fb679f6f287314 -U dan -Z ab7fcdd492fa54ea891500a7623bc431 +P 029ebcd30cb261d949f7587ac54c23d7479796b6716fd4ca7512361b8f32de3e +R 53bd80b0955af3fa0129a178c0e00c7d +U drh +Z 5dfc01b033acb2ff4461347ae9c97b23 diff --git a/manifest.uuid b/manifest.uuid index e4da9d78da..45ee5fd82d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -029ebcd30cb261d949f7587ac54c23d7479796b6716fd4ca7512361b8f32de3e \ No newline at end of file +7daa687340e475972d6849f84fe1419a1f5700934f53fd80656849adc73d6d04 \ No newline at end of file diff --git a/src/whereexpr.c b/src/whereexpr.c index 58f1908cf8..82cc519650 100644 --- a/src/whereexpr.c +++ b/src/whereexpr.c @@ -876,6 +876,9 @@ static Bitmask exprSelectUsage(WhereMaskSet *pMaskSet, Select *pS){ for(i=0; inSrc; i++){ mask |= exprSelectUsage(pMaskSet, pSrc->a[i].pSelect); mask |= sqlite3WhereExprUsage(pMaskSet, pSrc->a[i].pOn); + if( pSrc->a[i].fg.isTabFunc ){ + mask |= sqlite3WhereExprListUsage(pMaskSet, pSrc->a[i].u1.pFuncArg); + } } } pS = pS->pPrior; diff --git a/test/json101.test b/test/json101.test index 4bfcc2d80d..ac4e11ebc2 100644 --- a/test/json101.test +++ b/test/json101.test @@ -759,7 +759,30 @@ do_execsql_test json-12.120 { FROM t12; } {0} - +# 2018-01-26 +# ticket https://www.sqlite.org/src/tktview/80177f0c226ff54f6ddd41 +# Make sure the query planner knows about the arguments to table-valued functions. +# +do_execsql_test json-13.100 { + DROP TABLE IF EXISTS t1; + DROP TABLE IF EXISTS t2; + CREATE TABLE t1(id, json); + INSERT INTO t1(id,json) VALUES(1,'{"items":[3,5]}'); + CREATE TABLE t2(id, json); + INSERT INTO t2(id,json) VALUES(2,'{"value":2}'); + INSERT INTO t2(id,json) VALUES(3,'{"value":3}'); + INSERT INTO t2(id,json) VALUES(4,'{"value":4}'); + INSERT INTO t2(id,json) VALUES(5,'{"value":5}'); + INSERT INTO t2(id,json) VALUES(6,'{"value":6}'); + SELECT * FROM t1 CROSS JOIN t2 + WHERE EXISTS(SELECT 1 FROM json_each(t1.json,'$.items') AS Z + WHERE Z.value==t2.id); +} {1 {{"items":[3,5]}} 3 {{"value":3}} 1 {{"items":[3,5]}} 5 {{"value":5}}} +do_execsql_test json-13.110 { + SELECT * FROM t2 CROSS JOIN t1 + WHERE EXISTS(SELECT 1 FROM json_each(t1.json,'$.items') AS Z + WHERE Z.value==t2.id); +} {3 {{"value":3}} 1 {{"items":[3,5]}} 5 {{"value":5}} 1 {{"items":[3,5]}}} finish_test From c86caa58cd4f101417f6130f58a537c8cbaf6573 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 27 Jan 2018 03:26:10 +0000 Subject: [PATCH 013/107] Fix compiler warnings in zipfile.c. FossilOrigin-Name: bed610d99b5322ee209b06966c0086e8c55f8a82be8d7693cfdeb4d728679721 --- ext/misc/zipfile.c | 8 ++------ manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/ext/misc/zipfile.c b/ext/misc/zipfile.c index c199f8ec1b..61e60f63d2 100644 --- a/ext/misc/zipfile.c +++ b/ext/misc/zipfile.c @@ -504,10 +504,6 @@ static void zipfilePutU32(u8 *aBuf, u32 val){ #define zipfileWrite32(aBuf,val) { zipfilePutU32(aBuf,val); aBuf+=4; } #define zipfileWrite16(aBuf,val) { zipfilePutU16(aBuf,val); aBuf+=2; } -static u8* zipfileCsrBuffer(ZipfileCsr *pCsr){ - return ((ZipfileTab*)(pCsr->base.pVtab))->aBuffer; -} - /* ** Magic numbers used to read CDS records. */ @@ -649,7 +645,7 @@ static int zipfileGetEntry( int rc = SQLITE_OK; if( aBlob==0 ){ - aRead = pTab->aBuffer; + aRead = (u8*)pTab->aBuffer; rc = zipfileReadData(pFile, aRead, ZIPFILE_CDS_FIXED_SZ, iOff, pzErr); }else{ aRead = (u8*)&aBlob[iOff]; @@ -703,7 +699,7 @@ static int zipfileGetEntry( if( pFile ){ rc = zipfileReadData(pFile, aRead, szFix, pNew->cds.iOffset, pzErr); }else{ - aRead = &aBlob[pNew->cds.iOffset]; + aRead = (u8*)&aBlob[pNew->cds.iOffset]; } rc = zipfileReadLFH(aRead, &lfh); diff --git a/manifest b/manifest index d9b3091b28..0e0a71deef 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\squery\splanner\sso\sthat\sit\stakes\sinto\saccount\sdependencies\sin\sthe\narguments\sto\stable-valued\sfunctions\sin\ssubexpressions\sin\sthe\sWHERE\sclause.\nFix\sfor\sticket\s[80177f0c226ff54f6dd]. -D 2018-01-26T22:41:59.788 +C Fix\scompiler\swarnings\sin\szipfile.c. +D 2018-01-27T03:26:10.457 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -304,7 +304,7 @@ F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178 F ext/misc/vtablog.c 31d0d8f4406795679dcd3a67917c213d3a2a5fb3ea5de35f6e773491ed7e13c9 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 -F ext/misc/zipfile.c 071fb5af4d0e6403f01cfac755d78ed2008b04793ec6f87a1078755a6359a26f +F ext/misc/zipfile.c 40195c1cfb43e0ebdf176d37346fea1cb18246e35fbfce477eadc2ff5f7490df F ext/rbu/rbu.c ea7d1b7eb44c123a2a619332e19fe5313500705c4a58aaa1887905c0d83ffc2e F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842 F ext/rbu/rbu10.test 1846519a438697f45e9dcb246908af81b551c29e1078d0304fae83f1fed7e9ee @@ -1702,7 +1702,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 029ebcd30cb261d949f7587ac54c23d7479796b6716fd4ca7512361b8f32de3e -R 53bd80b0955af3fa0129a178c0e00c7d +P 7daa687340e475972d6849f84fe1419a1f5700934f53fd80656849adc73d6d04 +R db0d53a567e4f3bfdfebf8c5b2a2a369 U drh -Z 5dfc01b033acb2ff4461347ae9c97b23 +Z 1789fd20b7b2fb64dfc10fb2cef79caf diff --git a/manifest.uuid b/manifest.uuid index 45ee5fd82d..98506da2d8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7daa687340e475972d6849f84fe1419a1f5700934f53fd80656849adc73d6d04 \ No newline at end of file +bed610d99b5322ee209b06966c0086e8c55f8a82be8d7693cfdeb4d728679721 \ No newline at end of file From 33f10207aa0bc86a190e9ea75298895f508834dd Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 27 Jan 2018 05:40:10 +0000 Subject: [PATCH 014/107] Proposed fix for the query planner problem of ticket [ec32177c99ccac2b1]. FossilOrigin-Name: eef8cbef3c49e6331301a8f3f8a5fd12090b1e99e68487c2d405e53ef771f688 --- manifest | 14 +++++----- manifest.uuid | 2 +- src/where.c | 1 + test/whereF.test | 66 ++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 75 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 0e0a71deef..374e2479eb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\scompiler\swarnings\sin\szipfile.c. -D 2018-01-27T03:26:10.457 +C Proposed\sfix\sfor\sthe\squery\splanner\sproblem\sof\sticket\n[ec32177c99ccac2b1]. +D 2018-01-27T05:40:10.409 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -570,7 +570,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 5a3f464edd64596f601683ed321d12e6fd93c5fb9afdfb3653d6ffd0fee9c48f F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a F src/walker.c da987a20d40145c0a03c07d8fefcb2ed363becc7680d0500d9c79915591f5b1f -F src/where.c caf0b6c9d31f22f0b2c91aba723858de52b5d665aaa89034099015aaf9bb8219 +F src/where.c 10c7e13c909e4dc93057fae930733f5018cdf1a040f73b9e873a264edfc101e5 F src/whereInt.h 82c04c5075308abbac59180c8bad5ecb45b07453981f60a53f3c7dee21e1e971 F src/wherecode.c cb360c511f69294ddf00340d7b390e6b1d601a1cfb77b42c4d316fe2f4cd01c3 F src/whereexpr.c 22dbfd3bf5f6051a61523dd0ebef7a944fb29ee4aa7d2a62feb8aac6ffbbc0eb @@ -1574,7 +1574,7 @@ F test/whereB.test 0def95db3bdec220a731c7e4bec5930327c1d8c5 F test/whereC.test cae295158703cb3fc23bf1a108a9ab730efff0f6 F test/whereD.test 711d4df58d6d4fb9b3f5ce040b818564198be002 F test/whereE.test b3a055eef928c992b0a33198a7b8dc10eea5ad2f -F test/whereF.test d44b58338fe5ddd7286023e9bedb255aa264a6c4d2168b49591b167371c675c7 +F test/whereF.test 15552693e5965e096d58a325216bfcf102b6e2b38f4f5a0d64a728c2f3826714 F test/whereG.test dde4c52a97385a55be6a7cd46be8373f0cf35501 F test/whereH.test e4b07f7a3c2f5d31195cd33710054c78667573b2 F test/whereI.test eab5b226bbc344ac70d7dc09b963a064860ae6d7 @@ -1702,7 +1702,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7daa687340e475972d6849f84fe1419a1f5700934f53fd80656849adc73d6d04 -R db0d53a567e4f3bfdfebf8c5b2a2a369 +P bed610d99b5322ee209b06966c0086e8c55f8a82be8d7693cfdeb4d728679721 +R 5bfee097e7e4c4077c4171c76898db44 U drh -Z 1789fd20b7b2fb64dfc10fb2cef79caf +Z 3dd05a091b8a00a96f3fbc2c2cb36a31 diff --git a/manifest.uuid b/manifest.uuid index 98506da2d8..00da401ea7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bed610d99b5322ee209b06966c0086e8c55f8a82be8d7693cfdeb4d728679721 \ No newline at end of file +eef8cbef3c49e6331301a8f3f8a5fd12090b1e99e68487c2d405e53ef771f688 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 3152c8e9aa..7f45182d9a 100644 --- a/src/where.c +++ b/src/where.c @@ -4631,6 +4631,7 @@ WhereInfo *sqlite3WhereBegin( */ for(ii=0; iinTerm; ii++){ WhereTerm *pT = &sWLB.pWC->a[ii]; + if( pT->wtFlags & TERM_VIRTUAL ) continue; if( pT->prereqAll==0 && (nTabList==0 || exprIsDeterministic(pT->pExpr)) ){ sqlite3ExprIfFalse(pParse, pT->pExpr, pWInfo->iBreak, SQLITE_JUMPIFNULL); pT->wtFlags |= TERM_CODED; diff --git a/test/whereF.test b/test/whereF.test index 8b272a5c53..e5028969a1 100644 --- a/test/whereF.test +++ b/test/whereF.test @@ -215,4 +215,70 @@ ifcapable json1&&vtab { } {{{"foo":"meep","other":12345}}} } +# 2018-01-27 +# Ticket https://sqlite.org/src/tktview/ec32177c99ccac2b180fd3ea2083 +# Incorrect result when using the new OR clause factoring optimization +# +# This is the original test case as reported on the sqlite-users mailing +# list +# +do_execsql_test 7.1 { + DROP TABLE IF EXISTS cd; + CREATE TABLE cd ( cdid INTEGER PRIMARY KEY NOT NULL, genreid integer ); + CREATE INDEX cd_idx_genreid ON cd (genreid); + INSERT INTO cd ( cdid, genreid ) VALUES + ( 1, 1 ), + ( 2, NULL ), + ( 3, NULL ), + ( 4, NULL ), + ( 5, NULL ); + + SELECT cdid + FROM cd me + WHERE 2 > ( + SELECT COUNT( * ) + FROM cd rownum__emulation + WHERE + ( + me.genreid IS NOT NULL + AND + rownum__emulation.genreid IS NULL + ) + OR + ( + me.genreid IS NOT NULL + AND + rownum__emulation.genreid IS NOT NULL + AND + rownum__emulation.genreid < me.genreid + ) + OR + ( + ( me.genreid = rownum__emulation.genreid OR ( me.genreid IS NULL + AND rownum__emulation.genreid IS NULL ) ) + AND + rownum__emulation.cdid > me.cdid + ) + ); +} {4 5} + +# Simplified test cases from the ticket +# +do_execsql_test 7.2 { + DROP TABLE IF EXISTS t1; + DROP TABLE IF EXISTS t2; + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + INSERT INTO t1(a,b) VALUES(1,1); + CREATE TABLE t2(aa INTEGER PRIMARY KEY, bb); + INSERT INTO t2(aa,bb) VALUES(1,1),(2,NULL),(3,NULL); + SELECT ( + SELECT COUNT(*) FROM t2 + WHERE ( t1.b IS NOT NULL AND t2.bb IS NULL ) + OR ( t2.bb < t1.b ) + OR ( t1.b IS t2.bb AND t2.aa > t1.a ) + ) + FROM t1; +} {2} + + finish_test From c9f3db33d5547187f1ce129452d20e2b0b1e86ff Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 27 Jan 2018 13:55:56 +0000 Subject: [PATCH 015/107] New test case for ticket [ec32177c99ccac2b1] that works without the STAT4. FossilOrigin-Name: 5259d4847f2b73f26b2385f9d8cff8fe0cabc54b4deab8477c87c8d1bb5535b1 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/whereF.test | 28 ++++++++++++++++++++++++++++ 3 files changed, 35 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 374e2479eb..b88aeece1d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Proposed\sfix\sfor\sthe\squery\splanner\sproblem\sof\sticket\n[ec32177c99ccac2b1]. -D 2018-01-27T05:40:10.409 +C New\stest\scase\sfor\sticket\s[ec32177c99ccac2b1]\sthat\sworks\swithout\sthe\sSTAT4. +D 2018-01-27T13:55:56.815 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -1574,7 +1574,7 @@ F test/whereB.test 0def95db3bdec220a731c7e4bec5930327c1d8c5 F test/whereC.test cae295158703cb3fc23bf1a108a9ab730efff0f6 F test/whereD.test 711d4df58d6d4fb9b3f5ce040b818564198be002 F test/whereE.test b3a055eef928c992b0a33198a7b8dc10eea5ad2f -F test/whereF.test 15552693e5965e096d58a325216bfcf102b6e2b38f4f5a0d64a728c2f3826714 +F test/whereF.test 3d9412b1199d3e2bed34fcb76b4c48d0bf4df95d27e3f8dd27b6f8b4716d0d89 F test/whereG.test dde4c52a97385a55be6a7cd46be8373f0cf35501 F test/whereH.test e4b07f7a3c2f5d31195cd33710054c78667573b2 F test/whereI.test eab5b226bbc344ac70d7dc09b963a064860ae6d7 @@ -1702,7 +1702,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P bed610d99b5322ee209b06966c0086e8c55f8a82be8d7693cfdeb4d728679721 -R 5bfee097e7e4c4077c4171c76898db44 +P eef8cbef3c49e6331301a8f3f8a5fd12090b1e99e68487c2d405e53ef771f688 +R a57e94d6d2f924366bc9079a11d7258c U drh -Z 3dd05a091b8a00a96f3fbc2c2cb36a31 +Z 5c67b72d2f152607d54502c89af32425 diff --git a/manifest.uuid b/manifest.uuid index 00da401ea7..b2d9772040 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -eef8cbef3c49e6331301a8f3f8a5fd12090b1e99e68487c2d405e53ef771f688 \ No newline at end of file +5259d4847f2b73f26b2385f9d8cff8fe0cabc54b4deab8477c87c8d1bb5535b1 \ No newline at end of file diff --git a/test/whereF.test b/test/whereF.test index e5028969a1..121cc3cf22 100644 --- a/test/whereF.test +++ b/test/whereF.test @@ -280,5 +280,33 @@ do_execsql_test 7.2 { FROM t1; } {2} +# The fix for ticket ec32177c99ccac2b180fd3ea2083 only makes a difference +# in the output when there is a TERM_VNULL entry in the WhereClause array. +# And TERM_VNULL entries are only generated when compiling with +# SQLITE_ENABLE_STAT4. Nevertheless, it is correct that TERM_VIRTUAL terms +# should not participate in the factoring optimization. In all cases other +# than TERM_VNULL, participation is harmless, but it does consume a few +# extra CPU cycles. +# +# The following test verifies that the TERM_VIRTUAL terms resulting from +# a GLOB operator do not appear anywhere in the generated code. This +# confirms that the problem is fixed, even on builds that omit STAT4. +# +do_execsql_test 7.3 { + DROP TABLE IF EXISTS t1; + DROP TABLE IF EXISTS t2; + CREATE TABLE t1(a INTEGER PRIMARY KEY, b TEXT); + INSERT INTO t1(a,b) VALUES(1,'abcxyz'); + CREATE TABLE t2(aa INTEGER PRIMARY KEY, bb TEXT); + INSERT INTO t2(aa,bb) VALUES(1,'abc'),(2,'wxyz'),(3,'xyz'); + CREATE INDEX t2bb ON t2(bb); + EXPLAIN SELECT ( + SELECT COUNT(*) FROM t2 + WHERE ( t1.b GLOB 'a*z' AND t2.bb='xyz' ) + OR ( t2.bb = t1.b ) + OR ( t2.aa = t1.a ) + ) + FROM t1; +} {~/ (Lt|Ge) /} finish_test From 1822ebf9b1997c4bc8ed3371b065a494bac3d6d5 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 27 Jan 2018 14:25:27 +0000 Subject: [PATCH 016/107] Changes to avoid a harmless UB warning from clang. FossilOrigin-Name: 19f5c1400054df10688ab448e7e23afef97cab4a7c7a3e411f7527509b515dd8 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/util.c | 8 +++++++- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index b88aeece1d..faa8c1e356 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C New\stest\scase\sfor\sticket\s[ec32177c99ccac2b1]\sthat\sworks\swithout\sthe\sSTAT4. -D 2018-01-27T13:55:56.815 +C Changes\sto\savoid\sa\sharmless\sUB\swarning\sfrom\sclang. +D 2018-01-27T14:25:27.955 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -554,7 +554,7 @@ F src/treeview.c eae35972ff44f67064de2eaf35f04afe94e7aea3271a8b3bcebb3f954880fec F src/trigger.c a34539c69433276d37b0da9a89c117726ff2d292c0902895af1f393a983cd3a1 F src/update.c a90a32ffc0100265b0693dbbdbe490756447af181f5ea2c138cce515b08c8795 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5 -F src/util.c 8628fe1daa9f5db9fa3a70d6fcfd828b73e839feafe7d7faecd7073be56eaec2 +F src/util.c d9eb0a6c4aae1b00a7369eadd7ca0bbe946cb4c953b6751aa20d357c2f482157 F src/vacuum.c 90839322fd5f00df9617eb21b68beda9b6e2a2937576b0d65985e4aeb1c53739 F src/vdbe.c 0e13b8c33a9ecf236b3045f190fb58933049582307b621cbd924b6582a2b705f F src/vdbe.h 134beb7a12a6213c00eba58febaede33447cc4441bc568a0d9c144b33fc3720a @@ -1702,7 +1702,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P eef8cbef3c49e6331301a8f3f8a5fd12090b1e99e68487c2d405e53ef771f688 -R a57e94d6d2f924366bc9079a11d7258c +P 5259d4847f2b73f26b2385f9d8cff8fe0cabc54b4deab8477c87c8d1bb5535b1 +R dc25ffafa3daf6e245ccddfac4936cfd U drh -Z 5c67b72d2f152607d54502c89af32425 +Z 83f6cfb83d43c9463ef36c14a8c84938 diff --git a/manifest.uuid b/manifest.uuid index b2d9772040..3f53762e60 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5259d4847f2b73f26b2385f9d8cff8fe0cabc54b4deab8477c87c8d1bb5535b1 \ No newline at end of file +19f5c1400054df10688ab448e7e23afef97cab4a7c7a3e411f7527509b515dd8 \ No newline at end of file diff --git a/src/util.c b/src/util.c index 9954d709df..54f9b93887 100644 --- a/src/util.c +++ b/src/util.c @@ -641,7 +641,13 @@ int sqlite3Atoi64(const char *zNum, i64 *pNum, int length, u8 enc){ testcase( i==18*incr ); testcase( i==19*incr ); testcase( i==20*incr ); - if( neg ){ + if( u>LARGEST_INT64 ){ + /* This test and assignment is needed only to suppress UB warnings + ** from clang and -fsanitize=undefined. This test and assignment make + ** the code a little larger and slower, and no harm comes from omitting + ** them, but we must appaise the undefined-behavior pharisees. */ + *pNum = neg ? SMALLEST_INT64 : LARGEST_INT64; + }else if( neg ){ *pNum = -(i64)u; }else{ *pNum = (i64)u; From 26333ee3cae6491bf0f4ec2720446d726dfd3e2b Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 27 Jan 2018 16:29:59 +0000 Subject: [PATCH 017/107] If a zipfile virtual table is created with no argument - "CREATE VIRTUAL TABLE zzz USING zipfile()" - accumulate data in memory. Support "SELECT zipfile_blob(z) FROM zzz LIMIT 1" to retrieve a zip archive image. FossilOrigin-Name: e63185edfe0c316aa60c1fa085d032425ecc7db54536dfa5a977772eaf3c240e --- ext/misc/zipfile.c | 257 +++++++++++++++++++++++++++++---------------- manifest | 16 +-- manifest.uuid | 2 +- test/zipfile.test | 40 ++++++- 4 files changed, 213 insertions(+), 102 deletions(-) diff --git a/ext/misc/zipfile.c b/ext/misc/zipfile.c index 61e60f63d2..4e59675c1d 100644 --- a/ext/misc/zipfile.c +++ b/ext/misc/zipfile.c @@ -103,6 +103,8 @@ static const char ZIPFILE_SCHEMA[] = #define ZIPFILE_SIGNATURE_EOCD 0x06054b50 #define ZIPFILE_LFH_FIXED_SZ 30 +#define ZIPFILE_EOCD_FIXED_SZ 22 + /* ** Set the error message contained in context ctx to the results of ** vprintf(zFmt, ...). @@ -645,7 +647,7 @@ static int zipfileGetEntry( int rc = SQLITE_OK; if( aBlob==0 ){ - aRead = (u8*)pTab->aBuffer; + aRead = pTab->aBuffer; rc = zipfileReadData(pFile, aRead, ZIPFILE_CDS_FIXED_SZ, iOff, pzErr); }else{ aRead = (u8*)&aBlob[iOff]; @@ -1094,10 +1096,7 @@ static int zipfileFilter( if( pTab->zFile ){ zFile = pTab->zFile; }else if( idxNum==0 ){ - /* Error. This is an eponymous virtual table and the user has not - ** supplied a file name. */ - zipfileSetErrmsg(pCsr, "table function zipfile() requires an argument"); - return SQLITE_ERROR; + bInMemory = 1; }else if( sqlite3_value_type(argv[0])==SQLITE_BLOB ){ const u8 *aBlob = (const u8*)sqlite3_value_blob(argv[0]); int nBlob = sqlite3_value_bytes(argv[0]); @@ -1166,11 +1165,14 @@ static int zipfileBestIndex( return SQLITE_OK; } -static ZipfileEntry *zipfileNewEntry(const char *zPath){ +static ZipfileEntry *zipfileNewEntry(const char *zPath, int nData){ ZipfileEntry *pNew; - pNew = sqlite3_malloc(sizeof(ZipfileEntry)); + pNew = sqlite3_malloc(sizeof(ZipfileEntry) + nData); if( pNew ){ memset(pNew, 0, sizeof(ZipfileEntry)); + if( nData ){ + pNew->aData = (u8*)&pNew[1]; + } pNew->cds.zFile = sqlite3_mprintf("%s", zPath); if( pNew->cds.zFile==0 ){ sqlite3_free(pNew); @@ -1180,46 +1182,51 @@ static ZipfileEntry *zipfileNewEntry(const char *zPath){ return pNew; } -static int zipfileAppendEntry( - ZipfileTab *pTab, - ZipfileCDS *pCds, - const char *zPath, /* Path for new entry */ - int nPath, /* strlen(zPath) */ - const u8 *pData, - int nData, - u32 mTime -){ - u8 *aBuf = pTab->aBuffer; - int rc; +static int zipfileSerializeLFH(ZipfileEntry *pEntry, u8 *aBuf){ + ZipfileCDS *pCds = &pEntry->cds; + u8 *a = aBuf; pCds->nExtra = 9; - zipfileWrite32(aBuf, ZIPFILE_SIGNATURE_LFH); - zipfileWrite16(aBuf, pCds->iVersionExtract); - zipfileWrite16(aBuf, pCds->flags); - zipfileWrite16(aBuf, pCds->iCompression); - zipfileWrite16(aBuf, pCds->mTime); - zipfileWrite16(aBuf, pCds->mDate); - zipfileWrite32(aBuf, pCds->crc32); - zipfileWrite32(aBuf, pCds->szCompressed); - zipfileWrite32(aBuf, pCds->szUncompressed); - zipfileWrite16(aBuf, (u16)nPath); - zipfileWrite16(aBuf, pCds->nExtra); - assert( aBuf==&pTab->aBuffer[ZIPFILE_LFH_FIXED_SZ] ); - rc = zipfileAppendData(pTab, pTab->aBuffer, (int)(aBuf - pTab->aBuffer)); - if( rc==SQLITE_OK ){ - rc = zipfileAppendData(pTab, (const u8*)zPath, nPath); - } + /* Write the LFH itself */ + zipfileWrite32(a, ZIPFILE_SIGNATURE_LFH); + zipfileWrite16(a, pCds->iVersionExtract); + zipfileWrite16(a, pCds->flags); + zipfileWrite16(a, pCds->iCompression); + zipfileWrite16(a, pCds->mTime); + zipfileWrite16(a, pCds->mDate); + zipfileWrite32(a, pCds->crc32); + zipfileWrite32(a, pCds->szCompressed); + zipfileWrite32(a, pCds->szUncompressed); + zipfileWrite16(a, (u16)pCds->nFile); + zipfileWrite16(a, pCds->nExtra); + assert( a==&aBuf[ZIPFILE_LFH_FIXED_SZ] ); - if( rc==SQLITE_OK && pCds->nExtra ){ - aBuf = pTab->aBuffer; - zipfileWrite16(aBuf, ZIPFILE_EXTRA_TIMESTAMP); - zipfileWrite16(aBuf, 5); - *aBuf++ = 0x01; - zipfileWrite32(aBuf, mTime); - rc = zipfileAppendData(pTab, pTab->aBuffer, 9); - } + /* Add the file name */ + memcpy(a, pCds->zFile, (int)pCds->nFile); + a += (int)pCds->nFile; + /* The "extra" data */ + zipfileWrite16(a, ZIPFILE_EXTRA_TIMESTAMP); + zipfileWrite16(a, 5); + *a++ = 0x01; + zipfileWrite32(a, pEntry->mUnixTime); + + return a-aBuf; +} + +static int zipfileAppendEntry( + ZipfileTab *pTab, + ZipfileEntry *pEntry, + const u8 *pData, + int nData +){ + u8 *aBuf = pTab->aBuffer; + int nBuf; + int rc; + + nBuf = zipfileSerializeLFH(pEntry, aBuf); + rc = zipfileAppendData(pTab, aBuf, nBuf); if( rc==SQLITE_OK ){ rc = zipfileAppendData(pTab, pData, nData); } @@ -1303,8 +1310,7 @@ static int zipfileUpdate( int bIsDir = 0; u32 iCrc32 = 0; - assert( pTab->zFile ); - assert( pTab->pWriteFd ); + assert( (pTab->zFile==0)==(pTab->pWriteFd==0) ); /* If this is a DELETE or UPDATE, find the archive entry to delete. */ if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){ @@ -1407,7 +1413,7 @@ static int zipfileUpdate( if( rc==SQLITE_OK ){ /* Create the new CDS record. */ - pNew = zipfileNewEntry(zPath); + pNew = zipfileNewEntry(zPath, pTab->zFile ? 0 : (nData+1)); if( pNew==0 ){ rc = SQLITE_NOMEM; }else{ @@ -1423,9 +1429,11 @@ static int zipfileUpdate( pNew->cds.iOffset = (u32)pTab->szCurrent; pNew->cds.nFile = nPath; pNew->mUnixTime = (u32)mTime; - rc = zipfileAppendEntry( - pTab, &pNew->cds, zPath, nPath, pData, nData, pNew->mUnixTime - ); + if( pTab->zFile ){ + rc = zipfileAppendEntry(pTab, pNew, pData, nData); + }else{ + memcpy(pNew->aData, pData, nData); + } zipfileAddEntry(pTab, pOld, pNew); } } @@ -1450,20 +1458,24 @@ static int zipfileUpdate( return rc; } +static int zipfileSerializeEOCD(ZipfileEOCD *p, u8 *aBuf){ + u8 *a = aBuf; + zipfileWrite32(a, ZIPFILE_SIGNATURE_EOCD); + zipfileWrite16(a, p->iDisk); + zipfileWrite16(a, p->iFirstDisk); + zipfileWrite16(a, p->nEntry); + zipfileWrite16(a, p->nEntryTotal); + zipfileWrite32(a, p->nSize); + zipfileWrite32(a, p->iOffset); + zipfileWrite16(a, 0); /* Size of trailing comment in bytes*/ + + return a-aBuf; +} + static int zipfileAppendEOCD(ZipfileTab *pTab, ZipfileEOCD *p){ - u8 *aBuf = pTab->aBuffer; - - zipfileWrite32(aBuf, ZIPFILE_SIGNATURE_EOCD); - zipfileWrite16(aBuf, p->iDisk); - zipfileWrite16(aBuf, p->iFirstDisk); - zipfileWrite16(aBuf, p->nEntry); - zipfileWrite16(aBuf, p->nEntryTotal); - zipfileWrite32(aBuf, p->nSize); - zipfileWrite32(aBuf, p->iOffset); - zipfileWrite16(aBuf, 0); /* Size of trailing comment in bytes*/ - - assert( (aBuf-pTab->aBuffer)==22 ); - return zipfileAppendData(pTab, pTab->aBuffer, (int)(aBuf - pTab->aBuffer)); + int nBuf = zipfileSerializeEOCD(p, pTab->aBuffer); + assert( nBuf==ZIPFILE_EOCD_FIXED_SZ ); + return zipfileAppendData(pTab, pTab->aBuffer, nBuf); } static int zipfileBegin(sqlite3_vtab *pVtab){ @@ -1471,34 +1483,26 @@ static int zipfileBegin(sqlite3_vtab *pVtab){ int rc = SQLITE_OK; assert( pTab->pWriteFd==0 ); + if( pTab->zFile ){ + /* Open a write fd on the file. Also load the entire central directory + ** structure into memory. During the transaction any new file data is + ** appended to the archive file, but the central directory is accumulated + ** in main-memory until the transaction is committed. */ + pTab->pWriteFd = fopen(pTab->zFile, "ab+"); + if( pTab->pWriteFd==0 ){ + pTab->base.zErrMsg = sqlite3_mprintf( + "zipfile: failed to open file %s for writing", pTab->zFile + ); + rc = SQLITE_ERROR; + }else{ + fseek(pTab->pWriteFd, 0, SEEK_END); + pTab->szCurrent = pTab->szOrig = (i64)ftell(pTab->pWriteFd); + rc = zipfileLoadDirectory(pTab, 0, 0); + } - /* This table is only writable if a default archive path was specified - ** as part of the CREATE VIRTUAL TABLE statement. */ - if( pTab->zFile==0 ){ - pTab->base.zErrMsg = sqlite3_mprintf( - "zipfile: writing requires a default archive" - ); - return SQLITE_ERROR; - } - - /* Open a write fd on the file. Also load the entire central directory - ** structure into memory. During the transaction any new file data is - ** appended to the archive file, but the central directory is accumulated - ** in main-memory until the transaction is committed. */ - pTab->pWriteFd = fopen(pTab->zFile, "ab+"); - if( pTab->pWriteFd==0 ){ - pTab->base.zErrMsg = sqlite3_mprintf( - "zipfile: failed to open file %s for writing", pTab->zFile - ); - rc = SQLITE_ERROR; - }else{ - fseek(pTab->pWriteFd, 0, SEEK_END); - pTab->szCurrent = pTab->szOrig = (i64)ftell(pTab->pWriteFd); - rc = zipfileLoadDirectory(pTab, 0, 0); - } - - if( rc!=SQLITE_OK ){ - zipfileCleanupTransaction(pTab); + if( rc!=SQLITE_OK ){ + zipfileCleanupTransaction(pTab); + } } return rc; @@ -1643,6 +1647,73 @@ static void zipfileFunctionCds( } } +static void zipfileFree(void *p) { sqlite3_free(p); } + +static void zipfileFunctionBlob( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + ZipfileCsr *pCsr; + ZipfileTab *pTab = (ZipfileTab*)sqlite3_user_data(context); + ZipfileEntry *p; + int nBody = 0; + int nCds = 0; + int nEocd = ZIPFILE_EOCD_FIXED_SZ; + ZipfileEOCD eocd; + + u8 *aZip; + int nZip; + + u8 *aBody; + u8 *aCds; + + pCsr = zipfileFindCursor(pTab, sqlite3_value_int64(argv[0])); + if( pCsr->pFile || pTab->zFile ){ + sqlite3_result_error(context, "illegal use of zipfile_blob()", -1); + return; + } + + /* Figure out how large the final file will be */ + for(p=pTab->pFirstEntry; p; p=p->pNext){ + nBody += ZIPFILE_LFH_FIXED_SZ + p->cds.nFile + 9 + p->cds.szCompressed; + nCds += ZIPFILE_CDS_FIXED_SZ + p->cds.nFile + 9; + } + + /* Allocate space to create the serialized file */ + nZip = nBody + nCds + nEocd; + aZip = (u8*)sqlite3_malloc(nZip); + if( aZip==0 ){ + sqlite3_result_error_nomem(context); + return; + } + aBody = aZip; + aCds = &aZip[nBody]; + + /* Populate the body and CDS */ + memset(&eocd, 0, sizeof(eocd)); + for(p=pTab->pFirstEntry; p; p=p->pNext){ + p->cds.iOffset = (aBody - aZip); + aBody += zipfileSerializeLFH(p, aBody); + if( p->cds.szCompressed ){ + memcpy(aBody, p->aData, p->cds.szCompressed); + aBody += p->cds.szCompressed; + } + aCds += zipfileSerializeCDS(p, aCds); + eocd.nEntry++; + } + + /* Append the EOCD record */ + assert( aBody==&aZip[nBody] ); + assert( aCds==&aZip[nBody+nCds] ); + eocd.nEntryTotal = eocd.nEntry; + eocd.nSize = nCds; + eocd.iOffset = nBody; + zipfileSerializeEOCD(&eocd, aCds); + + sqlite3_result_blob(context, aZip, nZip, zipfileFree); +} + /* ** xFindFunction method. @@ -1660,6 +1731,11 @@ static int zipfileFindFunction( *ppArg = (void*)pVtab; return 1; } + if( sqlite3_stricmp("zipfile_blob", zName)==0 ){ + *pxFunc = zipfileFunctionBlob; + *ppArg = (void*)pVtab; + return 1; + } } return 0; @@ -1693,9 +1769,8 @@ static int zipfileRegister(sqlite3 *db){ }; int rc = sqlite3_create_module(db, "zipfile" , &zipfileModule, 0); - if( rc==SQLITE_OK ){ - rc = sqlite3_overload_function(db, "zipfile_cds", -1); - } + if( rc==SQLITE_OK ) rc = sqlite3_overload_function(db, "zipfile_cds", -1); + if( rc==SQLITE_OK ) rc = sqlite3_overload_function(db, "zipfile_blob", -1); return rc; } #else /* SQLITE_OMIT_VIRTUALTABLE */ diff --git a/manifest b/manifest index faa8c1e356..96a7ea5fe4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Changes\sto\savoid\sa\sharmless\sUB\swarning\sfrom\sclang. -D 2018-01-27T14:25:27.955 +C If\sa\szipfile\svirtual\stable\sis\screated\swith\sno\sargument\s-\s"CREATE\sVIRTUAL\sTABLE\nzzz\sUSING\szipfile()"\s-\saccumulate\sdata\sin\smemory.\sSupport\s"SELECT\nzipfile_blob(z)\sFROM\szzz\sLIMIT\s1"\sto\sretrieve\sa\szip\sarchive\simage. +D 2018-01-27T16:29:59.042 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -304,7 +304,7 @@ F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178 F ext/misc/vtablog.c 31d0d8f4406795679dcd3a67917c213d3a2a5fb3ea5de35f6e773491ed7e13c9 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 -F ext/misc/zipfile.c 40195c1cfb43e0ebdf176d37346fea1cb18246e35fbfce477eadc2ff5f7490df +F ext/misc/zipfile.c df57128d9ad2a1e60097d7971e787b582fb66ce0926577cb6f5978f7af210b8c F ext/rbu/rbu.c ea7d1b7eb44c123a2a619332e19fe5313500705c4a58aaa1887905c0d83ffc2e F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842 F ext/rbu/rbu10.test 1846519a438697f45e9dcb246908af81b551c29e1078d0304fae83f1fed7e9ee @@ -1603,7 +1603,7 @@ F test/wordcount.c cb589cec469a1d90add05b1f8cee75c7210338d87a5afd65260ed5c0f4bbf F test/writecrash.test f1da7f7adfe8d7f09ea79b42e5ca6dcc41102f27f8e334ad71539501ddd910cc F test/zeroblob.test 3857870fe681b8185654414a9bccfde80b62a0fa F test/zerodamage.test 9c41628db7e8d9e8a0181e59ea5f189df311a9f6ce99cc376dc461f66db6f8dc -F test/zipfile.test 0834b33e000991a80d94167e84af346162d212526d8efee0b9d4ba84589fe292 +F test/zipfile.test 1b213bdc31eddd4a41042875dbdb29b6ea12b7da7a372a8eb07f61d0d76d800f F tool/GetFile.cs a15e08acb5dd7539b75ba23501581d7c2b462cb5 F tool/GetTclKit.bat 8995df40c4209808b31f24de0b58f90930239a234f7591e3675d45bfbb990c5d F tool/Replace.cs 02c67258801c2fb5f63231e0ac0f220b4b36ba91 @@ -1702,7 +1702,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5259d4847f2b73f26b2385f9d8cff8fe0cabc54b4deab8477c87c8d1bb5535b1 -R dc25ffafa3daf6e245ccddfac4936cfd -U drh -Z 83f6cfb83d43c9463ef36c14a8c84938 +P 19f5c1400054df10688ab448e7e23afef97cab4a7c7a3e411f7527509b515dd8 +R 8b2b96d9b8d1e07c4df7e0d2971e5e57 +U dan +Z e468bfc5bd8b08bb5b9404b26ad000bc diff --git a/manifest.uuid b/manifest.uuid index 3f53762e60..a8aca2c7e5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -19f5c1400054df10688ab448e7e23afef97cab4a7c7a3e411f7527509b515dd8 \ No newline at end of file +e63185edfe0c316aa60c1fa085d032425ecc7db54536dfa5a977772eaf3c240e \ No newline at end of file diff --git a/test/zipfile.test b/test/zipfile.test index 60fde2ee15..e74f81ed1d 100644 --- a/test/zipfile.test +++ b/test/zipfile.test @@ -32,8 +32,23 @@ proc do_zipfile_blob_test {tn file} { set data [read $fd] close $fd - set res2 [db eval { SELECT name,mode,mtime,method,quote(data) FROM zipfile($data) }] - uplevel [list do_test $tn [list set {} $res2] $res1] + set res2 [db eval { + SELECT name,mode,mtime,method,quote(data) FROM zipfile($data) + }] + + uplevel [list do_test $tn.1 [list set {} $res2] $res1] + + set T "$file.test_zip" + set fd [open $T w] + fconfigure $fd -translation binary -encoding binary + puts -nonewline $fd $data + close $fd + + set res3 [ + db eval { SELECT name,mode,mtime,method,quote(data) FROM zipfile($T) } + ] + + uplevel [list do_test $tn.2 [list set {} $res3] $res1] } forcedelete test.zip @@ -271,6 +286,27 @@ do_catchsql_test 3.2 { SELECT rowid FROM x1 } {1 {no such column: rowid}} +#------------------------------------------------------------------------- +reset_db +forcedelete test.zip +load_static_extension db zipfile + +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE x2 USING zipfile(); + INSERT INTO x2(name, data) VALUES('dir1/', NULL); + INSERT INTO x2(name, data) VALUES('file1', '1234'); + INSERT INTO x2(name, data) VALUES('dir1/file2', '5678'); + SELECT name, data FROM x2 +} { + dir1/ {} file1 1234 dir1/file2 5678 +} + +do_test 4.1 { + set data [db one {SELECT zipfile_blob(z) FROM x2 LIMIT 1}] + db eval { SELECT name, data FROM zipfile($data) } +} {dir1/ {} file1 1234 dir1/file2 5678} + + finish_test From a07aa8d397bf1cbd490602283ea75385e1e81643 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 27 Jan 2018 18:55:18 +0000 Subject: [PATCH 018/107] Fix missing header comments and other code issues in zipfile.c. FossilOrigin-Name: 6ea8ba312c38365d3e28cfb2a367d729dd2751d1d853843eea0e18126777a320 --- ext/misc/zipfile.c | 284 +++++++++++++++++++++++++++++---------------- manifest | 12 +- manifest.uuid | 2 +- 3 files changed, 188 insertions(+), 110 deletions(-) diff --git a/ext/misc/zipfile.c b/ext/misc/zipfile.c index 4e59675c1d..efc25927d4 100644 --- a/ext/misc/zipfile.c +++ b/ext/misc/zipfile.c @@ -93,6 +93,9 @@ static const char ZIPFILE_SCHEMA[] = ** ** ZIPFILE_SIGNATURE_LFH: ** First 4 bytes of a valid LFH record. +** +** ZIPFILE_SIGNATURE_EOCD +** First 4 bytes of a valid EOCD record. */ #define ZIPFILE_EXTRA_TIMESTAMP 0x5455 #define ZIPFILE_NEWENTRY_MADEBY ((3<<8) + 30) @@ -101,24 +104,14 @@ static const char ZIPFILE_SCHEMA[] = #define ZIPFILE_SIGNATURE_CDS 0x02014b50 #define ZIPFILE_SIGNATURE_LFH 0x04034b50 #define ZIPFILE_SIGNATURE_EOCD 0x06054b50 -#define ZIPFILE_LFH_FIXED_SZ 30 - -#define ZIPFILE_EOCD_FIXED_SZ 22 /* -** Set the error message contained in context ctx to the results of -** vprintf(zFmt, ...). +** The sizes of the fixed-size part of each of the three main data +** structures in a zip archive. */ -static void zipfileCtxErrorMsg(sqlite3_context *ctx, const char *zFmt, ...){ - char *zMsg = 0; - va_list ap; - va_start(ap, zFmt); - zMsg = sqlite3_vmprintf(zFmt, ap); - sqlite3_result_error(ctx, zMsg, -1); - sqlite3_free(zMsg); - va_end(ap); -} - +#define ZIPFILE_LFH_FIXED_SZ 30 +#define ZIPFILE_EOCD_FIXED_SZ 22 +#define ZIPFILE_CDS_FIXED_SZ 46 /* *** 4.3.16 End of central directory record: @@ -227,13 +220,13 @@ struct ZipfileEntry { ZipfileCDS cds; /* Parsed CDS record */ u32 mUnixTime; /* Modification time, in UNIX format */ u8 *aExtra; /* cds.nExtra+cds.nComment bytes of extra data */ - i64 iDataOff; + i64 iDataOff; /* Offset to data in file (if aData==0) */ u8 *aData; /* cds.szCompressed bytes of compressed data */ ZipfileEntry *pNext; /* Next element in in-memory CDS */ }; /* -** Cursor type for recursively iterating through a directory structure. +** Cursor type for zipfile tables. */ typedef struct ZipfileCsr ZipfileCsr; struct ZipfileCsr { @@ -247,23 +240,11 @@ struct ZipfileCsr { i64 iNextOff; /* Offset of next record in central directory */ ZipfileEOCD eocd; /* Parse of central directory record */ - ZipfileEntry *pFreeEntry; - + ZipfileEntry *pFreeEntry; /* Free this list when cursor is closed or reset */ ZipfileEntry *pCurrent; /* Current entry */ ZipfileCsr *pCsrNext; /* Next cursor on same virtual table */ }; -/* -** Possible values for ZipfileCsr.eType. Set in zipfileFilter(). -*/ -#define ZIPFILE_CSR_LIST 1 /* Cursor reads from ZipfileTab.pFirstEntry */ -#define ZIPFILE_CSR_FILE 2 /* Cursor reads from file on disk */ - -/* -** Values for ZipfileCsr.flags. -*/ -#define ZIPFILE_NOOP_NEXT 0x0002 /* Next xNext() is a no-op */ - typedef struct ZipfileTab ZipfileTab; struct ZipfileTab { sqlite3_vtab base; /* Base class - must be first */ @@ -281,6 +262,24 @@ struct ZipfileTab { i64 szOrig; /* Size of archive at start of transaction */ }; +/* +** Set the error message contained in context ctx to the results of +** vprintf(zFmt, ...). +*/ +static void zipfileCtxErrorMsg(sqlite3_context *ctx, const char *zFmt, ...){ + char *zMsg = 0; + va_list ap; + va_start(ap, zFmt); + zMsg = sqlite3_vmprintf(zFmt, ap); + sqlite3_result_error(ctx, zMsg, -1); + sqlite3_free(zMsg); + va_end(ap); +} + +/* +** If string zIn is quoted, dequote it in place. Otherwise, if the string +** is not quoted, do nothing. +*/ static void zipfileDequote(char *zIn){ char q = zIn[0]; if( q=='"' || q=='\'' || q=='`' || q=='[' ){ @@ -340,6 +339,9 @@ static int zipfileConnect( return rc; } +/* +** Free the ZipfileEntry structure indicated by the only argument. +*/ static void zipfileEntryFree(ZipfileEntry *p){ if( p ){ sqlite3_free(p->cds.zFile); @@ -347,6 +349,10 @@ static void zipfileEntryFree(ZipfileEntry *p){ } } +/* +** Release resources that should be freed at the end of a write +** transaction. +*/ static void zipfileCleanupTransaction(ZipfileTab *pTab){ ZipfileEntry *pEntry; ZipfileEntry *pNext; @@ -446,6 +452,16 @@ static void zipfileSetErrmsg(ZipfileCsr *pCsr, const char *zFmt, ...){ va_end(ap); } +/* +** Read nRead bytes of data from offset iOff of file pFile into buffer +** aRead[]. Return SQLITE_OK if successful, or an SQLite error code +** otherwise. +** +** If an error does occur, output variable (*pzErrmsg) may be set to point +** to an English language error message. It is the responsibility of the +** caller to eventually free this buffer using +** sqlite3_free(). +*/ static int zipfileReadData( FILE *pFile, /* Read from this file */ u8 *aRead, /* Read into this buffer */ @@ -479,9 +495,16 @@ static int zipfileAppendData( return SQLITE_OK; } +/* +** Read and return a 16-bit little-endian unsigned integer from buffer aBuf. +*/ static u16 zipfileGetU16(const u8 *aBuf){ return (aBuf[1] << 8) + aBuf[0]; } + +/* +** Read and return a 32-bit little-endian unsigned integer from buffer aBuf. +*/ static u32 zipfileGetU32(const u8 *aBuf){ return ((u32)(aBuf[3]) << 24) + ((u32)(aBuf[2]) << 16) @@ -489,10 +512,17 @@ static u32 zipfileGetU32(const u8 *aBuf){ + ((u32)(aBuf[0]) << 0); } +/* +** Write a 16-bit little endiate integer into buffer aBuf. +*/ static void zipfilePutU16(u8 *aBuf, u16 val){ aBuf[0] = val & 0xFF; aBuf[1] = (val>>8) & 0xFF; } + +/* +** Write a 32-bit little endiate integer into buffer aBuf. +*/ static void zipfilePutU32(u8 *aBuf, u32 val){ aBuf[0] = val & 0xFF; aBuf[1] = (val>>8) & 0xFF; @@ -509,7 +539,6 @@ static void zipfilePutU32(u8 *aBuf, u32 val){ /* ** Magic numbers used to read CDS records. */ -#define ZIPFILE_CDS_FIXED_SZ 46 #define ZIPFILE_CDS_NFILE_OFF 28 #define ZIPFILE_CDS_SZCOMPRESSED_OFF 20 @@ -548,6 +577,44 @@ static int zipfileReadCDS(u8 *aBuf, ZipfileCDS *pCDS){ } /* +** Decode the LFH record in buffer aBuf into (*pLFH). Return SQLITE_ERROR +** if the record is not well-formed, or SQLITE_OK otherwise. +*/ +static int zipfileReadLFH( + u8 *aBuffer, + ZipfileLFH *pLFH +){ + u8 *aRead = aBuffer; + int rc = SQLITE_OK; + + u32 sig = zipfileRead32(aRead); + if( sig!=ZIPFILE_SIGNATURE_LFH ){ + rc = SQLITE_ERROR; + }else{ + pLFH->iVersionExtract = zipfileRead16(aRead); + pLFH->flags = zipfileRead16(aRead); + pLFH->iCompression = zipfileRead16(aRead); + pLFH->mTime = zipfileRead16(aRead); + pLFH->mDate = zipfileRead16(aRead); + pLFH->crc32 = zipfileRead32(aRead); + pLFH->szCompressed = zipfileRead32(aRead); + pLFH->szUncompressed = zipfileRead32(aRead); + pLFH->nFile = zipfileRead16(aRead); + pLFH->nExtra = zipfileRead16(aRead); + } + return rc; +} + + +/* +** Buffer aExtra (size nExtra bytes) contains zip archive "extra" fields. +** Scan through this buffer to find an "extra-timestamp" field. If one +** exists, extract the 32-bit modification-timestamp from it and store +** the value in output parameter *pmTime. +** +** Zero is returned if no extra-timestamp record could be found (and so +** *pmTime is left unchanged), or non-zero otherwise. +** ** The general format of an extra field is: ** ** Header ID 2 bytes @@ -580,6 +647,10 @@ static int zipfileScanExtra(u8 *aExtra, int nExtra, u32 *pmTime){ } /* +** Convert the standard MS-DOS timestamp stored in the mTime and mDate +** fields of the CDS structure passed as the only argument to a 32-bit +** UNIX seconds-since-the-epoch timestamp. Return the result. +** ** "Standard" MS-DOS time format: ** ** File modification time: @@ -590,6 +661,8 @@ static int zipfileScanExtra(u8 *aExtra, int nExtra, u32 *pmTime){ ** Bits 00-04: day ** Bits 05-08: month (1-12) ** Bits 09-15: years from 1980 +** +** https://msdn.microsoft.com/en-us/library/9kkf9tah.aspx */ static time_t zipfileMtime(ZipfileCDS *pCDS){ struct tm t; @@ -605,42 +678,50 @@ static time_t zipfileMtime(ZipfileCDS *pCDS){ return mktime(&t); } -static int zipfileReadLFH( - u8 *aBuffer, - ZipfileLFH *pLFH -){ - u8 *aRead = aBuffer; - int rc = SQLITE_OK; +/* +** The opposite of zipfileMtime(). This function populates the mTime and +** mDate fields of the CDS structure passed as the first argument according +** to the UNIX timestamp value passed as the second. +*/ +static void zipfileMtimeToDos(ZipfileCDS *pCds, u32 mUnixTime){ + time_t t = (time_t)mUnixTime; + struct tm res; - u32 sig = zipfileRead32(aRead); - if( sig!=ZIPFILE_SIGNATURE_LFH ){ - rc = SQLITE_ERROR; - }else{ - pLFH->iVersionExtract = zipfileRead16(aRead); - pLFH->flags = zipfileRead16(aRead); - pLFH->iCompression = zipfileRead16(aRead); - pLFH->mTime = zipfileRead16(aRead); - pLFH->mDate = zipfileRead16(aRead); - pLFH->crc32 = zipfileRead32(aRead); - pLFH->szCompressed = zipfileRead32(aRead); - pLFH->szUncompressed = zipfileRead32(aRead); - pLFH->nFile = zipfileRead16(aRead); - pLFH->nExtra = zipfileRead16(aRead); - } - return rc; +#if !defined(_WIN32) && !defined(WIN32) + localtime_r(&t, &res); +#else + memcpy(&res, localtime(&t), sizeof(struct tm)); +#endif + + pCds->mTime = (u16)( + (res.tm_sec / 2) + + (res.tm_min << 5) + + (res.tm_hour << 11)); + + pCds->mDate = (u16)( + (res.tm_mday-1) + + ((res.tm_mon+1) << 5) + + ((res.tm_year-80) << 9)); } /* -** Read a Zip archive CDS header from offset iOff of file pFile. Return -** SQLITE_OK if successful, or an SQLite error code otherwise. +** If aBlob is not NULL, then it is a pointer to a buffer (nBlob bytes in +** size) containing an entire zip archive image. Or, if aBlob is NULL, +** then pFile is a file-handle open on a zip file. In either case, this +** function creates a ZipfileEntry object based on the zip archive entry +** for which the CDS record is at offset iOff. +** +** If successful, SQLITE_OK is returned and (*ppEntry) set to point to +** the new object. Otherwise, an SQLite error code is returned and the +** final value of (*ppEntry) undefined. */ static int zipfileGetEntry( ZipfileTab *pTab, /* Store any error message here */ const u8 *aBlob, /* Pointer to in-memory file image */ int nBlob, /* Size of aBlob[] in bytes */ FILE *pFile, /* If aBlob==0, read from this file */ - i64 iOff, - ZipfileEntry **ppEntry + i64 iOff, /* Offset of CDS record */ + ZipfileEntry **ppEntry /* OUT: Pointer to new object */ ){ u8 *aRead; char **pzErr = &pTab->base.zErrMsg; @@ -729,12 +810,6 @@ static int zipfileGetEntry( return rc; } -static FILE *zipfileGetFd(ZipfileCsr *pCsr){ - if( pCsr->pFile ) return pCsr->pFile; - return ((ZipfileTab*)(pCsr->base.pVtab))->pWriteFd; -} - - /* ** Advance an ZipfileCsr to its next row of output. */ @@ -771,27 +846,17 @@ static int zipfileNext(sqlite3_vtab_cursor *cur){ return rc; } -static void zipfileMtimeToDos(ZipfileCDS *pCds, u32 mTime){ - time_t t = (time_t)mTime; - struct tm res; - -#if !defined(_WIN32) && !defined(WIN32) - localtime_r(&t, &res); -#else - memcpy(&res, localtime(&t), sizeof(struct tm)); -#endif - - pCds->mTime = (u16)( - (res.tm_sec / 2) + - (res.tm_min << 5) + - (res.tm_hour << 11)); - - pCds->mDate = (u16)( - (res.tm_mday-1) + - ((res.tm_mon+1) << 5) + - ((res.tm_year-80) << 9)); +static void zipfileFree(void *p) { + sqlite3_free(p); } +/* +** Buffer aIn (size nIn bytes) contains compressed data. Uncompressed, the +** size is nOut bytes. This function uncompresses the data and sets the +** return value in context pCtx to the result (a blob). +** +** If an error occurs, an error code is left in pCtx instead. +*/ static void zipfileInflate( sqlite3_context *pCtx, /* Store error here, if any */ const u8 *aIn, /* Compressed data */ @@ -819,7 +884,8 @@ static void zipfileInflate( if( err!=Z_STREAM_END ){ zipfileCtxErrorMsg(pCtx, "inflate() failed (%d)", err); }else{ - sqlite3_result_blob(pCtx, aRes, nOut, SQLITE_TRANSIENT); + sqlite3_result_blob(pCtx, aRes, nOut, zipfileFree); + aRes = 0; } } sqlite3_free(aRes); @@ -827,6 +893,18 @@ static void zipfileInflate( } } +/* +** Buffer aIn (size nIn bytes) contains uncompressed data. This function +** compresses it and sets (*ppOut) to point to a buffer containing the +** compressed data. The caller is responsible for eventually calling +** sqlite3_free() to release buffer (*ppOut). Before returning, (*pnOut) +** is set to the size of buffer (*ppOut) in bytes. +** +** If no error occurs, SQLITE_OK is returned. Otherwise, an SQLite error +** code is returned and an error message left in virtual-table handle +** pTab. The values of (*ppOut) and (*pnOut) are left unchanged in this +** case. +*/ static int zipfileDeflate( ZipfileTab *pTab, /* Set error message here */ const u8 *aIn, int nIn, /* Input */ @@ -913,12 +991,13 @@ static int zipfileColumn( if( aBuf==0 ){ rc = SQLITE_NOMEM; }else{ - FILE *pFile = zipfileGetFd(pCsr); - if( rc==SQLITE_OK ){ - rc = zipfileReadData(pFile, aBuf, sz, pCsr->pCurrent->iDataOff, - &pCsr->base.pVtab->zErrMsg - ); + FILE *pFile = pCsr->pFile; + if( pFile==0 ){ + pFile = ((ZipfileTab*)(pCsr->base.pVtab))->pWriteFd; } + rc = zipfileReadData(pFile, aBuf, sz, pCsr->pCurrent->iDataOff, + &pCsr->base.pVtab->zErrMsg + ); } } if( rc==SQLITE_OK ){ @@ -953,16 +1032,7 @@ static int zipfileColumn( } /* -** Return the rowid for the current row. -*/ -static int zipfileRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ - assert( 0 ); - return SQLITE_OK; -} - -/* -** Return TRUE if the cursor has been moved off of the last -** row of output. +** Return TRUE if the cursor is at EOF. */ static int zipfileEof(sqlite3_vtab_cursor *cur){ ZipfileCsr *pCsr = (ZipfileCsr*)cur; @@ -970,6 +1040,14 @@ static int zipfileEof(sqlite3_vtab_cursor *cur){ } /* +** If aBlob is not NULL, then it points to a buffer nBlob bytes in size +** containing an entire zip archive image. Or, if aBlob is NULL, then pFile +** is guaranteed to be a file-handle open on a zip file. +** +** This function attempts to locate the EOCD record within the zip archive +** and populate *pEOCD with the results of decoding it. SQLITE_OK is +** returned if successful. Otherwise, an SQLite error code is returned and +** an English language error message may be left in virtual-table pTab. */ static int zipfileReadEOCD( ZipfileTab *pTab, /* Return errors here */ @@ -1030,8 +1108,10 @@ static int zipfileReadEOCD( } /* -** Add object pNew to the end of the linked list that begins at -** ZipfileTab.pFirstEntry and ends with pLastEntry. +** Add object pNew to the linked list that begins at ZipfileTab.pFirstEntry +** and ends with pLastEntry. If argument pBefore is NULL, then pNew is added +** to the end of the list. Otherwise, it is added to the list immediately +** before pBefore (which is guaranteed to be a part of said list). */ static void zipfileAddEntry( ZipfileTab *pTab, @@ -1647,8 +1727,6 @@ static void zipfileFunctionCds( } } -static void zipfileFree(void *p) { sqlite3_free(p); } - static void zipfileFunctionBlob( sqlite3_context *context, int argc, @@ -1758,7 +1836,7 @@ static int zipfileRegister(sqlite3 *db){ zipfileNext, /* xNext - advance a cursor */ zipfileEof, /* xEof - check for end of scan */ zipfileColumn, /* xColumn - read data */ - zipfileRowid, /* xRowid - read data */ + 0, /* xRowid - read data */ zipfileUpdate, /* xUpdate */ zipfileBegin, /* xBegin */ 0, /* xSync */ diff --git a/manifest b/manifest index 96a7ea5fe4..e699a399ac 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C If\sa\szipfile\svirtual\stable\sis\screated\swith\sno\sargument\s-\s"CREATE\sVIRTUAL\sTABLE\nzzz\sUSING\szipfile()"\s-\saccumulate\sdata\sin\smemory.\sSupport\s"SELECT\nzipfile_blob(z)\sFROM\szzz\sLIMIT\s1"\sto\sretrieve\sa\szip\sarchive\simage. -D 2018-01-27T16:29:59.042 +C Fix\smissing\sheader\scomments\sand\sother\scode\sissues\sin\szipfile.c. +D 2018-01-27T18:55:18.394 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -304,7 +304,7 @@ F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178 F ext/misc/vtablog.c 31d0d8f4406795679dcd3a67917c213d3a2a5fb3ea5de35f6e773491ed7e13c9 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 -F ext/misc/zipfile.c df57128d9ad2a1e60097d7971e787b582fb66ce0926577cb6f5978f7af210b8c +F ext/misc/zipfile.c 398f53469ed90076dd5d69494eb7f1bd01a42d402fc5dbe92374178887244805 F ext/rbu/rbu.c ea7d1b7eb44c123a2a619332e19fe5313500705c4a58aaa1887905c0d83ffc2e F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842 F ext/rbu/rbu10.test 1846519a438697f45e9dcb246908af81b551c29e1078d0304fae83f1fed7e9ee @@ -1702,7 +1702,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 19f5c1400054df10688ab448e7e23afef97cab4a7c7a3e411f7527509b515dd8 -R 8b2b96d9b8d1e07c4df7e0d2971e5e57 +P e63185edfe0c316aa60c1fa085d032425ecc7db54536dfa5a977772eaf3c240e +R 347b5f95fee3abc6c3f7640c1a70a4d4 U dan -Z e468bfc5bd8b08bb5b9404b26ad000bc +Z faf05f7eafa9394c044e435efe4a986b diff --git a/manifest.uuid b/manifest.uuid index a8aca2c7e5..7c9ca14be3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e63185edfe0c316aa60c1fa085d032425ecc7db54536dfa5a977772eaf3c240e \ No newline at end of file +6ea8ba312c38365d3e28cfb2a367d729dd2751d1d853843eea0e18126777a320 \ No newline at end of file From 26e731cc8862999c595e31dae1dbcb698d08ebb3 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 29 Jan 2018 16:22:39 +0000 Subject: [PATCH 019/107] Ensure the "unique-not-null" flag is set for automatic indexes on columns declared with "col UNIQUE NOT NULL" (where the NOT NULL comes after the UNIQUE). FossilOrigin-Name: 8767f7b880f2e4112f75f0b6ef7be3f50ab1ae20e103e7d03d8bfe77e6c79438 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/build.c | 20 ++++++++++++++++++-- src/sqliteInt.h | 1 + test/notnull.test | 45 +++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 73 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index e699a399ac..7a5772a6de 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\smissing\sheader\scomments\sand\sother\scode\sissues\sin\szipfile.c. -D 2018-01-27T18:55:18.394 +C Ensure\sthe\s"unique-not-null"\sflag\sis\sset\sfor\sautomatic\sindexes\son\scolumns\ndeclared\swith\s"col\sUNIQUE\sNOT\sNULL"\s(where\sthe\sNOT\sNULL\scomes\safter\sthe\nUNIQUE). +D 2018-01-29T16:22:39.280 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -432,7 +432,7 @@ F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca F src/btree.c d711228cac336fb35fff21f3f4a0efe2ad58aa9a800dd02929cdf184be1e78a3 F src/btree.h 0866c0a08255142ea0e754aabd211c843cab32045c978a592a43152405ed0c84 F src/btreeInt.h 620ab4c7235f43572cf3ac2ac8723cbdf68073be4d29da24897c7b77dda5fd96 -F src/build.c 9f9647454f236cab097f266ae970f899b53c71cadab6756c47e2b2e81392c2a1 +F src/build.c 672022c06e1a5c2653f80c77a687de11f7e65ce81d20fe2825aadfa13a875c33 F src/callback.c fe677cb5f5abb02f7a772a62a98c2f516426081df68856e8f2d5f950929b966a F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c ff1be3eed7bdd75aaca61ca8dc848f7c9f850ef2fb9cb56f2734e922a098f9c0 @@ -492,7 +492,7 @@ F src/shell.c.in 7cea439c3f7f2e4ed6eb4b3a633cd93dccb1349241400de4da0c1291285ed51 F src/sqlite.h.in 51f9acf52c80113d793ddd38b3940ad6895d97b4752503b19291fb8fcbf54c5e F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 83a3c4ce93d650bedfd1aa558cb85a516bd6d094445ee989740827d0d944368d -F src/sqliteInt.h 9c70315598b34810a83e4894455acb18e95cf63ce4e6cbb451ac2d17eabc2544 +F src/sqliteInt.h 119e240796d23054148005524f1d54a61e27dbef53bef3bc7ab65001405f4751 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -1100,7 +1100,7 @@ F test/normalize.test 501630ab49b0b26b65c74124bf03e3374c1b57fa97aae750f848036091 F test/notify1.test 669b2b743618efdc18ca4b02f45423d5d2304abf F test/notify2.test 2ecabaa1305083856b7c39cf32816b612740c161 F test/notify3.test 10ff25cde502e72a92053a2f215d64bece4ef934 -F test/notnull.test f8fcf58669ddba79274daa2770d61dfad8274f62 +F test/notnull.test b6999231221df3534827e45e2005dd7a815fdd5f2c2e1afb9be21ead410816f8 F test/null.test 0dcce4f04284ec66108c503327ad6d224c0752b3 F test/numcast.test 5d126f7f581432e86a90d1e35cac625164aec4a1 F test/numindex1.test 20a5450d4b056e48cd5db30e659f13347a099823 @@ -1702,7 +1702,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e63185edfe0c316aa60c1fa085d032425ecc7db54536dfa5a977772eaf3c240e -R 347b5f95fee3abc6c3f7640c1a70a4d4 +P 6ea8ba312c38365d3e28cfb2a367d729dd2751d1d853843eea0e18126777a320 +R 3ff6b96757835180119359cd3166f741 U dan -Z faf05f7eafa9394c044e435efe4a986b +Z 6c5f0f4d34834e8c6ccf2da8ff3a018a diff --git a/manifest.uuid b/manifest.uuid index 7c9ca14be3..9eae811f9a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6ea8ba312c38365d3e28cfb2a367d729dd2751d1d853843eea0e18126777a320 \ No newline at end of file +8767f7b880f2e4112f75f0b6ef7be3f50ab1ae20e103e7d03d8bfe77e6c79438 \ No newline at end of file diff --git a/src/build.c b/src/build.c index 58b39d6475..3a656def3e 100644 --- a/src/build.c +++ b/src/build.c @@ -1118,10 +1118,24 @@ void sqlite3AddColumn(Parse *pParse, Token *pName, Token *pType){ */ void sqlite3AddNotNull(Parse *pParse, int onError){ Table *p; + Column *pCol; p = pParse->pNewTable; if( p==0 || NEVER(p->nCol<1) ) return; - p->aCol[p->nCol-1].notNull = (u8)onError; + pCol = &p->aCol[p->nCol-1]; + pCol->notNull = (u8)onError; p->tabFlags |= TF_HasNotNull; + + /* Set the uniqNotNull flag on any UNIQUE or PK indexes already created + ** on this column. */ + if( pCol->colFlags & COLFLAG_UNIQUE ){ + Index *pIdx; + for(pIdx=p->pIndex; pIdx; pIdx=pIdx->pNext){ + assert( pIdx->nKeyCol==1 && pIdx->onError!=OE_None ); + if( pIdx->aiColumn[0]==p->nCol-1 ){ + pIdx->uniqNotNull = 1; + } + } + } } /* @@ -3085,7 +3099,9 @@ void sqlite3CreateIndex( */ if( pList==0 ){ Token prevCol; - sqlite3TokenInit(&prevCol, pTab->aCol[pTab->nCol-1].zName); + Column *pCol = &pTab->aCol[pTab->nCol-1]; + pCol->colFlags |= COLFLAG_UNIQUE; + sqlite3TokenInit(&prevCol, pCol->zName); pList = sqlite3ExprListAppend(pParse, 0, sqlite3ExprAlloc(db, TK_ID, &prevCol, 0)); if( pList==0 ) goto exit_create_index; diff --git a/src/sqliteInt.h b/src/sqliteInt.h index b7b402b8e0..f408ff62a5 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1754,6 +1754,7 @@ struct Column { #define COLFLAG_PRIMKEY 0x0001 /* Column is part of the primary key */ #define COLFLAG_HIDDEN 0x0002 /* A hidden column in a virtual table */ #define COLFLAG_HASTYPE 0x0004 /* Type name follows column name */ +#define COLFLAG_UNIQUE 0x0008 /* Column def contains "UNIQUE" or "PK" */ /* ** A "Collating Sequence" is defined by an instance of the following diff --git a/test/notnull.test b/test/notnull.test index 23fd33d4ba..32d95eaf24 100644 --- a/test/notnull.test +++ b/test/notnull.test @@ -561,4 +561,49 @@ do_test notnull-5.5 { execsql { SELECT * FROM t1 } } {1 2} +#------------------------------------------------------------------------- +# Check that UNIQUE NOT NULL indexes are always recognized as such. +# +proc uses_op_next {sql} { + db eval "EXPLAIN $sql" a { + if {$a(opcode)=="Next"} { return 1 } + } + return 0 +} + +proc do_uses_op_next_test {tn sql res} { + uplevel [list do_test $tn [list uses_op_next $sql] $res] +} + +reset_db +do_execsql_test notnull-6.0 { + CREATE TABLE t1(a UNIQUE); + CREATE TABLE t2(a NOT NULL UNIQUE); + CREATE TABLE t3(a UNIQUE NOT NULL); + CREATE TABLE t4(a NOT NULL); + CREATE UNIQUE INDEX t4a ON t4(a); + + CREATE TABLE t5(a PRIMARY KEY); + CREATE TABLE t6(a PRIMARY KEY NOT NULL); + CREATE TABLE t7(a NOT NULL PRIMARY KEY); + CREATE TABLE t8(a PRIMARY KEY) WITHOUT ROWID; + + CREATE TABLE t9(a PRIMARY KEY UNIQUE NOT NULL); + CREATE TABLE t10(a UNIQUE PRIMARY KEY NOT NULL); +} + +do_uses_op_next_test notnull-6.1 "SELECT * FROM t1 WHERE a IS ?" 1 +do_uses_op_next_test notnull-6.2 "SELECT * FROM t2 WHERE a IS ?" 0 +do_uses_op_next_test notnull-6.3 "SELECT * FROM t3 WHERE a IS ?" 0 +do_uses_op_next_test notnull-6.4 "SELECT * FROM t4 WHERE a IS ?" 0 + +do_uses_op_next_test notnull-6.5 "SELECT * FROM t5 WHERE a IS ?" 1 +do_uses_op_next_test notnull-6.6 "SELECT * FROM t6 WHERE a IS ?" 0 +do_uses_op_next_test notnull-6.7 "SELECT * FROM t7 WHERE a IS ?" 0 +do_uses_op_next_test notnull-6.8 "SELECT * FROM t8 WHERE a IS ?" 0 + +do_uses_op_next_test notnull-6.9 "SELECT * FROM t8 WHERE a IS ?" 0 +do_uses_op_next_test notnull-6.10 "SELECT * FROM t8 WHERE a IS ?" 0 + finish_test + From 8433e7166b010ebcb727e3a7d85ae3a484fdc321 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 29 Jan 2018 17:08:52 +0000 Subject: [PATCH 020/107] Update the omit-left-join optimization so that it works in some cases when the RHS is subject to a UNIQUE but not NOT NULL constraint. FossilOrigin-Name: 88411a402b29f19d69e0d06d9099eabb98c238ddb7c07ce110236316f008753e --- manifest | 17 +++++++------ manifest.uuid | 2 +- src/where.c | 10 +++++--- test/join2.test | 66 +++++++++++++++++++++++++++++++++++++++++++++++-- 4 files changed, 82 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 7a5772a6de..7c15e392d1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Ensure\sthe\s"unique-not-null"\sflag\sis\sset\sfor\sautomatic\sindexes\son\scolumns\ndeclared\swith\s"col\sUNIQUE\sNOT\sNULL"\s(where\sthe\sNOT\sNULL\scomes\safter\sthe\nUNIQUE). -D 2018-01-29T16:22:39.280 +C Update\sthe\somit-left-join\soptimization\sso\sthat\sit\sworks\sin\ssome\scases\swhen\sthe\nRHS\sis\ssubject\sto\sa\sUNIQUE\sbut\snot\sNOT\sNULL\sconstraint. +D 2018-01-29T17:08:52.190 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -570,7 +570,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 5a3f464edd64596f601683ed321d12e6fd93c5fb9afdfb3653d6ffd0fee9c48f F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a F src/walker.c da987a20d40145c0a03c07d8fefcb2ed363becc7680d0500d9c79915591f5b1f -F src/where.c 10c7e13c909e4dc93057fae930733f5018cdf1a040f73b9e873a264edfc101e5 +F src/where.c eb37882608c30f7361cf3309b8128c3a2887ff0099886e3685ededb7a7e6b051 F src/whereInt.h 82c04c5075308abbac59180c8bad5ecb45b07453981f60a53f3c7dee21e1e971 F src/wherecode.c cb360c511f69294ddf00340d7b390e6b1d601a1cfb77b42c4d316fe2f4cd01c3 F src/whereexpr.c 22dbfd3bf5f6051a61523dd0ebef7a944fb29ee4aa7d2a62feb8aac6ffbbc0eb @@ -1001,7 +1001,7 @@ F test/ioerr4.test f130fe9e71008577b342b8874d52984bd04ede2c F test/ioerr5.test 2edfa4fb0f896f733071303b42224df8bedd9da4 F test/ioerr6.test a395a6ab144b26a9e3e21059a1ab6a7149cca65b F test/join.test 442c462eea85cf065d70a663c626b780a95af6e11585d909bb63b87598afe678 -F test/join2.test 1a0c26399910b015d9f8f95b884e9a079fd2cfdccd65f7b1603846508cae0dc6 +F test/join2.test 96f56ccf0a8fe6dbd7bb2e1a90bc61540febeaf9b6fdee14506f6fc140b93185 F test/join3.test 6f0c774ff1ba0489e6c88a3e77b9d3528fb4fda0 F test/join4.test 1a352e4e267114444c29266ce79e941af5885916 F test/join5.test bc98ea4b4e5003f5b1453701ebb8cd7d1c01a550 @@ -1702,7 +1702,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6ea8ba312c38365d3e28cfb2a367d729dd2751d1d853843eea0e18126777a320 -R 3ff6b96757835180119359cd3166f741 +P 8767f7b880f2e4112f75f0b6ef7be3f50ab1ae20e103e7d03d8bfe77e6c79438 +R f19810b3aa9bfaa7d3b666e31ece61ec +T *branch * omit-left-join-fix +T *sym-omit-left-join-fix * +T -sym-trunk * U dan -Z 6c5f0f4d34834e8c6ccf2da8ff3a018a +Z ff1cffb196461bca97e3f8e467ebcd67 diff --git a/manifest.uuid b/manifest.uuid index 9eae811f9a..84c8ec4673 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8767f7b880f2e4112f75f0b6ef7be3f50ab1ae20e103e7d03d8bfe77e6c79438 \ No newline at end of file +88411a402b29f19d69e0d06d9099eabb98c238ddb7c07ce110236316f008753e \ No newline at end of file diff --git a/src/where.c b/src/where.c index 7f45182d9a..3b18fbf5ff 100644 --- a/src/where.c +++ b/src/where.c @@ -2481,10 +2481,13 @@ static int whereLoopAddBtreeIndex( if( iCol==XN_ROWID || (iCol>=0 && nInMul==0 && saved_nEq==pProbe->nKeyCol-1) ){ - if( iCol>=0 && pProbe->uniqNotNull==0 ){ - pNew->wsFlags |= WHERE_UNQ_WANTED; - }else{ + assert( iCol!=XN_ROWID || pProbe->uniqNotNull ); + if( pProbe->uniqNotNull + || (pProbe->nKeyCol==1 && pProbe->onError && eOp==WO_EQ) + ){ pNew->wsFlags |= WHERE_ONEROW; + }else{ + pNew->wsFlags |= WHERE_UNQ_WANTED; } } }else if( eOp & WO_ISNULL ){ @@ -2825,6 +2828,7 @@ static int whereLoopAddBtree( sPk.onError = OE_Replace; sPk.pTable = pTab; sPk.szIdxRow = pTab->szTabRow; + sPk.uniqNotNull = 1; aiRowEstPk[0] = pTab->nRowLogEst; aiRowEstPk[1] = 0; pFirst = pSrc->pTab->pIndex; diff --git a/test/join2.test b/test/join2.test index 76006f1ecd..3dc067e508 100644 --- a/test/join2.test +++ b/test/join2.test @@ -155,20 +155,65 @@ do_execsql_test 4.1.4 { SELECT v1, v3 FROM c1 LEFT JOIN c2 LEFT JOIN c3 ON (c3.k=v1+1); } {2 v3 2 v3 1112 {} 1112 {}} -do_eqp_test 4.2.1 { +do_eqp_test 4.1.5 { SELECT v1, v3 FROM c1 LEFT JOIN c2 ON (c2.k=v1) LEFT JOIN c3 ON (c3.k=v2); } { 0 0 0 {SCAN TABLE c1} 0 1 1 {SEARCH TABLE c2 USING INTEGER PRIMARY KEY (rowid=?)} 0 2 2 {SEARCH TABLE c3 USING INTEGER PRIMARY KEY (rowid=?)} } -do_eqp_test 4.2.2 { +do_eqp_test 4.1.6 { SELECT v1, v3 FROM c1 LEFT JOIN c2 ON (c2.k=v1) LEFT JOIN c3 ON (c3.k=v1+1); } { 0 0 0 {SCAN TABLE c1} 0 1 2 {SEARCH TABLE c3 USING INTEGER PRIMARY KEY (rowid=?)} } +do_execsql_test 4.2.0 { + DROP TABLE c1; + DROP TABLE c2; + DROP TABLE c3; + CREATE TABLE c1(k UNIQUE, v1); + CREATE TABLE c2(k UNIQUE, v2); + CREATE TABLE c3(k UNIQUE, v3); + + INSERT INTO c1 VALUES(1, 2); + INSERT INTO c2 VALUES(2, 3); + INSERT INTO c3 VALUES(3, 'v3'); + + INSERT INTO c1 VALUES(111, 1112); + INSERT INTO c2 VALUES(112, 1113); + INSERT INTO c3 VALUES(113, 'v1113'); +} +do_execsql_test 4.2.1 { + SELECT v1, v3 FROM c1 LEFT JOIN c2 ON (c2.k=v1) LEFT JOIN c3 ON (c3.k=v2); +} {2 v3 1112 {}} +do_execsql_test 4.2.2 { + SELECT v1, v3 FROM c1 LEFT JOIN c2 ON (c2.k=v1) LEFT JOIN c3 ON (c3.k=v1+1); +} {2 v3 1112 {}} + +do_execsql_test 4.2.3 { + SELECT DISTINCT v1, v3 FROM c1 LEFT JOIN c2 LEFT JOIN c3 ON (c3.k=v1+1); +} {2 v3 1112 {}} + +do_execsql_test 4.2.4 { + SELECT v1, v3 FROM c1 LEFT JOIN c2 LEFT JOIN c3 ON (c3.k=v1+1); +} {2 v3 2 v3 1112 {} 1112 {}} + +do_eqp_test 4.2.5 { + SELECT v1, v3 FROM c1 LEFT JOIN c2 ON (c2.k=v1) LEFT JOIN c3 ON (c3.k=v2); +} { + 0 0 0 {SCAN TABLE c1} + 0 1 1 {SEARCH TABLE c2 USING INDEX sqlite_autoindex_c2_1 (k=?)} + 0 2 2 {SEARCH TABLE c3 USING INDEX sqlite_autoindex_c3_1 (k=?)} +} +do_eqp_test 4.2.6 { + SELECT v1, v3 FROM c1 LEFT JOIN c2 ON (c2.k=v1) LEFT JOIN c3 ON (c3.k=v1+1); +} { + 0 0 0 {SCAN TABLE c1} + 0 1 2 {SEARCH TABLE c3 USING INDEX sqlite_autoindex_c3_1 (k=?)} +} + # 2017-11-23 (Thanksgiving day) # OSSFuzz found an assertion fault in the new LEFT JOIN eliminator code. # @@ -192,4 +237,21 @@ do_execsql_test 4.3.1 { LEFT JOIN t2 AS c ON (a.x=c.x); } {1 {} 2 {} 3 {} 4 {} 5 {} 6 {} 7 {} 8 {} 9 {} 10 10} +do_execsql_test 5.0 { + CREATE TABLE s1 (a INTEGER PRIMARY KEY); + CREATE TABLE s2 (a INTEGER PRIMARY KEY); + CREATE TABLE s3 (a INTEGER); + CREATE UNIQUE INDEX ndx on s3(a); +} +do_eqp_test 5.1 { + SELECT s1.a FROM s1 left join s2 using (a); +} { + 0 0 0 {SCAN TABLE s1} +} +do_eqp_test 5.2 { + SELECT s1.a FROM s1 left join s3 using (a); +} { + 0 0 0 {SCAN TABLE s1} +} + finish_test From f8c4b99a0273297cd1779304444b99b79eae6ab4 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 29 Jan 2018 18:41:07 +0000 Subject: [PATCH 021/107] Add aggregate function zipfile() to the zipfile extension. For composing new zip archives in memory. FossilOrigin-Name: e364eeac76a8225146b37d801bc6cabe03e9abede5a1412ebe9d94a32d8838cc --- ext/misc/zipfile.c | 282 ++++++++++++++++++++++++++++++++++++++++++--- manifest | 14 +-- manifest.uuid | 2 +- test/zipfile.test | 48 +++++--- 4 files changed, 302 insertions(+), 44 deletions(-) diff --git a/ext/misc/zipfile.c b/ext/misc/zipfile.c index efc25927d4..f325868149 100644 --- a/ext/misc/zipfile.c +++ b/ext/misc/zipfile.c @@ -906,9 +906,9 @@ static void zipfileInflate( ** case. */ static int zipfileDeflate( - ZipfileTab *pTab, /* Set error message here */ const u8 *aIn, int nIn, /* Input */ - u8 **ppOut, int *pnOut /* Output */ + u8 **ppOut, int *pnOut, /* Output */ + char **pzErr /* OUT: Error message */ ){ int nAlloc = (int)compressBound(nIn); u8 *aOut; @@ -934,7 +934,7 @@ static int zipfileDeflate( *pnOut = (int)str.total_out; }else{ sqlite3_free(aOut); - pTab->base.zErrMsg = sqlite3_mprintf("zipfile: deflate() error"); + *pzErr = sqlite3_mprintf("zipfile: deflate() error"); rc = SQLITE_ERROR; } deflateEnd(&str); @@ -1315,15 +1315,15 @@ static int zipfileAppendEntry( } static int zipfileGetMode( - ZipfileTab *pTab, sqlite3_value *pVal, - u32 defaultMode, /* Value to use if pVal IS NULL */ - u32 *pMode + int bIsDir, /* If true, default to directory */ + u32 *pMode, /* OUT: Mode value */ + char **pzErr /* OUT: Error message */ ){ const char *z = (const char*)sqlite3_value_text(pVal); u32 mode = 0; if( z==0 ){ - mode = defaultMode; + mode = (bIsDir ? (S_IFDIR + 0755) : (S_IFREG + 0644)); }else if( z[0]>='0' && z[0]<='9' ){ mode = (unsigned int)sqlite3_value_int(pVal); }else{ @@ -1343,11 +1343,16 @@ static int zipfileGetMode( else if( z[i]!='-' ) goto parse_error; } } + if( (bIsDir == ((mode & S_IFDIR)==0)) ){ + /* The "mode" attribute is a directory, but data has been specified. + ** Or vice-versa - no data but "mode" is a file or symlink. */ + return SQLITE_CONSTRAINT; + } *pMode = mode; return SQLITE_OK; parse_error: - pTab->base.zErrMsg = sqlite3_mprintf("zipfile: parse error in mode: %s", z); + *pzErr = sqlite3_mprintf("zipfile: parse error in mode: %s", z); return SQLITE_ERROR; } @@ -1432,7 +1437,7 @@ static int zipfileUpdate( }else{ if( bAuto || iMethod ){ int nCmp; - rc = zipfileDeflate(pTab, aIn, nIn, &pFree, &nCmp); + rc = zipfileDeflate(aIn, nIn, &pFree, &nCmp, &pTab->base.zErrMsg); if( rc==SQLITE_OK ){ if( iMethod || nCmpbase.zErrMsg); } if( rc==SQLITE_OK ){ @@ -1819,6 +1817,251 @@ static int zipfileFindFunction( return 0; } +typedef struct ZipfileBuffer ZipfileBuffer; +struct ZipfileBuffer { + u8 *a; /* Pointer to buffer */ + int n; /* Size of buffer in bytes */ + int nAlloc; /* Byte allocated at a[] */ +}; + +typedef struct ZipfileCtx ZipfileCtx; +struct ZipfileCtx { + int nEntry; + ZipfileBuffer body; + ZipfileBuffer cds; +}; + +static int zipfileBufferGrow(ZipfileBuffer *pBuf, int nByte){ + if( pBuf->n+nByte>pBuf->nAlloc ){ + u8 *aNew; + int nNew = pBuf->n ? pBuf->n*2 : 512; + int nReq = pBuf->n + nByte; + + while( nNewa, nNew); + if( aNew==0 ) return SQLITE_NOMEM; + pBuf->a = aNew; + pBuf->nAlloc = nNew; + } + return SQLITE_OK; +} + +/* +** xStep() callback for the zipfile() aggregate. This can be called in +** any of the following ways: +** +** SELECT zipfile(name,data) ... +** SELECT zipfile(name,mode,mtime,data) ... +** SELECT zipfile(name,mode,mtime,data,method) ... +*/ +void zipfileStep(sqlite3_context *pCtx, int nVal, sqlite3_value **apVal){ + ZipfileCtx *p; /* Aggregate function context */ + ZipfileEntry e; /* New entry to add to zip archive */ + + sqlite3_value *pName = 0; + sqlite3_value *pMode = 0; + sqlite3_value *pMtime = 0; + sqlite3_value *pData = 0; + sqlite3_value *pMethod = 0; + + int bIsDir = 0; + u32 mode; + int rc = SQLITE_OK; + char *zErr = 0; + + int iMethod = -1; /* Compression method to use (0 or 8) */ + + const u8 *aData = 0; /* Possibly compressed data for new entry */ + int nData = 0; /* Size of aData[] in bytes */ + int szUncompressed = 0; /* Size of data before compression */ + u8 *aFree = 0; /* Free this before returning */ + u32 iCrc32 = 0; /* crc32 of uncompressed data */ + + char *zName = 0; /* Path (name) of new entry */ + int nName = 0; /* Size of zName in bytes */ + char *zFree = 0; /* Free this before returning */ + int nByte; + + memset(&e, 0, sizeof(e)); + p = (ZipfileCtx*)sqlite3_aggregate_context(pCtx, sizeof(ZipfileCtx)); + if( p==0 ) return; + + /* Martial the arguments into stack variables */ + if( nVal!=2 && nVal!=4 && nVal!=5 ){ + zErr = sqlite3_mprintf("wrong number of arguments to function zipfile()"); + rc = SQLITE_ERROR; + goto zipfile_step_out; + } + pName = apVal[0]; + if( nVal==2 ){ + pData = apVal[1]; + }else{ + pMode = apVal[1]; + pMtime = apVal[2]; + pData = apVal[3]; + if( nVal==5 ){ + pMethod = apVal[4]; + } + } + + /* Check that the 'name' parameter looks ok. */ + zName = (char*)sqlite3_value_text(pName); + nName = sqlite3_value_bytes(pName); + if( zName==0 ){ + zErr = sqlite3_mprintf("first argument to zipfile() must be non-NULL"); + rc = SQLITE_ERROR; + goto zipfile_step_out; + } + + /* Inspect the 'method' parameter. This must be either 0 (store), 8 (use + ** deflate compression) or NULL (choose automatically). */ + if( pMethod && SQLITE_NULL!=sqlite3_value_type(pMethod) ){ + iMethod = sqlite3_value_int64(pMethod); + if( iMethod!=0 && iMethod!=8 ){ + zErr = sqlite3_mprintf("illegal method value: %d", iMethod); + rc = SQLITE_ERROR; + goto zipfile_step_out; + } + } + + /* Now inspect the data. If this is NULL, then the new entry must be a + ** directory. Otherwise, figure out whether or not the data should + ** be deflated or simply stored in the zip archive. */ + if( sqlite3_value_type(pData)==SQLITE_NULL ){ + bIsDir = 1; + iMethod = 0; + }else{ + aData = sqlite3_value_blob(pData); + szUncompressed = nData = sqlite3_value_bytes(pData); + iCrc32 = crc32(0, aData, nData); + if( iMethod<0 || iMethod==8 ){ + int nOut = 0; + rc = zipfileDeflate(aData, nData, &aFree, &nOut, &zErr); + if( rc!=SQLITE_OK ){ + goto zipfile_step_out; + } + if( iMethod==8 || nOut1 && zName[nName-2]=='/' ) nName--; + } + } + + /* Assemble the ZipfileEntry object for the new zip archive entry */ + e.cds.iVersionMadeBy = ZIPFILE_NEWENTRY_MADEBY; + e.cds.iVersionExtract = ZIPFILE_NEWENTRY_REQUIRED; + e.cds.flags = ZIPFILE_NEWENTRY_FLAGS; + e.cds.iCompression = iMethod; + zipfileMtimeToDos(&e.cds, (u32)e.mUnixTime); + e.cds.crc32 = iCrc32; + e.cds.szCompressed = nData; + e.cds.szUncompressed = szUncompressed; + e.cds.iExternalAttr = (mode<<16); + e.cds.iOffset = p->body.n; + e.cds.nFile = nName; + e.cds.zFile = zName; + + /* Append the LFH to the body of the new archive */ + nByte = ZIPFILE_LFH_FIXED_SZ + e.cds.nFile + 9; + if( (rc = zipfileBufferGrow(&p->body, nByte)) ) goto zipfile_step_out; + p->body.n += zipfileSerializeLFH(&e, &p->body.a[p->body.n]); + + /* Append the data to the body of the new archive */ + if( (rc = zipfileBufferGrow(&p->body, nData)) ) goto zipfile_step_out; + memcpy(&p->body.a[p->body.n], aData, nData); + p->body.n += nData; + + /* Append the CDS record to the directory of the new archive */ + nByte = ZIPFILE_CDS_FIXED_SZ + e.cds.nFile + 9; + if( (rc = zipfileBufferGrow(&p->cds, nByte)) ) goto zipfile_step_out; + p->cds.n += zipfileSerializeCDS(&e, &p->cds.a[p->cds.n]); + + /* Increment the count of entries in the archive */ + p->nEntry++; + + zipfile_step_out: + sqlite3_free(aFree); + sqlite3_free(zFree); + if( rc ){ + if( zErr ){ + sqlite3_result_error(pCtx, zErr, -1); + }else{ + sqlite3_result_error_code(pCtx, rc); + } + } + sqlite3_free(zErr); +} + +/* +** xFinalize() callback for zipfile aggregate function. +*/ +void zipfileFinal(sqlite3_context *pCtx){ + ZipfileCtx *p; + ZipfileEOCD eocd; + int nZip; + u8 *aZip; + + p = (ZipfileCtx*)sqlite3_aggregate_context(pCtx, sizeof(ZipfileCtx)); + if( p==0 || p->nEntry==0 ) return; + + memset(&eocd, 0, sizeof(eocd)); + eocd.nEntry = p->nEntry; + eocd.nEntryTotal = p->nEntry; + eocd.nSize = p->cds.n; + eocd.iOffset = p->body.n; + + nZip = p->body.n + p->cds.n + ZIPFILE_EOCD_FIXED_SZ; + aZip = (u8*)sqlite3_malloc(nZip); + if( aZip==0 ){ + sqlite3_result_error_nomem(pCtx); + }else{ + memcpy(aZip, p->body.a, p->body.n); + memcpy(&aZip[p->body.n], p->cds.a, p->cds.n); + zipfileSerializeEOCD(&eocd, &aZip[p->body.n + p->cds.n]); + sqlite3_result_blob(pCtx, aZip, nZip, zipfileFree); + } + + sqlite3_free(p->body.a); + sqlite3_free(p->cds.a); +} + + /* ** Register the "zipfile" virtual table. */ @@ -1849,6 +2092,11 @@ static int zipfileRegister(sqlite3 *db){ int rc = sqlite3_create_module(db, "zipfile" , &zipfileModule, 0); if( rc==SQLITE_OK ) rc = sqlite3_overload_function(db, "zipfile_cds", -1); if( rc==SQLITE_OK ) rc = sqlite3_overload_function(db, "zipfile_blob", -1); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "zipfile", -1, SQLITE_UTF8, 0, 0, + zipfileStep, zipfileFinal + ); + } return rc; } #else /* SQLITE_OMIT_VIRTUALTABLE */ diff --git a/manifest b/manifest index 7a5772a6de..09731fddf8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Ensure\sthe\s"unique-not-null"\sflag\sis\sset\sfor\sautomatic\sindexes\son\scolumns\ndeclared\swith\s"col\sUNIQUE\sNOT\sNULL"\s(where\sthe\sNOT\sNULL\scomes\safter\sthe\nUNIQUE). -D 2018-01-29T16:22:39.280 +C Add\saggregate\sfunction\szipfile()\sto\sthe\szipfile\sextension.\sFor\scomposing\snew\nzip\sarchives\sin\smemory. +D 2018-01-29T18:41:07.175 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -304,7 +304,7 @@ F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178 F ext/misc/vtablog.c 31d0d8f4406795679dcd3a67917c213d3a2a5fb3ea5de35f6e773491ed7e13c9 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 -F ext/misc/zipfile.c 398f53469ed90076dd5d69494eb7f1bd01a42d402fc5dbe92374178887244805 +F ext/misc/zipfile.c 73d932caf69fea8ba42b58ce2cea0fbea2c126a355498fd215008f61651ec661 F ext/rbu/rbu.c ea7d1b7eb44c123a2a619332e19fe5313500705c4a58aaa1887905c0d83ffc2e F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842 F ext/rbu/rbu10.test 1846519a438697f45e9dcb246908af81b551c29e1078d0304fae83f1fed7e9ee @@ -1603,7 +1603,7 @@ F test/wordcount.c cb589cec469a1d90add05b1f8cee75c7210338d87a5afd65260ed5c0f4bbf F test/writecrash.test f1da7f7adfe8d7f09ea79b42e5ca6dcc41102f27f8e334ad71539501ddd910cc F test/zeroblob.test 3857870fe681b8185654414a9bccfde80b62a0fa F test/zerodamage.test 9c41628db7e8d9e8a0181e59ea5f189df311a9f6ce99cc376dc461f66db6f8dc -F test/zipfile.test 1b213bdc31eddd4a41042875dbdb29b6ea12b7da7a372a8eb07f61d0d76d800f +F test/zipfile.test 12199631759785b0185ceb1ff3cffdcc91985ef61475ab1c69139686b09a87c8 F tool/GetFile.cs a15e08acb5dd7539b75ba23501581d7c2b462cb5 F tool/GetTclKit.bat 8995df40c4209808b31f24de0b58f90930239a234f7591e3675d45bfbb990c5d F tool/Replace.cs 02c67258801c2fb5f63231e0ac0f220b4b36ba91 @@ -1702,7 +1702,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6ea8ba312c38365d3e28cfb2a367d729dd2751d1d853843eea0e18126777a320 -R 3ff6b96757835180119359cd3166f741 +P 8767f7b880f2e4112f75f0b6ef7be3f50ab1ae20e103e7d03d8bfe77e6c79438 +R 5f1a0bf3bc3038aabd0aee2df4b1bd63 U dan -Z 6c5f0f4d34834e8c6ccf2da8ff3a018a +Z eba1cb16b20a1e223d496488c50326dc diff --git a/manifest.uuid b/manifest.uuid index 9eae811f9a..78aea7435e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8767f7b880f2e4112f75f0b6ef7be3f50ab1ae20e103e7d03d8bfe77e6c79438 \ No newline at end of file +e364eeac76a8225146b37d801bc6cabe03e9abede5a1412ebe9d94a32d8838cc \ No newline at end of file diff --git a/test/zipfile.test b/test/zipfile.test index e74f81ed1d..04a475244b 100644 --- a/test/zipfile.test +++ b/test/zipfile.test @@ -22,33 +22,43 @@ if {[catch {load_static_extension db zipfile} error]} { finish_test; return } -proc do_zipfile_blob_test {tn file} { - set res1 [ - db eval { SELECT name,mode,mtime,method,quote(data) FROM zipfile($file) } - ] - - set fd [open $file] +proc readfile {f} { + set fd [open $f] fconfigure $fd -translation binary -encoding binary set data [read $fd] close $fd + set data +} - set res2 [db eval { - SELECT name,mode,mtime,method,quote(data) FROM zipfile($data) - }] +# Argument $file is the name of a zip archive on disk. This function +# executes test cases to check that the results of each of the following +# are the same: +# +# SELECT * FROM zipfile($file) +# SELECT * FROM zipfile( readfile($file) ) +# SELECT * FROM zipfile( +# (SELECT zipfile(name,mode,mtime,data,method) FROM zipfile($file)) +# ) +# +proc do_zipfile_blob_test {tn file} { - uplevel [list do_test $tn.1 [list set {} $res2] $res1] + db func r readfile + set q1 {SELECT name,mode,mtime,method,quote(data) FROM zipfile($file)} + set q2 {SELECT name,mode,mtime,method,quote(data) FROM zipfile( r($file) )} + set q3 {SELECT name,mode,mtime,method,quote(data) FROM zipfile( + ( SELECT zipfile(name,mode,mtime,data,method) FROM zipfile($file) ) + )} - set T "$file.test_zip" - set fd [open $T w] - fconfigure $fd -translation binary -encoding binary - puts -nonewline $fd $data - close $fd - set res3 [ - db eval { SELECT name,mode,mtime,method,quote(data) FROM zipfile($T) } - ] + set r1 [db eval $q1] + set r2 [db eval $q2] + set r3 [db eval $q3] + #puts $r1 + #puts $r2 + #puts $r3 - uplevel [list do_test $tn.2 [list set {} $res3] $res1] + uplevel [list do_test $tn.1 [list set {} $r2] $r1] + uplevel [list do_test $tn.1 [list set {} $r3] $r1] } forcedelete test.zip From ea4125bbefbed96310be4f3d0a742b257b812d8d Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 29 Jan 2018 19:47:32 +0000 Subject: [PATCH 022/107] Add unix-only tests to check that the "unzip" program can unpack archives generated by the zipfile extension. FossilOrigin-Name: 438c5c5237a801ae78809bf324bb9251fb50250addfc1f8e36659442b0e26ab6 --- ext/misc/fileio.c | 2 +- manifest | 14 ++++----- manifest.uuid | 2 +- test/zipfile.test | 75 ++++++++++++++++++++++++++++++++++++++++++----- 4 files changed, 77 insertions(+), 16 deletions(-) diff --git a/ext/misc/fileio.c b/ext/misc/fileio.c index e51b3e620b..7226ff18b3 100644 --- a/ext/misc/fileio.c +++ b/ext/misc/fileio.c @@ -491,6 +491,7 @@ static void fsdirResetCursor(fsdir_cursor *pCur){ sqlite3_free(pLvl->zDir); } sqlite3_free(pCur->zPath); + sqlite3_free(pCur->aLvl); pCur->aLvl = 0; pCur->zPath = 0; pCur->zBase = 0; @@ -506,7 +507,6 @@ static int fsdirClose(sqlite3_vtab_cursor *cur){ fsdir_cursor *pCur = (fsdir_cursor*)cur; fsdirResetCursor(pCur); - sqlite3_free(pCur->aLvl); sqlite3_free(pCur); return SQLITE_OK; } diff --git a/manifest b/manifest index 09731fddf8..0b73afb532 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\saggregate\sfunction\szipfile()\sto\sthe\szipfile\sextension.\sFor\scomposing\snew\nzip\sarchives\sin\smemory. -D 2018-01-29T18:41:07.175 +C Add\sunix-only\stests\sto\scheck\sthat\sthe\s"unzip"\sprogram\scan\sunpack\sarchives\ngenerated\sby\sthe\szipfile\sextension. +D 2018-01-29T19:47:32.262 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -277,7 +277,7 @@ F ext/misc/compress.c dd4f8a6d0baccff3c694757db5b430f3bbd821d8686d1fc24df55cf9f0 F ext/misc/csv.c 1a009b93650732e22334edc92459c4630b9fa703397cbb3c8ca279921a36ca11 F ext/misc/dbdump.c 3509fa6b8932d04e932d6b6b827b6a82ca362781b8e8f3c77336f416793e215e F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2 -F ext/misc/fileio.c bd2f717be63a9ae9ff85977a756c43a96d47a7763d98ae913636c64b714c232b +F ext/misc/fileio.c 4cad3a78bfbbea9b1414b4405fd704ee57fbe2621254b011888dae7b0ba8c513 F ext/misc/fuzzer.c 7c64b8197bb77b7d64eff7cac7848870235d4c25 F ext/misc/ieee754.c f190d0cc5182529acb15babd177781be1ac1718c F ext/misc/json1.c dbe086615b9546c156bf32b9378fc09383b58bd17513b866cfd24c1e15281984 @@ -1603,7 +1603,7 @@ F test/wordcount.c cb589cec469a1d90add05b1f8cee75c7210338d87a5afd65260ed5c0f4bbf F test/writecrash.test f1da7f7adfe8d7f09ea79b42e5ca6dcc41102f27f8e334ad71539501ddd910cc F test/zeroblob.test 3857870fe681b8185654414a9bccfde80b62a0fa F test/zerodamage.test 9c41628db7e8d9e8a0181e59ea5f189df311a9f6ce99cc376dc461f66db6f8dc -F test/zipfile.test 12199631759785b0185ceb1ff3cffdcc91985ef61475ab1c69139686b09a87c8 +F test/zipfile.test c40ae3a5d3fd0a31a8c6bdae1dbef55dd7140acb0d3b316c8edb744085ea6134 F tool/GetFile.cs a15e08acb5dd7539b75ba23501581d7c2b462cb5 F tool/GetTclKit.bat 8995df40c4209808b31f24de0b58f90930239a234f7591e3675d45bfbb990c5d F tool/Replace.cs 02c67258801c2fb5f63231e0ac0f220b4b36ba91 @@ -1702,7 +1702,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8767f7b880f2e4112f75f0b6ef7be3f50ab1ae20e103e7d03d8bfe77e6c79438 -R 5f1a0bf3bc3038aabd0aee2df4b1bd63 +P e364eeac76a8225146b37d801bc6cabe03e9abede5a1412ebe9d94a32d8838cc +R 78b9ab6ccbbf51cef503aff35913fad6 U dan -Z eba1cb16b20a1e223d496488c50326dc +Z 634d51e6967cb83c15fce21d55a23989 diff --git a/manifest.uuid b/manifest.uuid index 78aea7435e..63ba75784e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e364eeac76a8225146b37d801bc6cabe03e9abede5a1412ebe9d94a32d8838cc \ No newline at end of file +438c5c5237a801ae78809bf324bb9251fb50250addfc1f8e36659442b0e26ab6 \ No newline at end of file diff --git a/test/zipfile.test b/test/zipfile.test index 04a475244b..d43e685e15 100644 --- a/test/zipfile.test +++ b/test/zipfile.test @@ -30,6 +30,24 @@ proc readfile {f} { set data } +if {$::tcl_platform(platform)=="unix" && [catch {exec unzip}]==0} { + set ::UNZIP 1 + load_static_extension db fileio + proc do_unzip {file} { + forcedelete test_unzip + file mkdir test_unzip + exec unzip -d test_unzip $file + + set res [db eval { + SELECT replace(name,'test_unzip/',''),mode,mtime,data + FROM fsdir('test_unzip') + WHERE name!='test_unzip' + ORDER BY name + }] + set res + } +} + # Argument $file is the name of a zip archive on disk. This function # executes test cases to check that the results of each of the following # are the same: @@ -58,7 +76,49 @@ proc do_zipfile_blob_test {tn file} { #puts $r3 uplevel [list do_test $tn.1 [list set {} $r2] $r1] - uplevel [list do_test $tn.1 [list set {} $r3] $r1] + uplevel [list do_test $tn.2 [list set {} $r3] $r1] +} + +# Argument $file is a zip file on disk. This command runs tests to: +# +# 1. Unpack the archive with unix command [unzip] and compare the +# results to reading the same archive using the zipfile() table +# valued function. +# +# 2. Creates a new archive with the same contents using the zipfile() +# aggregate function as follows: +# +# SELECT writefile('test_unzip.zip', +# ( SELECT zipfile(name,mode,mtime,data,method) FROM zipfile($file) ) +# ); +# +# Then tests that unpacking the new archive using [unzip] produces +# the same results as in (1). +# +proc do_unzip_test {tn file} { + if {[info vars ::UNZIP]==""} { return } + db func sss strip_slash + + db eval { + SELECT writefile('test_unzip.zip', + ( SELECT zipfile(name,mode,mtime,data,method) FROM zipfile($file) ) + ); + } + + set r1 [db eval { + SELECT sss(name),mode,mtime,data FROM zipfile($file) ORDER BY name + }] + set r2 [do_unzip $file] + set r3 [do_unzip test_unzip.zip] + + uplevel [list do_test $tn.1 [list set {} $r2] $r1] + uplevel [list do_test $tn.2 [list set {} $r3] $r1] +} +proc strip_slash {in} { regsub {/$} $in {} } + +proc do_zip_tests {tn file} { + uplevel do_zipfile_blob_test $tn.1 $file + uplevel do_unzip_test $tn.2 $file } forcedelete test.zip @@ -99,14 +159,14 @@ do_execsql_test 1.2 { f.txt 1000000000 abcde g.txt 1000000002 12345 } -do_zipfile_blob_test 1.2.1 test.zip +do_zip_tests 1.2a test.zip do_execsql_test 1.3 { INSERT INTO zz(name, mode, mtime, data) VALUES('h.txt', '-rw-r--r--', 1000000004, 'aaaaaaaaaabbbbbbbbbb' ); } -do_zipfile_blob_test 1.3.1 test.zip +do_zip_tests 1.3a test.zip do_execsql_test 1.4 { SELECT name, mtime, data, method FROM zipfile('test.zip'); @@ -153,7 +213,7 @@ do_execsql_test 1.6.1 { h.txt 33188 1000000004 aaaaaaaaaabbbbbbbbbb 8 i.txt 33188 1000000006 zxcvb 0 } -do_zipfile_blob_test 1.6.1a test.zip +do_zip_tests 1.6.1a test.zip do_execsql_test 1.6.2 { UPDATE zz SET mtime=4 WHERE name='i.txt'; @@ -172,7 +232,7 @@ do_execsql_test 1.6.3 { h.txt 33189 1000000004 aaaaaaaaaabbbbbbbbbb 8 i.txt 33188 4 zxcvb 0 } -do_zipfile_blob_test 1.6.3a test.zip +do_zip_tests 1.6.3a test.zip do_execsql_test 1.6.4 { UPDATE zz SET name = 'blue.txt' WHERE name='f.txt'; @@ -182,7 +242,7 @@ do_execsql_test 1.6.4 { h.txt 33189 1000000004 aaaaaaaaaabbbbbbbbbb 8 i.txt 33188 4 zxcvb 0 } -do_zipfile_blob_test 1.6.4a test.zip +do_zip_tests 1.6.4a test.zip do_execsql_test 1.6.5 { UPDATE zz SET data = 'edcba' WHERE name='blue.txt'; @@ -217,6 +277,7 @@ do_execsql_test 1.6.8 { db close forcedelete test.zip reset_db +load_static_extension db fileio load_static_extension db zipfile do_execsql_test 2.1 { CREATE VIRTUAL TABLE zzz USING zipfile('test.zip'); @@ -243,7 +304,7 @@ do_execsql_test 2.4 { dirname2/ 16877 {} dirname2/file1.txt 33188 abcdefghijklmnop } -do_zipfile_blob_test 2.4.1 test.zip +do_zip_tests 2.4a test.zip # If on unix, check that the [unzip] utility can unpack our archive. # From fdcd9d4e6c4a08855660df26b84139105f863df3 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 30 Jan 2018 14:07:55 +0000 Subject: [PATCH 023/107] Prevent users from creating zipfile() virtual tables without an argument. FossilOrigin-Name: 81fdbe0cc5a360f818078d47a5888d0a29d555927da279a9a0213702d74ef09a --- ext/misc/zipfile.c | 87 ++++++++-------------------------------------- manifest | 14 ++++---- manifest.uuid | 2 +- test/zipfile.test | 26 +++++--------- 4 files changed, 31 insertions(+), 98 deletions(-) diff --git a/ext/misc/zipfile.c b/ext/misc/zipfile.c index f325868149..247d0b7355 100644 --- a/ext/misc/zipfile.c +++ b/ext/misc/zipfile.c @@ -318,6 +318,21 @@ static int zipfileConnect( ZipfileTab *pNew = 0; int rc; + /* If the table name is not "zipfile", require that the argument be + ** specified. This stops zipfile tables from being created as: + ** + ** CREATE VIRTUAL TABLE zzz USING zipfile(); + ** + ** It does not prevent: + ** + ** CREATE VIRTUAL TABLE zipfile USING zipfile(); + */ + assert( 0==sqlite3_stricmp(argv[0], "zipfile") ); + if( (0!=sqlite3_stricmp(argv[2], "zipfile") && argc<4) || argc>4 ){ + *pzErr = sqlite3_mprintf("zipfile constructor requires one argument"); + return SQLITE_ERROR; + } + if( argc>3 ){ zFile = argv[3]; nFile = (int)strlen(zFile)+1; @@ -1725,72 +1740,6 @@ static void zipfileFunctionCds( } } -static void zipfileFunctionBlob( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - ZipfileCsr *pCsr; - ZipfileTab *pTab = (ZipfileTab*)sqlite3_user_data(context); - ZipfileEntry *p; - int nBody = 0; - int nCds = 0; - int nEocd = ZIPFILE_EOCD_FIXED_SZ; - ZipfileEOCD eocd; - - u8 *aZip; - int nZip; - - u8 *aBody; - u8 *aCds; - - pCsr = zipfileFindCursor(pTab, sqlite3_value_int64(argv[0])); - if( pCsr->pFile || pTab->zFile ){ - sqlite3_result_error(context, "illegal use of zipfile_blob()", -1); - return; - } - - /* Figure out how large the final file will be */ - for(p=pTab->pFirstEntry; p; p=p->pNext){ - nBody += ZIPFILE_LFH_FIXED_SZ + p->cds.nFile + 9 + p->cds.szCompressed; - nCds += ZIPFILE_CDS_FIXED_SZ + p->cds.nFile + 9; - } - - /* Allocate space to create the serialized file */ - nZip = nBody + nCds + nEocd; - aZip = (u8*)sqlite3_malloc(nZip); - if( aZip==0 ){ - sqlite3_result_error_nomem(context); - return; - } - aBody = aZip; - aCds = &aZip[nBody]; - - /* Populate the body and CDS */ - memset(&eocd, 0, sizeof(eocd)); - for(p=pTab->pFirstEntry; p; p=p->pNext){ - p->cds.iOffset = (aBody - aZip); - aBody += zipfileSerializeLFH(p, aBody); - if( p->cds.szCompressed ){ - memcpy(aBody, p->aData, p->cds.szCompressed); - aBody += p->cds.szCompressed; - } - aCds += zipfileSerializeCDS(p, aCds); - eocd.nEntry++; - } - - /* Append the EOCD record */ - assert( aBody==&aZip[nBody] ); - assert( aCds==&aZip[nBody+nCds] ); - eocd.nEntryTotal = eocd.nEntry; - eocd.nSize = nCds; - eocd.iOffset = nBody; - zipfileSerializeEOCD(&eocd, aCds); - - sqlite3_result_blob(context, aZip, nZip, zipfileFree); -} - - /* ** xFindFunction method. */ @@ -1807,11 +1756,6 @@ static int zipfileFindFunction( *ppArg = (void*)pVtab; return 1; } - if( sqlite3_stricmp("zipfile_blob", zName)==0 ){ - *pxFunc = zipfileFunctionBlob; - *ppArg = (void*)pVtab; - return 1; - } } return 0; @@ -2091,7 +2035,6 @@ static int zipfileRegister(sqlite3 *db){ int rc = sqlite3_create_module(db, "zipfile" , &zipfileModule, 0); if( rc==SQLITE_OK ) rc = sqlite3_overload_function(db, "zipfile_cds", -1); - if( rc==SQLITE_OK ) rc = sqlite3_overload_function(db, "zipfile_blob", -1); if( rc==SQLITE_OK ){ rc = sqlite3_create_function(db, "zipfile", -1, SQLITE_UTF8, 0, 0, zipfileStep, zipfileFinal diff --git a/manifest b/manifest index 0b73afb532..f52d1dd656 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sunix-only\stests\sto\scheck\sthat\sthe\s"unzip"\sprogram\scan\sunpack\sarchives\ngenerated\sby\sthe\szipfile\sextension. -D 2018-01-29T19:47:32.262 +C Prevent\susers\sfrom\screating\szipfile()\svirtual\stables\swithout\san\sargument. +D 2018-01-30T14:07:55.726 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -304,7 +304,7 @@ F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178 F ext/misc/vtablog.c 31d0d8f4406795679dcd3a67917c213d3a2a5fb3ea5de35f6e773491ed7e13c9 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 -F ext/misc/zipfile.c 73d932caf69fea8ba42b58ce2cea0fbea2c126a355498fd215008f61651ec661 +F ext/misc/zipfile.c ead25f062cee790b7c764ce8d2c6ad32a7ac82fc31ea80f69be99948f96f2d19 F ext/rbu/rbu.c ea7d1b7eb44c123a2a619332e19fe5313500705c4a58aaa1887905c0d83ffc2e F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842 F ext/rbu/rbu10.test 1846519a438697f45e9dcb246908af81b551c29e1078d0304fae83f1fed7e9ee @@ -1603,7 +1603,7 @@ F test/wordcount.c cb589cec469a1d90add05b1f8cee75c7210338d87a5afd65260ed5c0f4bbf F test/writecrash.test f1da7f7adfe8d7f09ea79b42e5ca6dcc41102f27f8e334ad71539501ddd910cc F test/zeroblob.test 3857870fe681b8185654414a9bccfde80b62a0fa F test/zerodamage.test 9c41628db7e8d9e8a0181e59ea5f189df311a9f6ce99cc376dc461f66db6f8dc -F test/zipfile.test c40ae3a5d3fd0a31a8c6bdae1dbef55dd7140acb0d3b316c8edb744085ea6134 +F test/zipfile.test 368a5a0c97be0caaf8c3efa8293bfe18436d546805678fa00b6aa81bc98727ec F tool/GetFile.cs a15e08acb5dd7539b75ba23501581d7c2b462cb5 F tool/GetTclKit.bat 8995df40c4209808b31f24de0b58f90930239a234f7591e3675d45bfbb990c5d F tool/Replace.cs 02c67258801c2fb5f63231e0ac0f220b4b36ba91 @@ -1702,7 +1702,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e364eeac76a8225146b37d801bc6cabe03e9abede5a1412ebe9d94a32d8838cc -R 78b9ab6ccbbf51cef503aff35913fad6 +P 438c5c5237a801ae78809bf324bb9251fb50250addfc1f8e36659442b0e26ab6 +R 0f8eb13d715c2195926ed8684b288375 U dan -Z 634d51e6967cb83c15fce21d55a23989 +Z 6708ef6af1eb582ebfe61496df2754b7 diff --git a/manifest.uuid b/manifest.uuid index 63ba75784e..2d66824214 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -438c5c5237a801ae78809bf324bb9251fb50250addfc1f8e36659442b0e26ab6 \ No newline at end of file +81fdbe0cc5a360f818078d47a5888d0a29d555927da279a9a0213702d74ef09a \ No newline at end of file diff --git a/test/zipfile.test b/test/zipfile.test index d43e685e15..76e7d924df 100644 --- a/test/zipfile.test +++ b/test/zipfile.test @@ -358,24 +358,14 @@ do_catchsql_test 3.2 { } {1 {no such column: rowid}} #------------------------------------------------------------------------- -reset_db -forcedelete test.zip -load_static_extension db zipfile - -do_execsql_test 4.0 { - CREATE VIRTUAL TABLE x2 USING zipfile(); - INSERT INTO x2(name, data) VALUES('dir1/', NULL); - INSERT INTO x2(name, data) VALUES('file1', '1234'); - INSERT INTO x2(name, data) VALUES('dir1/file2', '5678'); - SELECT name, data FROM x2 -} { - dir1/ {} file1 1234 dir1/file2 5678 -} - -do_test 4.1 { - set data [db one {SELECT zipfile_blob(z) FROM x2 LIMIT 1}] - db eval { SELECT name, data FROM zipfile($data) } -} {dir1/ {} file1 1234 dir1/file2 5678} +# Test some error conditions. +# +do_catchsql_test 4.1 { + CREATE VIRTUAL TABLE yyy USING zipfile(); +} {1 {zipfile constructor requires one argument}} +do_catchsql_test 4.2 { + CREATE VIRTUAL TABLE yyy USING zipfile('test.zip', 'test.zip'); +} {1 {zipfile constructor requires one argument}} From 2f7260deba127ab34f3722035b9a3fb1ff3d0245 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 30 Jan 2018 17:43:22 +0000 Subject: [PATCH 024/107] Add tests cases for the zipfile extension. FossilOrigin-Name: 13b786dafd308b7aef42328f26fe64981c83d524e4bf82ceb7d58afc7146c7d3 --- ext/misc/zipfile.c | 33 ++++---- manifest | 14 ++-- manifest.uuid | 2 +- test/zipfile2.test | 167 +++++++++++++++++++++++++++++++++++++++++ test/zipfilefault.test | 67 +++++++++++++++++ 5 files changed, 263 insertions(+), 20 deletions(-) create mode 100644 test/zipfile2.test create mode 100644 test/zipfilefault.test diff --git a/ext/misc/zipfile.c b/ext/misc/zipfile.c index 247d0b7355..a52033e25b 100644 --- a/ext/misc/zipfile.c +++ b/ext/misc/zipfile.c @@ -48,13 +48,26 @@ SQLITE_EXTENSION_INIT1 #ifndef SQLITE_OMIT_VIRTUALTABLE #ifndef SQLITE_AMALGAMATION + typedef sqlite3_int64 i64; typedef unsigned char u8; typedef unsigned short u16; typedef unsigned long u32; #define MIN(a,b) ((a)<(b) ? (a) : (b)) + +#if defined(SQLITE_COVERAGE_TEST) || defined(SQLITE_MUTATION_TEST) +# define ALWAYS(X) (1) +# define NEVER(X) (0) +#elif !defined(NDEBUG) +# define ALWAYS(X) ((X)?1:(assert(0),0)) +# define NEVER(X) ((X)?(assert(0),1):0) +#else +# define ALWAYS(X) (X) +# define NEVER(X) (X) #endif +#endif /* SQLITE_AMALGAMATION */ + static const char ZIPFILE_SCHEMA[] = "CREATE TABLE y(" "name PRIMARY KEY," /* 0: Name of file in zip archive */ @@ -283,14 +296,12 @@ static void zipfileCtxErrorMsg(sqlite3_context *ctx, const char *zFmt, ...){ static void zipfileDequote(char *zIn){ char q = zIn[0]; if( q=='"' || q=='\'' || q=='`' || q=='[' ){ - char c; int iIn = 1; int iOut = 0; if( q=='[' ) q = ']'; - while( (c = zIn[iIn++]) ){ - if( c==q ){ - if( zIn[iIn++]!=q ) break; - } + while( ALWAYS(zIn[iIn]) ){ + char c = zIn[iIn++]; + if( c==q && zIn[iIn++]!=q ) break; zIn[iOut++] = c; } zIn[iOut] = '\0'; @@ -445,12 +456,8 @@ static int zipfileClose(sqlite3_vtab_cursor *cur){ zipfileResetCursor(pCsr); /* Remove this cursor from the ZipfileTab.pCsrList list. */ - for(pp=&pTab->pCsrList; *pp; pp=&((*pp)->pCsrNext)){ - if( *pp==pCsr ){ - *pp = pCsr->pCsrNext; - break; - } - } + for(pp=&pTab->pCsrList; *pp!=pCsr; pp=&((*pp)->pCsrNext)); + *pp = pCsr->pCsrNext; sqlite3_free(pCsr); return SQLITE_OK; @@ -873,7 +880,7 @@ static void zipfileFree(void *p) { ** If an error occurs, an error code is left in pCtx instead. */ static void zipfileInflate( - sqlite3_context *pCtx, /* Store error here, if any */ + sqlite3_context *pCtx, /* Store result here */ const u8 *aIn, /* Compressed data */ int nIn, /* Size of buffer aIn[] in bytes */ int nOut /* Expected output size */ @@ -1119,7 +1126,7 @@ static int zipfileReadEOCD( pEOCD->iOffset = zipfileRead32(aRead); } - return SQLITE_OK; + return rc; } /* diff --git a/manifest b/manifest index f52d1dd656..8cbe5d88c3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Prevent\susers\sfrom\screating\szipfile()\svirtual\stables\swithout\san\sargument. -D 2018-01-30T14:07:55.726 +C Add\stests\scases\sfor\sthe\szipfile\sextension. +D 2018-01-30T17:43:22.142 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -304,7 +304,7 @@ F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178 F ext/misc/vtablog.c 31d0d8f4406795679dcd3a67917c213d3a2a5fb3ea5de35f6e773491ed7e13c9 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 -F ext/misc/zipfile.c ead25f062cee790b7c764ce8d2c6ad32a7ac82fc31ea80f69be99948f96f2d19 +F ext/misc/zipfile.c aa10ec6a235030cd368b511e7c78e40251008dddddeb0eb006ad1f344f78b690 F ext/rbu/rbu.c ea7d1b7eb44c123a2a619332e19fe5313500705c4a58aaa1887905c0d83ffc2e F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842 F ext/rbu/rbu10.test 1846519a438697f45e9dcb246908af81b551c29e1078d0304fae83f1fed7e9ee @@ -1604,6 +1604,8 @@ F test/writecrash.test f1da7f7adfe8d7f09ea79b42e5ca6dcc41102f27f8e334ad71539501d F test/zeroblob.test 3857870fe681b8185654414a9bccfde80b62a0fa F test/zerodamage.test 9c41628db7e8d9e8a0181e59ea5f189df311a9f6ce99cc376dc461f66db6f8dc F test/zipfile.test 368a5a0c97be0caaf8c3efa8293bfe18436d546805678fa00b6aa81bc98727ec +F test/zipfile2.test 2855c993641935b51dfb3f9005173114538c7a2eb2a60fe3fc47a802e6794406 +F test/zipfilefault.test df4fa9e16116e0cb21d09037a6b0a7d93fecd8767d82154b66044b4ca43fca58 F tool/GetFile.cs a15e08acb5dd7539b75ba23501581d7c2b462cb5 F tool/GetTclKit.bat 8995df40c4209808b31f24de0b58f90930239a234f7591e3675d45bfbb990c5d F tool/Replace.cs 02c67258801c2fb5f63231e0ac0f220b4b36ba91 @@ -1702,7 +1704,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 438c5c5237a801ae78809bf324bb9251fb50250addfc1f8e36659442b0e26ab6 -R 0f8eb13d715c2195926ed8684b288375 +P 81fdbe0cc5a360f818078d47a5888d0a29d555927da279a9a0213702d74ef09a +R 63c2b54c1a4defd96e22435ea8f5b625 U dan -Z 6708ef6af1eb582ebfe61496df2754b7 +Z 6717becc48c95c87eb42213e9c88ecb3 diff --git a/manifest.uuid b/manifest.uuid index 2d66824214..97b7247606 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -81fdbe0cc5a360f818078d47a5888d0a29d555927da279a9a0213702d74ef09a \ No newline at end of file +13b786dafd308b7aef42328f26fe64981c83d524e4bf82ceb7d58afc7146c7d3 \ No newline at end of file diff --git a/test/zipfile2.test b/test/zipfile2.test new file mode 100644 index 0000000000..f3b7a84169 --- /dev/null +++ b/test/zipfile2.test @@ -0,0 +1,167 @@ +# 2018 January 30 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix zipfile2 + +ifcapable !vtab { + finish_test; return +} +if {[catch {load_static_extension db zipfile} error]} { + puts "Skipping zipfile2 tests, hit load error: $error" + finish_test; return +} + +proc blobliteral {str} { + set concat [string map {" " "" "\n" ""} $str] + return "X'$concat'" +} + +proc blob {str} { + binary decode hex $str +} + +proc findall {needle haystack} { + set L [list] + set start 0 + while { [set idx [string first $needle $haystack $start]]>=0 } { + lappend L $idx + set start [expr $idx+1] + } + set L +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE aaa USING zipfile('testzip'); + CREATE VIRTUAL TABLE bbb USING zipfile("testzip"); + CREATE VIRTUAL TABLE ccc USING zipfile(`testzip`); + CREATE VIRTUAL TABLE ddd USING zipfile([testzip]); + CREATE VIRTUAL TABLE eee USING zipfile(testzip); + CREATE VIRTUAL TABLE fff USING zipfile('test''zip'); +} + +do_test 2.0 { + forcedelete testdir + file mkdir testdir + execsql { CREATE VIRTUAL TABLE hhh USING zipfile('testdir') } + catchsql { SELECT * FROM hhh } +} {1 {error in fread()}} + + +set archive { + 504B0304140000080000D4A52BEC09F3B6E0110000001100000005000900612E + 747874555405000140420F00636F6E74656E7473206F6620612E747874504B03 + 04140000080000D4A52BECD98916A7110000001100000005000900622E747874 + 555405000140420F00636F6E74656E7473206F6620622E747874504B01021E03 + 140000080000D4A52BEC09F3B6E0110000001100000005000900000000000000 + 0000A48100000000612E747874555405000140420F00504B01021E0314000008 + 0000D4A52BECD98916A71100000011000000050009000000000000000000A481 + 3D000000622E747874555405000140420F00504B050600000000020002007800 + 00007A0000000000 +} + +do_execsql_test 3.1 { + WITH contents(name,mtime,data) AS ( + VALUES('a.txt', 1000000, 'contents of a.txt') UNION ALL + VALUES('b.txt', 1000000, 'contents of b.txt') + ) SELECT quote( zipfile(name,NULL,mtime,data) ) FROM contents; +} [blobliteral $archive] + +set blob [blob $archive] +do_execsql_test 3.2 { + SELECT name,mtime,data FROM zipfile($blob) +} { + a.txt 1000000 {contents of a.txt} + b.txt 1000000 {contents of b.txt} +} + +# Corrupt each of the 0x50 0x4B (ascii "PK") headers in the file +# Test that in each case this causes an error. +# +set L [findall 504B $archive] +for {set i 0} {$i < [llength $L]} {incr i} { + set idx [lindex $L $i] + set a [string replace $archive $idx [expr $idx+3] 0000] + set blob [blob $a] + do_catchsql_test 3.3.$i { + SELECT name,mtime,data FROM zipfile($blob) + } {/1 .*/} +} + +set L [findall 5554 $archive] +for {set i 0} {$i < [llength $L]} {incr i} { + set idx [lindex $L $i] + set a [string replace $archive $idx [expr $idx+3] 1234] + set blob [blob $a] + do_execsql_test 3.4.$i { + SELECT name,data FROM zipfile($blob) + } { + a.txt {contents of a.txt} + b.txt {contents of b.txt} + } +} + +for {set i 0} {$i < [llength $L]} {incr i} { + set idx [lindex $L $i] + set a [string replace $archive [expr $idx+8] [expr $idx+9] 00] + set blob [blob $a] + do_execsql_test 3.5.$i { + SELECT name,data FROM zipfile($blob) + } { + a.txt {contents of a.txt} + b.txt {contents of b.txt} + } +} + +if 0 { +set blob [db one { + WITH contents(name,mtime,data) AS ( + VALUES('a.txt', 1000000, 'aaaaaaaaaaaaaaaaaaaaaaa') + ) SELECT quote( zipfile(name,NULL,mtime,data) ) FROM contents; +}] +set blob [string range $blob 2 end] +set blob [string range $blob 0 end-1] +while {[string length $blob]>0} { + puts [string range $blob 0 63] + set blob [string range $blob 64 end] +} +exit +} + +set archive2 { + 504B0304140000080800D4A52BEC08F54C6E050000001700000005000900612E + 747874555405000140420F004B4CC40A00504B01021E03140000080800D4A52B + EC08F54C6E0500000017000000050009000000000000000000A4810000000061 + 2E747874555405000140420F00504B050600000000010001003C000000310000 + 000000 +} +set blob [blob $archive2] +do_execsql_test 4.0 { + SELECT name,mtime,data,method FROM zipfile($blob) +} { + a.txt 1000000 aaaaaaaaaaaaaaaaaaaaaaa 8 +} + +breakpoint +set L [findall 17000000 $archive2] +set a $archive2 +foreach i $L { set a [string replace $a $i [expr $i+7] 16000000] } +set blob [blob $a] +do_catchsql_test 4.1 { + SELECT name,mtime,data,method FROM zipfile($blob) +} {1 {SQL logic error}} + + + +finish_test + diff --git a/test/zipfilefault.test b/test/zipfilefault.test new file mode 100644 index 0000000000..9f0f4784dc --- /dev/null +++ b/test/zipfilefault.test @@ -0,0 +1,67 @@ +# 2018 January 30 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/malloc_common.tcl +set testprefix zipfilefault + +ifcapable !vtab { + finish_test; return +} +if {[catch {load_static_extension db zipfile} error]} { + puts "Skipping zipfile2 tests, hit load error: $error" + finish_test; return +} + +faultsim_save_and_close +do_faultsim_test 1 -prep { + faultsim_restore_and_reopen + load_static_extension db zipfile + execsql { DROP TABLE IF EXISTS aaa } +} -body { + execsql { CREATE VIRTUAL TABLE aaa USING zipfile('test.zip') } +} -test { + faultsim_test_result {0 {}} +} + +forcedelete test.zip +sqlite3 db test.db +load_static_extension db zipfile +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE setup USING zipfile('test.zip'); + INSERT INTO setup(name, data) VALUES('a.txt', '1234567890'); +} + +do_faultsim_test 2 -faults oom* -body { + execsql { SELECT name,data FROM zipfile('test.zip') } +} -test { + faultsim_test_result {0 {a.txt 1234567890}} +} + +forcedelete test.zip +reset_db +load_static_extension db zipfile +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE setup USING zipfile('test.zip'); + INSERT INTO setup(name, data) VALUES('a.txt', 'aaaaaaaaaaaaaaaaaaaaaaaaaaaa'); +} + +do_faultsim_test 2 -faults oom* -body { + execsql { SELECT name,data FROM zipfile('test.zip') } +} -test { + faultsim_test_result {0 {a.txt aaaaaaaaaaaaaaaaaaaaaaaaaaaa}} +} + + +finish_test + From 099fa84727a06f17809dfb90ad308a500a91ca40 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 30 Jan 2018 18:33:23 +0000 Subject: [PATCH 025/107] If a virtual table xColumn method sets an error message using sqlite3_result_error(), use that message in preference to any error message left in the sqlite3_vtab object. FossilOrigin-Name: 71e3b715ad36c2132a587b84221be6359c31d7a17c3fba201b7ed3baf17922f1 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbe.c | 1 + test/zipfile2.test | 4 +--- 4 files changed, 10 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 8cbe5d88c3..b8ff099373 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stests\scases\sfor\sthe\szipfile\sextension. -D 2018-01-30T17:43:22.142 +C If\sa\svirtual\stable\sxColumn\smethod\ssets\san\serror\smessage\susing\nsqlite3_result_error(),\suse\sthat\smessage\sin\spreference\sto\sany\serror\smessage\nleft\sin\sthe\ssqlite3_vtab\sobject. +D 2018-01-30T18:33:23.120 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -556,7 +556,7 @@ F src/update.c a90a32ffc0100265b0693dbbdbe490756447af181f5ea2c138cce515b08c8795 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5 F src/util.c d9eb0a6c4aae1b00a7369eadd7ca0bbe946cb4c953b6751aa20d357c2f482157 F src/vacuum.c 90839322fd5f00df9617eb21b68beda9b6e2a2937576b0d65985e4aeb1c53739 -F src/vdbe.c 0e13b8c33a9ecf236b3045f190fb58933049582307b621cbd924b6582a2b705f +F src/vdbe.c 619d385ff1798fab7c549deb04b078f5a8ec466e2ad315939bc824593a2f9fde F src/vdbe.h 134beb7a12a6213c00eba58febaede33447cc4441bc568a0d9c144b33fc3720a F src/vdbeInt.h 8d7d07f13cb3c4cbca91e22ba4a1920e542dda7c5d9299920432a0b3d5b009f5 F src/vdbeapi.c fea41171884a4de119f8b10ab514c788674eeeb7f27218bb6d008e1310bfd07f @@ -1604,7 +1604,7 @@ F test/writecrash.test f1da7f7adfe8d7f09ea79b42e5ca6dcc41102f27f8e334ad71539501d F test/zeroblob.test 3857870fe681b8185654414a9bccfde80b62a0fa F test/zerodamage.test 9c41628db7e8d9e8a0181e59ea5f189df311a9f6ce99cc376dc461f66db6f8dc F test/zipfile.test 368a5a0c97be0caaf8c3efa8293bfe18436d546805678fa00b6aa81bc98727ec -F test/zipfile2.test 2855c993641935b51dfb3f9005173114538c7a2eb2a60fe3fc47a802e6794406 +F test/zipfile2.test 1066dd9769028d6f0bf1c87303b24f087cc064d9bbcacefc95613380f642de18 F test/zipfilefault.test df4fa9e16116e0cb21d09037a6b0a7d93fecd8767d82154b66044b4ca43fca58 F tool/GetFile.cs a15e08acb5dd7539b75ba23501581d7c2b462cb5 F tool/GetTclKit.bat 8995df40c4209808b31f24de0b58f90930239a234f7591e3675d45bfbb990c5d @@ -1704,7 +1704,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 81fdbe0cc5a360f818078d47a5888d0a29d555927da279a9a0213702d74ef09a -R 63c2b54c1a4defd96e22435ea8f5b625 +P 13b786dafd308b7aef42328f26fe64981c83d524e4bf82ceb7d58afc7146c7d3 +R 48ccd51e910d5d576747af2d5e64e430 U dan -Z 6717becc48c95c87eb42213e9c88ecb3 +Z 9d867999d41b646c89b308fd5587d517 diff --git a/manifest.uuid b/manifest.uuid index 97b7247606..2c1ee2f33c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -13b786dafd308b7aef42328f26fe64981c83d524e4bf82ceb7d58afc7146c7d3 \ No newline at end of file +71e3b715ad36c2132a587b84221be6359c31d7a17c3fba201b7ed3baf17922f1 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 495b56aa1d..7d1cae0caf 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -6756,6 +6756,7 @@ case OP_VColumn: { rc = pModule->xColumn(pCur->uc.pVCur, &sContext, pOp->p2); sqlite3VtabImportErrmsg(p, pVtab); if( sContext.isError>0 ){ + sqlite3VdbeError(p, "%s", sqlite3_value_text(pDest)); rc = sContext.isError; } sqlite3VdbeChangeEncoding(pDest, encoding); diff --git a/test/zipfile2.test b/test/zipfile2.test index f3b7a84169..6d20e09301 100644 --- a/test/zipfile2.test +++ b/test/zipfile2.test @@ -152,15 +152,13 @@ do_execsql_test 4.0 { a.txt 1000000 aaaaaaaaaaaaaaaaaaaaaaa 8 } -breakpoint set L [findall 17000000 $archive2] set a $archive2 foreach i $L { set a [string replace $a $i [expr $i+7] 16000000] } set blob [blob $a] do_catchsql_test 4.1 { SELECT name,mtime,data,method FROM zipfile($blob) -} {1 {SQL logic error}} - +} {1 {inflate() failed (0)}} finish_test From 04a28c3b6d546a3fd97ee541cfcd0cac2fb60488 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 31 Jan 2018 01:38:44 +0000 Subject: [PATCH 026/107] Changes to the autoconf tarball so that it does not try to use system() when building on iOS. FossilOrigin-Name: 32ed9c106710c74a12d60ec33027fd6a9335627c95642ba608593b2735714da6 --- autoconf/Makefile.am | 2 +- autoconf/configure.ac | 3 +++ manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/shell.c.in | 15 ++++++++++++++- 5 files changed, 28 insertions(+), 12 deletions(-) diff --git a/autoconf/Makefile.am b/autoconf/Makefile.am index 0f0706873a..db246924fe 100644 --- a/autoconf/Makefile.am +++ b/autoconf/Makefile.am @@ -10,7 +10,7 @@ sqlite3_SOURCES = shell.c sqlite3.h EXTRA_sqlite3_SOURCES = sqlite3.c sqlite3_LDADD = @EXTRA_SHELL_OBJ@ @READLINE_LIBS@ sqlite3_DEPENDENCIES = @EXTRA_SHELL_OBJ@ -sqlite3_CFLAGS = $(AM_CFLAGS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_ENABLE_STMTVTAB -DSQLITE_ENABLE_DBSTAT_VTAB +sqlite3_CFLAGS = $(AM_CFLAGS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_ENABLE_STMTVTAB -DSQLITE_ENABLE_DBSTAT_VTAB $(SHELL_CFLAGS) include_HEADERS = sqlite3.h sqlite3ext.h diff --git a/autoconf/configure.ac b/autoconf/configure.ac index 19975e96c7..2680becbb0 100644 --- a/autoconf/configure.ac +++ b/autoconf/configure.ac @@ -169,6 +169,9 @@ AC_CHECK_HEADERS(zlib.h,[ ]) AC_SUBST(ZLIB_FLAGS) +AC_SEARCH_LIBS(system,,,[SHELL_CFLAGS="-DSQLITE_NOHAVE_SYSTEM"]) +AC_SUBST(SHELL_CFLAGS) + #----------------------------------------------------------------------- # UPDATE: Maybe it's better if users just set CFLAGS before invoking # configure. This option doesn't really add much... diff --git a/manifest b/manifest index b8ff099373..2ab0abc923 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C If\sa\svirtual\stable\sxColumn\smethod\ssets\san\serror\smessage\susing\nsqlite3_result_error(),\suse\sthat\smessage\sin\spreference\sto\sany\serror\smessage\nleft\sin\sthe\ssqlite3_vtab\sobject. -D 2018-01-30T18:33:23.120 +C Changes\sto\sthe\sautoconf\starball\sso\sthat\sit\sdoes\snot\stry\sto\suse\ssystem()\swhen\nbuilding\son\siOS. +D 2018-01-31T01:38:44.477 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -12,11 +12,11 @@ F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2 F art/sqlite370.ico af56c1d00fee7cd4753e8631ed60703ed0fc6e90 F art/sqlite370.jpg d512473dae7e378a67e28ff96a34da7cb331def2 F autoconf/INSTALL 83e4a25da9fd053c7b3665eaaaf7919707915903 -F autoconf/Makefile.am 6cca3f797c649b40c762484ce26491839fec54de72d376d774969e76ed13931f +F autoconf/Makefile.am 2c274948734e03c51790ff51468f91db8d570bcca864284d9c6d6e777264cd7e F autoconf/Makefile.msc 2c50a59319af7da4eaca8c13e3240881b1bc245fd175845a055faab7d03d6e67 F autoconf/README.first 6c4f34fe115ff55d4e8dbfa3cecf04a0188292f7 F autoconf/README.txt 4f04b0819303aabaa35fff5f7b257fb0c1ef95f1 -F autoconf/configure.ac aeeed858e5e54e79052ae44ba774e56595dcb787f23a2155aa98a8aa27327b66 +F autoconf/configure.ac 18fca06f884213be062dd5e07c5297079cc45893d9cd3f522ce426e715033e3d F autoconf/tea/Makefile.in b438a7020446c8a8156e8d97c8914a04833da6fd F autoconf/tea/README 3e9a3c060f29a44344ab50aec506f4db903fb873 F autoconf/tea/aclocal.m4 52c47aac44ce0ddb1f918b6993e8beb8eee88f43 @@ -488,7 +488,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c f02352ac5cbb6fad3804add825743b924cfb6c742ba2e8666d726828a9985d73 -F src/shell.c.in 7cea439c3f7f2e4ed6eb4b3a633cd93dccb1349241400de4da0c1291285ed514 +F src/shell.c.in ada6d184a5cdaada15a330ca97ad0126125f0aaed40b5c04385278fcaa78aafa F src/sqlite.h.in 51f9acf52c80113d793ddd38b3940ad6895d97b4752503b19291fb8fcbf54c5e F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 83a3c4ce93d650bedfd1aa558cb85a516bd6d094445ee989740827d0d944368d @@ -1704,7 +1704,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 13b786dafd308b7aef42328f26fe64981c83d524e4bf82ceb7d58afc7146c7d3 -R 48ccd51e910d5d576747af2d5e64e430 -U dan -Z 9d867999d41b646c89b308fd5587d517 +P 71e3b715ad36c2132a587b84221be6359c31d7a17c3fba201b7ed3baf17922f1 +R bcc68c8a2653dd62fee16c20bb3144f0 +U drh +Z ff1bd702823e7d15c0a453ffcea5582a diff --git a/manifest.uuid b/manifest.uuid index 2c1ee2f33c..4422b0dd56 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -71e3b715ad36c2132a587b84221be6359c31d7a17c3fba201b7ed3baf17922f1 \ No newline at end of file +32ed9c106710c74a12d60ec33027fd6a9335627c95642ba608593b2735714da6 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index a785517503..f683ce9868 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -1175,6 +1175,7 @@ static void shellPutsFunc( ** ** Also throw an error if the EDITOR program returns a non-zero exit code. */ +#ifndef SQLITE_NOHAVE_SYSTEM static void editFunc( sqlite3_context *context, int argc, @@ -1285,6 +1286,7 @@ edit_func_end: sqlite3_free(zTempFile); sqlite3_free(p); } +#endif /* SQLITE_NOHAVE_SYSTEM */ /* ** Save or restore the current output mode @@ -3273,10 +3275,14 @@ static char zHelp[] = ".session CMD ... Create or control sessions\n" #endif ".sha3sum ?OPTIONS...? Compute a SHA3 hash of database content\n" +#ifndef SQLITE_NOHAVE_SYSTEM ".shell CMD ARGS... Run CMD ARGS... in a system shell\n" +#endif ".show Show the current values for various settings\n" ".stats ?on|off? Show stats or turn stats on or off\n" +#ifndef SQLITE_NOHAVE_SYSTEM ".system CMD ARGS... Run CMD ARGS... in a system shell\n" +#endif ".tables ?TABLE? List names of tables\n" " If TABLE specified, only list tables matching\n" " LIKE pattern TABLE.\n" @@ -3477,10 +3483,12 @@ static void open_db(ShellState *p, int keepAlive){ shellModuleSchema, 0, 0); sqlite3_create_function(p->db, "shell_putsnl", 1, SQLITE_UTF8, p, shellPutsFunc, 0, 0); +#ifndef SQLITE_NOHAVE_SYSTEM sqlite3_create_function(p->db, "edit", 1, SQLITE_UTF8, 0, editFunc, 0, 0); sqlite3_create_function(p->db, "edit", 2, SQLITE_UTF8, 0, editFunc, 0, 0); +#endif if( p->openMode==SHELL_OPEN_ZIPFILE ){ char *zSql = sqlite3_mprintf( "CREATE VIRTUAL TABLE zip USING zipfile(%Q);", p->zDbFilename); @@ -4095,6 +4103,7 @@ static void output_reset(ShellState *p){ #endif }else{ output_file_close(p->out); +#ifndef SQLITE_NOHAVE_SYSTEM if( p->doXdgOpen ){ const char *zXdgOpenCmd = #if defined(_WIN32) @@ -4113,6 +4122,7 @@ static void output_reset(ShellState *p){ outputModePop(p); p->doXdgOpen = 0; } +#endif /* !defined(SQLITE_NOHAVE_SYSTEM) */ } p->outfile[0] = 0; p->out = stdout; @@ -6353,6 +6363,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } output_reset(p); if( zFile[0]=='-' && zFile[1]=='-' ) zFile++; +#ifndef SQLITE_NOHAVE_SYSTEM if( strcmp(zFile, "-e")==0 || strcmp(zFile, "-x")==0 ){ p->doXdgOpen = 1; outputModePush(p); @@ -6367,6 +6378,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } zFile = p->zTempFile; } +#endif /* SQLITE_NOHAVE_SYSTEM */ if( zFile[0]=='|' ){ #ifdef SQLITE_OMIT_POPEN raw_printf(stderr, "Error: pipes are not supported in this OS\n"); @@ -6486,7 +6498,6 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3_close(pSrc); }else - if( c=='s' && strncmp(azArg[0], "scanstats", n)==0 ){ if( nArg==2 ){ p->scanstatsOn = (u8)booleanValue(azArg[1]); @@ -7082,6 +7093,7 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3_free(zSql); }else +#ifndef SQLITE_NOHAVE_SYSTEM if( c=='s' && (strncmp(azArg[0], "shell", n)==0 || strncmp(azArg[0],"system",n)==0) ){ @@ -7101,6 +7113,7 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3_free(zCmd); if( x ) raw_printf(stderr, "System command returns %d\n", x); }else +#endif /* !defined(SQLITE_NOHAVE_SYSTEM) */ if( c=='s' && strncmp(azArg[0], "show", n)==0 ){ static const char *azBool[] = { "off", "on", "trigger", "full"}; From 4ea48144f91a65e82900cf8587380d8b451eaf1d Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 31 Jan 2018 14:07:01 +0000 Subject: [PATCH 027/107] Fix a failing assert() in the new code on this branch. FossilOrigin-Name: 74d857d178dfadea7e07ba47439fe11aa9d282b54caf78cd6961e593b37406d0 --- manifest | 17 +++++++---------- manifest.uuid | 2 +- src/where.c | 4 +--- test/join2.test | 11 +++++++++++ 4 files changed, 20 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 7c15e392d1..4b0592f36e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sthe\somit-left-join\soptimization\sso\sthat\sit\sworks\sin\ssome\scases\swhen\sthe\nRHS\sis\ssubject\sto\sa\sUNIQUE\sbut\snot\sNOT\sNULL\sconstraint. -D 2018-01-29T17:08:52.190 +C Fix\sa\sfailing\sassert()\sin\sthe\snew\scode\son\sthis\sbranch. +D 2018-01-31T14:07:01.020 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -570,7 +570,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 5a3f464edd64596f601683ed321d12e6fd93c5fb9afdfb3653d6ffd0fee9c48f F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a F src/walker.c da987a20d40145c0a03c07d8fefcb2ed363becc7680d0500d9c79915591f5b1f -F src/where.c eb37882608c30f7361cf3309b8128c3a2887ff0099886e3685ededb7a7e6b051 +F src/where.c 7cae47e813393d70c6d327fdf000fcb30f76b1b0b5a5b52ff6402e0c658de32c F src/whereInt.h 82c04c5075308abbac59180c8bad5ecb45b07453981f60a53f3c7dee21e1e971 F src/wherecode.c cb360c511f69294ddf00340d7b390e6b1d601a1cfb77b42c4d316fe2f4cd01c3 F src/whereexpr.c 22dbfd3bf5f6051a61523dd0ebef7a944fb29ee4aa7d2a62feb8aac6ffbbc0eb @@ -1001,7 +1001,7 @@ F test/ioerr4.test f130fe9e71008577b342b8874d52984bd04ede2c F test/ioerr5.test 2edfa4fb0f896f733071303b42224df8bedd9da4 F test/ioerr6.test a395a6ab144b26a9e3e21059a1ab6a7149cca65b F test/join.test 442c462eea85cf065d70a663c626b780a95af6e11585d909bb63b87598afe678 -F test/join2.test 96f56ccf0a8fe6dbd7bb2e1a90bc61540febeaf9b6fdee14506f6fc140b93185 +F test/join2.test f1b5ba94ba70a791df2d99e61adcebf62edacad1eca57881c97eb3e2ca718595 F test/join3.test 6f0c774ff1ba0489e6c88a3e77b9d3528fb4fda0 F test/join4.test 1a352e4e267114444c29266ce79e941af5885916 F test/join5.test bc98ea4b4e5003f5b1453701ebb8cd7d1c01a550 @@ -1702,10 +1702,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8767f7b880f2e4112f75f0b6ef7be3f50ab1ae20e103e7d03d8bfe77e6c79438 -R f19810b3aa9bfaa7d3b666e31ece61ec -T *branch * omit-left-join-fix -T *sym-omit-left-join-fix * -T -sym-trunk * +P 88411a402b29f19d69e0d06d9099eabb98c238ddb7c07ce110236316f008753e +R d723a0554461743d1cc2169a56c72b46 U dan -Z ff1cffb196461bca97e3f8e467ebcd67 +Z 54f22eae691d89833b1c2792d257ce0d diff --git a/manifest.uuid b/manifest.uuid index 84c8ec4673..15554a93f9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -88411a402b29f19d69e0d06d9099eabb98c238ddb7c07ce110236316f008753e \ No newline at end of file +74d857d178dfadea7e07ba47439fe11aa9d282b54caf78cd6961e593b37406d0 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 3b18fbf5ff..8c1e3cd01f 100644 --- a/src/where.c +++ b/src/where.c @@ -2481,8 +2481,7 @@ static int whereLoopAddBtreeIndex( if( iCol==XN_ROWID || (iCol>=0 && nInMul==0 && saved_nEq==pProbe->nKeyCol-1) ){ - assert( iCol!=XN_ROWID || pProbe->uniqNotNull ); - if( pProbe->uniqNotNull + if( iCol==XN_ROWID || pProbe->uniqNotNull || (pProbe->nKeyCol==1 && pProbe->onError && eOp==WO_EQ) ){ pNew->wsFlags |= WHERE_ONEROW; @@ -2828,7 +2827,6 @@ static int whereLoopAddBtree( sPk.onError = OE_Replace; sPk.pTable = pTab; sPk.szIdxRow = pTab->szTabRow; - sPk.uniqNotNull = 1; aiRowEstPk[0] = pTab->nRowLogEst; aiRowEstPk[1] = 0; pFirst = pSrc->pTab->pIndex; diff --git a/test/join2.test b/test/join2.test index 3dc067e508..256846ac91 100644 --- a/test/join2.test +++ b/test/join2.test @@ -254,4 +254,15 @@ do_eqp_test 5.2 { 0 0 0 {SCAN TABLE s1} } +do_execsql_test 6.0 { + CREATE TABLE u1(a INTEGER PRIMARY KEY, b, c); + CREATE TABLE u2(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX u1ab ON u1(b, c); +} +do_eqp_test 6.1 { + SELECT u2.* FROM u2 LEFT JOIN u1 ON( u1.a=u2.a AND u1.b=u2.b AND u1.c=u2.c ); +} { + 0 0 0 {SCAN TABLE u2} +} + finish_test From 44091ed3682ac115c108b3a1d201db76a78caf37 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 31 Jan 2018 19:13:31 +0000 Subject: [PATCH 028/107] When creating a new archive entry, have zipfile store UTC instead of local time in the legacy MS-DOS format timestamp field. FossilOrigin-Name: e2114df18383d111dd5fbac902e08b42a7f4b2b2d6f7bf29574a3722e4a4dad5 --- ext/misc/zipfile.c | 78 +++++++++++++++++++++++++++++++--------------- manifest | 22 +++++++------ manifest.uuid | 2 +- test/zipfile.test | 54 ++++++++++++++++++++++++++++++++ test/zipfile2.test | 45 ++++++++++++++------------ 5 files changed, 145 insertions(+), 56 deletions(-) diff --git a/ext/misc/zipfile.c b/ext/misc/zipfile.c index a52033e25b..067b6d4953 100644 --- a/ext/misc/zipfile.c +++ b/ext/misc/zipfile.c @@ -686,18 +686,34 @@ static int zipfileScanExtra(u8 *aExtra, int nExtra, u32 *pmTime){ ** ** https://msdn.microsoft.com/en-us/library/9kkf9tah.aspx */ -static time_t zipfileMtime(ZipfileCDS *pCDS){ - struct tm t; - memset(&t, 0, sizeof(t)); - t.tm_sec = (pCDS->mTime & 0x1F)*2; - t.tm_min = (pCDS->mTime >> 5) & 0x2F; - t.tm_hour = (pCDS->mTime >> 11) & 0x1F; +static u32 zipfileMtime(ZipfileCDS *pCDS){ + int Y = (1980 + ((pCDS->mDate >> 9) & 0x7F)); + int M = ((pCDS->mDate >> 5) & 0x0F); + int D = (pCDS->mDate & 0x1F); + int B = -13; - t.tm_mday = (pCDS->mDate & 0x1F); - t.tm_mon = ((pCDS->mDate >> 5) & 0x0F) - 1; - t.tm_year = 80 + ((pCDS->mDate >> 9) & 0x7F); + int sec = (pCDS->mTime & 0x1F)*2; + int min = (pCDS->mTime >> 5) & 0x3F; + int hr = (pCDS->mTime >> 11) & 0x1F; - return mktime(&t); + /* JD = INT(365.25 * (Y+4716)) + INT(30.6001 * (M+1)) + D + B - 1524.5 */ + + /* Calculate the JD in seconds for noon on the day in question */ + if( M<3 ){ + Y = Y-1; + M = M+12; + } + i64 JD = (i64)(24*60*60) * ( + (int)(365.25 * (Y + 4716)) + + (int)(30.6001 * (M + 1)) + + D + B - 1524 + ); + + /* Correct the JD for the time within the day */ + JD += (hr-12) * 3600 + min * 60 + sec; + + /* Convert JD to unix timestamp (the JD epoch is 2440587.5) */ + return (u32)(JD - (i64)(24405875) * 24*60*6); } /* @@ -706,24 +722,36 @@ static time_t zipfileMtime(ZipfileCDS *pCDS){ ** to the UNIX timestamp value passed as the second. */ static void zipfileMtimeToDos(ZipfileCDS *pCds, u32 mUnixTime){ - time_t t = (time_t)mUnixTime; - struct tm res; + /* Convert unix timestamp to JD (2440588 is noon on 1/1/1970) */ + i64 JD = (i64)2440588 + mUnixTime / (24*60*60); -#if !defined(_WIN32) && !defined(WIN32) - localtime_r(&t, &res); -#else - memcpy(&res, localtime(&t), sizeof(struct tm)); -#endif + int A, B, C, D, E; + int yr, mon, day; + int hr, min, sec; - pCds->mTime = (u16)( - (res.tm_sec / 2) + - (res.tm_min << 5) + - (res.tm_hour << 11)); + A = (int)((JD - 1867216.25)/36524.25); + A = JD + 1 + A - (A/4); + B = A + 1524; + C = (int)((B - 122.1)/365.25); + D = (36525*(C&32767))/100; + E = (int)((B-D)/30.6001); - pCds->mDate = (u16)( - (res.tm_mday-1) + - ((res.tm_mon+1) << 5) + - ((res.tm_year-80) << 9)); + day = B - D - (int)(30.6001*E); + mon = (E<14 ? E-1 : E-13); + yr = mon>2 ? C-4716 : C-4715; + + hr = (mUnixTime % (24*60*60)) / (60*60); + min = (mUnixTime % (60*60)) / 60; + sec = (mUnixTime % 60); + + pCds->mDate = (u16)(day + (mon << 5) + ((yr-1980) << 9)); + pCds->mTime = (u16)(sec/2 + (min<<5) + (hr<<11)); + + assert( mUnixTime<315507600 + || mUnixTime==zipfileMtime(pCds) + || ((mUnixTime % 2) && mUnixTime-1==zipfileMtime(pCds)) + /* || (mUnixTime % 2) */ + ); } /* diff --git a/manifest b/manifest index 5ab4a82cba..96d387ed83 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improve\sthe\somit-left-join\soptimization\sso\sthat\sit\sworks\sin\ssome\scases\s\nwhen\sthe\sRHS\sis\ssubject\sto\sa\sUNIQUE\sbut\snot\sNOT\sNULL\sconstraint. -D 2018-01-31T16:50:27.186 +C When\screating\sa\snew\sarchive\sentry,\shave\szipfile\sstore\sUTC\sinstead\sof\slocal\ntime\sin\sthe\slegacy\sMS-DOS\sformat\stimestamp\sfield. +D 2018-01-31T19:13:31.142 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -304,7 +304,7 @@ F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178 F ext/misc/vtablog.c 31d0d8f4406795679dcd3a67917c213d3a2a5fb3ea5de35f6e773491ed7e13c9 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 -F ext/misc/zipfile.c aa10ec6a235030cd368b511e7c78e40251008dddddeb0eb006ad1f344f78b690 +F ext/misc/zipfile.c 56028f7e74d948b7bef834624d128f563eb63f5950248d538afa9d0f6c3365dc F ext/rbu/rbu.c ea7d1b7eb44c123a2a619332e19fe5313500705c4a58aaa1887905c0d83ffc2e F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842 F ext/rbu/rbu10.test 1846519a438697f45e9dcb246908af81b551c29e1078d0304fae83f1fed7e9ee @@ -1603,8 +1603,8 @@ F test/wordcount.c cb589cec469a1d90add05b1f8cee75c7210338d87a5afd65260ed5c0f4bbf F test/writecrash.test f1da7f7adfe8d7f09ea79b42e5ca6dcc41102f27f8e334ad71539501ddd910cc F test/zeroblob.test 3857870fe681b8185654414a9bccfde80b62a0fa F test/zerodamage.test 9c41628db7e8d9e8a0181e59ea5f189df311a9f6ce99cc376dc461f66db6f8dc -F test/zipfile.test 368a5a0c97be0caaf8c3efa8293bfe18436d546805678fa00b6aa81bc98727ec -F test/zipfile2.test 1066dd9769028d6f0bf1c87303b24f087cc064d9bbcacefc95613380f642de18 +F test/zipfile.test 3695ab6d731720ca0b542fe21ded39d4a34891f4a590b2e6c909c9ff07e13148 +F test/zipfile2.test 5f93611307c131e83f226a471231d769b794b9e8c6a675cfa3d34b1a79df23fe F test/zipfilefault.test df4fa9e16116e0cb21d09037a6b0a7d93fecd8767d82154b66044b4ca43fca58 F tool/GetFile.cs a15e08acb5dd7539b75ba23501581d7c2b462cb5 F tool/GetTclKit.bat 8995df40c4209808b31f24de0b58f90930239a234f7591e3675d45bfbb990c5d @@ -1704,8 +1704,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 32ed9c106710c74a12d60ec33027fd6a9335627c95642ba608593b2735714da6 74d857d178dfadea7e07ba47439fe11aa9d282b54caf78cd6961e593b37406d0 -R ee3ce7babbd6fba601d683cb41b035ba -T +closed 74d857d178dfadea7e07ba47439fe11aa9d282b54caf78cd6961e593b37406d0 -U drh -Z 20db364b7139e3a10e7c112d48b03b80 +P 02ba8a7ba7ba71cd7abd5dd3093ea486f53a025f6972bb444f2da37e0e2fc3b2 +R 4025431a2ff076723fc2f621d10c0c47 +T *branch * zipfile-timestamp-fix +T *sym-zipfile-timestamp-fix * +T -sym-trunk * +U dan +Z 7679dfe9f8ef39802f520b9ec5eb4496 diff --git a/manifest.uuid b/manifest.uuid index 0940797d9a..555963b3cf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -02ba8a7ba7ba71cd7abd5dd3093ea486f53a025f6972bb444f2da37e0e2fc3b2 \ No newline at end of file +e2114df18383d111dd5fbac902e08b42a7f4b2b2d6f7bf29574a3722e4a4dad5 \ No newline at end of file diff --git a/test/zipfile.test b/test/zipfile.test index 76e7d924df..a6af4f8963 100644 --- a/test/zipfile.test +++ b/test/zipfile.test @@ -48,6 +48,18 @@ if {$::tcl_platform(platform)=="unix" && [catch {exec unzip}]==0} { } } + +# The argument is a blob (not a hex string) containing a zip archive. +# This proc removes the extended timestamp fields from the archive +# and returns the result. +# +proc remove_timestamps {blob} { + set hex [binary encode hex $blob] + set hex [string map {55540500 00000500} $hex] + binary decode hex $hex +} + + # Argument $file is the name of a zip archive on disk. This function # executes test cases to check that the results of each of the following # are the same: @@ -367,6 +379,48 @@ do_catchsql_test 4.2 { CREATE VIRTUAL TABLE yyy USING zipfile('test.zip', 'test.zip'); } {1 {zipfile constructor requires one argument}} +#-------------------------------------------------------------------------- + +db func rt remove_timestamps +do_execsql_test 5.0 { + WITH c(name,mtime,data) AS ( + SELECT 'a.txt', 946684800, 'abc' + ) + SELECT name,mtime,data FROM zipfile( + ( SELECT rt( zipfile(name,NULL,mtime,data) ) FROM c ) + ) +} { + a.txt 946684800 abc +} + +if {[info vars ::UNZIP]!=""} { + load_static_extension db fileio + forcedelete test.zip + do_test 6.0 { + execsql { + WITH c(name,mtime,data) AS ( + SELECT 'a.txt', 946684800, 'abc' UNION ALL + SELECT 'b.txt', 1000000000, 'abc' UNION ALL + SELECT 'c.txt', 1111111000, 'abc' + ) + SELECT writefile('test.zip', + ( SELECT rt ( zipfile(name,NULL,mtime,data) ) FROM c ) + ); + } + forcedelete test_unzip + file mkdir test_unzip + exec unzip -d test_unzip test.zip + + db eval { + SELECT name, mtime FROM fsdir('test_unzip') WHERE name!='test_unzip' + ORDER BY name + } + } [list {*}{ + test_unzip/a.txt 946684800 + test_unzip/b.txt 1000000000 + test_unzip/c.txt 1111111000 + }] +} finish_test diff --git a/test/zipfile2.test b/test/zipfile2.test index 6d20e09301..a6c3e7e3c4 100644 --- a/test/zipfile2.test +++ b/test/zipfile2.test @@ -70,12 +70,17 @@ set archive { 00007A0000000000 } -do_execsql_test 3.1 { - WITH contents(name,mtime,data) AS ( - VALUES('a.txt', 1000000, 'contents of a.txt') UNION ALL - VALUES('b.txt', 1000000, 'contents of b.txt') - ) SELECT quote( zipfile(name,NULL,mtime,data) ) FROM contents; -} [blobliteral $archive] +if 0 { + # This test is broken - the archive generated is slightly different + # depending on the zlib version used. + do_execsql_test 3.1 { + WITH contents(name,mtime,data) AS ( + VALUES('a.txt', 1000000, 'contents of a.txt') UNION ALL + VALUES('b.txt', 1000000, 'contents of b.txt') + ) SELECT quote( zipfile(name,NULL,mtime,data) ) FROM contents; + } [blobliteral $archive] +} + set blob [blob $archive] do_execsql_test 3.2 { @@ -98,6 +103,7 @@ for {set i 0} {$i < [llength $L]} {incr i} { } {/1 .*/} } +# Change the "extra info id" for all extended-timestamp fields. set L [findall 5554 $archive] for {set i 0} {$i < [llength $L]} {incr i} { set idx [lindex $L $i] @@ -123,20 +129,18 @@ for {set i 0} {$i < [llength $L]} {incr i} { } } -if 0 { -set blob [db one { - WITH contents(name,mtime,data) AS ( - VALUES('a.txt', 1000000, 'aaaaaaaaaaaaaaaaaaaaaaa') - ) SELECT quote( zipfile(name,NULL,mtime,data) ) FROM contents; -}] -set blob [string range $blob 2 end] -set blob [string range $blob 0 end-1] -while {[string length $blob]>0} { - puts [string range $blob 0 63] - set blob [string range $blob 64 end] -} -exit -} +# set blob [db one { +# WITH contents(name,mtime,data) AS ( +# VALUES('a.txt', 1000000, 'aaaaaaaaaaaaaaaaaaaaaaa') +# ) SELECT quote( zipfile(name,NULL,mtime,data) ) FROM contents; +# }] +# set blob [string range $blob 2 end] +# set blob [string range $blob 0 end-1] +# while {[string length $blob]>0} { +# puts [string range $blob 0 63] +# set blob [string range $blob 64 end] +# } +# exit set archive2 { 504B0304140000080800D4A52BEC08F54C6E050000001700000005000900612E @@ -161,5 +165,6 @@ do_catchsql_test 4.1 { } {1 {inflate() failed (0)}} + finish_test From cfaffad65c2e1ae7db956f3b177d55eed3bccb09 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 31 Jan 2018 19:45:58 +0000 Subject: [PATCH 029/107] Fix a test case in zipfile.test. FossilOrigin-Name: 4eb5b24c64ee5907c18371f563b79fa0caf58285a62b8f09e639a690e6727eaa --- manifest | 15 ++++++--------- manifest.uuid | 2 +- test/zipfile.test | 31 +++++++++++++++++++++++++------ 3 files changed, 32 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index 96d387ed83..b6843e249b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\screating\sa\snew\sarchive\sentry,\shave\szipfile\sstore\sUTC\sinstead\sof\slocal\ntime\sin\sthe\slegacy\sMS-DOS\sformat\stimestamp\sfield. -D 2018-01-31T19:13:31.142 +C Fix\sa\stest\scase\sin\szipfile.test. +D 2018-01-31T19:45:58.864 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -1603,7 +1603,7 @@ F test/wordcount.c cb589cec469a1d90add05b1f8cee75c7210338d87a5afd65260ed5c0f4bbf F test/writecrash.test f1da7f7adfe8d7f09ea79b42e5ca6dcc41102f27f8e334ad71539501ddd910cc F test/zeroblob.test 3857870fe681b8185654414a9bccfde80b62a0fa F test/zerodamage.test 9c41628db7e8d9e8a0181e59ea5f189df311a9f6ce99cc376dc461f66db6f8dc -F test/zipfile.test 3695ab6d731720ca0b542fe21ded39d4a34891f4a590b2e6c909c9ff07e13148 +F test/zipfile.test a16528022b08789168adc9c79646e41fb55c4f94b26a2aca97085ebc591ca84d F test/zipfile2.test 5f93611307c131e83f226a471231d769b794b9e8c6a675cfa3d34b1a79df23fe F test/zipfilefault.test df4fa9e16116e0cb21d09037a6b0a7d93fecd8767d82154b66044b4ca43fca58 F tool/GetFile.cs a15e08acb5dd7539b75ba23501581d7c2b462cb5 @@ -1704,10 +1704,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 02ba8a7ba7ba71cd7abd5dd3093ea486f53a025f6972bb444f2da37e0e2fc3b2 -R 4025431a2ff076723fc2f621d10c0c47 -T *branch * zipfile-timestamp-fix -T *sym-zipfile-timestamp-fix * -T -sym-trunk * +P e2114df18383d111dd5fbac902e08b42a7f4b2b2d6f7bf29574a3722e4a4dad5 +R 1aa884bc5be6161439d6cdee8401a11e U dan -Z 7679dfe9f8ef39802f520b9ec5eb4496 +Z 880157cddf7626c80e702257f31b9f96 diff --git a/manifest.uuid b/manifest.uuid index 555963b3cf..91dda60143 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e2114df18383d111dd5fbac902e08b42a7f4b2b2d6f7bf29574a3722e4a4dad5 \ No newline at end of file +4eb5b24c64ee5907c18371f563b79fa0caf58285a62b8f09e639a690e6727eaa \ No newline at end of file diff --git a/test/zipfile.test b/test/zipfile.test index a6af4f8963..92d8501ea8 100644 --- a/test/zipfile.test +++ b/test/zipfile.test @@ -394,8 +394,9 @@ do_execsql_test 5.0 { } if {[info vars ::UNZIP]!=""} { +ifcapable datetime { load_static_extension db fileio - forcedelete test.zip + forcedelete test1.zip test2.zip do_test 6.0 { execsql { WITH c(name,mtime,data) AS ( @@ -403,16 +404,33 @@ if {[info vars ::UNZIP]!=""} { SELECT 'b.txt', 1000000000, 'abc' UNION ALL SELECT 'c.txt', 1111111000, 'abc' ) - SELECT writefile('test.zip', - ( SELECT rt ( zipfile(name,NULL,mtime,data) ) FROM c ) - ); + SELECT writefile('test1.zip', rt( zipfile(name, NULL, mtime, data) ) ), + writefile('test2.zip', ( zipfile(name, NULL, mtime, data) ) ) + FROM c; } forcedelete test_unzip file mkdir test_unzip - exec unzip -d test_unzip test.zip + exec unzip -d test_unzip test1.zip db eval { - SELECT name, mtime FROM fsdir('test_unzip') WHERE name!='test_unzip' + SELECT name, strftime('%s', mtime, 'unixepoch', 'localtime') + FROM fsdir('test_unzip') WHERE name!='test_unzip' + ORDER BY name + } + } [list {*}{ + test_unzip/a.txt 946684800 + test_unzip/b.txt 1000000000 + test_unzip/c.txt 1111111000 + }] + + do_test 6.1 { + forcedelete test_unzip + file mkdir test_unzip + exec unzip -d test_unzip test2.zip + + db eval { + SELECT name, mtime + FROM fsdir('test_unzip') WHERE name!='test_unzip' ORDER BY name } } [list {*}{ @@ -421,6 +439,7 @@ if {[info vars ::UNZIP]!=""} { test_unzip/c.txt 1111111000 }] } +} finish_test From 06b3bd5b7add86bcf178abe008ba1189c973f6f3 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 1 Feb 2018 01:13:33 +0000 Subject: [PATCH 030/107] When an index is based on a text representation of a numeric column in the original table, make sure the indexed value uses the canonical text representation of the numeric value in the table. Proposed fix for ticket [343634942dd54ab57b70]. FossilOrigin-Name: 88e2ce916791d488076584f3795a89eb4277fcb812af9e4c2f383815d55ff6f1 --- manifest | 17 ++++++++--------- manifest.uuid | 2 +- src/vdbe.c | 5 +++++ test/indexexpr1.test | 21 +++++++++++++++++++++ 4 files changed, 35 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index adb223764e..96a5ce3aff 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\screating\sa\snew\sarchive\sentry,\shave\szipfile\sstore\sUTC\sinstead\sof\slocal\ntime\sin\sthe\slegacy\sMS-DOS\sformat\stimestamp\sfield. -D 2018-01-31T20:18:26.779 +C When\san\sindex\sis\sbased\son\sa\stext\srepresentation\sof\sa\snumeric\scolumn\sin\sthe\noriginal\stable,\smake\ssure\sthe\sindexed\svalue\suses\sthe\scanonical\stext\s\nrepresentation\sof\sthe\snumeric\svalue\sin\sthe\stable.\nProposed\sfix\sfor\sticket\s[343634942dd54ab57b70]. +D 2018-02-01T01:13:33.559 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -556,7 +556,7 @@ F src/update.c a90a32ffc0100265b0693dbbdbe490756447af181f5ea2c138cce515b08c8795 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5 F src/util.c d9eb0a6c4aae1b00a7369eadd7ca0bbe946cb4c953b6751aa20d357c2f482157 F src/vacuum.c 90839322fd5f00df9617eb21b68beda9b6e2a2937576b0d65985e4aeb1c53739 -F src/vdbe.c 619d385ff1798fab7c549deb04b078f5a8ec466e2ad315939bc824593a2f9fde +F src/vdbe.c a6892805df427bb9d554c37641406438bff568f99d26e72c1fb7e45f2ed6ce5b F src/vdbe.h 134beb7a12a6213c00eba58febaede33447cc4441bc568a0d9c144b33fc3720a F src/vdbeInt.h 8d7d07f13cb3c4cbca91e22ba4a1920e542dda7c5d9299920432a0b3d5b009f5 F src/vdbeapi.c fea41171884a4de119f8b10ab514c788674eeeb7f27218bb6d008e1310bfd07f @@ -978,7 +978,7 @@ F test/index7.test 7feababe16f2091b229c22aff2bcc1d4d6b9d2bb F test/index8.test bc2e3db70e8e62459aaa1bd7e4a9b39664f8f9d7 F test/index9.test 0aa3e509dddf81f93380396e40e9bb386904c1054924ba8fa9bcdfe85a8e7721 F test/indexedby.test faa585e315e868f09bce0eb39c41d6134649b13d2801638294d3ae616edf1609 -F test/indexexpr1.test ace1ad489adc25325ad298434f13b1a515b36bf5dca9fe2a4b66cdf17aea3fa0 +F test/indexexpr1.test 635261197bcdc19b9b2c59bbfa7227d525c00e9587faddb2d293c44d287ce60e F test/indexexpr2.test 13247bac49143196556eb3f65e97ef301bd3e993f4511558b5db322ddc370ea6 F test/indexfault.test 31d4ab9a7d2f6e9616933eb079722362a883eb1d F test/init.test 15c823093fdabbf7b531fe22cf037134d09587a7 @@ -1704,8 +1704,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 02ba8a7ba7ba71cd7abd5dd3093ea486f53a025f6972bb444f2da37e0e2fc3b2 4eb5b24c64ee5907c18371f563b79fa0caf58285a62b8f09e639a690e6727eaa -R 1aa884bc5be6161439d6cdee8401a11e -T +closed 4eb5b24c64ee5907c18371f563b79fa0caf58285a62b8f09e639a690e6727eaa -U dan -Z a1436cfe5518dde187c59fef646e7300 +P b730d187f2202e5b5d31ed6c94c9bb04d7c289f7086a9b44b3d9050ea3586d3a +R 03c296c6343c0711547e5b14fdc5c828 +U drh +Z 93daef2f8ab3c187bf34c937a8efe1d7 diff --git a/manifest.uuid b/manifest.uuid index b0255f791b..c0c07f33f0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b730d187f2202e5b5d31ed6c94c9bb04d7c289f7086a9b44b3d9050ea3586d3a \ No newline at end of file +88e2ce916791d488076584f3795a89eb4277fcb812af9e4c2f383815d55ff6f1 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 7d1cae0caf..1bcaedbfee 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -264,6 +264,11 @@ static void applyNumericAffinity(Mem *pRec, int bTryForInt){ pRec->flags |= MEM_Real; if( bTryForInt ) sqlite3VdbeIntegerAffinity(pRec); } + /* TEXT->NUMERIC is many->one. Hence, it is important to invalidate the + ** string representation after computing a numeric equivalent, because the + ** string representation might not be the canonical representation for the + ** numeric value. Ticket [343634942dd54ab57b7024] 2018-01-31. */ + pRec->flags &= ~MEM_Str; } /* diff --git a/test/indexexpr1.test b/test/indexexpr1.test index 28c23b9089..e93dcc0cd1 100644 --- a/test/indexexpr1.test +++ b/test/indexexpr1.test @@ -425,4 +425,25 @@ do_execsql_test indexexpr-1510 { REPLACE INTO t1 SELECT a, randomblob(a) FROM t1 } {} +# 2018-01-31 https://www.sqlite.org/src/tktview/343634942dd54ab57b702411 +# When an index on an expression depends on the string representation of +# a numeric table column, trouble can arise since there are multiple +# string that can map to the same numeric value. (Ex: 123, 0123, 000123). +# +do_execsql_test indexexpr-1600 { + DROP TABLE IF EXISTS t1; + CREATE TABLE t1 (a INTEGER, b); + CREATE INDEX idx1 ON t1 (lower(a)); + INSERT INTO t1 VALUES('0001234',3); + PRAGMA integrity_check; +} {ok} +do_execsql_test indexexpr-1610 { + INSERT INTO t1 VALUES('1234',0),('001234',2),('01234',1); + SELECT b FROM t1 WHERE lower(a)='1234' ORDER BY +b; +} {0 1 2 3} +do_execsql_test indexexpr-1620 { + SELECT b FROM t1 WHERE lower(a)='01234' ORDER BY +b; +} {} + + finish_test From c3ef23a1c9f71ae4eb9fbab9d0fa6da21de4da62 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 1 Feb 2018 15:19:54 +0000 Subject: [PATCH 031/107] Fix a memory leak that could follow an IO error in the zipfile extension. And add other tests. FossilOrigin-Name: e6bb750697c3c7ceb5ce41d216e8ef6a1d556822a3b55e0a007b4a03e194a7d9 --- ext/misc/zipfile.c | 33 +++++++++++++++++---------------- manifest | 18 +++++++++--------- manifest.uuid | 2 +- test/zipfile.test | 37 ++++++++++++++++++++++++++++++++++++- test/zipfilefault.test | 15 ++++++++++++++- 5 files changed, 77 insertions(+), 28 deletions(-) diff --git a/ext/misc/zipfile.c b/ext/misc/zipfile.c index 067b6d4953..dc2d831d37 100644 --- a/ext/misc/zipfile.c +++ b/ext/misc/zipfile.c @@ -2017,23 +2017,24 @@ void zipfileFinal(sqlite3_context *pCtx){ u8 *aZip; p = (ZipfileCtx*)sqlite3_aggregate_context(pCtx, sizeof(ZipfileCtx)); - if( p==0 || p->nEntry==0 ) return; + if( p==0 ) return; + if( p->nEntry>0 ){ + memset(&eocd, 0, sizeof(eocd)); + eocd.nEntry = p->nEntry; + eocd.nEntryTotal = p->nEntry; + eocd.nSize = p->cds.n; + eocd.iOffset = p->body.n; - memset(&eocd, 0, sizeof(eocd)); - eocd.nEntry = p->nEntry; - eocd.nEntryTotal = p->nEntry; - eocd.nSize = p->cds.n; - eocd.iOffset = p->body.n; - - nZip = p->body.n + p->cds.n + ZIPFILE_EOCD_FIXED_SZ; - aZip = (u8*)sqlite3_malloc(nZip); - if( aZip==0 ){ - sqlite3_result_error_nomem(pCtx); - }else{ - memcpy(aZip, p->body.a, p->body.n); - memcpy(&aZip[p->body.n], p->cds.a, p->cds.n); - zipfileSerializeEOCD(&eocd, &aZip[p->body.n + p->cds.n]); - sqlite3_result_blob(pCtx, aZip, nZip, zipfileFree); + nZip = p->body.n + p->cds.n + ZIPFILE_EOCD_FIXED_SZ; + aZip = (u8*)sqlite3_malloc(nZip); + if( aZip==0 ){ + sqlite3_result_error_nomem(pCtx); + }else{ + memcpy(aZip, p->body.a, p->body.n); + memcpy(&aZip[p->body.n], p->cds.a, p->cds.n); + zipfileSerializeEOCD(&eocd, &aZip[p->body.n + p->cds.n]); + sqlite3_result_blob(pCtx, aZip, nZip, zipfileFree); + } } sqlite3_free(p->body.a); diff --git a/manifest b/manifest index 96a5ce3aff..7f7cb2415b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\san\sindex\sis\sbased\son\sa\stext\srepresentation\sof\sa\snumeric\scolumn\sin\sthe\noriginal\stable,\smake\ssure\sthe\sindexed\svalue\suses\sthe\scanonical\stext\s\nrepresentation\sof\sthe\snumeric\svalue\sin\sthe\stable.\nProposed\sfix\sfor\sticket\s[343634942dd54ab57b70]. -D 2018-02-01T01:13:33.559 +C Fix\sa\smemory\sleak\sthat\scould\sfollow\san\sIO\serror\sin\sthe\szipfile\sextension.\sAnd\nadd\sother\stests. +D 2018-02-01T15:19:54.965 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -304,7 +304,7 @@ F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178 F ext/misc/vtablog.c 31d0d8f4406795679dcd3a67917c213d3a2a5fb3ea5de35f6e773491ed7e13c9 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 -F ext/misc/zipfile.c 56028f7e74d948b7bef834624d128f563eb63f5950248d538afa9d0f6c3365dc +F ext/misc/zipfile.c 28f06977290c28361dcb7279016194c8e632c841e8586eb1520bbf494f487a5f F ext/rbu/rbu.c ea7d1b7eb44c123a2a619332e19fe5313500705c4a58aaa1887905c0d83ffc2e F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842 F ext/rbu/rbu10.test 1846519a438697f45e9dcb246908af81b551c29e1078d0304fae83f1fed7e9ee @@ -1603,9 +1603,9 @@ F test/wordcount.c cb589cec469a1d90add05b1f8cee75c7210338d87a5afd65260ed5c0f4bbf F test/writecrash.test f1da7f7adfe8d7f09ea79b42e5ca6dcc41102f27f8e334ad71539501ddd910cc F test/zeroblob.test 3857870fe681b8185654414a9bccfde80b62a0fa F test/zerodamage.test 9c41628db7e8d9e8a0181e59ea5f189df311a9f6ce99cc376dc461f66db6f8dc -F test/zipfile.test a16528022b08789168adc9c79646e41fb55c4f94b26a2aca97085ebc591ca84d +F test/zipfile.test 37cc584afebc6b64691a5df13deef0623f4bb42be21f1421930c34d9817ba1f2 F test/zipfile2.test 5f93611307c131e83f226a471231d769b794b9e8c6a675cfa3d34b1a79df23fe -F test/zipfilefault.test df4fa9e16116e0cb21d09037a6b0a7d93fecd8767d82154b66044b4ca43fca58 +F test/zipfilefault.test e287d6783d95b7bbad3478701834a353675f48502f95d6d58e07d954b8278639 F tool/GetFile.cs a15e08acb5dd7539b75ba23501581d7c2b462cb5 F tool/GetTclKit.bat 8995df40c4209808b31f24de0b58f90930239a234f7591e3675d45bfbb990c5d F tool/Replace.cs 02c67258801c2fb5f63231e0ac0f220b4b36ba91 @@ -1704,7 +1704,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b730d187f2202e5b5d31ed6c94c9bb04d7c289f7086a9b44b3d9050ea3586d3a -R 03c296c6343c0711547e5b14fdc5c828 -U drh -Z 93daef2f8ab3c187bf34c937a8efe1d7 +P 88e2ce916791d488076584f3795a89eb4277fcb812af9e4c2f383815d55ff6f1 +R 59b5173dc10fbd02f5f4a286751c0af4 +U dan +Z 2f13a257a067155eca38f77c87e4d0a0 diff --git a/manifest.uuid b/manifest.uuid index c0c07f33f0..1c106ca118 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -88e2ce916791d488076584f3795a89eb4277fcb812af9e4c2f383815d55ff6f1 \ No newline at end of file +e6bb750697c3c7ceb5ce41d216e8ef6a1d556822a3b55e0a007b4a03e194a7d9 \ No newline at end of file diff --git a/test/zipfile.test b/test/zipfile.test index 92d8501ea8..bcfc4b2b1e 100644 --- a/test/zipfile.test +++ b/test/zipfile.test @@ -423,7 +423,15 @@ ifcapable datetime { test_unzip/c.txt 1111111000 }] - do_test 6.1 { + do_execsql_test 6.1 { + SELECT name, mtime, data FROM zipfile('test1.zip') + } { + a.txt 946684800 abc + b.txt 1000000000 abc + c.txt 1111111000 abc + } + + do_test 6.2 { forcedelete test_unzip file mkdir test_unzip exec unzip -d test_unzip test2.zip @@ -438,9 +446,36 @@ ifcapable datetime { test_unzip/b.txt 1000000000 test_unzip/c.txt 1111111000 }] + + do_execsql_test 6.3 { + SELECT name, mtime, data FROM zipfile('test2.zip') + } { + a.txt 946684800 abc + b.txt 1000000000 abc + c.txt 1111111000 abc + } } } +#------------------------------------------------------------------------- +forcedelete test.zip +do_test 7.0 { + execsql { + WITH c(name,data) AS ( + SELECT '1', randomblob(1000000) UNION ALL + SELECT '2', randomblob(1000000) UNION ALL + SELECT '3', randomblob(1000000) + ) + SELECT writefile('test.zip', zipfile(name, data) ) FROM c; + } + + list [catch { + db eval { SELECT name, data FROM zipfile('test.zip') } { + if {$name==2} { close [open test.zip w+] } + } + } msg] $msg +} {1 {error in fread()}} + finish_test diff --git a/test/zipfilefault.test b/test/zipfilefault.test index 9f0f4784dc..b2c6d7cfc4 100644 --- a/test/zipfilefault.test +++ b/test/zipfilefault.test @@ -56,12 +56,25 @@ do_execsql_test 3.0 { INSERT INTO setup(name, data) VALUES('a.txt', 'aaaaaaaaaaaaaaaaaaaaaaaaaaaa'); } -do_faultsim_test 2 -faults oom* -body { +do_faultsim_test 3 -faults oom* -body { execsql { SELECT name,data FROM zipfile('test.zip') } } -test { faultsim_test_result {0 {a.txt aaaaaaaaaaaaaaaaaaaaaaaaaaaa}} } +do_faultsim_test 4 -faults oom* -body { + execsql { + WITH c(n, d) AS ( + SELECT 1, 'aaaaaaaaaaabbbbbbbbbbaaaaaaaaaabbbbbbbbbb' + ) + SELECT name, data FROM zipfile( + (SELECT zipfile(n, d) FROM c) + ); + } +} -test { + faultsim_test_result {0 {1 aaaaaaaaaaabbbbbbbbbbaaaaaaaaaabbbbbbbbbb}} +} + finish_test From 563ddbe54d01079afa793a5eb25de2df0f1068ef Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 1 Feb 2018 15:57:00 +0000 Subject: [PATCH 032/107] New assert() statements to help ensure that no other errors similar to [343634942dd54ab57b7] ever appear in the code. FossilOrigin-Name: 5a70af1e9c567f12c997d25d0a305a8d42bf2cc92f2811e9d5fdde720665e213 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbemem.c | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 55 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 7f7cb2415b..d213885632 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\smemory\sleak\sthat\scould\sfollow\san\sIO\serror\sin\sthe\szipfile\sextension.\sAnd\nadd\sother\stests. -D 2018-02-01T15:19:54.965 +C New\sassert()\sstatements\sto\shelp\sensure\sthat\sno\sother\serrors\ssimilar\nto\s[343634942dd54ab57b7]\sever\sappear\sin\sthe\scode. +D 2018-02-01T15:57:00.046 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -562,7 +562,7 @@ F src/vdbeInt.h 8d7d07f13cb3c4cbca91e22ba4a1920e542dda7c5d9299920432a0b3d5b009f5 F src/vdbeapi.c fea41171884a4de119f8b10ab514c788674eeeb7f27218bb6d008e1310bfd07f F src/vdbeaux.c 2756ac68ac259c416554100598fc291870063288cd7e1af22847f57b3e130e56 F src/vdbeblob.c f5c70f973ea3a9e915d1693278a5f890dc78594300cf4d54e64f2b0917c94191 -F src/vdbemem.c 943e41881e6317c9f93c77c1d60d3b37ddc8d26a3f852233ce7423d3e581523e +F src/vdbemem.c 21c1d00f0b901dda0a2eccf56bf0d86648cae1bde37de0606ae4e7cdb0ff6d3d F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2f F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a @@ -1704,7 +1704,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 88e2ce916791d488076584f3795a89eb4277fcb812af9e4c2f383815d55ff6f1 -R 59b5173dc10fbd02f5f4a286751c0af4 -U dan -Z 2f13a257a067155eca38f77c87e4d0a0 +P e6bb750697c3c7ceb5ce41d216e8ef6a1d556822a3b55e0a007b4a03e194a7d9 +R 296950b2c8b3ce7a358fc83523609b8d +U drh +Z f52624ceb0e0d6c17658803080172034 diff --git a/manifest.uuid b/manifest.uuid index 1c106ca118..d7e32f11f1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e6bb750697c3c7ceb5ce41d216e8ef6a1d556822a3b55e0a007b4a03e194a7d9 \ No newline at end of file +5a70af1e9c567f12c997d25d0a305a8d42bf2cc92f2811e9d5fdde720665e213 \ No newline at end of file diff --git a/src/vdbemem.c b/src/vdbemem.c index e2912b3a6f..c02370c738 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -93,6 +93,51 @@ int sqlite3VdbeCheckMemInvariants(Mem *p){ } #endif +#ifdef SQLITE_DEBUG +/* +** Check that string value of pMem agrees with its integer or real value. +** +** A single int or real value always converts to the same strings. But +** many different strings can be converted into the same int or real. +** If a table contains a numeric value and an index is based on the +** corresponding string value, then it is important that the string be +** derived from the numeric value, not the other way around, to ensure +** that the index and table are consistent. See ticket +** https://www.sqlite.org/src/info/343634942dd54ab (2018-01-31) for +** an example. +** +** This routine looks at pMem to verify that if it has both a numeric +** representation and a string representation then the string rep has +** been derived from the numeric and not the other way around. It returns +** true if everything is ok and false if there is a problem. +** +** This routine is for use inside of assert() statements only. +*/ +int sqlite3VdbeMemConsistentDualRep(Mem *p){ + char zBuf[100]; + char *z; + int i, j, incr; + if( (p->flags & MEM_Str)==0 ) return 1; + if( (p->flags & (MEM_Int|MEM_Real))==0 ) return 1; + if( p->flags & MEM_Int ){ + sqlite3_snprintf(sizeof(zBuf),zBuf,"%lld",p->u.i); + }else{ + sqlite3_snprintf(sizeof(zBuf),zBuf,"%!.15g",p->u.r); + } + z = p->z; + i = j = 0; + incr = 1; + if( p->enc!=SQLITE_UTF8 ){ + incr = 2; + if( p->enc==SQLITE_UTF16BE ) z++; + } + while( zBuf[j] ){ + if( zBuf[j++]!=z[i] ) return 0; + i += incr; + } + return 1; +} +#endif /* SQLITE_DEBUG */ /* ** If pMem is an object with a valid string representation, this routine @@ -1096,6 +1141,7 @@ static SQLITE_NOINLINE const void *valueToText(sqlite3_value* pVal, u8 enc){ assert(pVal->enc==(enc & ~SQLITE_UTF16_ALIGNED) || pVal->db==0 || pVal->db->mallocFailed ); if( pVal->enc==(enc & ~SQLITE_UTF16_ALIGNED) ){ + assert( sqlite3VdbeMemConsistentDualRep(pVal) ); return pVal->z; }else{ return 0; @@ -1118,6 +1164,7 @@ const void *sqlite3ValueText(sqlite3_value* pVal, u8 enc){ assert( (enc&3)==(enc&~SQLITE_UTF16_ALIGNED) ); assert( (pVal->flags & MEM_RowSet)==0 ); if( (pVal->flags&(MEM_Str|MEM_Term))==(MEM_Str|MEM_Term) && pVal->enc==enc ){ + assert( sqlite3VdbeMemConsistentDualRep(pVal) ); return pVal->z; } if( pVal->flags&MEM_Null ){ From 15daa6b53ed3dd90b772deba903ab27ff790baf9 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 1 Feb 2018 19:41:23 +0000 Subject: [PATCH 033/107] Fix a problem triggered when a zipfile virtual table is created and written to within the same transaction. And add other zipfile test. FossilOrigin-Name: 48f1c556994d7f8f359c649a1da81eec02306106b68946a9a20b276742c4610d --- ext/misc/zipfile.c | 99 +++++++++++++++----------------- manifest | 20 +++---- manifest.uuid | 2 +- test/zipfile.test | 127 +++++++++++++++++++++++++++++++++++++++-- test/zipfile2.test | 38 ++++++++++++ test/zipfilefault.test | 54 +++++++++++++++++- 6 files changed, 270 insertions(+), 70 deletions(-) diff --git a/ext/misc/zipfile.c b/ext/misc/zipfile.c index dc2d831d37..441219565f 100644 --- a/ext/misc/zipfile.c +++ b/ext/misc/zipfile.c @@ -1073,7 +1073,8 @@ static int zipfileColumn( case 6: /* method */ sqlite3_result_int(ctx, pCDS->iCompression); break; - case 7: /* z */ + default: /* z */ + assert( i==7 ); sqlite3_result_int64(ctx, pCsr->iId); break; } @@ -1217,7 +1218,7 @@ static int zipfileFilter( ){ ZipfileTab *pTab = (ZipfileTab*)cur->pVtab; ZipfileCsr *pCsr = (ZipfileCsr*)cur; - const char *zFile; /* Zip file to scan */ + const char *zFile = 0; /* Zip file to scan */ int rc = SQLITE_OK; /* Return Code */ int bInMemory = 0; /* True for an in-memory zipfile */ @@ -1226,7 +1227,8 @@ static int zipfileFilter( if( pTab->zFile ){ zFile = pTab->zFile; }else if( idxNum==0 ){ - bInMemory = 1; + zipfileSetErrmsg(pCsr, "zipfile() function requires an argument"); + return SQLITE_ERROR; }else if( sqlite3_value_type(argv[0])==SQLITE_BLOB ){ const u8 *aBlob = (const u8*)sqlite3_value_blob(argv[0]); int nBlob = sqlite3_value_bytes(argv[0]); @@ -1295,14 +1297,11 @@ static int zipfileBestIndex( return SQLITE_OK; } -static ZipfileEntry *zipfileNewEntry(const char *zPath, int nData){ +static ZipfileEntry *zipfileNewEntry(const char *zPath){ ZipfileEntry *pNew; - pNew = sqlite3_malloc(sizeof(ZipfileEntry) + nData); + pNew = sqlite3_malloc(sizeof(ZipfileEntry)); if( pNew ){ memset(pNew, 0, sizeof(ZipfileEntry)); - if( nData ){ - pNew->aData = (u8*)&pNew[1]; - } pNew->cds.zFile = sqlite3_mprintf("%s", zPath); if( pNew->cds.zFile==0 ){ sqlite3_free(pNew); @@ -1358,6 +1357,7 @@ static int zipfileAppendEntry( nBuf = zipfileSerializeLFH(pEntry, aBuf); rc = zipfileAppendData(pTab, aBuf, nBuf); if( rc==SQLITE_OK ){ + pEntry->iDataOff = pTab->szCurrent; rc = zipfileAppendData(pTab, pData, nData); } @@ -1418,6 +1418,35 @@ static int zipfileComparePath(const char *zA, const char *zB, int nB){ return 1; } +static int zipfileBegin(sqlite3_vtab *pVtab){ + ZipfileTab *pTab = (ZipfileTab*)pVtab; + int rc = SQLITE_OK; + + assert( pTab->pWriteFd==0 ); + + /* Open a write fd on the file. Also load the entire central directory + ** structure into memory. During the transaction any new file data is + ** appended to the archive file, but the central directory is accumulated + ** in main-memory until the transaction is committed. */ + pTab->pWriteFd = fopen(pTab->zFile, "ab+"); + if( pTab->pWriteFd==0 ){ + pTab->base.zErrMsg = sqlite3_mprintf( + "zipfile: failed to open file %s for writing", pTab->zFile + ); + rc = SQLITE_ERROR; + }else{ + fseek(pTab->pWriteFd, 0, SEEK_END); + pTab->szCurrent = pTab->szOrig = (i64)ftell(pTab->pWriteFd); + rc = zipfileLoadDirectory(pTab, 0, 0); + } + + if( rc!=SQLITE_OK ){ + zipfileCleanupTransaction(pTab); + } + + return rc; +} + /* ** xUpdate method. */ @@ -1445,7 +1474,10 @@ static int zipfileUpdate( int bIsDir = 0; u32 iCrc32 = 0; - assert( (pTab->zFile==0)==(pTab->pWriteFd==0) ); + if( pTab->pWriteFd==0 ){ + rc = zipfileBegin(pVtab); + if( rc!=SQLITE_OK ) return rc; + } /* If this is a DELETE or UPDATE, find the archive entry to delete. */ if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){ @@ -1541,7 +1573,7 @@ static int zipfileUpdate( if( rc==SQLITE_OK ){ /* Create the new CDS record. */ - pNew = zipfileNewEntry(zPath, pTab->zFile ? 0 : (nData+1)); + pNew = zipfileNewEntry(zPath); if( pNew==0 ){ rc = SQLITE_NOMEM; }else{ @@ -1557,11 +1589,7 @@ static int zipfileUpdate( pNew->cds.iOffset = (u32)pTab->szCurrent; pNew->cds.nFile = nPath; pNew->mUnixTime = (u32)mTime; - if( pTab->zFile ){ - rc = zipfileAppendEntry(pTab, pNew, pData, nData); - }else{ - memcpy(pNew->aData, pData, nData); - } + rc = zipfileAppendEntry(pTab, pNew, pData, nData); zipfileAddEntry(pTab, pOld, pNew); } } @@ -1606,36 +1634,6 @@ static int zipfileAppendEOCD(ZipfileTab *pTab, ZipfileEOCD *p){ return zipfileAppendData(pTab, pTab->aBuffer, nBuf); } -static int zipfileBegin(sqlite3_vtab *pVtab){ - ZipfileTab *pTab = (ZipfileTab*)pVtab; - int rc = SQLITE_OK; - - assert( pTab->pWriteFd==0 ); - if( pTab->zFile ){ - /* Open a write fd on the file. Also load the entire central directory - ** structure into memory. During the transaction any new file data is - ** appended to the archive file, but the central directory is accumulated - ** in main-memory until the transaction is committed. */ - pTab->pWriteFd = fopen(pTab->zFile, "ab+"); - if( pTab->pWriteFd==0 ){ - pTab->base.zErrMsg = sqlite3_mprintf( - "zipfile: failed to open file %s for writing", pTab->zFile - ); - rc = SQLITE_ERROR; - }else{ - fseek(pTab->pWriteFd, 0, SEEK_END); - pTab->szCurrent = pTab->szOrig = (i64)ftell(pTab->pWriteFd); - rc = zipfileLoadDirectory(pTab, 0, 0); - } - - if( rc!=SQLITE_OK ){ - zipfileCleanupTransaction(pTab); - } - } - - return rc; -} - /* ** Serialize the CDS structure into buffer aBuf[]. Return the number ** of bytes written. @@ -1785,14 +1783,11 @@ static int zipfileFindFunction( void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), /* OUT: Result */ void **ppArg /* OUT: User data for *pxFunc */ ){ - if( nArg>0 ){ - if( sqlite3_stricmp("zipfile_cds", zName)==0 ){ - *pxFunc = zipfileFunctionCds; - *ppArg = (void*)pVtab; - return 1; - } + if( sqlite3_stricmp("zipfile_cds", zName)==0 ){ + *pxFunc = zipfileFunctionCds; + *ppArg = (void*)pVtab; + return 1; } - return 0; } diff --git a/manifest b/manifest index d213885632..e4aa3f1840 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C New\sassert()\sstatements\sto\shelp\sensure\sthat\sno\sother\serrors\ssimilar\nto\s[343634942dd54ab57b7]\sever\sappear\sin\sthe\scode. -D 2018-02-01T15:57:00.046 +C Fix\sa\sproblem\striggered\swhen\sa\szipfile\svirtual\stable\sis\screated\sand\swritten\sto\nwithin\sthe\ssame\stransaction.\sAnd\sadd\sother\szipfile\stest. +D 2018-02-01T19:41:23.066 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -304,7 +304,7 @@ F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178 F ext/misc/vtablog.c 31d0d8f4406795679dcd3a67917c213d3a2a5fb3ea5de35f6e773491ed7e13c9 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 -F ext/misc/zipfile.c 28f06977290c28361dcb7279016194c8e632c841e8586eb1520bbf494f487a5f +F ext/misc/zipfile.c 2856e99ce4b45669edbdc070649a7c6ea0348979d9297ab682f3c552078d47c7 F ext/rbu/rbu.c ea7d1b7eb44c123a2a619332e19fe5313500705c4a58aaa1887905c0d83ffc2e F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842 F ext/rbu/rbu10.test 1846519a438697f45e9dcb246908af81b551c29e1078d0304fae83f1fed7e9ee @@ -1603,9 +1603,9 @@ F test/wordcount.c cb589cec469a1d90add05b1f8cee75c7210338d87a5afd65260ed5c0f4bbf F test/writecrash.test f1da7f7adfe8d7f09ea79b42e5ca6dcc41102f27f8e334ad71539501ddd910cc F test/zeroblob.test 3857870fe681b8185654414a9bccfde80b62a0fa F test/zerodamage.test 9c41628db7e8d9e8a0181e59ea5f189df311a9f6ce99cc376dc461f66db6f8dc -F test/zipfile.test 37cc584afebc6b64691a5df13deef0623f4bb42be21f1421930c34d9817ba1f2 -F test/zipfile2.test 5f93611307c131e83f226a471231d769b794b9e8c6a675cfa3d34b1a79df23fe -F test/zipfilefault.test e287d6783d95b7bbad3478701834a353675f48502f95d6d58e07d954b8278639 +F test/zipfile.test 1f066994bd77493c87e8ee4bd94db0651d180cff19ffbbe0b70085eb9a2cb34c +F test/zipfile2.test 67d5f08a202796d4b7a71dfa4b8dcb74aa7a9d1f42c5f17bedff9855c1ba7aa5 +F test/zipfilefault.test 73b08e3d0bbeb275e325ee7e3678ca98781de0737f9153ca23bde1f48a93d728 F tool/GetFile.cs a15e08acb5dd7539b75ba23501581d7c2b462cb5 F tool/GetTclKit.bat 8995df40c4209808b31f24de0b58f90930239a234f7591e3675d45bfbb990c5d F tool/Replace.cs 02c67258801c2fb5f63231e0ac0f220b4b36ba91 @@ -1704,7 +1704,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e6bb750697c3c7ceb5ce41d216e8ef6a1d556822a3b55e0a007b4a03e194a7d9 -R 296950b2c8b3ce7a358fc83523609b8d -U drh -Z f52624ceb0e0d6c17658803080172034 +P 5a70af1e9c567f12c997d25d0a305a8d42bf2cc92f2811e9d5fdde720665e213 +R ec6ac070bff5134b5a74e8c3afc721f5 +U dan +Z 34f839c0a19e1ff6432f6acef976c132 diff --git a/manifest.uuid b/manifest.uuid index d7e32f11f1..be5881914d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5a70af1e9c567f12c997d25d0a305a8d42bf2cc92f2811e9d5fdde720665e213 \ No newline at end of file +48f1c556994d7f8f359c649a1da81eec02306106b68946a9a20b276742c4610d \ No newline at end of file diff --git a/test/zipfile.test b/test/zipfile.test index bcfc4b2b1e..b31077d33c 100644 --- a/test/zipfile.test +++ b/test/zipfile.test @@ -151,10 +151,18 @@ do_catchsql_test 1.1.0.1 { INSERT INTO zz(name, mode, mtime, sz, rawdata, method) VALUES('f.txt', '-rw-r--r--', 1000000000, 5, 'abcde', 0); } {1 {constraint failed}} -do_catchsql_test 1.1.0.1 { - INSERT INTO zz(name, mtime, sz, rawdata, method) +do_catchsql_test 1.1.0.2 { + INSERT INTO zz(name, mtime, sz, data, method) VALUES('g.txt', 1000000002, 5, '12345', 0); } {1 {constraint failed}} +do_catchsql_test 1.1.0.3 { + INSERT INTO zz(name, mtime, rawdata, method) + VALUES('g.txt', 1000000002, '12345', 0); +} {1 {constraint failed}} +do_catchsql_test 1.1.0.4 { + INSERT INTO zz(name, data, method) + VALUES('g.txt', '12345', 7); +} {1 {constraint failed}} do_execsql_test 1.1.1 { INSERT INTO zz(name, mode, mtime, data, method) @@ -198,6 +206,9 @@ ifcapable json1 { h.txt 1 } } +do_catchsql_test 1.4.2 { + SELECT zipfile_cds(mode) FROM zipfile('test.zip'); +} {0 {{} {} {}}} do_execsql_test 1.5.1 { BEGIN; @@ -285,6 +296,37 @@ do_execsql_test 1.6.8 { i.txt 33188 4 zxcvb 0 } +do_execsql_test 1.6.8 { + UPDATE zz SET data = '' WHERE name='i.txt'; + SELECT name,mode,mtime,data,method from zipfile('test.zip'); +} { + blue.txt/ 16877 1000000000 {} 0 + h.txt 33189 1000000004 aaaaaaaaaabbbbbbbbbb 8 + i.txt 33188 4 {} 0 +} + +do_execsql_test 1.6.9 { + SELECT a.name, a.data + FROM zz AS a, zz AS b + WHERE a.name=+b.name AND +a.mode=b.mode +} { + blue.txt/ {} + h.txt aaaaaaaaaabbbbbbbbbb + i.txt {} +} + +do_execsql_test 1.6.10 { + SELECT name, data FROM zz WHERE name LIKE '%txt' +} { + h.txt aaaaaaaaaabbbbbbbbbb + i.txt {} +} + +do_execsql_test 1.7 { + DELETE FROM zz; + SELECT * FROM zz; +} {} + #------------------------------------------------------------------------- db close forcedelete test.zip @@ -379,6 +421,25 @@ do_catchsql_test 4.2 { CREATE VIRTUAL TABLE yyy USING zipfile('test.zip', 'test.zip'); } {1 {zipfile constructor requires one argument}} +do_catchsql_test 4.3 { + SELECT * FROM zipfile() +} {1 {zipfile() function requires an argument}} + +do_catchsql_test 4.4 { + SELECT * FROM zipfile('/path/that/does/not/exist') +} {1 {cannot open file: /path/that/does/not/exist}} + +foreach {tn mode} { + 1 abcd + 2 brwxrwxrwx + 3 lrwxrrxrwx +} { + do_catchsql_test 4.5.$tn { + WITH m(m) AS ( SELECT $mode) + SELECT zipfile('a.txt', m, 1000, 'xyz') FROM m + } [list 1 "zipfile: parse error in mode: $mode"] +} + #-------------------------------------------------------------------------- db func rt remove_timestamps @@ -448,16 +509,18 @@ ifcapable datetime { }] do_execsql_test 6.3 { - SELECT name, mtime, data FROM zipfile('test2.zip') + SELECT name, mtime, sz, rawdata, data FROM zipfile('test2.zip') } { - a.txt 946684800 abc - b.txt 1000000000 abc - c.txt 1111111000 abc + a.txt 946684800 3 abc abc + b.txt 1000000000 3 abc abc + c.txt 1111111000 3 abc abc } } } #------------------------------------------------------------------------- +# Force an IO error by truncating the zip archive to zero bytes in size +# while it is being read. forcedelete test.zip do_test 7.0 { execsql { @@ -476,6 +539,58 @@ do_test 7.0 { } msg] $msg } {1 {error in fread()}} +forcedelete test.zip +do_execsql_test 8.0.1 { + CREATE VIRTUAL TABLE zz USING zipfile('test.zip'); + BEGIN; + INSERT INTO zz(name, data) VALUES('a.txt', '1'); + INSERT INTO zz(name, data) VALUES('b.txt', '2'); + INSERT INTO zz(name, data) VALUES('c.txt', '1'); + INSERT INTO zz(name, data) VALUES('d.txt', '2'); + SELECT name, data FROM zz; +} { + a.txt 1 b.txt 2 c.txt 1 d.txt 2 +} +do_test 8.0.2 { + db eval { SELECT name, data FROM zz } { + if { $data=="2" } { db eval { DELETE FROM zz WHERE name=$name } } + } + execsql { SELECT name, data FROM zz } +} {a.txt 1 c.txt 1} +do_test 8.0.3 { + db eval { SELECT name, data FROM zz } { + db eval { DELETE FROM zz WHERE name=$name } + } + execsql { SELECT name, data FROM zz } +} {} +execsql COMMIT + +do_execsql_test 8.1.1 { + CREATE VIRTUAL TABLE nogood USING zipfile('test_unzip'); +} +do_catchsql_test 8.1.2 { + INSERT INTO nogood(name, data) VALUES('abc', 'def'); +} {1 {zipfile: failed to open file test_unzip for writing}} + +do_execsql_test 8.2.1 { + DROP TABLE nogood; + BEGIN; + CREATE VIRTUAL TABLE nogood USING zipfile('test_unzip'); +} +do_catchsql_test 8.2.2 { + INSERT INTO nogood(name, data) VALUES('abc', 'def'); +} {1 {zipfile: failed to open file test_unzip for writing}} +do_execsql_test 8.2.3 { + COMMIT; +} + +forcedelete test.zip +do_execsql_test 8.3.1 { + BEGIN; + CREATE VIRTUAL TABLE ok USING zipfile('test.zip'); + INSERT INTO ok(name, data) VALUES ('sqlite3', 'elf'); + COMMIT; +} finish_test diff --git a/test/zipfile2.test b/test/zipfile2.test index a6c3e7e3c4..f3509d0a54 100644 --- a/test/zipfile2.test +++ b/test/zipfile2.test @@ -164,6 +164,44 @@ do_catchsql_test 4.1 { SELECT name,mtime,data,method FROM zipfile($blob) } {1 {inflate() failed (0)}} +# Check the response to an unknown compression method (set data to NULL). +set blob [blob [string map {0800 0900} $archive2]] +do_execsql_test 4.2 { + SELECT name,mtime,data IS NULL,method FROM zipfile($blob) +} {a.txt 1000000 1 9} + +# Corrupt the EOCDS signature bytes in various ways. +foreach {tn sub} { + 1 {504B0500} + 2 {504B0006} + 3 {50000506} + 4 {004B0506} +} { + set blob [blob [string map [list 504B0506 $sub] $archive2]] + do_catchsql_test 4.3.$tn { + SELECT * FROM zipfile($blob) + } {1 {cannot find end of central directory record}} +} + +#------------------------------------------------------------------------- +# Test that a zero-length file with a '/' at the end is treated as +# a directory (data IS NULL). Even if the mode doesn't indicate +# that it is a directory. + +do_test 5.0 { + set blob [db one { + WITH c(n, d) AS ( + SELECT 'notadir', '' + ) + SELECT zipfile(n, d) FROM c + }] + + set hex [binary encode hex $blob] + set hex [string map {6e6f7461646972 6e6f746164692f} $hex] + set blob2 [binary decode hex $hex] + + execsql { SELECT name, data IS NULL FROM zipfile($blob2) } +} {notadi/ 1} finish_test diff --git a/test/zipfilefault.test b/test/zipfilefault.test index b2c6d7cfc4..a850b5146a 100644 --- a/test/zipfilefault.test +++ b/test/zipfilefault.test @@ -42,11 +42,19 @@ do_execsql_test 2.0 { INSERT INTO setup(name, data) VALUES('a.txt', '1234567890'); } -do_faultsim_test 2 -faults oom* -body { +do_faultsim_test 2.1 -faults oom* -body { execsql { SELECT name,data FROM zipfile('test.zip') } } -test { faultsim_test_result {0 {a.txt 1234567890}} } +do_faultsim_test 2.2 -faults oom* -body { + execsql { + SELECT json_extract( zipfile_cds(z), '$.version-made-by' ) + FROM zipfile('test.zip') + } +} -test { + faultsim_test_result {0 798} +} forcedelete test.zip reset_db @@ -75,6 +83,50 @@ do_faultsim_test 4 -faults oom* -body { faultsim_test_result {0 {1 aaaaaaaaaaabbbbbbbbbbaaaaaaaaaabbbbbbbbbb}} } +reset_db +load_static_extension db zipfile + +do_execsql_test 5.0 { + CREATE VIRTUAL TABLE setup USING zipfile('test.zip') +} + +do_faultsim_test 5.1 -faults oom* -prep { + forcedelete test.zip +} -body { + execsql { + INSERT INTO setup(name, data) + VALUES('a.txt', 'aaaaaaaaaaaaaaaaaaaaaaaaaaaa'); + } +} -test { + faultsim_test_result {0 {}} +} + +do_faultsim_test 5.2 -faults oom* -prep { + forcedelete test.zip +} -body { + execsql { + INSERT INTO setup(name, data) VALUES('dir', NULL) + } +} -test { + faultsim_test_result {0 {}} +} + +do_faultsim_test 5.3 -faults oom* -prep { + forcedelete test.zip + execsql { + DROP TABLE IF EXISTS setup; + BEGIN; + CREATE VIRTUAL TABLE setup USING zipfile('test.zip') + } +} -body { + execsql { + INSERT INTO setup(name, data) VALUES('dir', NULL) + } +} -test { + catchsql { COMMIT } + faultsim_test_result {0 {}} +} + finish_test From 668845bb0bb831b98577b0261303a376d14ceeb0 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 1 Feb 2018 20:42:23 +0000 Subject: [PATCH 034/107] Add a few more zipfile tests. No changes to code. FossilOrigin-Name: 3f621545879ea39502cfaf2b61883f92c077070274b4fdf45524ded81cac3e63 --- manifest | 14 ++++++------- manifest.uuid | 2 +- test/zipfile.test | 46 +++++++++++++++++++++++++++++++++++++++++- test/zipfilefault.test | 33 ++++++++++++++++++++++++++++++ 4 files changed, 86 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index e4aa3f1840..868870f090 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\striggered\swhen\sa\szipfile\svirtual\stable\sis\screated\sand\swritten\sto\nwithin\sthe\ssame\stransaction.\sAnd\sadd\sother\szipfile\stest. -D 2018-02-01T19:41:23.066 +C Add\sa\sfew\smore\szipfile\stests.\sNo\schanges\sto\scode. +D 2018-02-01T20:42:23.543 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -1603,9 +1603,9 @@ F test/wordcount.c cb589cec469a1d90add05b1f8cee75c7210338d87a5afd65260ed5c0f4bbf F test/writecrash.test f1da7f7adfe8d7f09ea79b42e5ca6dcc41102f27f8e334ad71539501ddd910cc F test/zeroblob.test 3857870fe681b8185654414a9bccfde80b62a0fa F test/zerodamage.test 9c41628db7e8d9e8a0181e59ea5f189df311a9f6ce99cc376dc461f66db6f8dc -F test/zipfile.test 1f066994bd77493c87e8ee4bd94db0651d180cff19ffbbe0b70085eb9a2cb34c +F test/zipfile.test 44aa8af115cc3e8c905468768dc761260650a8fdfca57e10f9818f5f8008d340 F test/zipfile2.test 67d5f08a202796d4b7a71dfa4b8dcb74aa7a9d1f42c5f17bedff9855c1ba7aa5 -F test/zipfilefault.test 73b08e3d0bbeb275e325ee7e3678ca98781de0737f9153ca23bde1f48a93d728 +F test/zipfilefault.test 050be76778d2ec83566a542800fc3967f65ce719671a819016ada8b654bc14f7 F tool/GetFile.cs a15e08acb5dd7539b75ba23501581d7c2b462cb5 F tool/GetTclKit.bat 8995df40c4209808b31f24de0b58f90930239a234f7591e3675d45bfbb990c5d F tool/Replace.cs 02c67258801c2fb5f63231e0ac0f220b4b36ba91 @@ -1704,7 +1704,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5a70af1e9c567f12c997d25d0a305a8d42bf2cc92f2811e9d5fdde720665e213 -R ec6ac070bff5134b5a74e8c3afc721f5 +P 48f1c556994d7f8f359c649a1da81eec02306106b68946a9a20b276742c4610d +R cdbed97cc578ccd9e756b9f206f65092 U dan -Z 34f839c0a19e1ff6432f6acef976c132 +Z 07c3c59583bca5bfb67bba61fc8753c1 diff --git a/manifest.uuid b/manifest.uuid index be5881914d..3a4e6a6861 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -48f1c556994d7f8f359c649a1da81eec02306106b68946a9a20b276742c4610d \ No newline at end of file +3f621545879ea39502cfaf2b61883f92c077070274b4fdf45524ded81cac3e63 \ No newline at end of file diff --git a/test/zipfile.test b/test/zipfile.test index b31077d33c..2bb3f07892 100644 --- a/test/zipfile.test +++ b/test/zipfile.test @@ -440,6 +440,37 @@ foreach {tn mode} { } [list 1 "zipfile: parse error in mode: $mode"] } +do_catchsql_test 4.6 { + WITH c(name,data) AS ( SELECT 'a.txt', 'abc') + SELECT zipfile(name) FROM c +} {1 {wrong number of arguments to function zipfile()}} + +do_catchsql_test 4.7 { + WITH c(name,data) AS ( + SELECT 'a.txt', 'abc' UNION ALL + SELECT NULL, 'def' + ) + SELECT zipfile(name,data) FROM c +} {1 {first argument to zipfile() must be non-NULL}} + +do_catchsql_test 4.7 { + WITH c(name,data,method) AS ( + SELECT 'a.txt', 'abc', 0 + UNION SELECT 'b.txt', 'def', 8 + UNION SELECT 'c.txt', 'ghi', 16 + ) + SELECT zipfile(name,NULL,NULL,data,method) FROM c +} {1 {illegal method value: 16}} + +do_catchsql_test 4.8 { + WITH c(name,data) AS ( + SELECT 'a.txt', 'abc' + UNION SELECT 'b.txt', 'def' + UNION SELECT 'c.txt/', 'ghi' + ) + SELECT zipfile(name,NULL,NULL,data) FROM c +} {1 {non-directory name must not end with /}} + #-------------------------------------------------------------------------- db func rt remove_timestamps @@ -448,7 +479,7 @@ do_execsql_test 5.0 { SELECT 'a.txt', 946684800, 'abc' ) SELECT name,mtime,data FROM zipfile( - ( SELECT rt( zipfile(name,NULL,mtime,data) ) FROM c ) + ( SELECT rt( zipfile(name,NULL,mtime,data,NULL) ) FROM c ) ) } { a.txt 946684800 abc @@ -592,5 +623,18 @@ do_execsql_test 8.3.1 { COMMIT; } +#------------------------------------------------------------------------- +# Test that the zipfile aggregate correctly adds and removes "/" from +# the ends of directory file names. +do_execsql_test 9.0 { + WITH src(nm) AS ( + VALUES('dir1') UNION ALL + VALUES('dir2/') UNION ALL + VALUES('dir3//') UNION ALL + VALUES('dir4///') UNION ALL + VALUES('/') + ) + SELECT name FROM zipfile((SELECT zipfile(nm, NULL) FROM src)) +} {dir1/ dir2/ dir3/ dir4/ /} finish_test diff --git a/test/zipfilefault.test b/test/zipfilefault.test index a850b5146a..158370695e 100644 --- a/test/zipfilefault.test +++ b/test/zipfilefault.test @@ -84,6 +84,7 @@ do_faultsim_test 4 -faults oom* -body { } reset_db +sqlite3_db_config_lookaside db 0 0 0 load_static_extension db zipfile do_execsql_test 5.0 { @@ -127,6 +128,38 @@ do_faultsim_test 5.3 -faults oom* -prep { faultsim_test_result {0 {}} } +do_faultsim_test 6.1 -faults oom* -body { + execsql { + WITH c(n, d) AS ( + VALUES('a.txt', '1234567890') UNION ALL + VALUES('dir', NULL) + ) + SELECT zipfile(n, d) IS NULL FROM c; + } +} -test { + faultsim_test_result {0 0} +} + +set big [string repeat 0123456789 1000] +do_faultsim_test 6.2 -faults oom* -body { + execsql { + WITH c(n, d) AS ( + VALUES('a.txt', $big) + ) + SELECT zipfile(n, NULL, NULL, d, 0) IS NULL FROM c; + } +} -test { + faultsim_test_result {0 0} +} + +do_faultsim_test 7.0 -faults oom* -prep { + catch { db close } + sqlite3 db "" +} -body { + load_static_extension db zipfile +} -test { +} + finish_test From 1dff328119f076a1e4d63631fd57baefdc597be1 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 2 Feb 2018 16:20:41 +0000 Subject: [PATCH 035/107] Remove the time() dependency from the zipfile extension. Use SQLite VFS methods instead. FossilOrigin-Name: 0702fb5611c7cf0154c5490eeada66afc733731dff0fe030375bae6a72123a17 --- ext/misc/zipfile.c | 79 +++++++++++++++++++++++++++++++++------------- manifest | 12 +++---- manifest.uuid | 2 +- 3 files changed, 64 insertions(+), 29 deletions(-) diff --git a/ext/misc/zipfile.c b/ext/misc/zipfile.c index 441219565f..6c3f2b2373 100644 --- a/ext/misc/zipfile.c +++ b/ext/misc/zipfile.c @@ -30,18 +30,9 @@ SQLITE_EXTENSION_INIT1 #include #include -#include -#include -#include -#if !defined(_WIN32) && !defined(WIN32) -# include -# include -# include -#else +#if defined(_WIN32) || defined(WIN32) # include #endif -#include -#include #include @@ -68,6 +59,25 @@ typedef unsigned long u32; #endif /* SQLITE_AMALGAMATION */ +/* +** Definitions for mode bitmasks S_IFDIR, S_IFREG and S_IFLNK. +** +** In some ways it would be better to obtain these values from system +** header files. But, the dependency is undesirable and (a) these +** have been stable for decades, (b) the values are part of POSIX and +** are also made explicit in [man stat], and (c) are part of the +** file format for zip archives. +*/ +#ifndef S_IFDIR +# define S_IFDIR 0040000 +#endif +#ifndef S_IFREG +# define S_IFREG 0100000 +#endif +#ifndef S_IFLNK +# define S_IFLNK 0120000 +#endif + static const char ZIPFILE_SCHEMA[] = "CREATE TABLE y(" "name PRIMARY KEY," /* 0: Name of file in zip archive */ @@ -1447,6 +1457,39 @@ static int zipfileBegin(sqlite3_vtab *pVtab){ return rc; } +/* +** Return the current time as a 32-bit timestamp in UNIX epoch format (like +** time(2)). +*/ +static u32 zipfileTime(void){ + sqlite3_vfs *pVfs = sqlite3_vfs_find(0); + u32 ret; + if( pVfs->iVersion>=2 && pVfs->xCurrentTimeInt64 ){ + i64 ms; + pVfs->xCurrentTimeInt64(pVfs, &ms); + ret = (u32)((ms/1000) - ((i64)24405875 * 8640)); + }else{ + double day; + pVfs->xCurrentTime(pVfs, &day); + ret = (u32)((day - 2440587.5) * 86400); + } + return ret; +} + +/* +** Return a 32-bit timestamp in UNIX epoch format. +** +** If the value passed as the only argument is either NULL or an SQL NULL, +** return the current time. Otherwise, return the value stored in (*pVal) +** cast to a 32-bit unsigned integer. +*/ +static u32 zipfileGetTime(sqlite3_value *pVal){ + if( pVal==0 || sqlite3_value_type(pVal)==SQLITE_NULL ){ + return zipfileTime(); + } + return (u32)sqlite3_value_int64(pVal); +} + /* ** xUpdate method. */ @@ -1461,7 +1504,7 @@ static int zipfileUpdate( ZipfileEntry *pNew = 0; /* New in-memory CDS entry */ u32 mode = 0; /* Mode for new entry */ - i64 mTime = 0; /* Modification time for new entry */ + u32 mTime = 0; /* Modification time for new entry */ i64 sz = 0; /* Uncompressed size */ const char *zPath = 0; /* Path for new entry */ int nPath = 0; /* strlen(zPath) */ @@ -1540,11 +1583,7 @@ static int zipfileUpdate( if( rc==SQLITE_OK ){ zPath = (const char*)sqlite3_value_text(apVal[2]); nPath = (int)strlen(zPath); - if( sqlite3_value_type(apVal[4])==SQLITE_NULL ){ - mTime = (sqlite3_int64)time(0); - }else{ - mTime = sqlite3_value_int64(apVal[4]); - } + mTime = zipfileGetTime(apVal[4]); } if( rc==SQLITE_OK && bIsDir ){ @@ -1581,7 +1620,7 @@ static int zipfileUpdate( pNew->cds.iVersionExtract = ZIPFILE_NEWENTRY_REQUIRED; pNew->cds.flags = ZIPFILE_NEWENTRY_FLAGS; pNew->cds.iCompression = (u16)iMethod; - zipfileMtimeToDos(&pNew->cds, (u32)mTime); + zipfileMtimeToDos(&pNew->cds, mTime); pNew->cds.crc32 = iCrc32; pNew->cds.szCompressed = nData; pNew->cds.szUncompressed = (u32)sz; @@ -1929,11 +1968,7 @@ void zipfileStep(sqlite3_context *pCtx, int nVal, sqlite3_value **apVal){ if( rc ) goto zipfile_step_out; /* Decode the "mtime" argument. */ - if( pMtime==0 || sqlite3_value_type(pMtime)==SQLITE_NULL ){ - e.mUnixTime = (u32)time(0); - }else{ - e.mUnixTime = (u32)sqlite3_value_int64(pMtime); - } + e.mUnixTime = zipfileGetTime(pMtime); /* If this is a directory entry, ensure that there is exactly one '/' ** at the end of the path. Or, if this is not a directory and the path diff --git a/manifest b/manifest index 868870f090..f4b964ef4f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\sfew\smore\szipfile\stests.\sNo\schanges\sto\scode. -D 2018-02-01T20:42:23.543 +C Remove\sthe\stime()\sdependency\sfrom\sthe\szipfile\sextension.\sUse\sSQLite\sVFS\nmethods\sinstead. +D 2018-02-02T16:20:41.841 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -304,7 +304,7 @@ F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178 F ext/misc/vtablog.c 31d0d8f4406795679dcd3a67917c213d3a2a5fb3ea5de35f6e773491ed7e13c9 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 -F ext/misc/zipfile.c 2856e99ce4b45669edbdc070649a7c6ea0348979d9297ab682f3c552078d47c7 +F ext/misc/zipfile.c 8043b5efef0da4abba31ac021a9ea9aaf216c72547fb7f64eed9e08fd96107ff F ext/rbu/rbu.c ea7d1b7eb44c123a2a619332e19fe5313500705c4a58aaa1887905c0d83ffc2e F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842 F ext/rbu/rbu10.test 1846519a438697f45e9dcb246908af81b551c29e1078d0304fae83f1fed7e9ee @@ -1704,7 +1704,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 48f1c556994d7f8f359c649a1da81eec02306106b68946a9a20b276742c4610d -R cdbed97cc578ccd9e756b9f206f65092 +P 3f621545879ea39502cfaf2b61883f92c077070274b4fdf45524ded81cac3e63 +R 0244469f34b504c4ae85aff5c78a3c61 U dan -Z 07c3c59583bca5bfb67bba61fc8753c1 +Z ce324d9598828c350892743fba0ec306 diff --git a/manifest.uuid b/manifest.uuid index 3a4e6a6861..70ec2da9e1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3f621545879ea39502cfaf2b61883f92c077070274b4fdf45524ded81cac3e63 \ No newline at end of file +0702fb5611c7cf0154c5490eeada66afc733731dff0fe030375bae6a72123a17 \ No newline at end of file From 3944cf8de3073675c0bc30e136652db386584e0e Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 2 Feb 2018 16:28:19 +0000 Subject: [PATCH 036/107] Remove the "include " win32 dependency from the zipfile extension. FossilOrigin-Name: cb3feb3bc33718625159eddbf8ef608cd641af84e97d433d6b11ba6da94dad2e --- ext/misc/zipfile.c | 4 ---- manifest | 13 +++++++------ manifest.uuid | 2 +- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/ext/misc/zipfile.c b/ext/misc/zipfile.c index 6c3f2b2373..a94a8df3ce 100644 --- a/ext/misc/zipfile.c +++ b/ext/misc/zipfile.c @@ -30,10 +30,6 @@ SQLITE_EXTENSION_INIT1 #include #include -#if defined(_WIN32) || defined(WIN32) -# include -#endif - #include #ifndef SQLITE_OMIT_VIRTUALTABLE diff --git a/manifest b/manifest index f4b964ef4f..2443207a1f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sthe\stime()\sdependency\sfrom\sthe\szipfile\sextension.\sUse\sSQLite\sVFS\nmethods\sinstead. -D 2018-02-02T16:20:41.841 +C Remove\sthe\s"include\s"\swin32\sdependency\sfrom\sthe\szipfile\sextension. +D 2018-02-02T16:28:19.035 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -304,7 +304,7 @@ F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178 F ext/misc/vtablog.c 31d0d8f4406795679dcd3a67917c213d3a2a5fb3ea5de35f6e773491ed7e13c9 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 -F ext/misc/zipfile.c 8043b5efef0da4abba31ac021a9ea9aaf216c72547fb7f64eed9e08fd96107ff +F ext/misc/zipfile.c 8aa0d3da5ccb360dbca578a95597bd4892ccc56b63f01447dcbb0e035720fb9f F ext/rbu/rbu.c ea7d1b7eb44c123a2a619332e19fe5313500705c4a58aaa1887905c0d83ffc2e F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842 F ext/rbu/rbu10.test 1846519a438697f45e9dcb246908af81b551c29e1078d0304fae83f1fed7e9ee @@ -1704,7 +1704,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3f621545879ea39502cfaf2b61883f92c077070274b4fdf45524ded81cac3e63 -R 0244469f34b504c4ae85aff5c78a3c61 +P 0702fb5611c7cf0154c5490eeada66afc733731dff0fe030375bae6a72123a17 +R 343c0b60986622084d28e8dfa8a41cbf +T +closed db32506880e727597e8629c21d29a4b5e3360bec7548d98ecdbb09685fd6e7a8 U dan -Z ce324d9598828c350892743fba0ec306 +Z 0fa1b90106adf879fb72d19e962c14d7 diff --git a/manifest.uuid b/manifest.uuid index 70ec2da9e1..866db96ab5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0702fb5611c7cf0154c5490eeada66afc733731dff0fe030375bae6a72123a17 \ No newline at end of file +cb3feb3bc33718625159eddbf8ef608cd641af84e97d433d6b11ba6da94dad2e \ No newline at end of file From 001f52ed7d2ecd3b9c28a662234193b121318652 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Sun, 4 Feb 2018 01:29:01 +0000 Subject: [PATCH 037/107] Move variable declaration in zipfile extension to fix MSVC compiler error. FossilOrigin-Name: ac6a32482e675871c3ce5ecc727e50e4bba649e80e3c55bf99025fed5667da94 --- ext/misc/zipfile.c | 3 ++- manifest | 15 +++++++-------- manifest.uuid | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/ext/misc/zipfile.c b/ext/misc/zipfile.c index a94a8df3ce..f88b7e22d6 100644 --- a/ext/misc/zipfile.c +++ b/ext/misc/zipfile.c @@ -701,6 +701,7 @@ static u32 zipfileMtime(ZipfileCDS *pCDS){ int sec = (pCDS->mTime & 0x1F)*2; int min = (pCDS->mTime >> 5) & 0x3F; int hr = (pCDS->mTime >> 11) & 0x1F; + i64 JD; /* JD = INT(365.25 * (Y+4716)) + INT(30.6001 * (M+1)) + D + B - 1524.5 */ @@ -709,7 +710,7 @@ static u32 zipfileMtime(ZipfileCDS *pCDS){ Y = Y-1; M = M+12; } - i64 JD = (i64)(24*60*60) * ( + JD = (i64)(24*60*60) * ( (int)(365.25 * (Y + 4716)) + (int)(30.6001 * (M + 1)) + D + B - 1524 diff --git a/manifest b/manifest index 2443207a1f..757d26e001 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sthe\s"include\s"\swin32\sdependency\sfrom\sthe\szipfile\sextension. -D 2018-02-02T16:28:19.035 +C Move\svariable\sdeclaration\sin\szipfile\sextension\sto\sfix\sMSVC\scompiler\serror. +D 2018-02-04T01:29:01.007 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -304,7 +304,7 @@ F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178 F ext/misc/vtablog.c 31d0d8f4406795679dcd3a67917c213d3a2a5fb3ea5de35f6e773491ed7e13c9 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 -F ext/misc/zipfile.c 8aa0d3da5ccb360dbca578a95597bd4892ccc56b63f01447dcbb0e035720fb9f +F ext/misc/zipfile.c 3c3e21d0351a44c34c67ca7e833ba04e0de34421b4c9359aa764919ee36b37ef F ext/rbu/rbu.c ea7d1b7eb44c123a2a619332e19fe5313500705c4a58aaa1887905c0d83ffc2e F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842 F ext/rbu/rbu10.test 1846519a438697f45e9dcb246908af81b551c29e1078d0304fae83f1fed7e9ee @@ -1704,8 +1704,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0702fb5611c7cf0154c5490eeada66afc733731dff0fe030375bae6a72123a17 -R 343c0b60986622084d28e8dfa8a41cbf -T +closed db32506880e727597e8629c21d29a4b5e3360bec7548d98ecdbb09685fd6e7a8 -U dan -Z 0fa1b90106adf879fb72d19e962c14d7 +P cb3feb3bc33718625159eddbf8ef608cd641af84e97d433d6b11ba6da94dad2e +R 8b8d4f678534b218d44a7c7eccfb3100 +U mistachkin +Z e519063d0405ad8f0532efc454c9e10a diff --git a/manifest.uuid b/manifest.uuid index 866db96ab5..984b05d033 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cb3feb3bc33718625159eddbf8ef608cd641af84e97d433d6b11ba6da94dad2e \ No newline at end of file +ac6a32482e675871c3ce5ecc727e50e4bba649e80e3c55bf99025fed5667da94 \ No newline at end of file From 97f9059eee4c59afe21f9582293b24ad0d50930a Mon Sep 17 00:00:00 2001 From: mistachkin Date: Sun, 4 Feb 2018 01:30:54 +0000 Subject: [PATCH 038/107] Fix harmless compiler warning seen with MSVC. FossilOrigin-Name: 63a914410b5750bd08836864ae456ff09549487c5ce49706fedffa3d4f973059 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 757d26e001..b4577329bb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Move\svariable\sdeclaration\sin\szipfile\sextension\sto\sfix\sMSVC\scompiler\serror. -D 2018-02-04T01:29:01.007 +C Fix\sharmless\scompiler\swarning\sseen\swith\sMSVC. +D 2018-02-04T01:30:54.482 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -429,7 +429,7 @@ F src/auth.c 6277d63837357549fe14e723490d6dc1a38768d71c795c5eb5c0f8a99f918f73 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca -F src/btree.c d711228cac336fb35fff21f3f4a0efe2ad58aa9a800dd02929cdf184be1e78a3 +F src/btree.c f7bb37c27b3734d58796febb30b5d818b3b4eb592b57b88039e5d92b3ebd9dab F src/btree.h 0866c0a08255142ea0e754aabd211c843cab32045c978a592a43152405ed0c84 F src/btreeInt.h 620ab4c7235f43572cf3ac2ac8723cbdf68073be4d29da24897c7b77dda5fd96 F src/build.c 672022c06e1a5c2653f80c77a687de11f7e65ce81d20fe2825aadfa13a875c33 @@ -1704,7 +1704,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P cb3feb3bc33718625159eddbf8ef608cd641af84e97d433d6b11ba6da94dad2e -R 8b8d4f678534b218d44a7c7eccfb3100 +P ac6a32482e675871c3ce5ecc727e50e4bba649e80e3c55bf99025fed5667da94 +R 996f708603528b0dbd58b1571407dd89 U mistachkin -Z e519063d0405ad8f0532efc454c9e10a +Z c8cfe40d98eaa63e42dbf121b7e80b46 diff --git a/manifest.uuid b/manifest.uuid index 984b05d033..4640ba7b5d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ac6a32482e675871c3ce5ecc727e50e4bba649e80e3c55bf99025fed5667da94 \ No newline at end of file +63a914410b5750bd08836864ae456ff09549487c5ce49706fedffa3d4f973059 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 33f5842282..d3d2a41e26 100644 --- a/src/btree.c +++ b/src/btree.c @@ -4669,7 +4669,7 @@ static int accessPayload( if( (pCur->curFlags & BTCF_ValidOvfl)==0 ){ int nOvfl = (pCur->info.nPayload-pCur->info.nLocal+ovflSize-1)/ovflSize; if( pCur->aOverflow==0 - || nOvfl*sizeof(Pgno) > sqlite3MallocSize(pCur->aOverflow) + || nOvfl*(int)sizeof(Pgno) > sqlite3MallocSize(pCur->aOverflow) ){ Pgno *aNew = (Pgno*)sqlite3Realloc( pCur->aOverflow, nOvfl*2*sizeof(Pgno) From 4b18c1d53af50abfcbc79aeb29a88df175c58fab Mon Sep 17 00:00:00 2001 From: drh Date: Sun, 4 Feb 2018 20:33:13 +0000 Subject: [PATCH 039/107] In the CLI, make sure sqlite3_initialize() is called correctly even for non-UTF8 hosts. FossilOrigin-Name: a3591fb54178b8f124207ff16e3ef9e58a62585feb3a56ced2b862222aea19d8 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/shell.c.in | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index b4577329bb..fd2eb50f21 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarning\sseen\swith\sMSVC. -D 2018-02-04T01:30:54.482 +C In\sthe\sCLI,\nmake\ssure\ssqlite3_initialize()\sis\scalled\scorrectly\seven\sfor\snon-UTF8\shosts. +D 2018-02-04T20:33:13.165 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -488,7 +488,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c f02352ac5cbb6fad3804add825743b924cfb6c742ba2e8666d726828a9985d73 -F src/shell.c.in ada6d184a5cdaada15a330ca97ad0126125f0aaed40b5c04385278fcaa78aafa +F src/shell.c.in 0b2c8d55e3703723a22610d75ccd4037962c3bcb5ad5570efd781e8c521811c4 F src/sqlite.h.in 51f9acf52c80113d793ddd38b3940ad6895d97b4752503b19291fb8fcbf54c5e F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 83a3c4ce93d650bedfd1aa558cb85a516bd6d094445ee989740827d0d944368d @@ -1704,7 +1704,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ac6a32482e675871c3ce5ecc727e50e4bba649e80e3c55bf99025fed5667da94 -R 996f708603528b0dbd58b1571407dd89 -U mistachkin -Z c8cfe40d98eaa63e42dbf121b7e80b46 +P 63a914410b5750bd08836864ae456ff09549487c5ce49706fedffa3d4f973059 +R e1e5372ebf3735de49eb34feee943e70 +U drh +Z 839d4cdabfa93aa0b085a28f8c75bf19 diff --git a/manifest.uuid b/manifest.uuid index 4640ba7b5d..80271b1207 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -63a914410b5750bd08836864ae456ff09549487c5ce49706fedffa3d4f973059 \ No newline at end of file +a3591fb54178b8f124207ff16e3ef9e58a62585feb3a56ced2b862222aea19d8 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index f683ce9868..3e3f9ac8f9 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -8094,8 +8094,8 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ } #endif main_init(&data); -#if !SQLITE_SHELL_IS_UTF8 sqlite3_initialize(); +#if !SQLITE_SHELL_IS_UTF8 argv = sqlite3_malloc64(sizeof(argv[0])*argc); if( argv==0 ){ raw_printf(stderr, "out of memory\n"); From 60d72b9baba2ee0938324fb5e2d68d97dab27ffb Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 5 Feb 2018 13:28:51 +0000 Subject: [PATCH 040/107] Update test file walro2.test to account for systems with a page-size (getpagesize()) larger than 32KB. FossilOrigin-Name: d9e59cfb8476e1ec1ca458b8382172526c0557ab785f41e31971d03045c9344c --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/walro2.test | 18 +++++++++++++++--- 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index fd2eb50f21..53c8e27ad1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\sCLI,\nmake\ssure\ssqlite3_initialize()\sis\scalled\scorrectly\seven\sfor\snon-UTF8\shosts. -D 2018-02-04T20:33:13.165 +C Update\stest\sfile\swalro2.test\sto\saccount\sfor\ssystems\swith\sa\spage-size\n(getpagesize())\slarger\sthan\s32KB. +D 2018-02-05T13:28:51.262 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -1555,7 +1555,7 @@ F test/waloverwrite.test dad2f26567f1b45174e54fbf9a8dc1cb876a7f03 F test/walpersist.test 8c6b7e3ec1ba91b5e4dc4e0921d6d3f87cd356a6 F test/walprotocol.test a112aba0b79e3adeaa485fed09484b32c654e97df58e454aa8489ac2cd57bf84 F test/walro.test cb438d05ba0d191f10b688e39c4f0cd5b71569a1d1f4440e5bdf3c6880e08c20 -F test/walro2.test 6c73e8e4b5ccc55f907f4603ba36458b45c085fb6dfb04f30e3c0babbc1c2f41 +F test/walro2.test ff7e493ce65a4dadf878d89aab4c385f8ddd3c1919333a18890013fce3feeb68 F test/walrofault.test c70cb6e308c443867701856cce92ad8288cd99488fa52afab77cca6cfd51af68 F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417 F test/walslow.test c05c68d4dc2700a982f89133ce103a1a84cc285f @@ -1704,7 +1704,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 63a914410b5750bd08836864ae456ff09549487c5ce49706fedffa3d4f973059 -R e1e5372ebf3735de49eb34feee943e70 -U drh -Z 839d4cdabfa93aa0b085a28f8c75bf19 +P a3591fb54178b8f124207ff16e3ef9e58a62585feb3a56ced2b862222aea19d8 +R 6c73de726d0398e2d66d996a80d76264 +U dan +Z ca72fcd07de53b3665abd69681f9d461 diff --git a/manifest.uuid b/manifest.uuid index 80271b1207..486bb16f75 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a3591fb54178b8f124207ff16e3ef9e58a62585feb3a56ced2b862222aea19d8 \ No newline at end of file +d9e59cfb8476e1ec1ca458b8382172526c0557ab785f41e31971d03045c9344c \ No newline at end of file diff --git a/test/walro2.test b/test/walro2.test index 1c51e91ec5..248acf86cb 100644 --- a/test/walro2.test +++ b/test/walro2.test @@ -39,6 +39,18 @@ proc copy_to_test2 {bZeroShm} { } } +# Most systems allocate the *-shm file in 32KB trunks. But on UNIX systems +# for which the getpagesize() call returns greater than 32K, the *-shm +# file is allocated in page-sized units (since you cannot mmap part of +# a page). The following code sets variable $MINSHMSZ to the smallest +# possible *-shm file (i.e. the greater of 32KB and the system page-size). +# +do_execsql_test 0.0 { + PRAGMA journal_mode = wal; + CREATE TABLE t1(x); +} +set MINSHMSZ [file size test.db-shm] + foreach bZeroShm {0 1} { set TN [expr $bZeroShm+1] do_multiclient_test tn { @@ -169,7 +181,7 @@ do_multiclient_test tn { } {a b c d e f g h 1 2} do_test $TN.3.2.2 { list [file size test.db-wal] [file size test.db-shm] - } {0 32768} + } [list 0 $MINSHMSZ] do_test $TN.3.3.0 { code2 { sqlite3 db2 test.db } @@ -182,7 +194,7 @@ do_multiclient_test tn { code2 { db2 close } code1 { db close } list [file size test.db-wal] [file size test.db-shm] - } [list [wal_file_size 4 1024] 32768] + } [list [wal_file_size 4 1024] $MINSHMSZ] do_test $TN.3.3.1 { code1 { sqlite3 db file:test.db?readonly_shm=1 } sql1 { SELECT * FROM t1 } @@ -196,7 +208,7 @@ do_multiclient_test tn { } code2 { db2 close } list [file size test.db-wal] [file size test.db-shm] - } [list [wal_file_size 4 1024] 32768] + } [list [wal_file_size 4 1024] $MINSHMSZ] do_test $TN.3.3.3 { sql1 { SELECT * FROM t1 } } {i ii} From d1317095b58e87e9543fc35b418b59c70003deb0 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 5 Feb 2018 13:42:45 +0000 Subject: [PATCH 041/107] Fix another minor problem in walro2.test. FossilOrigin-Name: ba0631de60ca38bf7efa6dbd86ec8774bf6f438c804155968e97f17eabe3b20a --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/walro2.test | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 53c8e27ad1..5a2f7c7320 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\stest\sfile\swalro2.test\sto\saccount\sfor\ssystems\swith\sa\spage-size\n(getpagesize())\slarger\sthan\s32KB. -D 2018-02-05T13:28:51.262 +C Fix\sanother\sminor\sproblem\sin\swalro2.test. +D 2018-02-05T13:42:45.777 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -1555,7 +1555,7 @@ F test/waloverwrite.test dad2f26567f1b45174e54fbf9a8dc1cb876a7f03 F test/walpersist.test 8c6b7e3ec1ba91b5e4dc4e0921d6d3f87cd356a6 F test/walprotocol.test a112aba0b79e3adeaa485fed09484b32c654e97df58e454aa8489ac2cd57bf84 F test/walro.test cb438d05ba0d191f10b688e39c4f0cd5b71569a1d1f4440e5bdf3c6880e08c20 -F test/walro2.test ff7e493ce65a4dadf878d89aab4c385f8ddd3c1919333a18890013fce3feeb68 +F test/walro2.test 0e79dd15cbdb4f482c01ea248373669c732414a726b357d04846a816afafb768 F test/walrofault.test c70cb6e308c443867701856cce92ad8288cd99488fa52afab77cca6cfd51af68 F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417 F test/walslow.test c05c68d4dc2700a982f89133ce103a1a84cc285f @@ -1704,7 +1704,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a3591fb54178b8f124207ff16e3ef9e58a62585feb3a56ced2b862222aea19d8 -R 6c73de726d0398e2d66d996a80d76264 +P d9e59cfb8476e1ec1ca458b8382172526c0557ab785f41e31971d03045c9344c +R f3585ddda6ecfb38609942de6c6ce298 U dan -Z ca72fcd07de53b3665abd69681f9d461 +Z 93b8cc01dad51cf808b345d2ca0944f2 diff --git a/manifest.uuid b/manifest.uuid index 486bb16f75..b8f8b2f481 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d9e59cfb8476e1ec1ca458b8382172526c0557ab785f41e31971d03045c9344c \ No newline at end of file +ba0631de60ca38bf7efa6dbd86ec8774bf6f438c804155968e97f17eabe3b20a \ No newline at end of file diff --git a/test/walro2.test b/test/walro2.test index 248acf86cb..34408c1695 100644 --- a/test/walro2.test +++ b/test/walro2.test @@ -48,7 +48,7 @@ proc copy_to_test2 {bZeroShm} { do_execsql_test 0.0 { PRAGMA journal_mode = wal; CREATE TABLE t1(x); -} +} {wal} set MINSHMSZ [file size test.db-shm] foreach bZeroShm {0 1} { From 561158937bab6dcd057026dc554d28a35c7e0bd6 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 5 Feb 2018 16:39:12 +0000 Subject: [PATCH 042/107] Allocation the mutex used by the unix VFS only once at initialization, instead of every time it is needed. FossilOrigin-Name: 5764dc160783f5c4017204b3e26a89d31240c868484ced8214c9ad872bd77bd4 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_unix.c | 10 ++++++---- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 5a2f7c7320..265850e6d9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sanother\sminor\sproblem\sin\swalro2.test. -D 2018-02-05T13:42:45.777 +C Allocation\sthe\smutex\sused\sby\sthe\sunix\sVFS\sonly\sonce\sat\sinitialization,\sinstead\nof\severy\stime\sit\sis\sneeded. +D 2018-02-05T16:39:12.754 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -471,7 +471,7 @@ F src/os.c 22d31db3ca5a96a408fbf1ceeaaebcaf64c87024d2ff9fe1cf2ddbec3e75c104 F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 -F src/os_unix.c a82505be158d8ce42b38dcc9b426187d776904c12cdc68dc8925e1dfcc5cb6ce +F src/os_unix.c ce491421b3a54b63094a155eeac668a3bc8e5b86a5a58551d906e5b5affb443f F src/os_win.c 501dde1ee770f4ffa458bfe1cf376a556de3ab00bb8320d659c5984403991d62 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c cd194a8793ce061e184ddc369fadbc1020c6f431014d22093f6c5e55c9234033 @@ -1704,7 +1704,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d9e59cfb8476e1ec1ca458b8382172526c0557ab785f41e31971d03045c9344c -R f3585ddda6ecfb38609942de6c6ce298 -U dan -Z 93b8cc01dad51cf808b345d2ca0944f2 +P ba0631de60ca38bf7efa6dbd86ec8774bf6f438c804155968e97f17eabe3b20a +R 07658fd89f10e428329340f4fdba337c +U drh +Z f93cbf0dc0e4b2bfb043a63cd29cee7e diff --git a/manifest.uuid b/manifest.uuid index b8f8b2f481..2da9ca7d0a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ba0631de60ca38bf7efa6dbd86ec8774bf6f438c804155968e97f17eabe3b20a \ No newline at end of file +5764dc160783f5c4017204b3e26a89d31240c868484ced8214c9ad872bd77bd4 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 94b1efd87d..b24c6861d3 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -696,15 +696,16 @@ static int robust_open(const char *z, int f, mode_t m){ ** assert( unixMutexHeld() ); ** unixEnterLeave() */ +static sqlite3_mutex *unixBigLock = 0; static void unixEnterMutex(void){ - sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1)); + sqlite3_mutex_enter(unixBigLock); } static void unixLeaveMutex(void){ - sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1)); + sqlite3_mutex_leave(unixBigLock); } #ifdef SQLITE_DEBUG static int unixMutexHeld(void) { - return sqlite3_mutex_held(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1)); + return sqlite3_mutex_held(unixBigLock); } #endif @@ -5846,7 +5847,6 @@ static int unixOpen( randomnessPid = osGetpid(0); sqlite3_randomness(0,0); } - memset(p, 0, sizeof(unixFile)); if( eType==SQLITE_OPEN_MAIN_DB ){ @@ -7721,6 +7721,7 @@ int sqlite3_os_init(void){ for(i=0; i<(sizeof(aVfs)/sizeof(sqlite3_vfs)); i++){ sqlite3_vfs_register(&aVfs[i], i==0); } + unixBigLock = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1); return SQLITE_OK; } @@ -7732,6 +7733,7 @@ int sqlite3_os_init(void){ ** This routine is a no-op for unix. */ int sqlite3_os_end(void){ + unixBigLock = 0; return SQLITE_OK; } From 435666e26d906ed2b8c1022d63f714f8143be8bb Mon Sep 17 00:00:00 2001 From: mistachkin Date: Mon, 5 Feb 2018 20:42:50 +0000 Subject: [PATCH 043/107] Allocation the mutex used by the Win32 VFS only once at initialization, instead of every time it is needed. FossilOrigin-Name: 535ed0ac5e8728ec91fc0a4cb54b820923d161cfd4e0e6aed6df6cdae365bc7d --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_win.c | 9 ++++++--- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 265850e6d9..5469fa43ad 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Allocation\sthe\smutex\sused\sby\sthe\sunix\sVFS\sonly\sonce\sat\sinitialization,\sinstead\nof\severy\stime\sit\sis\sneeded. -D 2018-02-05T16:39:12.754 +C Allocation\sthe\smutex\sused\sby\sthe\sWin32\sVFS\sonly\sonce\sat\sinitialization,\sinstead\sof\severy\stime\sit\sis\sneeded. +D 2018-02-05T20:42:50.153 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -472,7 +472,7 @@ F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 F src/os_unix.c ce491421b3a54b63094a155eeac668a3bc8e5b86a5a58551d906e5b5affb443f -F src/os_win.c 501dde1ee770f4ffa458bfe1cf376a556de3ab00bb8320d659c5984403991d62 +F src/os_win.c ca6b6a557b9015d9ecf77003482558b04bbbec86d5065af30e89295dfc01fbb7 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c cd194a8793ce061e184ddc369fadbc1020c6f431014d22093f6c5e55c9234033 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a @@ -1704,7 +1704,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ba0631de60ca38bf7efa6dbd86ec8774bf6f438c804155968e97f17eabe3b20a -R 07658fd89f10e428329340f4fdba337c -U drh -Z f93cbf0dc0e4b2bfb043a63cd29cee7e +P 5764dc160783f5c4017204b3e26a89d31240c868484ced8214c9ad872bd77bd4 +R 7ec707deb63ac6534c16562af3ea60e2 +U mistachkin +Z 87b212ba57f1d2b4c01281b9a95b7258 diff --git a/manifest.uuid b/manifest.uuid index 2da9ca7d0a..12329d932f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5764dc160783f5c4017204b3e26a89d31240c868484ced8214c9ad872bd77bd4 \ No newline at end of file +535ed0ac5e8728ec91fc0a4cb54b820923d161cfd4e0e6aed6df6cdae365bc7d \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index 2b2b8ebd56..6132d321a1 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -3631,15 +3631,16 @@ static SYSTEM_INFO winSysInfo; ** assert( winShmMutexHeld() ); ** winShmLeaveMutex() */ +static sqlite3_mutex *winBigLock = 0; static void winShmEnterMutex(void){ - sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1)); + sqlite3_mutex_enter(winBigLock); } static void winShmLeaveMutex(void){ - sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1)); + sqlite3_mutex_leave(winBigLock); } #ifndef NDEBUG static int winShmMutexHeld(void) { - return sqlite3_mutex_held(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1)); + return sqlite3_mutex_held(winBigLock); } #endif @@ -6062,6 +6063,7 @@ int sqlite3_os_init(void){ sqlite3_vfs_register(&winLongPathNolockVfs, 0); #endif + winBigLock = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1); return SQLITE_OK; } @@ -6072,6 +6074,7 @@ int sqlite3_os_end(void){ sleepObj = NULL; } #endif + winBigLock = 0; return SQLITE_OK; } From bc6b8d73592ad72e674b10152012170a01c31c62 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Mon, 5 Feb 2018 21:02:47 +0000 Subject: [PATCH 044/107] Adjust the previous check-in, which modified the Win32 VFS, so that it works with SQLITE_OMIT_WAL. FossilOrigin-Name: 36c2e67e82626f8d0a187c6c286c133ed659889e3b577469261b9dcd3b3ab75b --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_win.c | 7 +++++++ 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 5469fa43ad..48ba691a5f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Allocation\sthe\smutex\sused\sby\sthe\sWin32\sVFS\sonly\sonce\sat\sinitialization,\sinstead\sof\severy\stime\sit\sis\sneeded. -D 2018-02-05T20:42:50.153 +C Adjust\sthe\sprevious\scheck-in,\swhich\smodified\sthe\sWin32\sVFS,\sso\sthat\sit\sworks\swith\sSQLITE_OMIT_WAL. +D 2018-02-05T21:02:47.090 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -472,7 +472,7 @@ F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 F src/os_unix.c ce491421b3a54b63094a155eeac668a3bc8e5b86a5a58551d906e5b5affb443f -F src/os_win.c ca6b6a557b9015d9ecf77003482558b04bbbec86d5065af30e89295dfc01fbb7 +F src/os_win.c eb03c6d52f893bcd7fdd4c6006674c13c1b5e49543fec98d605201af2997171c F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c cd194a8793ce061e184ddc369fadbc1020c6f431014d22093f6c5e55c9234033 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a @@ -1704,7 +1704,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5764dc160783f5c4017204b3e26a89d31240c868484ced8214c9ad872bd77bd4 -R 7ec707deb63ac6534c16562af3ea60e2 +P 535ed0ac5e8728ec91fc0a4cb54b820923d161cfd4e0e6aed6df6cdae365bc7d +R 610345e12e18345d79d861c16dfe01d9 U mistachkin -Z 87b212ba57f1d2b4c01281b9a95b7258 +Z 7a54a037e46133481e56c9143fb3122e diff --git a/manifest.uuid b/manifest.uuid index 12329d932f..899bc00cb8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -535ed0ac5e8728ec91fc0a4cb54b820923d161cfd4e0e6aed6df6cdae365bc7d \ No newline at end of file +36c2e67e82626f8d0a187c6c286c133ed659889e3b577469261b9dcd3b3ab75b \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index 6132d321a1..534426977f 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -6063,7 +6063,10 @@ int sqlite3_os_init(void){ sqlite3_vfs_register(&winLongPathNolockVfs, 0); #endif +#ifndef SQLITE_OMIT_WAL winBigLock = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1); +#endif + return SQLITE_OK; } @@ -6074,7 +6077,11 @@ int sqlite3_os_end(void){ sleepObj = NULL; } #endif + +#ifndef SQLITE_OMIT_WAL winBigLock = 0; +#endif + return SQLITE_OK; } From fa68815fa326af4b9bcdc61df39185170985f078 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 7 Feb 2018 16:14:41 +0000 Subject: [PATCH 045/107] When the final connection disconnects from a wal mode database, check that the database file has not been moved or unlinked before deleting the wal and shm files. FossilOrigin-Name: 4761db83b6d3d57f281370899403c102e39ad0021d315dd6a6912d250436782a --- manifest | 16 +++++----- manifest.uuid | 2 +- src/pager.c | 67 ++++++++++++++++++++++------------------- test/nockpt.test | 78 ++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 123 insertions(+), 40 deletions(-) diff --git a/manifest b/manifest index 48ba691a5f..80740ef60b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Adjust\sthe\sprevious\scheck-in,\swhich\smodified\sthe\sWin32\sVFS,\sso\sthat\sit\sworks\swith\sSQLITE_OMIT_WAL. -D 2018-02-05T21:02:47.090 +C When\sthe\sfinal\sconnection\sdisconnects\sfrom\sa\swal\smode\sdatabase,\scheck\sthat\sthe\ndatabase\sfile\shas\snot\sbeen\smoved\sor\sunlinked\sbefore\sdeleting\sthe\swal\sand\sshm\nfiles. +D 2018-02-07T16:14:41.573 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -474,7 +474,7 @@ F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 F src/os_unix.c ce491421b3a54b63094a155eeac668a3bc8e5b86a5a58551d906e5b5affb443f F src/os_win.c eb03c6d52f893bcd7fdd4c6006674c13c1b5e49543fec98d605201af2997171c F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a -F src/pager.c cd194a8793ce061e184ddc369fadbc1020c6f431014d22093f6c5e55c9234033 +F src/pager.c a3834a40acc2f3ab247d846f850d8c9313587d9c99c57a4dc194f2d4d7bf9d15 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a F src/parse.y 4e750e1b261ff9f1d0b6b5d40a829c66d691899f48953fde839d8b52d41aa148 F src/pcache.c 7ae91a4557a43d77d449accbfdc68846e6516f8e2eda46e8bbe4536fb669b201 @@ -1094,7 +1094,7 @@ F test/multiplex4.test e8ae4c4bd70606a5727743241f13b5701990abe4 F test/mutex1.test ea2cc74d97f077b9e74c84cbd024f14d79a8126f F test/mutex2.test bfeaeac2e73095b2ac32285d2756e3a65e681660 F test/nan.test 437d40e6d0778b050d7750726c0cbd2c9936b81962926e8f8c48ca698f00f4d1 -F test/nockpt.test 9a436a7213ba5ef7a32304998d386d3ea3f76c9d +F test/nockpt.test d291d618c934a453683cb2eff95f633d406f7147fa0403e10055db19dcc3842a F test/nolock.test f196cf8b8fbea4e2ca345140a2b3f3b0da45c76e F test/normalize.test 501630ab49b0b26b65c74124bf03e3374c1b57fa97aae750f84803609141d167 F test/notify1.test 669b2b743618efdc18ca4b02f45423d5d2304abf @@ -1704,7 +1704,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 535ed0ac5e8728ec91fc0a4cb54b820923d161cfd4e0e6aed6df6cdae365bc7d -R 610345e12e18345d79d861c16dfe01d9 -U mistachkin -Z 7a54a037e46133481e56c9143fb3122e +P 36c2e67e82626f8d0a187c6c286c133ed659889e3b577469261b9dcd3b3ab75b +R ed31a2562d55c76884009a210d222cf0 +U dan +Z 0e612251c296ff4ce7a47fbd872de74d diff --git a/manifest.uuid b/manifest.uuid index 899bc00cb8..f1767d6dc9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -36c2e67e82626f8d0a187c6c286c133ed659889e3b577469261b9dcd3b3ab75b \ No newline at end of file +4761db83b6d3d57f281370899403c102e39ad0021d315dd6a6912d250436782a \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index e4680bfbfe..93a9b3bd3e 100644 --- a/src/pager.c +++ b/src/pager.c @@ -4102,6 +4102,30 @@ static void pagerFreeMapHdrs(Pager *pPager){ } } +/* Verify that the database file has not be deleted or renamed out from +** under the pager. Return SQLITE_OK if the database is still were it ought +** to be on disk. Return non-zero (SQLITE_READONLY_DBMOVED or some other error +** code from sqlite3OsAccess()) if the database has gone missing. +*/ +static int databaseIsUnmoved(Pager *pPager){ + int bHasMoved = 0; + int rc; + + if( pPager->tempFile ) return SQLITE_OK; + if( pPager->dbSize==0 ) return SQLITE_OK; + assert( pPager->zFilename && pPager->zFilename[0] ); + rc = sqlite3OsFileControl(pPager->fd, SQLITE_FCNTL_HAS_MOVED, &bHasMoved); + if( rc==SQLITE_NOTFOUND ){ + /* If the HAS_MOVED file-control is unimplemented, assume that the file + ** has not been moved. That is the historical behavior of SQLite: prior to + ** version 3.8.3, it never checked */ + rc = SQLITE_OK; + }else if( rc==SQLITE_OK && bHasMoved ){ + rc = SQLITE_READONLY_DBMOVED; + } + return rc; +} + /* ** Shutdown the page cache. Free all memory and close all files. @@ -4118,8 +4142,7 @@ static void pagerFreeMapHdrs(Pager *pPager){ ** to the caller. */ int sqlite3PagerClose(Pager *pPager, sqlite3 *db){ - u8 *pTmp = (u8 *)pPager->pTmpSpace; - + u8 *pTmp = (u8*)pPager->pTmpSpace; assert( db || pagerUseWal(pPager)==0 ); assert( assert_pager_state(pPager) ); disable_simulated_io_errors(); @@ -4128,11 +4151,17 @@ int sqlite3PagerClose(Pager *pPager, sqlite3 *db){ /* pPager->errCode = 0; */ pPager->exclusiveMode = 0; #ifndef SQLITE_OMIT_WAL - assert( db || pPager->pWal==0 ); - sqlite3WalClose(pPager->pWal, db, pPager->walSyncFlags, pPager->pageSize, - (db && (db->flags & SQLITE_NoCkptOnClose) ? 0 : pTmp) - ); - pPager->pWal = 0; + { + u8 *a = 0; + assert( db || pPager->pWal==0 ); + if( db && 0==(db->flags & SQLITE_NoCkptOnClose) + && SQLITE_OK==databaseIsUnmoved(pPager) + ){ + a = pTmp; + } + sqlite3WalClose(pPager->pWal, db, pPager->walSyncFlags, pPager->pageSize,a); + pPager->pWal = 0; + } #endif pager_reset(pPager); if( MEMDB ){ @@ -4967,30 +4996,6 @@ act_like_temp_file: } -/* Verify that the database file has not be deleted or renamed out from -** under the pager. Return SQLITE_OK if the database is still were it ought -** to be on disk. Return non-zero (SQLITE_READONLY_DBMOVED or some other error -** code from sqlite3OsAccess()) if the database has gone missing. -*/ -static int databaseIsUnmoved(Pager *pPager){ - int bHasMoved = 0; - int rc; - - if( pPager->tempFile ) return SQLITE_OK; - if( pPager->dbSize==0 ) return SQLITE_OK; - assert( pPager->zFilename && pPager->zFilename[0] ); - rc = sqlite3OsFileControl(pPager->fd, SQLITE_FCNTL_HAS_MOVED, &bHasMoved); - if( rc==SQLITE_NOTFOUND ){ - /* If the HAS_MOVED file-control is unimplemented, assume that the file - ** has not been moved. That is the historical behavior of SQLite: prior to - ** version 3.8.3, it never checked */ - rc = SQLITE_OK; - }else if( rc==SQLITE_OK && bHasMoved ){ - rc = SQLITE_READONLY_DBMOVED; - } - return rc; -} - /* ** This function is called after transitioning from PAGER_UNLOCK to diff --git a/test/nockpt.test b/test/nockpt.test index bd3953f1ee..8f6b5e3be4 100644 --- a/test/nockpt.test +++ b/test/nockpt.test @@ -61,6 +61,84 @@ do_test 1.14 { sqlite3_db_config db NO_CKPT_ON_CLOSE 1 } {1} do_execsql_test 1.14 { PRAGMA main.journal_mode = delete } {delete} do_test 1.15 { file exists test.db-wal } {0} +#------------------------------------------------------------------------- +# Test an unusual scenario: +# +# 1. A wal mode db is opened and written. Then sqlite3_close_v2() used +# to close the db handle while there is still an unfinalized +# statement (so the db handle stays open). +# +# 2. The db, wal and *-shm files are deleted from the file system. +# +# 3. Another connection creates a new wal mode db at the same file-system +# location as the previous one. +# +# 4. The statement left unfinalized in (1) is finalized. +# +# The test is to ensure that the connection left open in step (1) does +# not try to delete the wal file from the file-system as part of step +# 4. +# +reset_db +db close + +# Open a connection on a wal database. Write to it a bit. Then prepare +# a statement and call sqlite3_close_v2() (so that the statement handle +# holds the db connection open). +# +set ::db1 [sqlite3_open_v2 test.db SQLITE_OPEN_READWRITE ""] +do_test 2.0 { + lindex [ + sqlite3_exec $::db1 { + PRAGMA journal_mode = wal; + CREATE TABLE t1(x PRIMARY KEY, y UNIQUE, z); + INSERT INTO t1 VALUES(1, 2, 3); + PRAGMA wal_checkpoint; + }] 0 +} {0} +set ::stmt [sqlite3_prepare $::db1 "SELECT * FROM t1" -1 dummy] +sqlite3_close_v2 $::db1 + +# Delete the database, wal and shm files. +# +forcedelete test.db test.db-wal test.db-shm + +# Open and populate a new database file at the same file-system location +# as the one just deleted. Contrive a partial checkpoint on it. +# +sqlite3 db test.db +sqlite3 db2 test.db +do_execsql_test 2.1 { + PRAGMA journal_mode = wal; + CREATE TABLE y1(a PRIMARY KEY, b UNIQUE, c); + INSERT INTO y1 VALUES('a', 'b', 'c'); + INSERT INTO y1 VALUES('d', 'e', 'f'); +} {wal} +do_execsql_test -db db2 2.2 { + BEGIN; + SELECT * FROM y1; +} {a b c d e f} +do_execsql_test 2.3 { + UPDATE y1 SET c='g' WHERE a='d'; + PRAGMA wal_checkpoint; +} {0 11 10} +do_execsql_test -db db2 2.4 { + COMMIT +} + +# Finalize the statement handle, causing the first connection to be +# closed. Test that this has not corrupted the database file by +# deleting the new wal file from the file-system. If it has, this +# test should fail with an IO or corruption error. +# +do_test 2.5 { + sqlite3_finalize $::stmt + sqlite3 db3 test.db + execsql { + PRAGMA integrity_check; + SELECT * FROM y1; + } db3 +} {ok a b c d e g} finish_test From eab0e103042143fe5712ea067c86f7b8acd545df Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 7 Feb 2018 18:02:50 +0000 Subject: [PATCH 046/107] In extensions rtree, fts3 and fts5, ensure that when dynamic buffers are bound to persistent SQL statements using SQLITE_STATIC, the binding is replaced with an SQL NULL before the buffer is freed. Otherwise, a user may obtain a pointer to the persistent statement using sqlite3_next_stmt() and attempt to access the freed buffer using sqlite3_expanded_sql() or similar. FossilOrigin-Name: 2a5f813bc61f9e780f2ccbda425611f65ad523b6d486a1e5e2b9d5e9f1d260a2 --- ext/fts3/fts3_write.c | 5 +++++ ext/fts5/fts5_index.c | 4 ++++ ext/fts5/fts5_storage.c | 2 ++ ext/fts5/test/fts5aa.test | 2 +- ext/rtree/rtree.c | 1 + ext/rtree/rtree1.test | 1 + ext/rtree/rtree4.test | 1 + ext/rtree/rtree5.test | 1 + ext/rtree/rtree6.test | 2 +- ext/rtree/rtreeG.test | 1 + manifest | 34 +++++++++++++++++----------------- manifest.uuid | 2 +- test/fts3aa.test | 1 + test/tester.tcl | 10 ++++++++++ 14 files changed, 47 insertions(+), 20 deletions(-) diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index daf3399a43..5bca766c20 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -1908,6 +1908,7 @@ static int fts3WriteSegment( sqlite3_bind_blob(pStmt, 2, z, n, SQLITE_STATIC); sqlite3_step(pStmt); rc = sqlite3_reset(pStmt); + sqlite3_bind_null(pStmt, 2); } return rc; } @@ -1964,6 +1965,7 @@ static int fts3WriteSegdir( sqlite3_bind_blob(pStmt, 6, zRoot, nRoot, SQLITE_STATIC); sqlite3_step(pStmt); rc = sqlite3_reset(pStmt); + sqlite3_bind_null(pStmt, 6); } return rc; } @@ -3443,6 +3445,7 @@ static void fts3UpdateDocTotals( sqlite3_bind_blob(pStmt, 2, pBlob, nBlob, SQLITE_STATIC); sqlite3_step(pStmt); *pRC = sqlite3_reset(pStmt); + sqlite3_bind_null(pStmt, 2); sqlite3_free(a); } @@ -4631,6 +4634,7 @@ static int fts3TruncateSegment( sqlite3_bind_int(pChomp, 4, iIdx); sqlite3_step(pChomp); rc = sqlite3_reset(pChomp); + sqlite3_bind_null(pChomp, 2); } } @@ -4710,6 +4714,7 @@ static int fts3IncrmergeHintStore(Fts3Table *p, Blob *pHint){ sqlite3_bind_blob(pReplace, 2, pHint->a, pHint->n, SQLITE_STATIC); sqlite3_step(pReplace); rc = sqlite3_reset(pReplace); + sqlite3_bind_null(pReplace, 2); } return rc; diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index a75bf0fd42..412a04faca 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -758,6 +758,7 @@ static void fts5DataWrite(Fts5Index *p, i64 iRowid, const u8 *pData, int nData){ sqlite3_bind_blob(p->pWriter, 2, pData, nData, SQLITE_STATIC); sqlite3_step(p->pWriter); p->rc = sqlite3_reset(p->pWriter); + sqlite3_bind_null(p->pWriter, 2); } /* @@ -2386,6 +2387,7 @@ static void fts5SegIterSeekInit( bDlidx = (val & 0x0001); } p->rc = sqlite3_reset(pIdxSelect); + sqlite3_bind_null(pIdxSelect, 2); if( iPgpgnoFirst ){ iPg = pSeg->pgnoFirst; @@ -3598,6 +3600,7 @@ static int fts5AllocateSegid(Fts5Index *p, Fts5Structure *pStruct){ sqlite3_bind_blob(pIdxSelect, 2, aBlob, 2, SQLITE_STATIC); assert( sqlite3_step(pIdxSelect)!=SQLITE_ROW ); p->rc = sqlite3_reset(pIdxSelect); + sqlite3_bind_null(pIdxSelect, 2); } } #endif @@ -3724,6 +3727,7 @@ static void fts5WriteFlushBtree(Fts5Index *p, Fts5SegWriter *pWriter){ sqlite3_bind_int64(p->pIdxWriter, 3, bFlag + ((i64)pWriter->iBtPage<<1)); sqlite3_step(p->pIdxWriter); p->rc = sqlite3_reset(p->pIdxWriter); + sqlite3_bind_null(p->pIdxWriter, 2); } pWriter->iBtPage = 0; } diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c index 59336fc7ac..70d7135113 100644 --- a/ext/fts5/fts5_storage.c +++ b/ext/fts5/fts5_storage.c @@ -458,6 +458,7 @@ static int fts5StorageInsertDocsize( sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC); sqlite3_step(pReplace); rc = sqlite3_reset(pReplace); + sqlite3_bind_null(pReplace, 2); } } return rc; @@ -1118,6 +1119,7 @@ int sqlite3Fts5StorageConfigValue( } sqlite3_step(pReplace); rc = sqlite3_reset(pReplace); + sqlite3_bind_null(pReplace, 1); } if( rc==SQLITE_OK && pVal ){ int iNew = p->pConfig->iCookie + 1; diff --git a/ext/fts5/test/fts5aa.test b/ext/fts5/test/fts5aa.test index a3ea0afc28..67cb62012a 100644 --- a/ext/fts5/test/fts5aa.test +++ b/ext/fts5/test/fts5aa.test @@ -593,5 +593,5 @@ do_execsql_test 22.1 { } - +expand_all_sql db finish_test diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c index c0fd8c1819..00513d4005 100644 --- a/ext/rtree/rtree.c +++ b/ext/rtree/rtree.c @@ -785,6 +785,7 @@ static int nodeWrite(Rtree *pRtree, RtreeNode *pNode){ sqlite3_step(p); pNode->isDirty = 0; rc = sqlite3_reset(p); + sqlite3_bind_null(p, 2); if( pNode->iNode==0 && rc==SQLITE_OK ){ pNode->iNode = sqlite3_last_insert_rowid(pRtree->db); nodeHashInsert(pRtree, pNode); diff --git a/ext/rtree/rtree1.test b/ext/rtree/rtree1.test index 0deee6635b..ac6e8d9d96 100644 --- a/ext/rtree/rtree1.test +++ b/ext/rtree/rtree1.test @@ -609,4 +609,5 @@ do_execsql_test 15.2 { COMMIT; } +expand_all_sql db finish_test diff --git a/ext/rtree/rtree4.test b/ext/rtree/rtree4.test index af3f8d3995..a73921d8d5 100644 --- a/ext/rtree/rtree4.test +++ b/ext/rtree/rtree4.test @@ -250,4 +250,5 @@ for {set nDim 1} {$nDim<=5} {incr nDim} { do_rtree_integrity_test rtree4-$nDim.3 rx } +expand_all_sql db finish_test diff --git a/ext/rtree/rtree5.test b/ext/rtree/rtree5.test index 749385e882..92bb6905c7 100644 --- a/ext/rtree/rtree5.test +++ b/ext/rtree/rtree5.test @@ -79,4 +79,5 @@ do_test rtree5-1.13 { } {2 2147483643 2147483647 -2147483648 -2147483643} do_rtree_integrity_test rtree5-1.14 t1 +expand_all_sql db finish_test diff --git a/ext/rtree/rtree6.test b/ext/rtree/rtree6.test index c9c87e8ad9..406604810b 100644 --- a/ext/rtree/rtree6.test +++ b/ext/rtree/rtree6.test @@ -158,5 +158,5 @@ do_execsql_test rtree6-3.5 { x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>1.1 } {} - +expand_all_sql db finish_test diff --git a/ext/rtree/rtreeG.test b/ext/rtree/rtreeG.test index 3bef89c8e7..12225d5832 100644 --- a/ext/rtree/rtreeG.test +++ b/ext/rtree/rtreeG.test @@ -59,6 +59,7 @@ do_test rtreeG-1.4log { set ::log } {} +expand_all_sql db db close sqlite3_shutdown test_sqlite3_log diff --git a/manifest b/manifest index 80740ef60b..047f9dc234 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\sthe\sfinal\sconnection\sdisconnects\sfrom\sa\swal\smode\sdatabase,\scheck\sthat\sthe\ndatabase\sfile\shas\snot\sbeen\smoved\sor\sunlinked\sbefore\sdeleting\sthe\swal\sand\sshm\nfiles. -D 2018-02-07T16:14:41.573 +C In\sextensions\srtree,\sfts3\sand\sfts5,\sensure\sthat\swhen\sdynamic\sbuffers\sare\sbound\nto\spersistent\sSQL\sstatements\susing\sSQLITE_STATIC,\sthe\sbinding\sis\sreplaced\swith\nan\sSQL\sNULL\sbefore\sthe\sbuffer\sis\sfreed.\sOtherwise,\sa\suser\smay\sobtain\sa\spointer\nto\sthe\spersistent\sstatement\susing\ssqlite3_next_stmt()\sand\sattempt\sto\saccess\nthe\sfreed\sbuffer\susing\ssqlite3_expanded_sql()\sor\ssimilar. +D 2018-02-07T18:02:50.375 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -96,7 +96,7 @@ F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 F ext/fts3/fts3_unicode.c 525a3bd9a7564603c5c061b7de55403a565307758a94600e8a2f6b00d1c40d9d F ext/fts3/fts3_unicode2.c cc04fc672bfd42b1e650398cb0bf71f64f9aae032cfe75bbcfe75b9cf966029c -F ext/fts3/fts3_write.c a3f7bf869622d1d0aa66661ba71d88e6f9646d69a2c335f40a0addf25974db47 +F ext/fts3/fts3_write.c 7a7cf93c02ebe0ee4211e4aa07da77586c2dcf7d381c1382f81d29c9aa8cae8c F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/fts3/tool/fts3cov.sh c331d006359456cf6f8f953e37f2b9c7d568f3863f00bb5f7eb87fea4ac01b73 @@ -113,9 +113,9 @@ F ext/fts5/fts5_buffer.c 1dd1ec0446b3acfc2d7d407eb894762a461613e2695273f48e449bf F ext/fts5/fts5_config.c 5af9c360e99669d29f06492c370892394aba0857 F ext/fts5/fts5_expr.c 01048018d21524e2c302b063ff5c3cdcf546e03297215e577205d85b47499deb F ext/fts5/fts5_hash.c 32be400cf761868c9db33efe81a06eb19a17c5402ad477ee9efb51301546dd55 -F ext/fts5/fts5_index.c 5fe14375a29e8a7aa8f3e863babe180a19269206c254c8f47b216821d4ac1e15 +F ext/fts5/fts5_index.c 22b71d0e9e4b3ddd123a39ae27174e0012da2806f91b64087a68584f13f189de F ext/fts5/fts5_main.c 24868f88ab2a865defbba7a92eebeb726cc991eb092b71b5f5508f180c72605b -F ext/fts5/fts5_storage.c fb5ef3c27073f67ade2e1bea08405f9e43f68f5f3676ed0ab7013bce5ba10be6 +F ext/fts5/fts5_storage.c 4bec8a1b3905978b22a67bca5f4a3cfdb94af234cf51efb36f4f2d733d278634 F ext/fts5/fts5_tcl.c 39bcbae507f594aad778172fa914cad0f585bf92fd3b078c686e249282db0d95 F ext/fts5/fts5_test_mi.c 65864ba1e5c34a61d409c4c587e0bbe0466eb4f8f478d85dc42a92caad1338e6 F ext/fts5/fts5_test_tok.c ffd657dd67e7fcdb31bf63fb60b6d867299a581d0f46e97086abacd66c2a9b26 @@ -126,7 +126,7 @@ F ext/fts5/fts5_vocab.c 1cd79854cb21543e66507b25b0578bc1b20aa6a1349b7feceb8e8fed F ext/fts5/fts5parse.y eb526940f892ade5693f22ffd6c4f2702543a9059942772526eac1fde256bb05 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba F ext/fts5/test/fts5_common.tcl b01c584144b5064f30e6c648145a2dd6bc440841 -F ext/fts5/test/fts5aa.test cba3fae6466446980caf1b9f5f26df77f95a999d35db7d932d6e82ae7ba0ede9 +F ext/fts5/test/fts5aa.test 6e2fdb0ee667c05f41921e7ec345cae874be651670900918e9ccc539514b9356 F ext/fts5/test/fts5ab.test 9205c839332c908aaad2b01ab8670ece8b161e8f2ec8a9fabf18ca9385880bb7 F ext/fts5/test/fts5ac.test a7aa7e1fefc6e1918aa4d3111d5c44a09177168e962c5fd2cca9620de8a7ed6d F ext/fts5/test/fts5ad.test e8cf959dfcd57c8e46d6f5f25665686f3b6627130a9a981371dafdf6482790de @@ -351,14 +351,14 @@ F ext/repair/test/checkfreelist01.test 3e8aa6aeb4007680c94a8d07b41c339aa635cc782 F ext/repair/test/checkindex01.test 6945d0ffc0c1dc993b2ce88036b26e0f5d6fcc65da70fc9df27c2647bb358b0f F ext/repair/test/test.tcl 686d76d888dffd021f64260abf29a55c57b2cedfa7fc69150b42b1d6119aac3c F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 -F ext/rtree/rtree.c d941e44ad901da039caebb9f9fa99d81f2a4fc822e67cafe33fa4f6f789074a0 +F ext/rtree/rtree.c bc61010e978b5b8ae6dbb90274a2fbb5db5ff5e2880b5c6e8abd48eea77264db F ext/rtree/rtree.h 4a690463901cb5e6127cf05eb8e642f127012fd5003830dbc974eca5802d9412 -F ext/rtree/rtree1.test 82a353747fcab1083d114b2ac84723dfefdbf86c1a6e1df57bf588c7d4285436 +F ext/rtree/rtree1.test 47e2095bebea6813754fd7afa6a20e2b7b4ebcd5cb7dbcb6932b6c9f86bbf972 F ext/rtree/rtree2.test 5f25b01acd03470067a2d52783b2eb0a50bf836803d4342d20ca39e541220fe2 F ext/rtree/rtree3.test 2cafe8265d1ff28f206fce88d114f208349df482 -F ext/rtree/rtree4.test 67b021858ba4334c8d49b3449476942c2ce0e5ef7123538f2e9dd508ed03a12d -F ext/rtree/rtree5.test 8aaa4bcdc42f718fe165572f5623e4732831aca95a2bc32482d33d4d2cf1325d -F ext/rtree/rtree6.test 773a90db2dce6a8353dd0d5b64bca69b29761196 +F ext/rtree/rtree4.test 304de65d484540111b896827e4261815e5dca4ce28eeecd58be648cd73452c4b +F ext/rtree/rtree5.test 49c9041d713d54560b315c2c7ef7207ee287eba1b20f8266968a06f2e55d3142 +F ext/rtree/rtree6.test 916a641d2beac01b9880871ff07612d56c1e466190a27c82ab36ffd58be03b9f F ext/rtree/rtree7.test c8fb2e555b128dd0f0bdb520c61380014f497f8a23c40f2e820acc9f9e4fdce5 F ext/rtree/rtree8.test 649f5a37ec656028a4a32674b9b1183104285a7625a09d2a8f52a1cef72c93f2 F ext/rtree/rtree9.test c646f12c8c1c68ef015c6c043d86a0c42488e2e68ed1bb1b0771a7ca246cbabf @@ -368,7 +368,7 @@ F ext/rtree/rtreeC.test d9d06dda1aee68b4dc227dfcc899f335f8b621e9d1920ee3d4e5dab8 F ext/rtree/rtreeD.test fe46aa7f012e137bd58294409b16c0d43976c3bb92c8f710481e577c4a1100dc F ext/rtree/rtreeE.test e65d3fc625da1800b412fc8785817327d43ccfec5f5973912d8c9e471928caa9 F ext/rtree/rtreeF.test 81ffa7ef51c4e4618d497a57328c265bf576990c7070633b623b23cd450ed331 -F ext/rtree/rtreeG.test fd3af1ca944a0bdb0cbb5455a4905c9f012e2fffcab6b791f07afa0dcbbcae0e +F ext/rtree/rtreeG.test 1b9ca6e3effb48f4161edaa463ddeaa8fca4b2526d084f9cbf5dbe4e0184939c F ext/rtree/rtree_perf.tcl 6c18c1f23cd48e0f948930c98dfdd37dfccb5195 F ext/rtree/rtree_util.tcl db734b4c5e75fed6acc56d9701f2235345acfdec750b5fc7b587936f5f6bceed F ext/rtree/rtreecheck.test 4d29103d1e16fcbf90135d1c637b833688492b063b2971dfb5dc6ba76555cfee @@ -845,7 +845,7 @@ F test/fts2r.test b154c30b63061d8725e320fba1a39e2201cadd5e F test/fts2token.test d8070b241a15ff13592a9ae4a8b7c171af6f445a F test/fts3.test 672a040ea57036fb4b6fdc09027c18d7d24ab654 F test/fts3_common.tcl 99cf6659b87c0f74f55963c2aea03b3a7d66ceb0 -F test/fts3aa.test 39b65c11913d277c91d7426c62cfc1d147d1b4e9a48fecd9e38f60d0b5a5f505 +F test/fts3aa.test f267fcd6aca30fc70b81e5d82b68b34b38f581896020b57ed49e9777c7ebd85f F test/fts3ab.test 7f6cf260ae80dda064023df8e8e503e9a412b91f F test/fts3ac.test 636ed7486043055d4f126a0e385f2d5a82ebbf63 F test/fts3ad.test e40570cb6f74f059129ad48bcef3d7cbc20dda49 @@ -1299,7 +1299,7 @@ F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30 F test/temptable2.test cd396beb41117a5302fff61767c35fa4270a0d5e F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc -F test/tester.tcl 3ed81b9e1d9718a8d9603596c8a877793d054294053c4277a3d3897eabab3866 +F test/tester.tcl 94901a4625d9a2229666dd5c44120ddf7f0fb639470710ef74a4cefc7b039e07 F test/thread001.test 9f22fd3525a307ff42a326b6bc7b0465be1745a5 F test/thread002.test e630504f8a06c00bf8bbe68528774dd96aeb2e58 F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7 @@ -1704,7 +1704,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 36c2e67e82626f8d0a187c6c286c133ed659889e3b577469261b9dcd3b3ab75b -R ed31a2562d55c76884009a210d222cf0 +P 4761db83b6d3d57f281370899403c102e39ad0021d315dd6a6912d250436782a +R 43f41f7fb61c314e4ad3b5665d35c1a5 U dan -Z 0e612251c296ff4ce7a47fbd872de74d +Z 58e2db6aa4428ce3cec59e835a85dfcd diff --git a/manifest.uuid b/manifest.uuid index f1767d6dc9..6993509eea 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4761db83b6d3d57f281370899403c102e39ad0021d315dd6a6912d250436782a \ No newline at end of file +2a5f813bc61f9e780f2ccbda425611f65ad523b6d486a1e5e2b9d5e9f1d260a2 \ No newline at end of file diff --git a/test/fts3aa.test b/test/fts3aa.test index 10ec273cbf..d5f96d81a7 100644 --- a/test/fts3aa.test +++ b/test/fts3aa.test @@ -250,4 +250,5 @@ do_execsql_test 9.2 { CREATE VIRTUAL TABLE t10 USING fts3(<, b, c); } +expand_all_sql db finish_test diff --git a/test/tester.tcl b/test/tester.tcl index d0d6c92a60..6021ce72be 100644 --- a/test/tester.tcl +++ b/test/tester.tcl @@ -2309,6 +2309,16 @@ proc test_find_sqldiff {} { return $prog } +# Call sqlite3_expanded_sql() on all statements associated with database +# connection $db. This sometimes finds use-after-free bugs if run with +# valgrind or address-sanitizer. +proc expand_all_sql {db} { + set stmt "" + while {[set stmt [sqlite3_next_stmt $db $stmt]]!=""} { + sqlite3_expanded_sql $stmt + } +} + # If the library is compiled with the SQLITE_DEFAULT_AUTOVACUUM macro set # to non-zero, then set the global variable $AUTOVACUUM to 1. From b189e4101c2a8a6651222901f41afc00ff41d33f Mon Sep 17 00:00:00 2001 From: mistachkin Date: Wed, 7 Feb 2018 18:45:00 +0000 Subject: [PATCH 047/107] Fix typo in comment. Skip tests added by check-in [4761db83b6] when running on Windows. FossilOrigin-Name: 468a389cdc8b6f92c77b11f682e210ad8cce7e3dbc0df308a898f4115e85fce2 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/pager.c | 2 +- test/nockpt.test | 2 ++ 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 047f9dc234..a9179281b9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sextensions\srtree,\sfts3\sand\sfts5,\sensure\sthat\swhen\sdynamic\sbuffers\sare\sbound\nto\spersistent\sSQL\sstatements\susing\sSQLITE_STATIC,\sthe\sbinding\sis\sreplaced\swith\nan\sSQL\sNULL\sbefore\sthe\sbuffer\sis\sfreed.\sOtherwise,\sa\suser\smay\sobtain\sa\spointer\nto\sthe\spersistent\sstatement\susing\ssqlite3_next_stmt()\sand\sattempt\sto\saccess\nthe\sfreed\sbuffer\susing\ssqlite3_expanded_sql()\sor\ssimilar. -D 2018-02-07T18:02:50.375 +C Fix\stypo\sin\scomment.\s\sSkip\stests\sadded\sby\scheck-in\s[4761db83b6]\swhen\srunning\son\sWindows. +D 2018-02-07T18:45:00.417 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -474,7 +474,7 @@ F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 F src/os_unix.c ce491421b3a54b63094a155eeac668a3bc8e5b86a5a58551d906e5b5affb443f F src/os_win.c eb03c6d52f893bcd7fdd4c6006674c13c1b5e49543fec98d605201af2997171c F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a -F src/pager.c a3834a40acc2f3ab247d846f850d8c9313587d9c99c57a4dc194f2d4d7bf9d15 +F src/pager.c 0b6bd5442733b2e08d0673de6cdafe3e7ab0b5715e4844ac836ab346b1d9ed89 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a F src/parse.y 4e750e1b261ff9f1d0b6b5d40a829c66d691899f48953fde839d8b52d41aa148 F src/pcache.c 7ae91a4557a43d77d449accbfdc68846e6516f8e2eda46e8bbe4536fb669b201 @@ -1094,7 +1094,7 @@ F test/multiplex4.test e8ae4c4bd70606a5727743241f13b5701990abe4 F test/mutex1.test ea2cc74d97f077b9e74c84cbd024f14d79a8126f F test/mutex2.test bfeaeac2e73095b2ac32285d2756e3a65e681660 F test/nan.test 437d40e6d0778b050d7750726c0cbd2c9936b81962926e8f8c48ca698f00f4d1 -F test/nockpt.test d291d618c934a453683cb2eff95f633d406f7147fa0403e10055db19dcc3842a +F test/nockpt.test fd5473e30a84848b25ab06524750c05095383e191be83ccb4e6951c8beddfb5b F test/nolock.test f196cf8b8fbea4e2ca345140a2b3f3b0da45c76e F test/normalize.test 501630ab49b0b26b65c74124bf03e3374c1b57fa97aae750f84803609141d167 F test/notify1.test 669b2b743618efdc18ca4b02f45423d5d2304abf @@ -1704,7 +1704,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4761db83b6d3d57f281370899403c102e39ad0021d315dd6a6912d250436782a -R 43f41f7fb61c314e4ad3b5665d35c1a5 -U dan -Z 58e2db6aa4428ce3cec59e835a85dfcd +P 2a5f813bc61f9e780f2ccbda425611f65ad523b6d486a1e5e2b9d5e9f1d260a2 +R c941c05feb1d90fa13b7661ff1c5c31d +U mistachkin +Z 34b282be56160e3c52f85502f9e831c9 diff --git a/manifest.uuid b/manifest.uuid index 6993509eea..56adc03264 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2a5f813bc61f9e780f2ccbda425611f65ad523b6d486a1e5e2b9d5e9f1d260a2 \ No newline at end of file +468a389cdc8b6f92c77b11f682e210ad8cce7e3dbc0df308a898f4115e85fce2 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 93a9b3bd3e..b98487f333 100644 --- a/src/pager.c +++ b/src/pager.c @@ -4103,7 +4103,7 @@ static void pagerFreeMapHdrs(Pager *pPager){ } /* Verify that the database file has not be deleted or renamed out from -** under the pager. Return SQLITE_OK if the database is still were it ought +** under the pager. Return SQLITE_OK if the database is still where it ought ** to be on disk. Return non-zero (SQLITE_READONLY_DBMOVED or some other error ** code from sqlite3OsAccess()) if the database has gone missing. */ diff --git a/test/nockpt.test b/test/nockpt.test index 8f6b5e3be4..6da3313ad8 100644 --- a/test/nockpt.test +++ b/test/nockpt.test @@ -61,6 +61,7 @@ do_test 1.14 { sqlite3_db_config db NO_CKPT_ON_CLOSE 1 } {1} do_execsql_test 1.14 { PRAGMA main.journal_mode = delete } {delete} do_test 1.15 { file exists test.db-wal } {0} +if {$::tcl_platform(platform)!="windows"} { #------------------------------------------------------------------------- # Test an unusual scenario: # @@ -139,6 +140,7 @@ do_test 2.5 { SELECT * FROM y1; } db3 } {ok a b c d e g} +} finish_test From 2a86110a6c9b409be5f465717d3a7cd3519aedfe Mon Sep 17 00:00:00 2001 From: mistachkin Date: Thu, 8 Feb 2018 01:00:11 +0000 Subject: [PATCH 048/107] Enhance sqlite3ErrStr() to include several more error codes. FossilOrigin-Name: ad5d3bdc739a0997786f94fb5789b726b9f53ff883226093924338fe5000922b --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/main.c | 10 ++++++++++ 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index a9179281b9..84c8db0cbd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\stypo\sin\scomment.\s\sSkip\stests\sadded\sby\scheck-in\s[4761db83b6]\swhen\srunning\son\sWindows. -D 2018-02-07T18:45:00.417 +C Enhance\ssqlite3ErrStr()\sto\sinclude\sseveral\smore\serror\scodes. +D 2018-02-08T01:00:11.430 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -452,7 +452,7 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 14686083cedc198540b15a79586cdd4be2acf6d5fa97627e355f817ab07e9fee F src/legacy.c 134ab3e3fae00a0f67a5187981d6935b24b337bcf0f4b3e5c9fa5763da95bf4e F src/loadext.c f6e4e416a736369f9e80eba609f0acda97148a8b0453784d670c78d3eed2f302 -F src/main.c 26918d50dd4a61b8f6f210320a522f46b5e7e592335b6aa664ab15b80b7c239b +F src/main.c dc75b3585bab31a88962eda2abe35a38b58c2e4d648e1a11e249fe43174c783f F src/malloc.c 07295435093ce354c6d9063ac05a2eeae28bd251d2e63c48b3d67c12c76f7e18 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de @@ -1704,7 +1704,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 2a5f813bc61f9e780f2ccbda425611f65ad523b6d486a1e5e2b9d5e9f1d260a2 -R c941c05feb1d90fa13b7661ff1c5c31d +P 468a389cdc8b6f92c77b11f682e210ad8cce7e3dbc0df308a898f4115e85fce2 +R 1a5e19bb7fec633fe466598d51b2a7e2 U mistachkin -Z 34b282be56160e3c52f85502f9e831c9 +Z 7e5222f5afb9dd9d9137e335880f687f diff --git a/manifest.uuid b/manifest.uuid index 56adc03264..e5baaa9133 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -468a389cdc8b6f92c77b11f682e210ad8cce7e3dbc0df308a898f4115e85fce2 \ No newline at end of file +ad5d3bdc739a0997786f94fb5789b726b9f53ff883226093924338fe5000922b \ No newline at end of file diff --git a/src/main.c b/src/main.c index ed41f98947..4534631da7 100644 --- a/src/main.c +++ b/src/main.c @@ -1438,6 +1438,8 @@ const char *sqlite3ErrStr(int rc){ /* SQLITE_FORMAT */ 0, /* SQLITE_RANGE */ "column index out of range", /* SQLITE_NOTADB */ "file is not a database", + /* SQLITE_NOTICE */ "notification message", + /* SQLITE_WARNING */ "warning message", }; const char *zErr = "unknown error"; switch( rc ){ @@ -1445,6 +1447,14 @@ const char *sqlite3ErrStr(int rc){ zErr = "abort due to ROLLBACK"; break; } + case SQLITE_ROW: { + zErr = "another row available"; + break; + } + case SQLITE_DONE: { + zErr = "no more rows available"; + break; + } default: { rc &= 0xff; if( ALWAYS(rc>=0) && rc Date: Fri, 9 Feb 2018 15:04:51 +0000 Subject: [PATCH 049/107] Fix a harmless compiler warning. FossilOrigin-Name: a6c3115483d597fc77ab19fdcfd1d3437cad7e467081ad8c5315fb98c115eed9 --- manifest | 14 +++++++------- manifest.uuid | 2 +- tool/lempar.c | 3 ++- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 84c8db0cbd..7bf30b74e1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\ssqlite3ErrStr()\sto\sinclude\sseveral\smore\serror\scodes. -D 2018-02-08T01:00:11.430 +C Fix\sa\sharmless\scompiler\swarning. +D 2018-02-09T15:04:51.897 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -1624,7 +1624,7 @@ F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce F tool/kvtest-speed.sh 4761a9c4b3530907562314d7757995787f7aef8f F tool/lemon.c 7f7735326ca9c3b48327b241063cee52d35d44e20ebe1b3624a81658052a4d39 -F tool/lempar.c da840fc8a6fbac23599a65ff075e6e3d01320417c794ff577088e09f5d74b689 +F tool/lempar.c 468a155e8729cfbccfe1d85bf60d064f1dab76167a51149ec5c7928a2de63953 F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9 F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862 F tool/logest.c 11346aa019e2e77a00902aa7d0cabd27bd2e8cca @@ -1704,7 +1704,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 468a389cdc8b6f92c77b11f682e210ad8cce7e3dbc0df308a898f4115e85fce2 -R 1a5e19bb7fec633fe466598d51b2a7e2 -U mistachkin -Z 7e5222f5afb9dd9d9137e335880f687f +P ad5d3bdc739a0997786f94fb5789b726b9f53ff883226093924338fe5000922b +R b0cb95e907260417df10582a6a66e391 +U drh +Z cf86aa36db1eb294d8bf4ad196d065d9 diff --git a/manifest.uuid b/manifest.uuid index e5baaa9133..f234bbcf32 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ad5d3bdc739a0997786f94fb5789b726b9f53ff883226093924338fe5000922b \ No newline at end of file +a6c3115483d597fc77ab19fdcfd1d3437cad7e467081ad8c5315fb98c115eed9 \ No newline at end of file diff --git a/tool/lempar.c b/tool/lempar.c index ecc0e6389b..1ade666916 100644 --- a/tool/lempar.c +++ b/tool/lempar.c @@ -511,7 +511,8 @@ static unsigned int yy_find_shift_action( #endif do{ i = yy_shift_ofst[stateno]; - assert( i>=0 && i+YYNTOKEN<=sizeof(yy_lookahead)/sizeof(yy_lookahead[0]) ); + assert( i>=0 ); + assert( i+YYNTOKEN<=(int)sizeof(yy_lookahead)/sizeof(yy_lookahead[0]) ); assert( iLookAhead!=YYNOCODE ); assert( iLookAhead < YYNTOKEN ); i += iLookAhead; From 91347c066b96a9704b4c1a01c1a91b7a5a0765e6 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 9 Feb 2018 15:42:40 +0000 Subject: [PATCH 050/107] Make the tests in func6.test more robust against implementation changes. FossilOrigin-Name: b685d3231097fb90e7d61d9ac01cc560e8bf2671d49390ae7af5bfdbd6d04f11 --- manifest | 14 ++--- manifest.uuid | 2 +- test/func6.test | 147 +++++++++++++++++++++++++++++++++++------------- 3 files changed, 117 insertions(+), 46 deletions(-) diff --git a/manifest b/manifest index 7bf30b74e1..a0ff58b5eb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sharmless\scompiler\swarning. -D 2018-02-09T15:04:51.897 +C Make\sthe\stests\sin\sfunc6.test\smore\srobust\sagainst\simplementation\schanges. +D 2018-02-09T15:42:40.632 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -926,7 +926,7 @@ F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f F test/func3.test d202a7606d23f90988a664e88e268aed1087c11c F test/func4.test 6beacdfcb0e18c358e6c2dcacf1b65d1fa80955f F test/func5.test cdd224400bc3e48d891827cc913a57051a426fa4 -F test/func6.test a4281c8fcd42b56f7a60f28e8e4d444e8b2256f9e82658b7ab87699f8318f564 +F test/func6.test 612311a51adad23326d15353eea8d90394798acb26dd1f7c7cad59bd8ac982d2 F test/fuzz-oss1.test e58330d01cbbd8215ee636b17a03fe220b37dbfa F test/fuzz.test 96083052bf5765e4518c1ba686ce2bab785670d1 F test/fuzz2.test 76dc35b32b6d6f965259508508abce75a6c4d7e1 @@ -1704,7 +1704,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ad5d3bdc739a0997786f94fb5789b726b9f53ff883226093924338fe5000922b -R b0cb95e907260417df10582a6a66e391 -U drh -Z cf86aa36db1eb294d8bf4ad196d065d9 +P a6c3115483d597fc77ab19fdcfd1d3437cad7e467081ad8c5315fb98c115eed9 +R f6c1b3a83bf1fd9e6dbd920e348c4245 +U dan +Z baeba07272c3f575c3e3036aa614a6dd diff --git a/manifest.uuid b/manifest.uuid index f234bbcf32..3dcc2798f8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a6c3115483d597fc77ab19fdcfd1d3437cad7e467081ad8c5315fb98c115eed9 \ No newline at end of file +b685d3231097fb90e7d61d9ac01cc560e8bf2671d49390ae7af5bfdbd6d04f11 \ No newline at end of file diff --git a/test/func6.test b/test/func6.test index 02e1998cd0..50415e7963 100644 --- a/test/func6.test +++ b/test/func6.test @@ -33,54 +33,125 @@ do_execsql_test func6-100 { CREATE TABLE t2(x TEXT PRIMARY KEY, y) WITHOUT ROWID; INSERT INTO t2(x,y) SELECT a, b FROM t1; } + +# Load the contents of $file from disk and return it encoded as a hex +# string. +proc loadhex {file} { + set fd [open $file] + fconfigure $fd -translation binary -encoding binary + set data [read $fd] + close $fd + binary encode hex $data +} + +# Each argument is either an integer between 0 and 65535, a text value, or +# an empty string representing an SQL NULL. This command builds an SQLite +# record containing the values passed as arguments and returns it encoded +# as a hex string. +proc hexrecord {args} { + set hdr "" + set body "" + foreach x $args { + if {$x==""} { + append hdr 00 + } elseif {[string is integer $x]==0} { + set n [string length $x] + append hdr [format %02x [expr $n*2 + 13]] + append body [binary encode hex $x] + } elseif {$x == 0} { + append hdr 08 + } elseif {$x == 1} { + append hdr 09 + } elseif {$x <= 127} { + append hdr 01 + append body [format %02x $x] + } else { + append hdr 02 + append body [format %04x $x] + } + } + set res [format %02x [expr 1 + [string length $hdr]/2]] + append res $hdr + append res $body +} + +# Argument $off is an offset into the database image encoded as a hex string +# in argument $hexdb. This command returns 0 if the offset contains the hex +# $hexrec, or throws an exception otherwise. +# +proc offset_contains_record {off hexdb hexrec} { + set n [string length $hexrec] + set off [expr $off*2] + if { [string compare $hexrec [string range $hexdb $off [expr $off+$n-1]]] } { + error "record not found!" + } + return 0 +} + +# This command is the implementation of SQL function "offrec()". The first +# argument to this is an offset value. The remaining values are used to +# formulate an SQLite record. If database file test.db does not contain +# an equivalent record at the specified offset, an exception is thrown. +# Otherwise, 0 is returned. +# +proc offrec {args} { + set offset [lindex $args 0] + set rec [hexrecord {*}[lrange $args 1 end]] + offset_contains_record $offset $::F $rec +} +set F [loadhex test.db] +db func offrec offrec + +# Test the sanity of the tests. +do_execsql_test func6-105 { + SELECT sqlite_offset(d) FROM t1 ORDER BY rowid LIMIT 1; +} {8179} +do_test func6-106 { + set r [hexrecord abc001 1 999 {}] + offset_contains_record 8179 $F $r +} 0 + +set z100 [string trim [string repeat "0 " 100]] + +# Test offsets within table b-tree t1. do_execsql_test func6-110 { - SELECT a, sqlite_offset(d)/4096 + 1, - sqlite_offset(d)%4096 FROM t1 - ORDER BY rowid LIMIT 2; -} {abc001 2 4084 abc002 2 4069} + SELECT offrec(sqlite_offset(d), a, b, c, d) FROM t1 ORDER BY rowid +} $z100 + do_execsql_test func6-120 { SELECT a, typeof(sqlite_offset(+a)) FROM t1 ORDER BY rowid LIMIT 2; } {abc001 null abc002 null} + +# Test offsets within index b-tree t1a. do_execsql_test func6-130 { - SELECT a, sqlite_offset(a)/4096+1, - sqlite_offset(a)%4096 - FROM t1 - ORDER BY a LIMIT 2; -} {abc001 3 4087 abc002 3 4076} + SELECT offrec(sqlite_offset(a), a, rowid) FROM t1 ORDER BY a +} $z100 + +# Test offsets within table b-tree t1 with a temp b-tree ORDER BY. do_execsql_test func6-140 { - SELECT a, sqlite_offset(d)/4096+1, - sqlite_offset(d)%4096 - FROM t1 - ORDER BY a LIMIT 2; -} {abc001 2 4084 abc002 2 4069} + SELECT offrec(sqlite_offset(d), a, b, c, d) FROM t1 ORDER BY a +} $z100 + +# Test offsets from both index t1a and table t1 in the same query. do_execsql_test func6-150 { - SELECT a, - sqlite_offset(a)/4096+1, - sqlite_offset(a)%4096, - sqlite_offset(d)/4096+1, - sqlite_offset(d)%4096 - FROM t1 - ORDER BY a LIMIT 2; -} {abc001 3 4087 2 4084 abc002 3 4076 2 4069} + SELECT offrec(sqlite_offset(a), a, rowid), + offrec(sqlite_offset(d), a, b, c, d) + FROM t1 ORDER BY a +} [concat $z100 $z100] + +# Test offsets from both index t1bc and table t1 in the same query. do_execsql_test func6-160 { - SELECT b, - sqlite_offset(b)/4096+1, - sqlite_offset(b)%4096, - sqlite_offset(c)/4096+1, - sqlite_offset(c)%4096, - sqlite_offset(d)/4096+1, - sqlite_offset(d)%4096 - FROM t1 - ORDER BY b LIMIT 2; -} {1 4 4090 4 4090 2 4084 2 4 4081 4 4081 2 4069} - + SELECT offrec(sqlite_offset(b), b, c, rowid), + offrec(sqlite_offset(c), b, c, rowid), + offrec(sqlite_offset(d), a, b, c, d) + FROM t1 + ORDER BY b +} [concat $z100 $z100 $z100] +# Test offsets in WITHOUT ROWID table t2. do_execsql_test func6-200 { - SELECT y, sqlite_offset(y)/4096+1, - sqlite_offset(y)%4096 - FROM t2 - ORDER BY x LIMIT 2; -} {1 5 4087 2 5 4076} + SELECT offrec( sqlite_offset(y), x, y ) FROM t2 ORDER BY x +} $z100 finish_test From 510fea885e5b4841a57d13c21958ce650029f408 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 9 Feb 2018 20:49:15 +0000 Subject: [PATCH 051/107] Add the zorder.c extension implementing zorder() and unzorder() SQL functions. FossilOrigin-Name: a57a77dc0cc9fbaa9d5b134422f7a8cc8d4c2851ed3c2bdd449800c6a5d2aae0 --- ext/misc/zorder.c | 102 ++++++++++++++++++++++++++++++++++++++++++++++ manifest | 13 +++--- manifest.uuid | 2 +- 3 files changed, 110 insertions(+), 7 deletions(-) create mode 100644 ext/misc/zorder.c diff --git a/ext/misc/zorder.c b/ext/misc/zorder.c new file mode 100644 index 0000000000..c385d3c3c3 --- /dev/null +++ b/ext/misc/zorder.c @@ -0,0 +1,102 @@ +/* +** 2018-02-09 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** SQL functions for z-order (Morton code) transformations. +** +** zorder(X0,X0,..,xN) Generate an N+1 dimension Morton code +** +** unzorder(Z,N,I) Extract the I-th dimension from N-dimensional +** Morton code Z. +*/ +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 +#include +#include + +/* +** Functions: zorder(X0,X1,....) +** +** Convert integers X0, X1, ... into morton code. +** +** The output is a signed 64-bit integer. If any argument is too large, +** an error is thrown. +*/ +static void zorderFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + sqlite3_int64 z, x[63]; + int i, j; + z = 0; + for(i=0; i0 ){ + for(i=0; i<63; i++){ + j = i%argc; + z |= (x[j]&1)<>= 1; + } + } + sqlite3_result_int64(context, z); + for(i=0; i>j)&1)< Date: Fri, 9 Feb 2018 23:25:14 +0000 Subject: [PATCH 052/107] Improve the performance of the built-in REPLACE() function in cases where it does many substitutions that make the string larger. OSSFuzz is reporting intermittant timeouts when running a test where it does a REPLACE() on a 930KB random blob. Perhaps this enhancement will fix that. FossilOrigin-Name: fab2c2b07b5d3cd851db3e6f5c8a44155e32b0df22905ea33412b153b825a928 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/func.c | 43 ++++++++++++++++++++++++++----------------- test/func.test | 11 +++++++++++ 4 files changed, 45 insertions(+), 25 deletions(-) diff --git a/manifest b/manifest index 6cda78885d..e41ea214cc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\szorder.c\sextension\simplementing\szorder()\sand\sunzorder()\sSQL\sfunctions. -D 2018-02-09T20:49:15.905 +C Improve\sthe\sperformance\sof\sthe\sbuilt-in\sREPLACE()\sfunction\sin\scases\swhere\nit\sdoes\smany\ssubstitutions\sthat\smake\sthe\sstring\slarger.\s\sOSSFuzz\sis\sreporting\nintermittant\stimeouts\swhen\srunning\sa\stest\swhere\sit\sdoes\sa\sREPLACE()\son\sa\s\n930KB\srandom\sblob.\sPerhaps\sthis\senhancement\swill\sfix\sthat. +D 2018-02-09T23:25:14.764 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -444,7 +444,7 @@ F src/delete.c 20c8788451dc737a967c87ea53ad43544d617f5b57d32ccce8bd52a0daf9e89b F src/expr.c 9e06de431c09f144438aa6895ea4d4290fa3c6875bfcc3ba331012ca78deadf0 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c d617daf66b5515e2b42c1405b2b4984c30ca50fb705ab164271a9bf66c69e331 -F src/func.c 9970db37cc004136996a5c9b966b86f06583bcf3f275449b977fbb06d75e7300 +F src/func.c 385b9b01851f55c6547f2592ac378572298eb979b02516387ec6f21379d85507 F src/global.c ac3094f1dc59fbeb919aef7cc0cc827a8459d1fb1adb7972ef75bd9e0c10b75b F src/hash.c a12580e143f10301ed5166ea4964ae2853d3905a511d4e0c44497245c7ce1f7a F src/hash.h ab34c5c54a9e9de2e790b24349ba5aab3dbb4fd4 @@ -922,7 +922,7 @@ F test/fts4onepass.test 7319d61a2ed1325fc54afd0c060a0513b462303a F test/fts4opt.test fd6a11684b965e1999564ae763797b7fb9e34c96 F test/fts4unicode.test ceca76422abc251818cb25dabe33d3c3970da5f7c90e1540f190824e6b3a7c95 F test/full.test 6b3c8fb43c6beab6b95438c1675374b95fab245d -F test/func.test ae97561957aba6ca9e3a7b8a13aac41830d701ef +F test/func.test 09dda479bcfc568f99f3070413e9672a8eeedc1be9c5d819bf55d4788c2583b7 F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f F test/func3.test d202a7606d23f90988a664e88e268aed1087c11c F test/func4.test 6beacdfcb0e18c358e6c2dcacf1b65d1fa80955f @@ -1705,7 +1705,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b685d3231097fb90e7d61d9ac01cc560e8bf2671d49390ae7af5bfdbd6d04f11 -R 574de61b6452baa1e71eeeef413a6e94 +P a57a77dc0cc9fbaa9d5b134422f7a8cc8d4c2851ed3c2bdd449800c6a5d2aae0 +R adb83e680df996ddba44e43de1a05c6b U drh -Z 5935918b7bd1bd1e30b211bf68154f29 +Z 6ead8882aba40c327286a8b049f64ee5 diff --git a/manifest.uuid b/manifest.uuid index c0133280c0..fcc9af0088 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a57a77dc0cc9fbaa9d5b134422f7a8cc8d4c2851ed3c2bdd449800c6a5d2aae0 \ No newline at end of file +fab2c2b07b5d3cd851db3e6f5c8a44155e32b0df22905ea33412b153b825a928 \ No newline at end of file diff --git a/src/func.c b/src/func.c index 4e80535e81..399f6dba35 100644 --- a/src/func.c +++ b/src/func.c @@ -1199,6 +1199,8 @@ static void replaceFunc( i64 nOut; /* Maximum size of zOut */ int loopLimit; /* Last zStr[] that might match zPattern[] */ int i, j; /* Loop counters */ + unsigned cntExpand; /* Number zOut expansions */ + sqlite3 *db = sqlite3_context_db_handle(context); assert( argc==3 ); UNUSED_PARAMETER(argc); @@ -1230,33 +1232,40 @@ static void replaceFunc( return; } loopLimit = nStr - nPattern; + cntExpand = 0; for(i=j=0; i<=loopLimit; i++){ if( zStr[i]!=zPattern[0] || memcmp(&zStr[i], zPattern, nPattern) ){ zOut[j++] = zStr[i]; }else{ - u8 *zOld; - sqlite3 *db = sqlite3_context_db_handle(context); - nOut += nRep - nPattern; - testcase( nOut-1==db->aLimit[SQLITE_LIMIT_LENGTH] ); - testcase( nOut-2==db->aLimit[SQLITE_LIMIT_LENGTH] ); - if( nOut-1>db->aLimit[SQLITE_LIMIT_LENGTH] ){ - sqlite3_result_error_toobig(context); - sqlite3_free(zOut); - return; - } - zOld = zOut; - zOut = sqlite3_realloc64(zOut, (int)nOut); - if( zOut==0 ){ - sqlite3_result_error_nomem(context); - sqlite3_free(zOld); - return; + if( nRep>nPattern ){ + nOut += nRep - nPattern; + if( nOut-1>db->aLimit[SQLITE_LIMIT_LENGTH] ){ + sqlite3_result_error_toobig(context); + sqlite3_free(zOut); + return; + } + testcase( nOut-1==db->aLimit[SQLITE_LIMIT_LENGTH] ); + testcase( nOut-2==db->aLimit[SQLITE_LIMIT_LENGTH] ); + cntExpand++; + if( (cntExpand&(cntExpand-1))==0 ){ + /* Grow the size of the output buffer only on substitutions + ** whose index is a power of two: 1, 2, 4, 8, 16, 32, ... */ + u8 *zOld; + zOld = zOut; + zOut = sqlite3_realloc64(zOut, (int)nOut + (nOut - nStr - 1)); + if( zOut==0 ){ + sqlite3_result_error_nomem(context); + sqlite3_free(zOld); + return; + } + } } memcpy(&zOut[j], zRep, nRep); j += nRep; i += nPattern-1; } } - assert( j+nStr-i+1==nOut ); + assert( j+nStr-i+1<=nOut ); memcpy(&zOut[j], &zStr[i], nStr-i); j += nStr - i; assert( j<=nOut ); diff --git a/test/func.test b/test/func.test index 98ae8ddeb5..23a3ae4392 100644 --- a/test/func.test +++ b/test/func.test @@ -507,6 +507,17 @@ if {$encoding=="UTF-16le"} { execsql {SELECT hex(replace('aabcdefg','a','aaa'))} } {616161616161626364656667} } +do_execsql_test func-9.14 { + WITH RECURSIVE c(x) AS ( + VALUES(1) + UNION ALL + SELECT x+1 FROM c WHERE x<1040 + ) + SELECT + count(*), + sum(length(replace(printf('abc%.*cxyz',x,'m'),'m','nnnn'))-(6+x*4)) + FROM c; +} {1040 0} # Use the "sqlite_register_test_function" TCL command which is part of # the text fixture in order to verify correct operation of some of From c86d82f27d3dc1450a73a7eae83c9cb69a9ca0c7 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 10 Feb 2018 02:31:30 +0000 Subject: [PATCH 053/107] Fix misplaced testcase() macros from the previous check-in. FossilOrigin-Name: 3aed949a18a251c5795f21f0385c205a127502b7e9cf06bc7f4c763951cd7984 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/func.c | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index e41ea214cc..bb81d9a18e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improve\sthe\sperformance\sof\sthe\sbuilt-in\sREPLACE()\sfunction\sin\scases\swhere\nit\sdoes\smany\ssubstitutions\sthat\smake\sthe\sstring\slarger.\s\sOSSFuzz\sis\sreporting\nintermittant\stimeouts\swhen\srunning\sa\stest\swhere\sit\sdoes\sa\sREPLACE()\son\sa\s\n930KB\srandom\sblob.\sPerhaps\sthis\senhancement\swill\sfix\sthat. -D 2018-02-09T23:25:14.764 +C Fix\smisplaced\stestcase()\smacros\sfrom\sthe\sprevious\scheck-in. +D 2018-02-10T02:31:30.872 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -444,7 +444,7 @@ F src/delete.c 20c8788451dc737a967c87ea53ad43544d617f5b57d32ccce8bd52a0daf9e89b F src/expr.c 9e06de431c09f144438aa6895ea4d4290fa3c6875bfcc3ba331012ca78deadf0 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c d617daf66b5515e2b42c1405b2b4984c30ca50fb705ab164271a9bf66c69e331 -F src/func.c 385b9b01851f55c6547f2592ac378572298eb979b02516387ec6f21379d85507 +F src/func.c 94f42cba2cc1c34aeaa441022ba0170ec3fec4bba54db4e0ded085c6dc0fdc51 F src/global.c ac3094f1dc59fbeb919aef7cc0cc827a8459d1fb1adb7972ef75bd9e0c10b75b F src/hash.c a12580e143f10301ed5166ea4964ae2853d3905a511d4e0c44497245c7ce1f7a F src/hash.h ab34c5c54a9e9de2e790b24349ba5aab3dbb4fd4 @@ -1705,7 +1705,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a57a77dc0cc9fbaa9d5b134422f7a8cc8d4c2851ed3c2bdd449800c6a5d2aae0 -R adb83e680df996ddba44e43de1a05c6b +P fab2c2b07b5d3cd851db3e6f5c8a44155e32b0df22905ea33412b153b825a928 +R ec6c39c5f311f4ee60e0b56f01a82df2 U drh -Z 6ead8882aba40c327286a8b049f64ee5 +Z 0f7daa30b8fefb9cfc3cc8e0bb044dd7 diff --git a/manifest.uuid b/manifest.uuid index fcc9af0088..611a8eb076 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fab2c2b07b5d3cd851db3e6f5c8a44155e32b0df22905ea33412b153b825a928 \ No newline at end of file +3aed949a18a251c5795f21f0385c205a127502b7e9cf06bc7f4c763951cd7984 \ No newline at end of file diff --git a/src/func.c b/src/func.c index 399f6dba35..deb7e74928 100644 --- a/src/func.c +++ b/src/func.c @@ -1239,13 +1239,13 @@ static void replaceFunc( }else{ if( nRep>nPattern ){ nOut += nRep - nPattern; + testcase( nOut-1==db->aLimit[SQLITE_LIMIT_LENGTH] ); + testcase( nOut-2==db->aLimit[SQLITE_LIMIT_LENGTH] ); if( nOut-1>db->aLimit[SQLITE_LIMIT_LENGTH] ){ sqlite3_result_error_toobig(context); sqlite3_free(zOut); return; } - testcase( nOut-1==db->aLimit[SQLITE_LIMIT_LENGTH] ); - testcase( nOut-2==db->aLimit[SQLITE_LIMIT_LENGTH] ); cntExpand++; if( (cntExpand&(cntExpand-1))==0 ){ /* Grow the size of the output buffer only on substitutions From 00c3ef2df906a6e05ce7635f65a49c67a5d391f3 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 12 Feb 2018 13:30:34 +0000 Subject: [PATCH 054/107] Fix a possible infinite loop in VACUUM that can come up when the schema has been carefully corrupted. Problem discovered by OSSFuzz. Test cases in TH3. FossilOrigin-Name: 27754b74ddf6463c6f9c48ea0df3637945b5ea086b92a41ddeb55b9aaefea4c2 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vacuum.c | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index bb81d9a18e..6364a68c58 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\smisplaced\stestcase()\smacros\sfrom\sthe\sprevious\scheck-in. -D 2018-02-10T02:31:30.872 +C Fix\sa\spossible\sinfinite\sloop\sin\sVACUUM\sthat\scan\scome\sup\swhen\sthe\sschema\nhas\sbeen\scarefully\scorrupted.\s\sProblem\sdiscovered\sby\sOSSFuzz.\sTest\scases\nin\sTH3. +D 2018-02-12T13:30:34.512 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -556,7 +556,7 @@ F src/trigger.c a34539c69433276d37b0da9a89c117726ff2d292c0902895af1f393a983cd3a1 F src/update.c a90a32ffc0100265b0693dbbdbe490756447af181f5ea2c138cce515b08c8795 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5 F src/util.c d9eb0a6c4aae1b00a7369eadd7ca0bbe946cb4c953b6751aa20d357c2f482157 -F src/vacuum.c 90839322fd5f00df9617eb21b68beda9b6e2a2937576b0d65985e4aeb1c53739 +F src/vacuum.c 762ee9bbf8733d87d8cd06f58d950e881982e416f8c767334a40ffd341b6bff5 F src/vdbe.c a6892805df427bb9d554c37641406438bff568f99d26e72c1fb7e45f2ed6ce5b F src/vdbe.h 134beb7a12a6213c00eba58febaede33447cc4441bc568a0d9c144b33fc3720a F src/vdbeInt.h 8d7d07f13cb3c4cbca91e22ba4a1920e542dda7c5d9299920432a0b3d5b009f5 @@ -1705,7 +1705,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P fab2c2b07b5d3cd851db3e6f5c8a44155e32b0df22905ea33412b153b825a928 -R ec6c39c5f311f4ee60e0b56f01a82df2 +P 3aed949a18a251c5795f21f0385c205a127502b7e9cf06bc7f4c763951cd7984 +R fd19a34320fb110422297cf4ffe4adf5 U drh -Z 0f7daa30b8fefb9cfc3cc8e0bb044dd7 +Z e49790afb4374ff1f3bcb3262a0392c1 diff --git a/manifest.uuid b/manifest.uuid index 611a8eb076..01c0f92c3d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3aed949a18a251c5795f21f0385c205a127502b7e9cf06bc7f4c763951cd7984 \ No newline at end of file +27754b74ddf6463c6f9c48ea0df3637945b5ea086b92a41ddeb55b9aaefea4c2 \ No newline at end of file diff --git a/src/vacuum.c b/src/vacuum.c index fde08ddc2a..c7a14a675d 100644 --- a/src/vacuum.c +++ b/src/vacuum.c @@ -39,8 +39,8 @@ static int execSql(sqlite3 *db, char **pzErrMsg, const char *zSql){ while( SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){ const char *zSubSql = (const char*)sqlite3_column_text(pStmt,0); assert( sqlite3_strnicmp(zSql,"SELECT",6)==0 ); - if( zSubSql ){ - assert( zSubSql[0]!='S' ); + assert( sqlite3_strnicmp(zSubSql,"SELECT",6)!=0 || CORRUPT_DB ); + if( zSubSql && zSubSql[0]!='S' ){ rc = execSql(db, pzErrMsg, zSubSql); if( rc!=SQLITE_OK ) break; } From 1db7db1f03303b798ac572dbc2de8a2d7beea745 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 12 Feb 2018 15:27:32 +0000 Subject: [PATCH 055/107] Update test file func6.test so that it works with SQLITE_ENABLE_NULL_TRIM builds. FossilOrigin-Name: e128d72f1a565551d3b6fdb3807fcb1c2a2bf355f0c09d04562b62f49d01fdbc --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/test_config.c | 6 ++++++ test/func6.test | 21 +++++++++++++++++++-- 4 files changed, 34 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 6364a68c58..8f42c5f8a4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\spossible\sinfinite\sloop\sin\sVACUUM\sthat\scan\scome\sup\swhen\sthe\sschema\nhas\sbeen\scarefully\scorrupted.\s\sProblem\sdiscovered\sby\sOSSFuzz.\sTest\scases\nin\sTH3. -D 2018-02-12T13:30:34.512 +C Update\stest\sfile\sfunc6.test\sso\sthat\sit\sworks\swith\sSQLITE_ENABLE_NULL_TRIM\nbuilds. +D 2018-02-12T15:27:32.327 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -513,7 +513,7 @@ F src/test_backup.c bf5da90c9926df0a4b941f2d92825a01bbe090a0 F src/test_bestindex.c 78809f11026f18a93fcfd798d9479cba37e1201c830260bf1edc674b2fa9b857 F src/test_blob.c ae4a0620b478548afb67963095a7417cd06a4ec0a56adb453542203bfdcb31ce F src/test_btree.c 8b2dc8b8848cf3a4db93f11578f075e82252a274 -F src/test_config.c cc8a1d44648d9392a14f4ecfc841d027daaf61f952b9f70792edf11373aaa3dd +F src/test_config.c 03df8eb9dc8d5ee96588dfa34fd989c75625d5a0daf59491e32b966da4010870 F src/test_delete.c e2fe07646dff6300b48d49b2fee2fe192ed389e834dd635e3b3bac0ce0bf9f8f F src/test_demovfs.c a0c3bdd45ed044115c2c9f7779e56eafff18741e F src/test_devsym.c 1960abbb234b97e9b920f07e99503fc04b443f62bbc3c6ff2c2cea2133e3b8a2 @@ -927,7 +927,7 @@ F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f F test/func3.test d202a7606d23f90988a664e88e268aed1087c11c F test/func4.test 6beacdfcb0e18c358e6c2dcacf1b65d1fa80955f F test/func5.test cdd224400bc3e48d891827cc913a57051a426fa4 -F test/func6.test 612311a51adad23326d15353eea8d90394798acb26dd1f7c7cad59bd8ac982d2 +F test/func6.test 90e42b64c4f9fb6f04f44cb8a1da586c8542502e926b19c76504fe74ff2a9b7c F test/fuzz-oss1.test e58330d01cbbd8215ee636b17a03fe220b37dbfa F test/fuzz.test 96083052bf5765e4518c1ba686ce2bab785670d1 F test/fuzz2.test 76dc35b32b6d6f965259508508abce75a6c4d7e1 @@ -1705,7 +1705,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3aed949a18a251c5795f21f0385c205a127502b7e9cf06bc7f4c763951cd7984 -R fd19a34320fb110422297cf4ffe4adf5 -U drh -Z e49790afb4374ff1f3bcb3262a0392c1 +P 27754b74ddf6463c6f9c48ea0df3637945b5ea086b92a41ddeb55b9aaefea4c2 +R 6efccf0da31b2c149ed049f6fe56fc5d +U dan +Z 5bfb70e60829ab8825e3358e0b864cd4 diff --git a/manifest.uuid b/manifest.uuid index 01c0f92c3d..029b7dbbe1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -27754b74ddf6463c6f9c48ea0df3637945b5ea086b92a41ddeb55b9aaefea4c2 \ No newline at end of file +e128d72f1a565551d3b6fdb3807fcb1c2a2bf355f0c09d04562b62f49d01fdbc \ No newline at end of file diff --git a/src/test_config.c b/src/test_config.c index ad63016baf..c3961ad928 100644 --- a/src/test_config.c +++ b/src/test_config.c @@ -501,6 +501,12 @@ Tcl_SetVar2(interp, "sqlite_options", "long_double", Tcl_SetVar2(interp, "sqlite_options", "mergesort", "1", TCL_GLOBAL_ONLY); +#ifdef SQLITE_ENABLE_NULL_TRIM + Tcl_SetVar2(interp, "sqlite_options", "null_trim", "1", TCL_GLOBAL_ONLY); +#else + Tcl_SetVar2(interp, "sqlite_options", "null_trim", "0", TCL_GLOBAL_ONLY); +#endif + #ifdef SQLITE_OMIT_OR_OPTIMIZATION Tcl_SetVar2(interp, "sqlite_options", "or_opt", "0", TCL_GLOBAL_ONLY); #else diff --git a/test/func6.test b/test/func6.test index 50415e7963..1e16a7ca38 100644 --- a/test/func6.test +++ b/test/func6.test @@ -22,6 +22,11 @@ ifcapable !offset_sql_func { return } +set bNullTrim 0 +ifcapable null_trim { + set bNullTrim 1 +} + do_execsql_test func6-100 { PRAGMA page_size=4096; PRAGMA auto_vacuum=NONE; @@ -51,6 +56,13 @@ proc loadhex {file} { proc hexrecord {args} { set hdr "" set body "" + + if {$::bNullTrim} { + while {[llength $args] && [lindex $args end]=={}} { + set args [lrange $args 0 end-1] + } + } + foreach x $args { if {$x==""} { append hdr 00 @@ -103,12 +115,17 @@ set F [loadhex test.db] db func offrec offrec # Test the sanity of the tests. +if {$bNullTrim} { + set offset 8180 +} else { + set offset 8179 +} do_execsql_test func6-105 { SELECT sqlite_offset(d) FROM t1 ORDER BY rowid LIMIT 1; -} {8179} +} $offset do_test func6-106 { set r [hexrecord abc001 1 999 {}] - offset_contains_record 8179 $F $r + offset_contains_record $offset $F $r } 0 set z100 [string trim [string repeat "0 " 100]] From 35043cc32a4717edcc36153b2a1ced67d8750232 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 12 Feb 2018 20:27:34 +0000 Subject: [PATCH 056/107] When compiling with VDBE_PROFILE, add the sqlite3NProfileCnt global variable which can be used to measure per-opcode values other than elapse time, if non-zero. FossilOrigin-Name: d44d5936428859b6d3e089d9835acb8ca1f3c0a85b59eeaa619170623e7dcc30 --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/global.c | 7 +++++++ src/sqliteInt.h | 3 +++ src/vdbe.c | 4 ++-- 5 files changed, 22 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 8f42c5f8a4..b9699c0d2d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\stest\sfile\sfunc6.test\sso\sthat\sit\sworks\swith\sSQLITE_ENABLE_NULL_TRIM\nbuilds. -D 2018-02-12T15:27:32.327 +C When\scompiling\swith\sVDBE_PROFILE,\sadd\sthe\ssqlite3NProfileCnt\sglobal\svariable\nwhich\scan\sbe\sused\sto\smeasure\sper-opcode\svalues\sother\sthan\selapse\stime,\sif\nnon-zero. +D 2018-02-12T20:27:34.197 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -445,7 +445,7 @@ F src/expr.c 9e06de431c09f144438aa6895ea4d4290fa3c6875bfcc3ba331012ca78deadf0 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c d617daf66b5515e2b42c1405b2b4984c30ca50fb705ab164271a9bf66c69e331 F src/func.c 94f42cba2cc1c34aeaa441022ba0170ec3fec4bba54db4e0ded085c6dc0fdc51 -F src/global.c ac3094f1dc59fbeb919aef7cc0cc827a8459d1fb1adb7972ef75bd9e0c10b75b +F src/global.c 01506976bd75e5e7b977207a6a05062e2dd0050012f8071be06bbea22ec6d69a F src/hash.c a12580e143f10301ed5166ea4964ae2853d3905a511d4e0c44497245c7ce1f7a F src/hash.h ab34c5c54a9e9de2e790b24349ba5aab3dbb4fd4 F src/hwtime.h 747c1bbe9df21a92e9c50f3bbec1de841dc5e5da @@ -493,7 +493,7 @@ F src/shell.c.in 0b2c8d55e3703723a22610d75ccd4037962c3bcb5ad5570efd781e8c521811c F src/sqlite.h.in 51f9acf52c80113d793ddd38b3940ad6895d97b4752503b19291fb8fcbf54c5e F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 83a3c4ce93d650bedfd1aa558cb85a516bd6d094445ee989740827d0d944368d -F src/sqliteInt.h 119e240796d23054148005524f1d54a61e27dbef53bef3bc7ab65001405f4751 +F src/sqliteInt.h fd8943e9d6b8bd04b3a22ef806b843c2c06f022db3f37f1436ac3a9b04e5b8a1 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -557,7 +557,7 @@ F src/update.c a90a32ffc0100265b0693dbbdbe490756447af181f5ea2c138cce515b08c8795 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5 F src/util.c d9eb0a6c4aae1b00a7369eadd7ca0bbe946cb4c953b6751aa20d357c2f482157 F src/vacuum.c 762ee9bbf8733d87d8cd06f58d950e881982e416f8c767334a40ffd341b6bff5 -F src/vdbe.c a6892805df427bb9d554c37641406438bff568f99d26e72c1fb7e45f2ed6ce5b +F src/vdbe.c 72f5a4b95be486170104779aeef000423acabf55ac0c661afdae889be72c8922 F src/vdbe.h 134beb7a12a6213c00eba58febaede33447cc4441bc568a0d9c144b33fc3720a F src/vdbeInt.h 8d7d07f13cb3c4cbca91e22ba4a1920e542dda7c5d9299920432a0b3d5b009f5 F src/vdbeapi.c fea41171884a4de119f8b10ab514c788674eeeb7f27218bb6d008e1310bfd07f @@ -1705,7 +1705,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 27754b74ddf6463c6f9c48ea0df3637945b5ea086b92a41ddeb55b9aaefea4c2 -R 6efccf0da31b2c149ed049f6fe56fc5d -U dan -Z 5bfb70e60829ab8825e3358e0b864cd4 +P e128d72f1a565551d3b6fdb3807fcb1c2a2bf355f0c09d04562b62f49d01fdbc +R ea7b0f8bed7883637ff159eaee70912b +U drh +Z 7df15e7989dc925e831e6a33868c90fa diff --git a/manifest.uuid b/manifest.uuid index 029b7dbbe1..c9cd92d93c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e128d72f1a565551d3b6fdb3807fcb1c2a2bf355f0c09d04562b62f49d01fdbc \ No newline at end of file +d44d5936428859b6d3e089d9835acb8ca1f3c0a85b59eeaa619170623e7dcc30 \ No newline at end of file diff --git a/src/global.c b/src/global.c index e66be81495..04a3d185a8 100644 --- a/src/global.c +++ b/src/global.c @@ -258,6 +258,13 @@ const Token sqlite3IntTokens[] = { { "1", 1 } }; +#ifdef VDBE_PROFILE +/* +** The following performance counter can be used in place of +** sqlite3Hwtime() for profiling. This is a no-op on standard builds. +*/ +sqlite3_uint64 sqlite3NProfileCnt = 0; +#endif /* ** The value of the "pending" byte must be 0x40000000 (1 byte past the diff --git a/src/sqliteInt.h b/src/sqliteInt.h index f408ff62a5..f4c82cdb68 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -4069,6 +4069,9 @@ extern FuncDefHash sqlite3BuiltinFunctions; extern int sqlite3PendingByte; #endif #endif +#ifdef VDBE_PROFILE +extern sqlite3_uint64 sqlite3NProfileCnt; +#endif void sqlite3RootPageMoved(sqlite3*, int, int, int); void sqlite3Reindex(Parse*, Token*, Token*); void sqlite3AlterFunctions(void); diff --git a/src/vdbe.c b/src/vdbe.c index 1bcaedbfee..ebf221d02f 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -648,7 +648,7 @@ int sqlite3VdbeExec( assert( pOp>=aOp && pOp<&aOp[p->nOp]); #ifdef VDBE_PROFILE - start = sqlite3Hwtime(); + start = sqlite3NProfileCnt ? sqlite3NProfileCnt : sqlite3Hwtime(); #endif nVmStep++; #ifdef SQLITE_ENABLE_STMT_SCANSTATUS @@ -7233,7 +7233,7 @@ default: { /* This is really OP_Noop and OP_Explain */ #ifdef VDBE_PROFILE { - u64 endTime = sqlite3Hwtime(); + u64 endTime = sqlite3NProfileCnt ? sqlite3NProfileCnt : sqlite3Hwtime(); if( endTime>start ) pOrigOp->cycles += endTime - start; pOrigOp->cnt++; } From c083cb4422a602f1c1a5c08ec3d9d2460ee8716a Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 12 Feb 2018 20:50:28 +0000 Subject: [PATCH 057/107] Updates to the speed-check.sh script for improvements to the --vdbeprofile option. FossilOrigin-Name: f89c8f8ecd67f72ee730a2a334dff69b406a3dd00f50abbb54c4dc86cfab1bd1 --- manifest | 12 ++++++------ manifest.uuid | 2 +- tool/speed-check.sh | 11 ++++++++++- 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index b9699c0d2d..240d519aa1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\scompiling\swith\sVDBE_PROFILE,\sadd\sthe\ssqlite3NProfileCnt\sglobal\svariable\nwhich\scan\sbe\sused\sto\smeasure\sper-opcode\svalues\sother\sthan\selapse\stime,\sif\nnon-zero. -D 2018-02-12T20:27:34.197 +C Updates\sto\sthe\sspeed-check.sh\sscript\sfor\simprovements\sto\sthe\s--vdbeprofile\noption. +D 2018-02-12T20:50:28.825 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -1663,7 +1663,7 @@ F tool/showstat4.c 0682ebea7abf4d3657f53c4a243f2e7eab48eab344ed36a94bb75dcd19a5c F tool/showwal.c ad9d768f96ca6199ad3a8c9562d679680bd032dd01204ea3e5ea6fb931d81847 F tool/soak1.tcl 8d407956e1a45b485a8e072470a3e629a27037fe F tool/spaceanal.tcl 4bfd19aad7eb3ce0372ef0255f58035e0bba4ff5e9acfd763a10c6fb365c8dec -F tool/speed-check.sh 9ae425da8819e54e780cf494fc6d8175dfb16e109ae3214a45a5c9bb2b74e2c4 +F tool/speed-check.sh c6239c71ba6951ac5d093bbea55aadf019246c8e7a0c079422e62312a6ab1247 F tool/speedtest.tcl 06c76698485ccf597b9e7dbb1ac70706eb873355 F tool/speedtest16.c ecb6542862151c3e6509bbc00509b234562ae81e F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff @@ -1705,7 +1705,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e128d72f1a565551d3b6fdb3807fcb1c2a2bf355f0c09d04562b62f49d01fdbc -R ea7b0f8bed7883637ff159eaee70912b +P d44d5936428859b6d3e089d9835acb8ca1f3c0a85b59eeaa619170623e7dcc30 +R ed478e21f56ba8798873dd31125e1435 U drh -Z 7df15e7989dc925e831e6a33868c90fa +Z b17fd438653dec89029ec54019e16fa1 diff --git a/manifest.uuid b/manifest.uuid index c9cd92d93c..a9e9aba772 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d44d5936428859b6d3e089d9835acb8ca1f3c0a85b59eeaa619170623e7dcc30 \ No newline at end of file +f89c8f8ecd67f72ee730a2a334dff69b406a3dd00f50abbb54c4dc86cfab1bd1 \ No newline at end of file diff --git a/tool/speed-check.sh b/tool/speed-check.sh index 6cc3018981..cc31e6aa0a 100644 --- a/tool/speed-check.sh +++ b/tool/speed-check.sh @@ -39,6 +39,7 @@ LEAN_OPTS="$LEAN_OPTS -DSQLITE_USE_ALLOCA" BASELINE="trunk" doExplain=0 doCachegrind=1 +doVdbeProfile=0 while test "$1" != ""; do case $1 in --reprepare) @@ -78,6 +79,7 @@ while test "$1" != ""; do rm -f vdbe_profile.out CC_OPTS="$CC_OPTS -DVDBE_PROFILE" doCachegrind=0 + doVdbeProfile=1 ;; --lean) CC_OPTS="$CC_OPTS $LEAN_OPTS" @@ -137,6 +139,9 @@ echo "NAME = $NAME" | tee summary-$NAME.txt echo "SPEEDTEST_OPTS = $SPEEDTEST_OPTS" | tee -a summary-$NAME.txt echo "CC_OPTS = $CC_OPTS" | tee -a summary-$NAME.txt rm -f cachegrind.out.* speedtest1 speedtest1.db sqlite3.o +if test $doVdbeProfile -eq 1; then + rm -f vdbe_profile.out +fi $CC -g -Os -Wall -I. $CC_OPTS -c sqlite3.c size sqlite3.o | tee -a summary-$NAME.txt if test $doExplain -eq 1; then @@ -163,6 +168,10 @@ fi if test $doExplain -eq 1; then ./speedtest1 --explain $SPEEDTEST_OPTS | ./sqlite3 >explain-$NAME.txt fi -if test "$NAME" != "$BASELINE"; then +if test $doVdbeProfile -eq 1; then + tclsh ../sqlite/tool/vdbe_profile.tcl >vdbeprofile-$NAME.txt + open vdbeprofile-$NAME.txt +fi +if test "$NAME" != "$BASELINE" -a $doVdbeProfile -ne 1; then fossil test-diff --tk -c 20 cout-$BASELINE.txt cout-$NAME.txt fi From e84c498be19511245f2acd5c6f71dad441d908fb Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 13 Feb 2018 16:21:32 +0000 Subject: [PATCH 058/107] Ensure that multiple updates of the same FTS4 row (i.e. the row with the same rowid) within a single transaction are written to separate segments. Fix for [d6ec09ec]. FossilOrigin-Name: d2a81a3752a8298ee65fddcf32dee2147ba6467f770315a41966b3533977aa64 --- ext/fts3/fts3_write.c | 2 +- manifest | 16 ++++++++-------- manifest.uuid | 2 +- test/fts4onepass.test | 14 ++++++++++++++ 4 files changed, 24 insertions(+), 10 deletions(-) diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 5bca766c20..792a498a6f 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -5639,7 +5639,7 @@ int sqlite3Fts3UpdateMethod( rc = FTS_CORRUPT_VTAB; } } - if( rc==SQLITE_OK && (!isRemove || *pRowid!=p->iPrevDocid ) ){ + if( rc==SQLITE_OK ){ rc = fts3PendingTermsDocid(p, 0, iLangid, *pRowid); } if( rc==SQLITE_OK ){ diff --git a/manifest b/manifest index 240d519aa1..ecd5e1242b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Updates\sto\sthe\sspeed-check.sh\sscript\sfor\simprovements\sto\sthe\s--vdbeprofile\noption. -D 2018-02-12T20:50:28.825 +C Ensure\sthat\smultiple\supdates\sof\sthe\ssame\sFTS4\srow\s(i.e.\sthe\srow\swith\sthe\ssame\nrowid)\swithin\sa\ssingle\stransaction\sare\swritten\sto\sseparate\ssegments.\sFix\sfor\n[d6ec09ec]. +D 2018-02-13T16:21:32.960 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -96,7 +96,7 @@ F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 F ext/fts3/fts3_unicode.c 525a3bd9a7564603c5c061b7de55403a565307758a94600e8a2f6b00d1c40d9d F ext/fts3/fts3_unicode2.c cc04fc672bfd42b1e650398cb0bf71f64f9aae032cfe75bbcfe75b9cf966029c -F ext/fts3/fts3_write.c 7a7cf93c02ebe0ee4211e4aa07da77586c2dcf7d381c1382f81d29c9aa8cae8c +F ext/fts3/fts3_write.c 434fe5eaf6b848e9fa18dd269ab2450732f776ffb4d8051cfd0eef2c510ec5da F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/fts3/tool/fts3cov.sh c331d006359456cf6f8f953e37f2b9c7d568f3863f00bb5f7eb87fea4ac01b73 @@ -918,7 +918,7 @@ F test/fts4merge2.test 5faa558d1b672f82b847d2a337465fa745e46891 F test/fts4merge3.test 8d9ccb4a3d41c4c617a149d6c4b13ad02de797d0 F test/fts4merge4.test d895b1057a7798b67e03455d0fa50e9ea836c47b F test/fts4noti.test 5553d7bb2e20bf4a06b23e849352efc022ce6309 -F test/fts4onepass.test 7319d61a2ed1325fc54afd0c060a0513b462303a +F test/fts4onepass.test d69ddc4ee3415e40b0c5d1d0408488a87614d4f63ba9c44f3e52db541d6b7cc7 F test/fts4opt.test fd6a11684b965e1999564ae763797b7fb9e34c96 F test/fts4unicode.test ceca76422abc251818cb25dabe33d3c3970da5f7c90e1540f190824e6b3a7c95 F test/full.test 6b3c8fb43c6beab6b95438c1675374b95fab245d @@ -1705,7 +1705,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d44d5936428859b6d3e089d9835acb8ca1f3c0a85b59eeaa619170623e7dcc30 -R ed478e21f56ba8798873dd31125e1435 -U drh -Z b17fd438653dec89029ec54019e16fa1 +P f89c8f8ecd67f72ee730a2a334dff69b406a3dd00f50abbb54c4dc86cfab1bd1 +R 6201bc04be1c8ea59cb7f6218616fce1 +U dan +Z a105e3a27f3cee541040c34c5a8f6a03 diff --git a/manifest.uuid b/manifest.uuid index a9e9aba772..a809b05e72 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f89c8f8ecd67f72ee730a2a334dff69b406a3dd00f50abbb54c4dc86cfab1bd1 \ No newline at end of file +d2a81a3752a8298ee65fddcf32dee2147ba6467f770315a41966b3533977aa64 \ No newline at end of file diff --git a/test/fts4onepass.test b/test/fts4onepass.test index 46cb4b794b..344be4b1df 100644 --- a/test/fts4onepass.test +++ b/test/fts4onepass.test @@ -143,4 +143,18 @@ foreach {tn tcl1 tcl2} { eval $tcl2 } +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE zt USING fts4(a, b); + INSERT INTO zt(rowid, a, b) VALUES(1, 'unus duo', NULL); + INSERT INTO zt(rowid, a, b) VALUES(2, NULL, NULL); + + BEGIN; + UPDATE zt SET b='septum' WHERE rowid = 1; + UPDATE zt SET b='octo' WHERE rowid = 1; + COMMIT; + + SELECT count(*) FROM zt_segdir; +} {3} + + finish_test From d446e79545c8cf260869e2436a85a43ec4722a40 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 13 Feb 2018 16:30:19 +0000 Subject: [PATCH 059/107] Remove an unused variable from FTS4. FossilOrigin-Name: 27ea783b281606196ad1e7e6e6dcbd80c75d4bafcf3e58756a71c213320e4db2 --- ext/fts3/fts3_write.c | 2 -- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 792a498a6f..0baf82b76e 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -5529,7 +5529,6 @@ int sqlite3Fts3UpdateMethod( ){ Fts3Table *p = (Fts3Table *)pVtab; int rc = SQLITE_OK; /* Return Code */ - int isRemove = 0; /* True for an UPDATE or DELETE */ u32 *aSzIns = 0; /* Sizes of inserted documents */ u32 *aSzDel = 0; /* Sizes of deleted documents */ int nChng = 0; /* Net change in number of documents */ @@ -5627,7 +5626,6 @@ int sqlite3Fts3UpdateMethod( if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){ assert( sqlite3_value_type(apVal[0])==SQLITE_INTEGER ); rc = fts3DeleteByRowid(p, apVal[0], &nChng, aSzDel); - isRemove = 1; } /* If this is an INSERT or UPDATE operation, insert the new record. */ diff --git a/manifest b/manifest index ecd5e1242b..a075d14e40 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Ensure\sthat\smultiple\supdates\sof\sthe\ssame\sFTS4\srow\s(i.e.\sthe\srow\swith\sthe\ssame\nrowid)\swithin\sa\ssingle\stransaction\sare\swritten\sto\sseparate\ssegments.\sFix\sfor\n[d6ec09ec]. -D 2018-02-13T16:21:32.960 +C Remove\san\sunused\svariable\sfrom\sFTS4. +D 2018-02-13T16:30:19.084 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -96,7 +96,7 @@ F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 F ext/fts3/fts3_unicode.c 525a3bd9a7564603c5c061b7de55403a565307758a94600e8a2f6b00d1c40d9d F ext/fts3/fts3_unicode2.c cc04fc672bfd42b1e650398cb0bf71f64f9aae032cfe75bbcfe75b9cf966029c -F ext/fts3/fts3_write.c 434fe5eaf6b848e9fa18dd269ab2450732f776ffb4d8051cfd0eef2c510ec5da +F ext/fts3/fts3_write.c b583dede85eb0c3c3026f8d7ccb781ea4e845ae583754fecb2ca425b5907d87d F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/fts3/tool/fts3cov.sh c331d006359456cf6f8f953e37f2b9c7d568f3863f00bb5f7eb87fea4ac01b73 @@ -1705,7 +1705,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f89c8f8ecd67f72ee730a2a334dff69b406a3dd00f50abbb54c4dc86cfab1bd1 -R 6201bc04be1c8ea59cb7f6218616fce1 -U dan -Z a105e3a27f3cee541040c34c5a8f6a03 +P d2a81a3752a8298ee65fddcf32dee2147ba6467f770315a41966b3533977aa64 +R 24596befc5c0ec11ef119fbee4890efb +U drh +Z ec945a84da77652b5b990130e3b70870 diff --git a/manifest.uuid b/manifest.uuid index a809b05e72..80c33318ca 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d2a81a3752a8298ee65fddcf32dee2147ba6467f770315a41966b3533977aa64 \ No newline at end of file +27ea783b281606196ad1e7e6e6dcbd80c75d4bafcf3e58756a71c213320e4db2 \ No newline at end of file From 4d1c6845ff65486beff0d35255694664dcd50686 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 13 Feb 2018 18:48:08 +0000 Subject: [PATCH 060/107] Fix an incorrect table lookup used to find the appropriate search operator for a WHERE clause on a row-value inequality. The incorrect table lookup was causing an incorrect answer for the less-than operator. Fix for ticket [f484b65f3d6230593c34f11] FossilOrigin-Name: f3112e67cdb27c1aec8d2cee3cb91ade061d093e13505894698e26336898b336 --- manifest | 14 +++--- manifest.uuid | 2 +- src/wherecode.c | 10 ++++- test/rowvalue.test | 103 ++++++++++++++++++++++++++++++++++++++++++++- 4 files changed, 119 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index a075d14e40..378392f9f0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\san\sunused\svariable\sfrom\sFTS4. -D 2018-02-13T16:30:19.084 +C Fix\san\sincorrect\stable\slookup\sused\sto\sfind\sthe\sappropriate\ssearch\soperator\nfor\sa\sWHERE\sclause\son\sa\srow-value\sinequality.\s\sThe\sincorrect\stable\slookup\nwas\scausing\san\sincorrect\sanswer\sfor\sthe\sless-than\soperator.\nFix\sfor\sticket\s[f484b65f3d6230593c34f11] +D 2018-02-13T18:48:08.593 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -573,7 +573,7 @@ F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a F src/walker.c da987a20d40145c0a03c07d8fefcb2ed363becc7680d0500d9c79915591f5b1f F src/where.c 7cae47e813393d70c6d327fdf000fcb30f76b1b0b5a5b52ff6402e0c658de32c F src/whereInt.h 82c04c5075308abbac59180c8bad5ecb45b07453981f60a53f3c7dee21e1e971 -F src/wherecode.c cb360c511f69294ddf00340d7b390e6b1d601a1cfb77b42c4d316fe2f4cd01c3 +F src/wherecode.c e1aaadd8fec650037cfbf27d1b3470338fb3b58fec34d11082df16fe9a08fbd7 F src/whereexpr.c 22dbfd3bf5f6051a61523dd0ebef7a944fb29ee4aa7d2a62feb8aac6ffbbc0eb F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test a6d901b436328bd67a79b41bb0ac2663918fe3bd @@ -1166,7 +1166,7 @@ F test/rollbackfault.test 0e646aeab8840c399cfbfa43daab46fd609cf04a F test/rowallock.test 3f88ec6819489d0b2341c7a7528ae17c053ab7cc F test/rowhash.test 0bc1d31415e4575d10cacf31e1a66b5cc0f8be81 F test/rowid.test 5b7509f384f4f6fae1af3c8c104c8ca299fea18d -F test/rowvalue.test e1bd1690d891abff6138f9612241615d2de7671cb28eb58ebdd591a5b01089a4 +F test/rowvalue.test a786af333ab4d64e2ac91e4a234851a959ab61ea74b4fc9db333f5f8a9cb24f0 F test/rowvalue2.test 060d238b7e5639a7c5630cb5e63e311b44efef2b F test/rowvalue3.test 3068f508753af69884b12125995f023da0dbb256 F test/rowvalue4.test 4b556d7de161a0dd8cff095c336e913986398bea @@ -1705,7 +1705,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d2a81a3752a8298ee65fddcf32dee2147ba6467f770315a41966b3533977aa64 -R 24596befc5c0ec11ef119fbee4890efb +P 27ea783b281606196ad1e7e6e6dcbd80c75d4bafcf3e58756a71c213320e4db2 +R db63889646c3d3e78e93aa0d3ea314fa U drh -Z ec945a84da77652b5b990130e3b70870 +Z e60bc3e5bf72f574703087920d14128f diff --git a/manifest.uuid b/manifest.uuid index 80c33318ca..4a351d48d9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -27ea783b281606196ad1e7e6e6dcbd80c75d4bafcf3e58756a71c213320e4db2 \ No newline at end of file +f3112e67cdb27c1aec8d2cee3cb91ade061d093e13505894698e26336898b336 \ No newline at end of file diff --git a/src/wherecode.c b/src/wherecode.c index 8e8672ff9f..e40a940ac1 100644 --- a/src/wherecode.c +++ b/src/wherecode.c @@ -1395,7 +1395,15 @@ Bitmask sqlite3WhereCodeOneLoopStart( if( sqlite3ExprIsVector(pX->pRight) ){ r1 = rTemp = sqlite3GetTempReg(pParse); codeExprOrVector(pParse, pX->pRight, r1, 1); - op = aMoveOp[(pX->op - TK_GT) | 0x0001]; + testcase( pX->op==TK_GT ); + testcase( pX->op==TK_GE ); + testcase( pX->op==TK_LT ); + testcase( pX->op==TK_LE ); + op = aMoveOp[((pX->op - TK_GT - 1) & 0x3) | 0x1]; + assert( pX->op!=TK_GT || op==OP_SeekGE ); + assert( pX->op!=TK_GE || op==OP_SeekGE ); + assert( pX->op!=TK_LT || op==OP_SeekLE ); + assert( pX->op!=TK_LE || op==OP_SeekLE ); }else{ r1 = sqlite3ExprCodeTemp(pParse, pX->pRight, &rTemp); disableTerm(pLevel, pStart); diff --git a/test/rowvalue.test b/test/rowvalue.test index d900e0daa3..8a5f6f3ef4 100644 --- a/test/rowvalue.test +++ b/test/rowvalue.test @@ -437,6 +437,107 @@ do_execsql_test 18.6 { } {1 1 1 1 2 1} -finish_test +# 2018-02-13 Ticket https://www.sqlite.org/src/tktview/f484b65f3d6230593c3 +# Incorrect result from a row-value comparison in the WHERE clause. +# +do_execsql_test 19.1 { + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(a INTEGER PRIMARY KEY,b); + INSERT INTO t1(a,b) VALUES(1,11),(2,22),(3,33),(4,44); + SELECT * FROM t1 WHERE (a,b)>(0,0) ORDER BY +a; +} {1 11 2 22 3 33 4 44} +do_execsql_test 19.2 { + SELECT * FROM t1 WHERE (a,b)>=(0,0) ORDER BY +a; +} {1 11 2 22 3 33 4 44} +do_execsql_test 19.3 { + SELECT * FROM t1 WHERE (a,b)<(5,0) ORDER BY +a; +} {1 11 2 22 3 33 4 44} +do_execsql_test 19.4 { + SELECT * FROM t1 WHERE (a,b)<=(5,0) ORDER BY +a; +} {1 11 2 22 3 33 4 44} +do_execsql_test 19.5 { + SELECT * FROM t1 WHERE (a,b)>(3,0) ORDER BY +a; +} {3 33 4 44} +do_execsql_test 19.6 { + SELECT * FROM t1 WHERE (a,b)>=(3,0) ORDER BY +a; +} {3 33 4 44} +do_execsql_test 19.7 { + SELECT * FROM t1 WHERE (a,b)<(3,0) ORDER BY +a; +} {1 11 2 22} +do_execsql_test 19.8 { + SELECT * FROM t1 WHERE (a,b)<=(3,0) ORDER BY +a; +} {1 11 2 22} +do_execsql_test 19.9 { + SELECT * FROM t1 WHERE (a,b)>(3,32) ORDER BY +a; +} {3 33 4 44} +do_execsql_test 19.10 { + SELECT * FROM t1 WHERE (a,b)>(3,33) ORDER BY +a; +} {4 44} +do_execsql_test 19.11 { + SELECT * FROM t1 WHERE (a,b)>=(3,33) ORDER BY +a; +} {3 33 4 44} +do_execsql_test 19.12 { + SELECT * FROM t1 WHERE (a,b)>=(3,34) ORDER BY +a; +} {4 44} +do_execsql_test 19.13 { + SELECT * FROM t1 WHERE (a,b)<(3,34) ORDER BY +a; +} {1 11 2 22 3 33} +do_execsql_test 19.14 { + SELECT * FROM t1 WHERE (a,b)<(3,33) ORDER BY +a; +} {1 11 2 22} +do_execsql_test 19.15 { + SELECT * FROM t1 WHERE (a,b)<=(3,33) ORDER BY +a; +} {1 11 2 22 3 33} +do_execsql_test 19.16 { + SELECT * FROM t1 WHERE (a,b)<=(3,32) ORDER BY +a; +} {1 11 2 22} +do_execsql_test 19.21 { + SELECT * FROM t1 WHERE (0,0)<(a,b) ORDER BY +a; +} {1 11 2 22 3 33 4 44} +do_execsql_test 19.22 { + SELECT * FROM t1 WHERE (0,0)<=(a,b) ORDER BY +a; +} {1 11 2 22 3 33 4 44} +do_execsql_test 19.23 { + SELECT * FROM t1 WHERE (5,0)>(a,b) ORDER BY +a; +} {1 11 2 22 3 33 4 44} +do_execsql_test 19.24 { + SELECT * FROM t1 WHERE (5,0)>=(a,b) ORDER BY +a; +} {1 11 2 22 3 33 4 44} +do_execsql_test 19.25 { + SELECT * FROM t1 WHERE (3,0)<(a,b) ORDER BY +a; +} {3 33 4 44} +do_execsql_test 19.26 { + SELECT * FROM t1 WHERE (3,0)<=(a,b) ORDER BY +a; +} {3 33 4 44} +do_execsql_test 19.27 { + SELECT * FROM t1 WHERE (3,0)>(a,b) ORDER BY +a; +} {1 11 2 22} +do_execsql_test 19.28 { + SELECT * FROM t1 WHERE (3,0)>=(a,b) ORDER BY +a; +} {1 11 2 22} +do_execsql_test 19.29 { + SELECT * FROM t1 WHERE (3,32)<(a,b) ORDER BY +a; +} {3 33 4 44} +do_execsql_test 19.30 { + SELECT * FROM t1 WHERE (3,33)<(a,b) ORDER BY +a; +} {4 44} +do_execsql_test 19.31 { + SELECT * FROM t1 WHERE (3,33)<=(a,b) ORDER BY +a; +} {3 33 4 44} +do_execsql_test 19.32 { + SELECT * FROM t1 WHERE (3,34)<=(a,b) ORDER BY +a; +} {4 44} +do_execsql_test 19.33 { + SELECT * FROM t1 WHERE (3,34)>(a,b) ORDER BY +a; +} {1 11 2 22 3 33} +do_execsql_test 19.34 { + SELECT * FROM t1 WHERE (3,33)>(a,b) ORDER BY +a; +} {1 11 2 22} +do_execsql_test 19.35 { + SELECT * FROM t1 WHERE (3,33)>=(a,b) ORDER BY +a; +} {1 11 2 22 3 33} +do_execsql_test 19.36 { + SELECT * FROM t1 WHERE (3,32)>=(a,b) ORDER BY +a; +} {1 11 2 22} finish_test From a9abfb1e6c4f6d7a37613c52da9e2316a5333341 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 13 Feb 2018 19:13:05 +0000 Subject: [PATCH 061/107] Improved test cases for the [f484b65f3d6230593c34f] bug. FossilOrigin-Name: 1f300514e6cdd32c755dc1b05dfa125b10eda2eb55a1024b66f66dbf5cf96289 --- manifest | 12 +++--- manifest.uuid | 2 +- test/rowvalue.test | 96 +++++++++++++++++++++++----------------------- 3 files changed, 55 insertions(+), 55 deletions(-) diff --git a/manifest b/manifest index 378392f9f0..7a723a9a49 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sincorrect\stable\slookup\sused\sto\sfind\sthe\sappropriate\ssearch\soperator\nfor\sa\sWHERE\sclause\son\sa\srow-value\sinequality.\s\sThe\sincorrect\stable\slookup\nwas\scausing\san\sincorrect\sanswer\sfor\sthe\sless-than\soperator.\nFix\sfor\sticket\s[f484b65f3d6230593c34f11] -D 2018-02-13T18:48:08.593 +C Improved\stest\scases\sfor\sthe\s[f484b65f3d6230593c34f]\sbug. +D 2018-02-13T19:13:05.932 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -1166,7 +1166,7 @@ F test/rollbackfault.test 0e646aeab8840c399cfbfa43daab46fd609cf04a F test/rowallock.test 3f88ec6819489d0b2341c7a7528ae17c053ab7cc F test/rowhash.test 0bc1d31415e4575d10cacf31e1a66b5cc0f8be81 F test/rowid.test 5b7509f384f4f6fae1af3c8c104c8ca299fea18d -F test/rowvalue.test a786af333ab4d64e2ac91e4a234851a959ab61ea74b4fc9db333f5f8a9cb24f0 +F test/rowvalue.test 0bc33483f2ef5e69ff4bdd2ae58e36fc598bfd1605fb718c8329bcfc0c10cfd1 F test/rowvalue2.test 060d238b7e5639a7c5630cb5e63e311b44efef2b F test/rowvalue3.test 3068f508753af69884b12125995f023da0dbb256 F test/rowvalue4.test 4b556d7de161a0dd8cff095c336e913986398bea @@ -1705,7 +1705,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 27ea783b281606196ad1e7e6e6dcbd80c75d4bafcf3e58756a71c213320e4db2 -R db63889646c3d3e78e93aa0d3ea314fa +P f3112e67cdb27c1aec8d2cee3cb91ade061d093e13505894698e26336898b336 +R 5562ebb89ed1db5735eb0527fd19eb1a U drh -Z e60bc3e5bf72f574703087920d14128f +Z 5c7cba3bc5ff7309961a0180cc01b405 diff --git a/manifest.uuid b/manifest.uuid index 4a351d48d9..f101211b79 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f3112e67cdb27c1aec8d2cee3cb91ade061d093e13505894698e26336898b336 \ No newline at end of file +1f300514e6cdd32c755dc1b05dfa125b10eda2eb55a1024b66f66dbf5cf96289 \ No newline at end of file diff --git a/test/rowvalue.test b/test/rowvalue.test index 8a5f6f3ef4..5805178301 100644 --- a/test/rowvalue.test +++ b/test/rowvalue.test @@ -444,100 +444,100 @@ do_execsql_test 19.1 { DROP TABLE IF EXISTS t1; CREATE TABLE t1(a INTEGER PRIMARY KEY,b); INSERT INTO t1(a,b) VALUES(1,11),(2,22),(3,33),(4,44); - SELECT * FROM t1 WHERE (a,b)>(0,0) ORDER BY +a; + SELECT * FROM t1 WHERE (a,b)>(0,0) ORDER BY a; } {1 11 2 22 3 33 4 44} do_execsql_test 19.2 { - SELECT * FROM t1 WHERE (a,b)>=(0,0) ORDER BY +a; + SELECT * FROM t1 WHERE (a,b)>=(0,0) ORDER BY a; } {1 11 2 22 3 33 4 44} do_execsql_test 19.3 { - SELECT * FROM t1 WHERE (a,b)<(5,0) ORDER BY +a; -} {1 11 2 22 3 33 4 44} + SELECT * FROM t1 WHERE (a,b)<(5,0) ORDER BY a DESC; +} {4 44 3 33 2 22 1 11} do_execsql_test 19.4 { - SELECT * FROM t1 WHERE (a,b)<=(5,0) ORDER BY +a; -} {1 11 2 22 3 33 4 44} + SELECT * FROM t1 WHERE (a,b)<=(5,0) ORDER BY a DESC; +} {4 44 3 33 2 22 1 11} do_execsql_test 19.5 { - SELECT * FROM t1 WHERE (a,b)>(3,0) ORDER BY +a; + SELECT * FROM t1 WHERE (a,b)>(3,0) ORDER BY a; } {3 33 4 44} do_execsql_test 19.6 { - SELECT * FROM t1 WHERE (a,b)>=(3,0) ORDER BY +a; + SELECT * FROM t1 WHERE (a,b)>=(3,0) ORDER BY a; } {3 33 4 44} do_execsql_test 19.7 { - SELECT * FROM t1 WHERE (a,b)<(3,0) ORDER BY +a; -} {1 11 2 22} + SELECT * FROM t1 WHERE (a,b)<(3,0) ORDER BY a DESC; +} {2 22 1 11} do_execsql_test 19.8 { - SELECT * FROM t1 WHERE (a,b)<=(3,0) ORDER BY +a; -} {1 11 2 22} + SELECT * FROM t1 WHERE (a,b)<=(3,0) ORDER BY a DESC; +} {2 22 1 11} do_execsql_test 19.9 { - SELECT * FROM t1 WHERE (a,b)>(3,32) ORDER BY +a; + SELECT * FROM t1 WHERE (a,b)>(3,32) ORDER BY a; } {3 33 4 44} do_execsql_test 19.10 { - SELECT * FROM t1 WHERE (a,b)>(3,33) ORDER BY +a; + SELECT * FROM t1 WHERE (a,b)>(3,33) ORDER BY a; } {4 44} do_execsql_test 19.11 { - SELECT * FROM t1 WHERE (a,b)>=(3,33) ORDER BY +a; + SELECT * FROM t1 WHERE (a,b)>=(3,33) ORDER BY a; } {3 33 4 44} do_execsql_test 19.12 { - SELECT * FROM t1 WHERE (a,b)>=(3,34) ORDER BY +a; + SELECT * FROM t1 WHERE (a,b)>=(3,34) ORDER BY a; } {4 44} do_execsql_test 19.13 { - SELECT * FROM t1 WHERE (a,b)<(3,34) ORDER BY +a; -} {1 11 2 22 3 33} + SELECT * FROM t1 WHERE (a,b)<(3,34) ORDER BY a DESC; +} {3 33 2 22 1 11} do_execsql_test 19.14 { - SELECT * FROM t1 WHERE (a,b)<(3,33) ORDER BY +a; -} {1 11 2 22} + SELECT * FROM t1 WHERE (a,b)<(3,33) ORDER BY a DESC; +} {2 22 1 11} do_execsql_test 19.15 { - SELECT * FROM t1 WHERE (a,b)<=(3,33) ORDER BY +a; -} {1 11 2 22 3 33} + SELECT * FROM t1 WHERE (a,b)<=(3,33) ORDER BY a DESC; +} {3 33 2 22 1 11} do_execsql_test 19.16 { - SELECT * FROM t1 WHERE (a,b)<=(3,32) ORDER BY +a; -} {1 11 2 22} + SELECT * FROM t1 WHERE (a,b)<=(3,32) ORDER BY a DESC; +} {2 22 1 11} do_execsql_test 19.21 { - SELECT * FROM t1 WHERE (0,0)<(a,b) ORDER BY +a; + SELECT * FROM t1 WHERE (0,0)<(a,b) ORDER BY a; } {1 11 2 22 3 33 4 44} do_execsql_test 19.22 { - SELECT * FROM t1 WHERE (0,0)<=(a,b) ORDER BY +a; + SELECT * FROM t1 WHERE (0,0)<=(a,b) ORDER BY a; } {1 11 2 22 3 33 4 44} do_execsql_test 19.23 { - SELECT * FROM t1 WHERE (5,0)>(a,b) ORDER BY +a; -} {1 11 2 22 3 33 4 44} + SELECT * FROM t1 WHERE (5,0)>(a,b) ORDER BY a DESC; +} {4 44 3 33 2 22 1 11} do_execsql_test 19.24 { - SELECT * FROM t1 WHERE (5,0)>=(a,b) ORDER BY +a; -} {1 11 2 22 3 33 4 44} + SELECT * FROM t1 WHERE (5,0)>=(a,b) ORDER BY a DESC; +} {4 44 3 33 2 22 1 11} do_execsql_test 19.25 { - SELECT * FROM t1 WHERE (3,0)<(a,b) ORDER BY +a; + SELECT * FROM t1 WHERE (3,0)<(a,b) ORDER BY a; } {3 33 4 44} do_execsql_test 19.26 { - SELECT * FROM t1 WHERE (3,0)<=(a,b) ORDER BY +a; + SELECT * FROM t1 WHERE (3,0)<=(a,b) ORDER BY a; } {3 33 4 44} do_execsql_test 19.27 { - SELECT * FROM t1 WHERE (3,0)>(a,b) ORDER BY +a; -} {1 11 2 22} + SELECT * FROM t1 WHERE (3,0)>(a,b) ORDER BY a DESC; +} {2 22 1 11} do_execsql_test 19.28 { - SELECT * FROM t1 WHERE (3,0)>=(a,b) ORDER BY +a; -} {1 11 2 22} + SELECT * FROM t1 WHERE (3,0)>=(a,b) ORDER BY a DESC; +} {2 22 1 11} do_execsql_test 19.29 { - SELECT * FROM t1 WHERE (3,32)<(a,b) ORDER BY +a; + SELECT * FROM t1 WHERE (3,32)<(a,b) ORDER BY a; } {3 33 4 44} do_execsql_test 19.30 { - SELECT * FROM t1 WHERE (3,33)<(a,b) ORDER BY +a; + SELECT * FROM t1 WHERE (3,33)<(a,b) ORDER BY a; } {4 44} do_execsql_test 19.31 { - SELECT * FROM t1 WHERE (3,33)<=(a,b) ORDER BY +a; + SELECT * FROM t1 WHERE (3,33)<=(a,b) ORDER BY a; } {3 33 4 44} do_execsql_test 19.32 { - SELECT * FROM t1 WHERE (3,34)<=(a,b) ORDER BY +a; + SELECT * FROM t1 WHERE (3,34)<=(a,b) ORDER BY a; } {4 44} do_execsql_test 19.33 { - SELECT * FROM t1 WHERE (3,34)>(a,b) ORDER BY +a; -} {1 11 2 22 3 33} + SELECT * FROM t1 WHERE (3,34)>(a,b) ORDER BY a DESC; +} {3 33 2 22 1 11} do_execsql_test 19.34 { - SELECT * FROM t1 WHERE (3,33)>(a,b) ORDER BY +a; -} {1 11 2 22} + SELECT * FROM t1 WHERE (3,33)>(a,b) ORDER BY a DESC; +} {2 22 1 11} do_execsql_test 19.35 { - SELECT * FROM t1 WHERE (3,33)>=(a,b) ORDER BY +a; -} {1 11 2 22 3 33} + SELECT * FROM t1 WHERE (3,33)>=(a,b) ORDER BY a DESC; +} {3 33 2 22 1 11} do_execsql_test 19.36 { - SELECT * FROM t1 WHERE (3,32)>=(a,b) ORDER BY +a; -} {1 11 2 22} + SELECT * FROM t1 WHERE (3,32)>=(a,b) ORDER BY a DESC; +} {2 22 1 11} finish_test From e2d27e02590f1ab0fb10681e5e5425e59d81b593 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 14 Feb 2018 14:13:09 +0000 Subject: [PATCH 062/107] Try to optimize spellfix1 by storing a NULL in the k1 column of %_vocab if it would otherwise have the same value as the word column. FossilOrigin-Name: b76ec7cb5841f9ad02b342271e91c5f5a598006f69df7953a6213c706e01f7a2 --- ext/misc/spellfix.c | 14 +++++++------- manifest | 14 +++++++------- manifest.uuid | 2 +- test/spellfix.test | 2 +- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/ext/misc/spellfix.c b/ext/misc/spellfix.c index 2d7ef1b02b..6e3a3bca16 100644 --- a/ext/misc/spellfix.c +++ b/ext/misc/spellfix.c @@ -2492,7 +2492,7 @@ static int spellfix1FilterForMatch( nPattern = (int)strlen(zPattern); if( zPattern[nPattern-1]=='*' ) nPattern--; zSql = sqlite3_mprintf( - "SELECT id, word, rank, k1" + "SELECT id, word, rank, coalesce(k1,word)" " FROM \"%w\".\"%w_vocab\"" " WHERE langid=%d AND k2>=?1 AND k2zDbName, p->zTableName, iLang @@ -2826,17 +2826,17 @@ static int spellfix1Update( if( sqlite3_value_type(argv[1])==SQLITE_NULL ){ spellfix1DbExec(&rc, db, "INSERT INTO \"%w\".\"%w_vocab\"(rank,langid,word,k1,k2) " - "VALUES(%d,%d,%Q,%Q,%Q)", + "VALUES(%d,%d,%Q,nullif(%Q,%Q),%Q)", p->zDbName, p->zTableName, - iRank, iLang, zWord, zK1, zK2 + iRank, iLang, zWord, zK1, zWord, zK2 ); }else{ newRowid = sqlite3_value_int64(argv[1]); spellfix1DbExec(&rc, db, "INSERT OR %s INTO \"%w\".\"%w_vocab\"(id,rank,langid,word,k1,k2) " - "VALUES(%lld,%d,%d,%Q,%Q,%Q)", + "VALUES(%lld,%d,%d,%Q,nullif(%Q,%Q),%Q)", zConflict, p->zDbName, p->zTableName, - newRowid, iRank, iLang, zWord, zK1, zK2 + newRowid, iRank, iLang, zWord, zK1, zWord, zK2 ); } *pRowid = sqlite3_last_insert_rowid(db); @@ -2845,9 +2845,9 @@ static int spellfix1Update( newRowid = *pRowid = sqlite3_value_int64(argv[1]); spellfix1DbExec(&rc, db, "UPDATE OR %s \"%w\".\"%w_vocab\" SET id=%lld, rank=%d, langid=%d," - " word=%Q, k1=%Q, k2=%Q WHERE id=%lld", + " word=%Q, k1=nullif(%Q,%Q), k2=%Q WHERE id=%lld", zConflict, p->zDbName, p->zTableName, newRowid, iRank, iLang, - zWord, zK1, zK2, rowid + zWord, zK1, zWord, zK2, rowid ); } sqlite3_free(zK1); diff --git a/manifest b/manifest index 7a723a9a49..8bac7cb68b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\stest\scases\sfor\sthe\s[f484b65f3d6230593c34f]\sbug. -D 2018-02-13T19:13:05.932 +C Try\sto\soptimize\sspellfix1\sby\sstoring\sa\sNULL\sin\sthe\sk1\scolumn\sof\s%_vocab\nif\sit\swould\sotherwise\shave\sthe\ssame\svalue\sas\sthe\sword\scolumn. +D 2018-02-14T14:13:09.238 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -294,7 +294,7 @@ F ext/misc/series.c f3c0dba5c5c749ce1782b53076108f87cf0b71041eb6023f727a9c50681d F ext/misc/sha1.c 0b9e9b855354910d3ca467bf39099d570e73db56 F ext/misc/shathree.c 9e960ba50483214c6a7a4b1517f8d8cef799e9db381195178c3fd3ad207e10c0 F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52 -F ext/misc/spellfix.c 705ce1ed701b38f43c8ee7dd848877ff51c09eda322e39ccac6873adb4b74735 +F ext/misc/spellfix.c efe188bdd84d283e1c82e4bcf7058ffc1856176d9ba2ba14fd5efc4aa2d8661f F ext/misc/sqlar.c 57d5bc45cd5492208e451f697404be88f8612527d64c9d42f96b325b64983d74 F ext/misc/stmt.c 6f16443abb3551e3f5813bb13ba19a30e7032830015b0f92fe0c0453045c0a11 F ext/misc/totype.c 4a167594e791abeed95e0a8db028822b5e8fe512 @@ -1262,7 +1262,7 @@ F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715 F test/speed4p.explain 6b5f104ebeb34a038b2f714150f51d01143e59aa F test/speed4p.test 0e51908951677de5a969b723e03a27a1c45db38b F test/speedtest1.c a5faf4cbe5769eee4b721b3875cb3f12520a9b99d9026b1063b47c39603375b8 -F test/spellfix.test f9c1f431e2c096c8775fec032952320c0e4700db +F test/spellfix.test 951a6405d49d1a23d6b78027d3877b4a33eeb8221dcab5704b499755bb4f552e F test/spellfix2.test dfc8f519a3fc204cb2dfa8b4f29821ae90f6f8c3 F test/spellfix3.test 0f9efaaa502a0e0a09848028518a6fb096c8ad33 F test/sqldiff1.test 28cd737cf1b0078b1ec1bbf425e674c47785835e @@ -1705,7 +1705,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f3112e67cdb27c1aec8d2cee3cb91ade061d093e13505894698e26336898b336 -R 5562ebb89ed1db5735eb0527fd19eb1a +P 1f300514e6cdd32c755dc1b05dfa125b10eda2eb55a1024b66f66dbf5cf96289 +R 255e72ba75e81cf4ca1d4ef198b4e2a9 U drh -Z 5c7cba3bc5ff7309961a0180cc01b405 +Z 21f5c49e8ec82780f55a5436de6c983b diff --git a/manifest.uuid b/manifest.uuid index f101211b79..91ef23e4ef 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1f300514e6cdd32c755dc1b05dfa125b10eda2eb55a1024b66f66dbf5cf96289 \ No newline at end of file +b76ec7cb5841f9ad02b342271e91c5f5a598006f69df7953a6213c706e01f7a2 \ No newline at end of file diff --git a/test/spellfix.test b/test/spellfix.test index 8128bb59d2..68bcfd5adb 100644 --- a/test/spellfix.test +++ b/test/spellfix.test @@ -279,7 +279,7 @@ ifcapable trace { do_tracesql_test 6.2.3 { SELECT word, distance FROM t3 WHERE rowid = 10 AND word MATCH 'kiiner'; } {keener 300 - {SELECT id, word, rank, k1 FROM "main"."t3_vocab" WHERE langid=0 AND k2>=?1 AND k2=?1 AND k2 Date: Wed, 14 Feb 2018 15:30:48 +0000 Subject: [PATCH 063/107] New test cases for the editdist3() function in the spellfix extension. FossilOrigin-Name: 2d3f1f22d53034576a0ec8d9c360fb19efbb05b2e95ad1b09d120fb1b7da33fd --- manifest | 11 +- manifest.uuid | 2 +- test/spellfix4.test | 346 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 353 insertions(+), 6 deletions(-) create mode 100644 test/spellfix4.test diff --git a/manifest b/manifest index 8bac7cb68b..0b9be9c6dd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Try\sto\soptimize\sspellfix1\sby\sstoring\sa\sNULL\sin\sthe\sk1\scolumn\sof\s%_vocab\nif\sit\swould\sotherwise\shave\sthe\ssame\svalue\sas\sthe\sword\scolumn. -D 2018-02-14T14:13:09.238 +C New\stest\scases\sfor\sthe\seditdist3()\sfunction\sin\sthe\sspellfix\sextension. +D 2018-02-14T15:30:48.110 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -1265,6 +1265,7 @@ F test/speedtest1.c a5faf4cbe5769eee4b721b3875cb3f12520a9b99d9026b1063b47c396033 F test/spellfix.test 951a6405d49d1a23d6b78027d3877b4a33eeb8221dcab5704b499755bb4f552e F test/spellfix2.test dfc8f519a3fc204cb2dfa8b4f29821ae90f6f8c3 F test/spellfix3.test 0f9efaaa502a0e0a09848028518a6fb096c8ad33 +F test/spellfix4.test eaae7a334516e09896a684fb552d338fa2844c65db9817d900701fe7598327e6 F test/sqldiff1.test 28cd737cf1b0078b1ec1bbf425e674c47785835e F test/sqllimits1.test a74ee2a3740b9f9c2437c246d8fb77354862a142 F test/sqllog.test 6af6cb0b09f4e44e1917e06ce85be7670302517a @@ -1705,7 +1706,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1f300514e6cdd32c755dc1b05dfa125b10eda2eb55a1024b66f66dbf5cf96289 -R 255e72ba75e81cf4ca1d4ef198b4e2a9 +P b76ec7cb5841f9ad02b342271e91c5f5a598006f69df7953a6213c706e01f7a2 +R 955df0b8bc6bab58cd406ae281f344ae U drh -Z 21f5c49e8ec82780f55a5436de6c983b +Z d19bf6cab5bd800e73ca3fefa0386288 diff --git a/manifest.uuid b/manifest.uuid index 91ef23e4ef..ebe643ee66 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b76ec7cb5841f9ad02b342271e91c5f5a598006f69df7953a6213c706e01f7a2 \ No newline at end of file +2d3f1f22d53034576a0ec8d9c360fb19efbb05b2e95ad1b09d120fb1b7da33fd \ No newline at end of file diff --git a/test/spellfix4.test b/test/spellfix4.test new file mode 100644 index 0000000000..7054bb98ef --- /dev/null +++ b/test/spellfix4.test @@ -0,0 +1,346 @@ +# 2018-02-14 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Test cases for the editdist3() function in the spellfix extension. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix spellfix4 + +ifcapable !vtab { finish_test ; return } + +load_static_extension db spellfix + +do_execsql_test 100 { + CREATE TABLE cost1(iLang, cFrom, cTo, iCost); + INSERT INTO cost1 VALUES + (0, '', '?', 97), + (0, '?', '', 98), + (0, '?', '?', 99), + (0, 'm', 'n', 50), + (0, 'n', 'm', 50) + ; + SELECT editdist3('cost1'); + SELECT editdist3('anchor','amchor'); +} {{} 50} +do_execsql_test 110 { + SELECT editdist3('anchor','anchoxr'); +} {97} +do_execsql_test 111 { + SELECT editdist3('anchor','xanchor'); +} {97} +do_execsql_test 112 { + SELECT editdist3('anchor','anchorx'); +} {97} +do_execsql_test 120 { + SELECT editdist3('anchor','anchr'); +} {98} +do_execsql_test 121 { + SELECT editdist3('anchor','ancho'); +} {98} +do_execsql_test 122 { + SELECT editdist3('anchor','nchor'); +} {98} +do_execsql_test 130 { + SELECT editdist3('anchor','anchur'); +} {99} +do_execsql_test 131 { + SELECT editdist3('anchor','onchor'); +} {99} +do_execsql_test 132 { + SELECT editdist3('anchor','anchot'); +} {99} +do_execsql_test 140 { + SELECT editdist3('anchor','omchor'); +} {149} + +do_execsql_test 200 { + INSERT INTO cost1 VALUES + (0, 'a', 'ä', 5), + (0, 'ss', 'ß', 8) + ; + SELECT editdist3('cost1'); + SELECT editdist3('strasse','straße'); + SELECT editdist3('straße','strasse'); +} {{} 8 196} +do_execsql_test 210 { + SELECT editdist3('baume','bäume'); +} {5} +do_execsql_test 220 { + SELECT editdist3('baum','bäume'); +} {102} +do_execsql_test 230 { + INSERT INTO cost1 VALUES + (0, 'ä', 'a', 5), + (0, 'ß', 'ss', 8) + ; + SELECT editdist3('cost1'); + SELECT editdist3('strasse','straße'); + SELECT editdist3('straße','strasse'); +} {{} 8 8} + +do_execsql_test 300 { + DELETE FROM cost1; + INSERT INTO cost1 VALUES + (0, '', '?', 97), + (0, '?', '', 98), + (0, '?', '?', 99), + (0, 'a', 'e', 50), + (0, 'a', 'i', 70), + (0, 'a', 'o', 75), + (0, 'a', 'u', 85), + (0, 'e', 'a', 50), + (0, 'e', 'i', 50), + (0, 'e', 'o', 75), + (0, 'e', 'u', 85), + (0, 'i', 'a', 70), + (0, 'i', 'e', 50), + (0, 'i', 'o', 75), + (0, 'i', 'u', 85), + (0, 'o', 'a', 75), + (0, 'o', 'e', 75), + (0, 'o', 'i', 75), + (0, 'o', 'u', 40), + (0, 'u', 'a', 85), + (0, 'u', 'e', 85), + (0, 'u', 'i', 85), + (0, 'u', 'o', 40), + (0, 'm', 'n', 45), + (0, 'n', 'm', 45) + ; + CREATE TABLE words(x TEXT); + INSERT INTO words VALUES + ('abraham'), + ('action'), + ('africa'), + ('aladdin'), + ('alert'), + ('alien'), + ('amazon'), + ('analog'), + ('animal'), + ('apollo'), + ('archive'), + ('arnold'), + ('aspirin'), + ('august'), + ('average'), + ('bahama'), + ('bambino'), + ('barcode'), + ('bazooka'), + ('belgium'), + ('between'), + ('biology'), + ('blonde'), + ('border'), + ('brave'), + ('british'), + ('bucket'), + ('button'), + ('caesar'), + ('camilla'), + ('cannon'), + ('caramel'), + ('carpet'), + ('catalog'), + ('century'), + ('chaos'), + ('chef'), + ('china'), + ('circus'), + ('classic'), + ('clinic'), + ('coconut'), + ('combine'), + ('complex'), + ('congo'), + ('convert'), + ('cosmos'), + ('crack'), + ('crown'), + ('cyclone'), + ('deal'), + ('delete'), + ('denver'), + ('detail'), + ('diana'), + ('direct'), + ('dolby'), + ('double'), + ('dublin'), + ('echo'), + ('edition'), + ('electra'), + ('emotion'), + ('enjoy'), + ('escape'), + ('everest'), + ('exile'), + ('express'), + ('family'), + ('ferrari'), + ('filter'), + ('fish'), + ('florida'), + ('ford'), + ('forum'), + ('frank'), + ('frozen'), + ('gallery'), + ('garlic'), + ('geneva'), + ('gibson'), + ('gloria'), + ('gordon'), + ('gravity'), + ('ground'), + ('habitat'), + ('harlem'), + ('hazard'), + ('herbert'), + ('hobby'), + ('house'), + ('icon'), + ('immune'), + ('india'), + ('inside'), + ('isotope'), + ('jamaica'), + ('jazz'), + ('joker'), + ('juliet'), + ('jupiter'), + ('kevin'), + ('korea'), + ('latin'), + ('legal'), + ('lexicon'), + ('limbo'), + ('lithium'), + ('logo'), + ('lucas'), + ('madrid'), + ('major'), + ('manual'), + ('mars'), + ('maximum'), + ('medical'), + ('mental'), + ('meter'), + ('miguel'), + ('mimosa'), + ('miranda'), + ('modern'), + ('money'), + ('morgan'), + ('motor'), + ('mystic'), + ('nebula'), + ('network'), + ('nice'), + ('nitro'), + ('norway'), + ('nurse'), + ('octavia'), + ('olympic'), + ('opus'), + ('orient'), + ('othello'), + ('pacific'), + ('panama'), + ('paper'), + ('parking'), + ('pasta'), + ('paul'), + ('people'), + ('permit'), + ('phrase'), + ('pilgrim'), + ('planet'), + ('pocket'), + ('police'), + ('popular'), + ('prefer'), + ('presto'), + ('private'), + ('project'), + ('proxy'), + ('python'), + ('quota'), + ('rainbow'), + ('raymond'), + ('region'), + ('report'), + ('reward'), + ('risk'), + ('robot'), + ('rose'), + ('russian'), + ('sailor'), + ('salt'), + ('saturn'), + ('scorpio'), + ('second'), + ('seminar'), + ('shadow'), + ('shave'), + ('shock'), + ('silence'), + ('sinatra'), + ('sleep'), + ('social'), + ('sonata'), + ('spain'), + ('sphere'), + ('spray'), + ('state'), + ('stone'), + ('strong'), + ('sugar'), + ('supreme'), + ('swing'), + ('talent'), + ('telecom'), + ('thermos'), + ('tina'), + ('tommy'), + ('torso'), + ('trade'), + ('trick'), + ('tropic'), + ('turtle'), + ('uniform'), + ('user'), + ('vega'), + ('vertigo'), + ('village'), + ('visible'), + ('vocal'), + ('voyage'), + ('weekend'), + ('winter'), + ('year'), + ('zipper') + ; + SELECT editdist3('cost1'); +} {{}} +do_execsql_test 310 { + SELECT editdist3(a.x,b.x), a.x, b.x + FROM words a, words b + WHERE a.x Date: Wed, 14 Feb 2018 20:25:33 +0000 Subject: [PATCH 064/107] Disable assert() in the spellfix extension if not compiled with SQLITE_DEBUG. FossilOrigin-Name: 3c53ee0fdea4cbf2590e2b289b021b0ef8b7ead2945db1bdfc767432d9447acb --- ext/misc/spellfix.c | 6 ++++++ manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/ext/misc/spellfix.c b/ext/misc/spellfix.c index 6e3a3bca16..1966d554e7 100644 --- a/ext/misc/spellfix.c +++ b/ext/misc/spellfix.c @@ -18,6 +18,12 @@ SQLITE_EXTENSION_INIT1 #ifndef SQLITE_AMALGAMATION +# if !defined(NDEBUG) && !defined(SQLITE_DEBUG) +# define NDEBUG 1 +# endif +# if defined(NDEBUG) && defined(SQLITE_DEBUG) +# undef NDEBUG +# endif # include # include # include diff --git a/manifest b/manifest index 0b9be9c6dd..09759a8b0f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C New\stest\scases\sfor\sthe\seditdist3()\sfunction\sin\sthe\sspellfix\sextension. -D 2018-02-14T15:30:48.110 +C Disable\sassert()\sin\sthe\sspellfix\sextension\sif\snot\scompiled\swith\sSQLITE_DEBUG. +D 2018-02-14T20:25:33.844 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -294,7 +294,7 @@ F ext/misc/series.c f3c0dba5c5c749ce1782b53076108f87cf0b71041eb6023f727a9c50681d F ext/misc/sha1.c 0b9e9b855354910d3ca467bf39099d570e73db56 F ext/misc/shathree.c 9e960ba50483214c6a7a4b1517f8d8cef799e9db381195178c3fd3ad207e10c0 F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52 -F ext/misc/spellfix.c efe188bdd84d283e1c82e4bcf7058ffc1856176d9ba2ba14fd5efc4aa2d8661f +F ext/misc/spellfix.c bb73be37491fd5007efab8dfd8ac416315d47f8aba99179b8adb4255e85c7ff5 F ext/misc/sqlar.c 57d5bc45cd5492208e451f697404be88f8612527d64c9d42f96b325b64983d74 F ext/misc/stmt.c 6f16443abb3551e3f5813bb13ba19a30e7032830015b0f92fe0c0453045c0a11 F ext/misc/totype.c 4a167594e791abeed95e0a8db028822b5e8fe512 @@ -1706,7 +1706,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b76ec7cb5841f9ad02b342271e91c5f5a598006f69df7953a6213c706e01f7a2 -R 955df0b8bc6bab58cd406ae281f344ae +P 2d3f1f22d53034576a0ec8d9c360fb19efbb05b2e95ad1b09d120fb1b7da33fd +R 4aff58b3b0f19d8824c9d78629ce8a88 U drh -Z d19bf6cab5bd800e73ca3fefa0386288 +Z bdffc86a194b9f42419156834dd68b1f diff --git a/manifest.uuid b/manifest.uuid index ebe643ee66..129d553977 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2d3f1f22d53034576a0ec8d9c360fb19efbb05b2e95ad1b09d120fb1b7da33fd \ No newline at end of file +3c53ee0fdea4cbf2590e2b289b021b0ef8b7ead2945db1bdfc767432d9447acb \ No newline at end of file From d9274a8ae35a5f195a5d4a3629ea74978e69cf13 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 14 Feb 2018 20:58:36 +0000 Subject: [PATCH 065/107] Performance optimizations to the editdist3() function in the spellfix extension. FossilOrigin-Name: 70d304dcbac4c3fd5e3b96108bffea2ce6c0db19c847397d5c5e268bb90a981d --- ext/misc/spellfix.c | 14 ++++++-------- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/ext/misc/spellfix.c b/ext/misc/spellfix.c index 1966d554e7..28374b8cc2 100644 --- a/ext/misc/spellfix.c +++ b/ext/misc/spellfix.c @@ -691,6 +691,7 @@ static int editDist3ConfigLoad( assert( zTo!=0 || nTo==0 ); if( nFrom>100 || nTo>100 ) continue; if( iCost<0 ) continue; + if( iCost>10000 ) continue; /* Costs above 10K are considered infinite */ if( pLang==0 || iLang!=iLangPrev ){ EditDist3Lang *pNew; pNew = sqlite3_realloc64(p->a, (p->nLang+1)*sizeof(p->a[0])); @@ -782,7 +783,7 @@ static int matchFromTo( ){ int b1 = pStr->a[n1].nByte; if( b1>n2 ) return 0; - if( memcmp(pStr->z+n1, z2, b1)!=0 ) return 0; + if( strncmp(pStr->z+n1, z2, b1)!=0 ) return 0; return 1; } @@ -864,9 +865,6 @@ static EditDist3FromString *editDist3FromStringNew( /* ** Update entry m[i] such that it is the minimum of its current value ** and m[j]+iCost. -** -** If the iCost is 1,000,000 or greater, then consider the cost to be -** infinite and skip the update. */ static void updateCost( unsigned int *m, @@ -874,11 +872,11 @@ static void updateCost( int j, int iCost ){ + unsigned int b; assert( iCost>=0 ); - if( iCost<10000 ){ - unsigned int b = m[j] + iCost; - if( b Date: Wed, 14 Feb 2018 23:27:43 +0000 Subject: [PATCH 066/107] Add the --readonly option to the ".open" command in the CLI. FossilOrigin-Name: 06870bb15656b50b0e14d4364bb21afac76500e313ecf67aaef3688d603fd076 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c.in | 11 +++++++++++ 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 13a742a2be..a6b76d1492 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Performance\soptimizations\sto\sthe\seditdist3()\sfunction\sin\sthe\sspellfix\nextension. -D 2018-02-14T20:58:36.311 +C Add\sthe\s--readonly\soption\sto\sthe\s".open"\scommand\sin\sthe\sCLI. +D 2018-02-14T23:27:43.700 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -489,7 +489,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c f02352ac5cbb6fad3804add825743b924cfb6c742ba2e8666d726828a9985d73 -F src/shell.c.in 0b2c8d55e3703723a22610d75ccd4037962c3bcb5ad5570efd781e8c521811c4 +F src/shell.c.in 6fdccd0a2879dab2e0fc2ffedccddebc1f84bd88a806d28c2774ec5e4feca479 F src/sqlite.h.in 51f9acf52c80113d793ddd38b3940ad6895d97b4752503b19291fb8fcbf54c5e F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 83a3c4ce93d650bedfd1aa558cb85a516bd6d094445ee989740827d0d944368d @@ -1706,7 +1706,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3c53ee0fdea4cbf2590e2b289b021b0ef8b7ead2945db1bdfc767432d9447acb -R 52a6749d61b6207535a693308aa98655 +P 70d304dcbac4c3fd5e3b96108bffea2ce6c0db19c847397d5c5e268bb90a981d +R 2fdde9b3aa61b61468b1828a4e04c2be U drh -Z c6d9011f82c1274dfef70231bb03befb +Z 881e2b520c52d585df5898197d96b448 diff --git a/manifest.uuid b/manifest.uuid index 8a40d27ca4..10c7702d81 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -70d304dcbac4c3fd5e3b96108bffea2ce6c0db19c847397d5c5e268bb90a981d \ No newline at end of file +06870bb15656b50b0e14d4364bb21afac76500e313ecf67aaef3688d603fd076 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 3e3f9ac8f9..1ef414f4c2 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -1069,6 +1069,7 @@ struct ShellState { #define SHELL_OPEN_NORMAL 1 /* Normal database file */ #define SHELL_OPEN_APPENDVFS 2 /* Use appendvfs */ #define SHELL_OPEN_ZIPFILE 3 /* Use the zipfile virtual table */ +#define SHELL_OPEN_READONLY 4 /* Open a normal database read-only */ /* ** These are the allowed shellFlgs values @@ -3258,6 +3259,7 @@ static char zHelp[] = " on the output.\n" ".open ?OPTIONS? ?FILE? Close existing database and reopen FILE\n" " The --new option starts with an empty file\n" + " Other options: --readonly --append --zip\n" ".output ?FILE? Send output to FILE or stdout\n" ".print STRING... Print literal STRING\n" ".prompt MAIN CONTINUE Replace the standard prompts\n" @@ -3454,6 +3456,10 @@ static void open_db(ShellState *p, int keepAlive){ sqlite3_open(":memory:", &p->db); break; } + case SHELL_OPEN_READONLY: { + sqlite3_open_v2(p->zDbFilename, &p->db, SQLITE_OPEN_READONLY, 0); + break; + } case SHELL_OPEN_UNSPEC: case SHELL_OPEN_NORMAL: { sqlite3_open(p->zDbFilename, &p->db); @@ -6307,6 +6313,8 @@ static int do_meta_command(char *zLine, ShellState *p){ #endif }else if( optionMatch(z, "append") ){ p->openMode = SHELL_OPEN_APPENDVFS; + }else if( optionMatch(z, "readonly") ){ + p->openMode = SHELL_OPEN_READONLY; }else if( z[0]=='-' ){ utf8_printf(stderr, "unknown option: %s\n", z); rc = 1; @@ -7985,6 +7993,7 @@ static const char zOptions[] = " -nullvalue TEXT set text string for NULL values. Default ''\n" " -pagecache SIZE N use N slots of SZ bytes each for page cache memory\n" " -quote set output mode to 'quote'\n" + " -readonly open the database read-only\n" " -separator SEP set output column separator. Default: '|'\n" " -stats print memory stats before each finalize\n" " -version show SQLite version\n" @@ -8234,6 +8243,8 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ #endif }else if( strcmp(z,"-append")==0 ){ data.openMode = SHELL_OPEN_APPENDVFS; + }else if( strcmp(z,"-readonly")==0 ){ + data.openMode = SHELL_OPEN_READONLY; } } if( data.zDbFilename==0 ){ From 501ea056309a623303e5cfdbeda6c55d5fadfe0b Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 15 Feb 2018 01:03:37 +0000 Subject: [PATCH 067/107] Fix the --heap, --pagecache, --lookaside and other options to the command-line shell that invoke sqlite3_config() so that they work again. FossilOrigin-Name: 00707f2f2f746a6421f3e2de995e68cc8adba7225a04db6b28db52944e7e988e --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c.in | 24 +++++++++++++++++++++--- 3 files changed, 28 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index a6b76d1492..98d58723e4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s--readonly\soption\sto\sthe\s".open"\scommand\sin\sthe\sCLI. -D 2018-02-14T23:27:43.700 +C Fix\sthe\s--heap,\s--pagecache,\s--lookaside\sand\sother\soptions\sto\sthe\scommand-line\nshell\sthat\sinvoke\ssqlite3_config()\sso\sthat\sthey\swork\sagain. +D 2018-02-15T01:03:37.596 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -489,7 +489,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c f02352ac5cbb6fad3804add825743b924cfb6c742ba2e8666d726828a9985d73 -F src/shell.c.in 6fdccd0a2879dab2e0fc2ffedccddebc1f84bd88a806d28c2774ec5e4feca479 +F src/shell.c.in c1b14bc0bc1fe73a97dfaad55ca09f5d6e44f6019977d94e7e3d1861616035e1 F src/sqlite.h.in 51f9acf52c80113d793ddd38b3940ad6895d97b4752503b19291fb8fcbf54c5e F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 83a3c4ce93d650bedfd1aa558cb85a516bd6d094445ee989740827d0d944368d @@ -1706,7 +1706,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 70d304dcbac4c3fd5e3b96108bffea2ce6c0db19c847397d5c5e268bb90a981d -R 2fdde9b3aa61b61468b1828a4e04c2be +P 06870bb15656b50b0e14d4364bb21afac76500e313ecf67aaef3688d603fd076 +R 4e71ff1498e1cda7e0d47e08c283419f U drh -Z 881e2b520c52d585df5898197d96b448 +Z 76196b3bed6ed8589ed1fea48a14df1f diff --git a/manifest.uuid b/manifest.uuid index 10c7702d81..4756fcb048 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -06870bb15656b50b0e14d4364bb21afac76500e313ecf67aaef3688d603fd076 \ No newline at end of file +00707f2f2f746a6421f3e2de995e68cc8adba7225a04db6b28db52944e7e988e \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 1ef414f4c2..c074b5ca7b 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -8103,21 +8103,39 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ } #endif main_init(&data); - sqlite3_initialize(); + + /* On Windows, we must translate command-line arguments into UTF-8. + ** The SQLite memory allocator subsystem has to be enabled in order to + ** do this. But we want to run an sqlite3_shutdown() afterwards so that + ** subsequent sqlite3_config() calls will work. So copy all results into + ** memory that does not come from the SQLite memory allocator. + */ #if !SQLITE_SHELL_IS_UTF8 - argv = sqlite3_malloc64(sizeof(argv[0])*argc); + sqlite3_initialize(); + argv = malloc(sizeof(argv[0])*argc); if( argv==0 ){ raw_printf(stderr, "out of memory\n"); exit(1); } for(i=0; i=1 && argv && argv[0] ); Argv0 = argv[0]; From 46e835a2c42f8322e27fe00d4d6bd4525e207712 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 15 Feb 2018 03:05:56 +0000 Subject: [PATCH 068/107] Reduce the number of calls to strncmp() required to run editDist3Core(). FossilOrigin-Name: afd6fbc01052ccfc9bd29fb8f934b291b8f56af44fcae870da7e1355fe95c29a --- ext/misc/spellfix.c | 3 +++ manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/ext/misc/spellfix.c b/ext/misc/spellfix.c index 28374b8cc2..8b528562a8 100644 --- a/ext/misc/spellfix.c +++ b/ext/misc/spellfix.c @@ -756,6 +756,7 @@ static int utf8Len(unsigned char c, int N){ ** the given string. */ static int matchTo(EditDist3Cost *p, const char *z, int n){ + if( p->a[p->nFrom]!=z[0] ) return 0; if( p->nTo>n ) return 0; if( strncmp(p->a+p->nFrom, z, p->nTo)!=0 ) return 0; return 1; @@ -767,6 +768,7 @@ static int matchTo(EditDist3Cost *p, const char *z, int n){ */ static int matchFrom(EditDist3Cost *p, const char *z, int n){ assert( p->nFrom<=n ); + if( p->a[0]!=z[0] ) return 0; if( strncmp(p->a, z, p->nFrom)!=0 ) return 0; return 1; } @@ -783,6 +785,7 @@ static int matchFromTo( ){ int b1 = pStr->a[n1].nByte; if( b1>n2 ) return 0; + if( pStr->z[n1]!=z2[0] ) return 0; if( strncmp(pStr->z+n1, z2, b1)!=0 ) return 0; return 1; } diff --git a/manifest b/manifest index 98d58723e4..411cad769d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\s--heap,\s--pagecache,\s--lookaside\sand\sother\soptions\sto\sthe\scommand-line\nshell\sthat\sinvoke\ssqlite3_config()\sso\sthat\sthey\swork\sagain. -D 2018-02-15T01:03:37.596 +C Reduce\sthe\snumber\sof\scalls\sto\sstrncmp()\srequired\sto\srun\seditDist3Core(). +D 2018-02-15T03:05:56.872 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -294,7 +294,7 @@ F ext/misc/series.c f3c0dba5c5c749ce1782b53076108f87cf0b71041eb6023f727a9c50681d F ext/misc/sha1.c 0b9e9b855354910d3ca467bf39099d570e73db56 F ext/misc/shathree.c 9e960ba50483214c6a7a4b1517f8d8cef799e9db381195178c3fd3ad207e10c0 F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52 -F ext/misc/spellfix.c fcf23b0746bfefbec46a54cfb2cd0720c442a6e382247b4049748f882ad26b94 +F ext/misc/spellfix.c ad435d8d84a64a7287f67ba618d5fc4a775e73d8489b1c7ae85a339ed5d20c34 F ext/misc/sqlar.c 57d5bc45cd5492208e451f697404be88f8612527d64c9d42f96b325b64983d74 F ext/misc/stmt.c 6f16443abb3551e3f5813bb13ba19a30e7032830015b0f92fe0c0453045c0a11 F ext/misc/totype.c 4a167594e791abeed95e0a8db028822b5e8fe512 @@ -1706,7 +1706,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 06870bb15656b50b0e14d4364bb21afac76500e313ecf67aaef3688d603fd076 -R 4e71ff1498e1cda7e0d47e08c283419f +P 00707f2f2f746a6421f3e2de995e68cc8adba7225a04db6b28db52944e7e988e +R bf16a0c80a0f655a42b9a2dc09fe803f U drh -Z 76196b3bed6ed8589ed1fea48a14df1f +Z 2ab397c1e9ab0ece50ec14a24b630ce7 diff --git a/manifest.uuid b/manifest.uuid index 4756fcb048..df37324255 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -00707f2f2f746a6421f3e2de995e68cc8adba7225a04db6b28db52944e7e988e \ No newline at end of file +afd6fbc01052ccfc9bd29fb8f934b291b8f56af44fcae870da7e1355fe95c29a \ No newline at end of file From f4bc6c43d7e45580f5ee4cc40586829890afa155 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 15 Feb 2018 03:56:33 +0000 Subject: [PATCH 069/107] Improve performance of editdist3() by keeping the costs in sorted order. Also add a new regression test to editdist3(). FossilOrigin-Name: dc734c5b61464dfd6bfa7963f2ecce32e405a0c2ba1ef6f453ec9389da080256 --- ext/misc/spellfix.c | 82 ++++++++++++++++++++++++++++++++++++++++++++- manifest | 14 ++++---- manifest.uuid | 2 +- test/spellfix4.test | 31 ++++++++++------- 4 files changed, 108 insertions(+), 21 deletions(-) diff --git a/ext/misc/spellfix.c b/ext/misc/spellfix.c index 8b528562a8..8651bb7a9d 100644 --- a/ext/misc/spellfix.c +++ b/ext/misc/spellfix.c @@ -658,6 +658,79 @@ static void editDist3ConfigDelete(void *pIn){ sqlite3_free(p); } +/* Compare the FROM values of two EditDist3Cost objects, for sorting. +** Return negative, zero, or positive if the A is less than, equal to, +** or greater than B. +*/ +static int editDist3CostCompare(EditDist3Cost *pA, EditDist3Cost *pB){ + int n = pA->nFrom; + int rc; + if( n>pB->nFrom ) n = pB->nFrom; + rc = strncmp(pA->a, pB->a, n); + if( rc==0 ) rc = pA->nFrom - pB->nFrom; + return rc; +} + +/* +** Merge together two sorted lists of EditDist3Cost objects, in order +** of increasing FROM. +*/ +static EditDist3Cost *editDist3CostMerge( + EditDist3Cost *pA, + EditDist3Cost *pB +){ + EditDist3Cost *pHead = 0; + EditDist3Cost **ppTail = &pHead; + EditDist3Cost *p; + while( pA && pB ){ + if( editDist3CostCompare(pA,pB)<=0 ){ + p = pA; + pA = pA->pNext; + }else{ + p = pB; + pB = pB->pNext; + } + *ppTail = p; + ppTail = &p->pNext; + } + if( pA ){ + *ppTail = pA; + }else{ + *ppTail = pB; + } + return pHead; +} + +/* +** Sort a list of EditDist3Cost objects into order of increasing FROM +*/ +static EditDist3Cost *editDist3CostSort(EditDist3Cost *pList){ + EditDist3Cost *ap[60], *p; + int i; + int mx = 0; + ap[0] = 0; + ap[1] = 0; + while( pList ){ + p = pList; + pList = p->pNext; + p->pNext = 0; + for(i=0; ap[i]; i++){ + p = editDist3CostMerge(ap[i],p); + ap[i] = 0; + } + ap[i] = p; + if( i>mx ){ + mx = i; + ap[i+1] = 0; + } + } + p = 0; + for(i=0; i<=mx; i++){ + if( ap[i] ) p = editDist3CostMerge(p,ap[i]); + } + return p; +} + /* ** Load all edit-distance weights from a table. */ @@ -729,6 +802,12 @@ static int editDist3ConfigLoad( } rc2 = sqlite3_finalize(pStmt); if( rc==SQLITE_OK ) rc = rc2; + if( rc==SQLITE_OK ){ + int iLang; + for(iLang=0; iLangnLang; iLang++){ + p->a[iLang].pCost = editDist3CostSort(p->a[iLang].pCost); + } + } return rc; } @@ -943,8 +1022,9 @@ static int editDist3Core( a2[i2].nByte = utf8Len((unsigned char)z2[i2], n2-i2); for(p=pLang->pCost; p; p=p->pNext){ EditDist3Cost **apNew; - if( p->nFrom>0 ) continue; + if( p->nFrom>0 ) break; if( i2+p->nTo>n2 ) continue; + if( p->a[0]>z2[i2] ) break; if( matchTo(p, z2+i2, n2-i2)==0 ) continue; a2[i2].nIns++; apNew = sqlite3_realloc64(a2[i2].apIns, sizeof(*apNew)*a2[i2].nIns); diff --git a/manifest b/manifest index 411cad769d..dc00c62eff 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Reduce\sthe\snumber\sof\scalls\sto\sstrncmp()\srequired\sto\srun\seditDist3Core(). -D 2018-02-15T03:05:56.872 +C Improve\sperformance\sof\seditdist3()\sby\skeeping\sthe\scosts\sin\ssorted\sorder.\nAlso\sadd\sa\snew\sregression\stest\sto\seditdist3(). +D 2018-02-15T03:56:33.574 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -294,7 +294,7 @@ F ext/misc/series.c f3c0dba5c5c749ce1782b53076108f87cf0b71041eb6023f727a9c50681d F ext/misc/sha1.c 0b9e9b855354910d3ca467bf39099d570e73db56 F ext/misc/shathree.c 9e960ba50483214c6a7a4b1517f8d8cef799e9db381195178c3fd3ad207e10c0 F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52 -F ext/misc/spellfix.c ad435d8d84a64a7287f67ba618d5fc4a775e73d8489b1c7ae85a339ed5d20c34 +F ext/misc/spellfix.c b3a644285cb008f3c10ed4cf04e17adcbc7d62c3911c79d786dfc91c177534f0 F ext/misc/sqlar.c 57d5bc45cd5492208e451f697404be88f8612527d64c9d42f96b325b64983d74 F ext/misc/stmt.c 6f16443abb3551e3f5813bb13ba19a30e7032830015b0f92fe0c0453045c0a11 F ext/misc/totype.c 4a167594e791abeed95e0a8db028822b5e8fe512 @@ -1265,7 +1265,7 @@ F test/speedtest1.c a5faf4cbe5769eee4b721b3875cb3f12520a9b99d9026b1063b47c396033 F test/spellfix.test 951a6405d49d1a23d6b78027d3877b4a33eeb8221dcab5704b499755bb4f552e F test/spellfix2.test dfc8f519a3fc204cb2dfa8b4f29821ae90f6f8c3 F test/spellfix3.test 0f9efaaa502a0e0a09848028518a6fb096c8ad33 -F test/spellfix4.test eaae7a334516e09896a684fb552d338fa2844c65db9817d900701fe7598327e6 +F test/spellfix4.test 51c7c26514ade169855c66bcf130bd5acfb4d7fd090cc624645ab275ae6a41fb F test/sqldiff1.test 28cd737cf1b0078b1ec1bbf425e674c47785835e F test/sqllimits1.test a74ee2a3740b9f9c2437c246d8fb77354862a142 F test/sqllog.test 6af6cb0b09f4e44e1917e06ce85be7670302517a @@ -1706,7 +1706,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 00707f2f2f746a6421f3e2de995e68cc8adba7225a04db6b28db52944e7e988e -R bf16a0c80a0f655a42b9a2dc09fe803f +P afd6fbc01052ccfc9bd29fb8f934b291b8f56af44fcae870da7e1355fe95c29a +R 8c5288c6ebac589573d48d7520074806 U drh -Z 2ab397c1e9ab0ece50ec14a24b630ce7 +Z 65b4bdcfd9180aa4d82e9c5cf2ce0fa5 diff --git a/manifest.uuid b/manifest.uuid index df37324255..a1afa09049 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -afd6fbc01052ccfc9bd29fb8f934b291b8f56af44fcae870da7e1355fe95c29a \ No newline at end of file +dc734c5b61464dfd6bfa7963f2ecce32e405a0c2ba1ef6f453ec9389da080256 \ No newline at end of file diff --git a/test/spellfix4.test b/test/spellfix4.test index 7054bb98ef..caf6d5139a 100644 --- a/test/spellfix4.test +++ b/test/spellfix4.test @@ -97,22 +97,22 @@ do_execsql_test 300 { (0, 'a', 'e', 50), (0, 'a', 'i', 70), (0, 'a', 'o', 75), - (0, 'a', 'u', 85), + (0, 'a', 'u', 81), (0, 'e', 'a', 50), - (0, 'e', 'i', 50), - (0, 'e', 'o', 75), - (0, 'e', 'u', 85), + (0, 'e', 'i', 52), + (0, 'e', 'o', 72), + (0, 'e', 'u', 82), (0, 'i', 'a', 70), - (0, 'i', 'e', 50), + (0, 'i', 'e', 52), (0, 'i', 'o', 75), - (0, 'i', 'u', 85), + (0, 'i', 'u', 83), (0, 'o', 'a', 75), - (0, 'o', 'e', 75), + (0, 'o', 'e', 72), (0, 'o', 'i', 75), (0, 'o', 'u', 40), - (0, 'u', 'a', 85), - (0, 'u', 'e', 85), - (0, 'u', 'i', 85), + (0, 'u', 'a', 81), + (0, 'u', 'e', 82), + (0, 'u', 'i', 83), (0, 'u', 'o', 40), (0, 'm', 'n', 45), (0, 'n', 'm', 45) @@ -340,7 +340,14 @@ do_execsql_test 310 { WHERE a.x Date: Thu, 15 Feb 2018 21:00:37 +0000 Subject: [PATCH 070/107] Do not allow parameters or schema references inside of WITH clause of triggers and views. This fixes a bug discovered by OSSFuzz and present since common-table-expressions were first added in 2014-02-03. FossilOrigin-Name: b918d4b4e546d3903ff20efc3c8ca26dd9761cc8ed9ef7d2799b66ff9ae1ae51 --- manifest | 13 +++++++------ manifest.uuid | 2 +- src/attach.c | 10 ++++++++++ test/with4.test | 52 +++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 70 insertions(+), 7 deletions(-) create mode 100644 test/with4.test diff --git a/manifest b/manifest index dc00c62eff..b3d50466f6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improve\sperformance\sof\seditdist3()\sby\skeeping\sthe\scosts\sin\ssorted\sorder.\nAlso\sadd\sa\snew\sregression\stest\sto\seditdist3(). -D 2018-02-15T03:56:33.574 +C Do\snot\sallow\sparameters\sor\sschema\sreferences\sinside\sof\sWITH\sclause\sof\ntriggers\sand\sviews.\s\sThis\sfixes\sa\sbug\sdiscovered\sby\sOSSFuzz\sand\spresent\nsince\scommon-table-expressions\swere\sfirst\sadded\sin\s2014-02-03. +D 2018-02-15T21:00:37.044 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -425,7 +425,7 @@ F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c cf7a8af45cb0ace672f47a1b29ab24092a9e8cd8d945a9974e3b5d925f548594 F src/analyze.c 6b42e36a5dcc2703a771f2411bd5e99524bd62c7ecde209bb88dfb04c72f046e -F src/attach.c 84c477e856b24c2b9a0983b438a707c0cf4d616cee7a425401d418e58afec24c +F src/attach.c e1c0d72dd0747da84aa1352f8729906b1d8d2a162ba7f6c475980be202b37c46 F src/auth.c 6277d63837357549fe14e723490d6dc1a38768d71c795c5eb5c0f8a99f918f73 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33 @@ -1594,6 +1594,7 @@ F test/win32nolock.test ac4f08811a562e45a5755e661f45ca85892bdbbc F test/with1.test ca08e291249a810a2ec9b72ceef5575e07d5925b360fcf6652ae6fe06ac4dced F test/with2.test e0030e2f0267a910d6c0e4f46f2dfe941c1cc0d4f659ba69b3597728e7e8f1ab F test/with3.test e71604a0e53cba82bc04c703987cb1d6751ec0b6 +F test/with4.test 257be66c0c67fee1defbbac0f685c3465e2cad037f21ce65f23f86084f198205 F test/withM.test 693b61765f2b387b5e3e24a4536e2e82de15ff64 F test/without_rowid1.test 06b7215130882d6a072233820dd364c874c4fd69221e8fc756ec471009192874 F test/without_rowid2.test af260339f79d13cb220288b67cd287fbcf81ad99 @@ -1706,7 +1707,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P afd6fbc01052ccfc9bd29fb8f934b291b8f56af44fcae870da7e1355fe95c29a -R 8c5288c6ebac589573d48d7520074806 +P dc734c5b61464dfd6bfa7963f2ecce32e405a0c2ba1ef6f453ec9389da080256 +R 8b09aca00d46d50dd4b70270c44a2f05 U drh -Z 65b4bdcfd9180aa4d82e9c5cf2ce0fa5 +Z bc5ab0c5d5dc21d5926f6e1994fc1380 diff --git a/manifest.uuid b/manifest.uuid index a1afa09049..636ea46456 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -dc734c5b61464dfd6bfa7963f2ecce32e405a0c2ba1ef6f453ec9389da080256 \ No newline at end of file +b918d4b4e546d3903ff20efc3c8ca26dd9761cc8ed9ef7d2799b66ff9ae1ae51 \ No newline at end of file diff --git a/src/attach.c b/src/attach.c index fa38e84159..67fc4afa82 100644 --- a/src/attach.c +++ b/src/attach.c @@ -504,6 +504,16 @@ int sqlite3FixSelect( if( sqlite3FixExpr(pFix, pSelect->pLimit) ){ return 1; } +#if 1 + if( pSelect->pWith ){ + int i; + for(i=0; ipWith->nCte; i++){ + if( sqlite3FixSelect(pFix, pSelect->pWith->a[i].pSelect) ){ + return 1; + } + } + } +#endif pSelect = pSelect->pPrior; } return 0; diff --git a/test/with4.test b/test/with4.test new file mode 100644 index 0000000000..b0eeba6d14 --- /dev/null +++ b/test/with4.test @@ -0,0 +1,52 @@ +# 2018-02-15 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing the WITH clause in TRIGGERs and VIEWs. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set ::testprefix with4 + +ifcapable {!cte} { + finish_test + return +} + +do_execsql_test 100 { + ATTACH ':memory:' AS aux; + CREATE TABLE main.t1(a,b); + CREATE TABLE aux.t2(x,y); + INSERT INTO t1 VALUES(1,2); + INSERT INTO t2 VALUES(3,4); +} {} +do_catchsql_test 110 { + CREATE VIEW v1 AS SELECT * FROM t1, aux.t2; +} {1 {view v1 cannot reference objects in database aux}} +do_catchsql_test 120 { + CREATE VIEW v2 AS WITH v(m,n) AS (SELECT x,y FROM aux.t2) SELECT * FROM t1, v; +} {1 {view v2 cannot reference objects in database aux}} +do_catchsql_test 130 { + CREATE VIEW v2 AS WITH v(m,n) AS (SELECT 5,?2) SELECT * FROM t1, v; +} {1 {parameters are not allowed in views}} + +do_catchsql_test 200 { + CREATE TRIGGER r1 AFTER INSERT ON t1 BEGIN + WITH v(m,n) AS (SELECT x,y FROM aux.t2) SELECT * FROM t1, v; + END; +} {1 {trigger r1 cannot reference objects in database aux}} +do_catchsql_test 210 { + CREATE TRIGGER r1 AFTER INSERT ON t1 BEGIN + WITH v(m,n) AS (SELECT 5,?2) SELECT * FROM t1, v; + END; +} {1 {trigger cannot use variables}} + +finish_test From c5515508081c94e96d3e9adb9ac8579e4dc9335b Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 17 Feb 2018 07:38:50 +0000 Subject: [PATCH 071/107] Fix a data race causing a tsan complaint with SQLITE_ENABLE_API_ARMOR builds on unix. The race condition is not dangerous in practice, it just upsets tsan. FossilOrigin-Name: f53b8a573bfbb48780243d56ca8372165adb1b979731c43d46cd1f8eb7e593f3 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/mutex_unix.c | 39 +++++++++++++++++++++++---------------- 3 files changed, 31 insertions(+), 24 deletions(-) diff --git a/manifest b/manifest index b3d50466f6..3d3100acc3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\sallow\sparameters\sor\sschema\sreferences\sinside\sof\sWITH\sclause\sof\ntriggers\sand\sviews.\s\sThis\sfixes\sa\sbug\sdiscovered\sby\sOSSFuzz\sand\spresent\nsince\scommon-table-expressions\swere\sfirst\sadded\sin\s2014-02-03. -D 2018-02-15T21:00:37.044 +C Fix\sa\sdata\srace\scausing\sa\stsan\scomplaint\swith\sSQLITE_ENABLE_API_ARMOR\sbuilds\non\sunix.\sThe\srace\scondition\sis\snot\sdangerous\sin\spractice,\sit\sjust\supsets\stsan. +D 2018-02-17T07:38:50.155 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -465,7 +465,7 @@ F src/msvc.h 4942752b6a253116baaa8de75256c51a459a5e81 F src/mutex.c b021263554c8a3995e9d53193b8194b96d1ed28e06c3b532dd7f7d29cf0c7d53 F src/mutex.h 779d588e3b7756ec3ecf7d78cde1d84aba414f85 F src/mutex_noop.c 9d4309c075ba9cc7249e19412d3d62f7f94839c4 -F src/mutex_unix.c 27bb6cc49485ee46711a6580ab7b3f1402211d23 +F src/mutex_unix.c aaf9ebc3f89df28483c52208497a99a02cc3650011422fc9d4c57e4392f7fe58 F src/mutex_w32.c a898fa969823b100c0f5fdc57e54c9a1e419ab4d F src/notify.c 9711a7575036f0d3040ba61bc6e217f13a9888e7 F src/os.c 22d31db3ca5a96a408fbf1ceeaaebcaf64c87024d2ff9fe1cf2ddbec3e75c104 @@ -1707,7 +1707,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P dc734c5b61464dfd6bfa7963f2ecce32e405a0c2ba1ef6f453ec9389da080256 -R 8b09aca00d46d50dd4b70270c44a2f05 -U drh -Z bc5ab0c5d5dc21d5926f6e1994fc1380 +P b918d4b4e546d3903ff20efc3c8ca26dd9761cc8ed9ef7d2799b66ff9ae1ae51 +R f2c9c538fa5937452e735a3a9fd538f8 +U dan +Z 36b8ef4562da3a84a51e82bfb896975f diff --git a/manifest.uuid b/manifest.uuid index 636ea46456..caa3d48cb7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b918d4b4e546d3903ff20efc3c8ca26dd9761cc8ed9ef7d2799b66ff9ae1ae51 \ No newline at end of file +f53b8a573bfbb48780243d56ca8372165adb1b979731c43d46cd1f8eb7e593f3 \ No newline at end of file diff --git a/src/mutex_unix.c b/src/mutex_unix.c index 55d08c8052..9282d28016 100644 --- a/src/mutex_unix.c +++ b/src/mutex_unix.c @@ -50,11 +50,12 @@ struct sqlite3_mutex { #endif }; #if SQLITE_MUTEX_NREF -#define SQLITE3_MUTEX_INITIALIZER {PTHREAD_MUTEX_INITIALIZER,0,0,(pthread_t)0,0} +# define SQLITE3_MUTEX_INITIALIZER(id) \ + {PTHREAD_MUTEX_INITIALIZER,id,0,(pthread_t)0,0} #elif defined(SQLITE_ENABLE_API_ARMOR) -#define SQLITE3_MUTEX_INITIALIZER { PTHREAD_MUTEX_INITIALIZER, 0 } +# define SQLITE3_MUTEX_INITIALIZER(id) { PTHREAD_MUTEX_INITIALIZER, id } #else -#define SQLITE3_MUTEX_INITIALIZER { PTHREAD_MUTEX_INITIALIZER } +#define SQLITE3_MUTEX_INITIALIZER(id) { PTHREAD_MUTEX_INITIALIZER } #endif /* @@ -151,18 +152,18 @@ static int pthreadMutexEnd(void){ return SQLITE_OK; } */ static sqlite3_mutex *pthreadMutexAlloc(int iType){ static sqlite3_mutex staticMutexes[] = { - SQLITE3_MUTEX_INITIALIZER, - SQLITE3_MUTEX_INITIALIZER, - SQLITE3_MUTEX_INITIALIZER, - SQLITE3_MUTEX_INITIALIZER, - SQLITE3_MUTEX_INITIALIZER, - SQLITE3_MUTEX_INITIALIZER, - SQLITE3_MUTEX_INITIALIZER, - SQLITE3_MUTEX_INITIALIZER, - SQLITE3_MUTEX_INITIALIZER, - SQLITE3_MUTEX_INITIALIZER, - SQLITE3_MUTEX_INITIALIZER, - SQLITE3_MUTEX_INITIALIZER + SQLITE3_MUTEX_INITIALIZER(2), + SQLITE3_MUTEX_INITIALIZER(3), + SQLITE3_MUTEX_INITIALIZER(4), + SQLITE3_MUTEX_INITIALIZER(5), + SQLITE3_MUTEX_INITIALIZER(6), + SQLITE3_MUTEX_INITIALIZER(7), + SQLITE3_MUTEX_INITIALIZER(8), + SQLITE3_MUTEX_INITIALIZER(9), + SQLITE3_MUTEX_INITIALIZER(10), + SQLITE3_MUTEX_INITIALIZER(11), + SQLITE3_MUTEX_INITIALIZER(12), + SQLITE3_MUTEX_INITIALIZER(13) }; sqlite3_mutex *p; switch( iType ){ @@ -180,6 +181,9 @@ static sqlite3_mutex *pthreadMutexAlloc(int iType){ pthread_mutexattr_settype(&recursiveAttr, PTHREAD_MUTEX_RECURSIVE); pthread_mutex_init(&p->mutex, &recursiveAttr); pthread_mutexattr_destroy(&recursiveAttr); +#endif +#if SQLITE_MUTEX_NREF || defined(SQLITE_ENABLE_API_ARMOR) + p->id = SQLITE_MUTEX_RECURSIVE; #endif } break; @@ -188,6 +192,9 @@ static sqlite3_mutex *pthreadMutexAlloc(int iType){ p = sqlite3MallocZero( sizeof(*p) ); if( p ){ pthread_mutex_init(&p->mutex, 0); +#if SQLITE_MUTEX_NREF || defined(SQLITE_ENABLE_API_ARMOR) + p->id = SQLITE_MUTEX_FAST; +#endif } break; } @@ -203,7 +210,7 @@ static sqlite3_mutex *pthreadMutexAlloc(int iType){ } } #if SQLITE_MUTEX_NREF || defined(SQLITE_ENABLE_API_ARMOR) - if( p ) p->id = iType; + assert( p==0 || p->id==iType ); #endif return p; } From 091881bbb030581af513b879bb8adc2878d63592 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Sun, 18 Feb 2018 00:54:06 +0000 Subject: [PATCH 072/107] Port mutex enhancments from check-in [f53b8a573b] to the Win32 implementation. FossilOrigin-Name: 74bb7225d132c80fd5758bb8c120448e3b3e951d0ca2fa0c57cac0a9c6c27045 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/mutex_w32.c | 34 +++++++++++++++++----------------- 3 files changed, 25 insertions(+), 25 deletions(-) diff --git a/manifest b/manifest index 3d3100acc3..7d33baf3f3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sdata\srace\scausing\sa\stsan\scomplaint\swith\sSQLITE_ENABLE_API_ARMOR\sbuilds\non\sunix.\sThe\srace\scondition\sis\snot\sdangerous\sin\spractice,\sit\sjust\supsets\stsan. -D 2018-02-17T07:38:50.155 +C Port\smutex\senhancments\sfrom\scheck-in\s[f53b8a573b]\sto\sthe\sWin32\simplementation. +D 2018-02-18T00:54:06.860 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -466,7 +466,7 @@ F src/mutex.c b021263554c8a3995e9d53193b8194b96d1ed28e06c3b532dd7f7d29cf0c7d53 F src/mutex.h 779d588e3b7756ec3ecf7d78cde1d84aba414f85 F src/mutex_noop.c 9d4309c075ba9cc7249e19412d3d62f7f94839c4 F src/mutex_unix.c aaf9ebc3f89df28483c52208497a99a02cc3650011422fc9d4c57e4392f7fe58 -F src/mutex_w32.c a898fa969823b100c0f5fdc57e54c9a1e419ab4d +F src/mutex_w32.c 7670d770c94bbfe8289bec9d7f1394c5a00a57c37f892aab6b6612d085255235 F src/notify.c 9711a7575036f0d3040ba61bc6e217f13a9888e7 F src/os.c 22d31db3ca5a96a408fbf1ceeaaebcaf64c87024d2ff9fe1cf2ddbec3e75c104 F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432 @@ -1707,7 +1707,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b918d4b4e546d3903ff20efc3c8ca26dd9761cc8ed9ef7d2799b66ff9ae1ae51 -R f2c9c538fa5937452e735a3a9fd538f8 -U dan -Z 36b8ef4562da3a84a51e82bfb896975f +P f53b8a573bfbb48780243d56ca8372165adb1b979731c43d46cd1f8eb7e593f3 +R 46ee6e1b6f9dc8c46dcea8bb3cc2eb52 +U mistachkin +Z 55a8fbd8babcd295b13a70346a882792 diff --git a/manifest.uuid b/manifest.uuid index caa3d48cb7..28ef8a23c6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f53b8a573bfbb48780243d56ca8372165adb1b979731c43d46cd1f8eb7e593f3 \ No newline at end of file +74bb7225d132c80fd5758bb8c120448e3b3e951d0ca2fa0c57cac0a9c6c27045 \ No newline at end of file diff --git a/src/mutex_w32.c b/src/mutex_w32.c index 9da93cf319..8a8ae289ba 100644 --- a/src/mutex_w32.c +++ b/src/mutex_w32.c @@ -40,7 +40,7 @@ struct sqlite3_mutex { #ifdef SQLITE_DEBUG volatile int nRef; /* Number of enterances */ volatile DWORD owner; /* Thread holding this mutex */ - volatile int trace; /* True to trace changes */ + volatile LONG trace; /* True to trace changes */ #endif }; @@ -52,10 +52,10 @@ struct sqlite3_mutex { #define SQLITE_W32_MUTEX_INITIALIZER { 0 } #ifdef SQLITE_DEBUG -#define SQLITE3_MUTEX_INITIALIZER { SQLITE_W32_MUTEX_INITIALIZER, 0, \ +#define SQLITE3_MUTEX_INITIALIZER(id) { SQLITE_W32_MUTEX_INITIALIZER, id, \ 0L, (DWORD)0, 0 } #else -#define SQLITE3_MUTEX_INITIALIZER { SQLITE_W32_MUTEX_INITIALIZER, 0 } +#define SQLITE3_MUTEX_INITIALIZER(id) { SQLITE_W32_MUTEX_INITIALIZER, id } #endif #ifdef SQLITE_DEBUG @@ -98,18 +98,18 @@ void sqlite3MemoryBarrier(void){ ** Initialize and deinitialize the mutex subsystem. */ static sqlite3_mutex winMutex_staticMutexes[] = { - SQLITE3_MUTEX_INITIALIZER, - SQLITE3_MUTEX_INITIALIZER, - SQLITE3_MUTEX_INITIALIZER, - SQLITE3_MUTEX_INITIALIZER, - SQLITE3_MUTEX_INITIALIZER, - SQLITE3_MUTEX_INITIALIZER, - SQLITE3_MUTEX_INITIALIZER, - SQLITE3_MUTEX_INITIALIZER, - SQLITE3_MUTEX_INITIALIZER, - SQLITE3_MUTEX_INITIALIZER, - SQLITE3_MUTEX_INITIALIZER, - SQLITE3_MUTEX_INITIALIZER + SQLITE3_MUTEX_INITIALIZER(2), + SQLITE3_MUTEX_INITIALIZER(3), + SQLITE3_MUTEX_INITIALIZER(4), + SQLITE3_MUTEX_INITIALIZER(5), + SQLITE3_MUTEX_INITIALIZER(6), + SQLITE3_MUTEX_INITIALIZER(7), + SQLITE3_MUTEX_INITIALIZER(8), + SQLITE3_MUTEX_INITIALIZER(9), + SQLITE3_MUTEX_INITIALIZER(10), + SQLITE3_MUTEX_INITIALIZER(11), + SQLITE3_MUTEX_INITIALIZER(12), + SQLITE3_MUTEX_INITIALIZER(13) }; static int winMutex_isInit = 0; @@ -239,15 +239,15 @@ static sqlite3_mutex *winMutexAlloc(int iType){ } #endif p = &winMutex_staticMutexes[iType-2]; - p->id = iType; #ifdef SQLITE_DEBUG #ifdef SQLITE_WIN32_MUTEX_TRACE_STATIC - p->trace = 1; + InterlockedCompareExchange(&p->trace, 1, 0); #endif #endif break; } } + assert( p==0 || p->id==iType ); return p; } From e28eb64d09e3615d2695c2f6972d0c01b472bc2f Mon Sep 17 00:00:00 2001 From: drh Date: Sun, 18 Feb 2018 17:50:03 +0000 Subject: [PATCH 073/107] Fix a memory leak in the processing of nested row values. This problem has existed every since row values support was added (version 3.15.0, 2016-10-14) but was only just now detected by OSSFuzz. FossilOrigin-Name: 2df6bbf1b8ca881c8a465d6624de66fde4c5975ccae6b2f2dda392b137f577de --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/whereexpr.c | 2 +- test/rowvalue.test | 6 ++++++ 4 files changed, 16 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 7d33baf3f3..646fefe6c5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Port\smutex\senhancments\sfrom\scheck-in\s[f53b8a573b]\sto\sthe\sWin32\simplementation. -D 2018-02-18T00:54:06.860 +C Fix\sa\smemory\sleak\sin\sthe\sprocessing\sof\snested\srow\svalues.\s\sThis\sproblem\shas\nexisted\severy\ssince\srow\svalues\ssupport\swas\sadded\s(version\s3.15.0,\s2016-10-14)\nbut\swas\sonly\sjust\snow\sdetected\sby\sOSSFuzz. +D 2018-02-18T17:50:03.135 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -574,7 +574,7 @@ F src/walker.c da987a20d40145c0a03c07d8fefcb2ed363becc7680d0500d9c79915591f5b1f F src/where.c 7cae47e813393d70c6d327fdf000fcb30f76b1b0b5a5b52ff6402e0c658de32c F src/whereInt.h 82c04c5075308abbac59180c8bad5ecb45b07453981f60a53f3c7dee21e1e971 F src/wherecode.c e1aaadd8fec650037cfbf27d1b3470338fb3b58fec34d11082df16fe9a08fbd7 -F src/whereexpr.c 22dbfd3bf5f6051a61523dd0ebef7a944fb29ee4aa7d2a62feb8aac6ffbbc0eb +F src/whereexpr.c 53532be687e12f3cd314f1e204cd4fbdac7ad250e918a182b048121e16e828ae F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test a6d901b436328bd67a79b41bb0ac2663918fe3bd F test/affinity3.test 6a101af2fc945ce2912f6fe54dd646018551710d @@ -1166,7 +1166,7 @@ F test/rollbackfault.test 0e646aeab8840c399cfbfa43daab46fd609cf04a F test/rowallock.test 3f88ec6819489d0b2341c7a7528ae17c053ab7cc F test/rowhash.test 0bc1d31415e4575d10cacf31e1a66b5cc0f8be81 F test/rowid.test 5b7509f384f4f6fae1af3c8c104c8ca299fea18d -F test/rowvalue.test 0bc33483f2ef5e69ff4bdd2ae58e36fc598bfd1605fb718c8329bcfc0c10cfd1 +F test/rowvalue.test 32861d6a933ded868035f2ec79aeb993a2a46eb7a6d282ae13415a4c2e369463 F test/rowvalue2.test 060d238b7e5639a7c5630cb5e63e311b44efef2b F test/rowvalue3.test 3068f508753af69884b12125995f023da0dbb256 F test/rowvalue4.test 4b556d7de161a0dd8cff095c336e913986398bea @@ -1707,7 +1707,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f53b8a573bfbb48780243d56ca8372165adb1b979731c43d46cd1f8eb7e593f3 -R 46ee6e1b6f9dc8c46dcea8bb3cc2eb52 -U mistachkin -Z 55a8fbd8babcd295b13a70346a882792 +P 74bb7225d132c80fd5758bb8c120448e3b3e951d0ca2fa0c57cac0a9c6c27045 +R 31bbf28c288f447f1b599c906d481780 +U drh +Z 61189c02e04bfbbf029a1ddaa686693e diff --git a/manifest.uuid b/manifest.uuid index 28ef8a23c6..cc0ffaa4fc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -74bb7225d132c80fd5758bb8c120448e3b3e951d0ca2fa0c57cac0a9c6c27045 \ No newline at end of file +2df6bbf1b8ca881c8a465d6624de66fde4c5975ccae6b2f2dda392b137f577de \ No newline at end of file diff --git a/src/whereexpr.c b/src/whereexpr.c index 82cc519650..313c5ee9bc 100644 --- a/src/whereexpr.c +++ b/src/whereexpr.c @@ -1291,7 +1291,7 @@ static void exprAnalyze( exprAnalyze(pSrc, pWC, idxNew); } pTerm = &pWC->a[idxTerm]; - pTerm->wtFlags = TERM_CODED|TERM_VIRTUAL; /* Disable the original */ + pTerm->wtFlags |= TERM_CODED|TERM_VIRTUAL; /* Disable the original */ pTerm->eOperator = 0; } diff --git a/test/rowvalue.test b/test/rowvalue.test index 5805178301..00d9395331 100644 --- a/test/rowvalue.test +++ b/test/rowvalue.test @@ -540,4 +540,10 @@ do_execsql_test 19.36 { SELECT * FROM t1 WHERE (3,32)>=(a,b) ORDER BY a DESC; } {2 22 1 11} +# 2018-02-18: Memory leak nexted row-value. Detected by OSSFuzz. +# +do_catchsql_test 20.1 { + SELECT 1 WHERE (2,(2,0)) IS (2,(2,0)); +} {0 1} + finish_test From a224ee2a1b12450014aa4ae1e0e85316efcf39c4 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 19 Feb 2018 13:53:56 +0000 Subject: [PATCH 074/107] Fix an assert so that it compares two CellInfo objects field by field instead of using memcmp(). Memcmp() does not work on x86 because of uninitialized padding bytes. FossilOrigin-Name: 88258770adead70fa101c74e266a37bb9aaffac0ba738a4b345617feb8c46477 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 10 +++++++++- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 646fefe6c5..d73a577eff 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\smemory\sleak\sin\sthe\sprocessing\sof\snested\srow\svalues.\s\sThis\sproblem\shas\nexisted\severy\ssince\srow\svalues\ssupport\swas\sadded\s(version\s3.15.0,\s2016-10-14)\nbut\swas\sonly\sjust\snow\sdetected\sby\sOSSFuzz. -D 2018-02-18T17:50:03.135 +C Fix\san\sassert\sso\sthat\sit\scompares\stwo\sCellInfo\sobjects\sfield\sby\sfield\ninstead\sof\susing\smemcmp().\s\sMemcmp()\sdoes\snot\swork\son\sx86\nbecause\sof\suninitialized\spadding\sbytes. +D 2018-02-19T13:53:56.396 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -430,7 +430,7 @@ F src/auth.c 6277d63837357549fe14e723490d6dc1a38768d71c795c5eb5c0f8a99f918f73 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca -F src/btree.c f7bb37c27b3734d58796febb30b5d818b3b4eb592b57b88039e5d92b3ebd9dab +F src/btree.c 18a53540aa35dbdf77f715ea928422a4ed9011dc16ea7b50f803fd1617fcc4f5 F src/btree.h 0866c0a08255142ea0e754aabd211c843cab32045c978a592a43152405ed0c84 F src/btreeInt.h 620ab4c7235f43572cf3ac2ac8723cbdf68073be4d29da24897c7b77dda5fd96 F src/build.c 672022c06e1a5c2653f80c77a687de11f7e65ce81d20fe2825aadfa13a875c33 @@ -1707,7 +1707,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 74bb7225d132c80fd5758bb8c120448e3b3e951d0ca2fa0c57cac0a9c6c27045 -R 31bbf28c288f447f1b599c906d481780 +P 2df6bbf1b8ca881c8a465d6624de66fde4c5975ccae6b2f2dda392b137f577de +R e03e92857318b640fae3aae1181693fa U drh -Z 61189c02e04bfbbf029a1ddaa686693e +Z 8b8a951294db4a887bc90d59e44c7680 diff --git a/manifest.uuid b/manifest.uuid index cc0ffaa4fc..d969c3165e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2df6bbf1b8ca881c8a465d6624de66fde4c5975ccae6b2f2dda392b137f577de \ No newline at end of file +88258770adead70fa101c74e266a37bb9aaffac0ba738a4b345617feb8c46477 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index d3d2a41e26..22f8a50e59 100644 --- a/src/btree.c +++ b/src/btree.c @@ -4388,11 +4388,19 @@ int sqlite3BtreeCloseCursor(BtCursor *pCur){ ** Using this cache reduces the number of calls to btreeParseCell(). */ #ifndef NDEBUG + static int cellInfoEqual(CellInfo *a, CellInfo *b){ + if( a->nKey!=b->nKey ) return 0; + if( a->pPayload!=b->pPayload ) return 0; + if( a->nPayload!=b->nPayload ) return 0; + if( a->nLocal!=b->nLocal ) return 0; + if( a->nSize!=b->nSize ) return 0; + return 1; + } static void assertCellInfo(BtCursor *pCur){ CellInfo info; memset(&info, 0, sizeof(info)); btreeParseCell(pCur->pPage, pCur->ix, &info); - assert( CORRUPT_DB || memcmp(&info, &pCur->info, sizeof(info))==0 ); + assert( CORRUPT_DB || cellInfoEqual(&info, &pCur->info) ); } #else #define assertCellInfo(x) From 26f625fbf4b293012f2a3ff4b8127faadc787c66 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 19 Feb 2018 16:34:31 +0000 Subject: [PATCH 075/107] Only try to use the geteuid() interface on unix if HAVE_FCHOWN is defined. This fixes the build for vxWorks, we are told. FossilOrigin-Name: 38f654dc117bb11d61407327884b5c913cfaf920547bebc4ce5a4cf983e5c20e --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_unix.c | 4 ++++ 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index d73a577eff..ea203fe2f3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sassert\sso\sthat\sit\scompares\stwo\sCellInfo\sobjects\sfield\sby\sfield\ninstead\sof\susing\smemcmp().\s\sMemcmp()\sdoes\snot\swork\son\sx86\nbecause\sof\suninitialized\spadding\sbytes. -D 2018-02-19T13:53:56.396 +C Only\stry\sto\suse\sthe\sgeteuid()\sinterface\son\sunix\sif\sHAVE_FCHOWN\sis\sdefined.\nThis\sfixes\sthe\sbuild\sfor\svxWorks,\swe\sare\stold. +D 2018-02-19T16:34:31.891 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -472,7 +472,7 @@ F src/os.c 22d31db3ca5a96a408fbf1ceeaaebcaf64c87024d2ff9fe1cf2ddbec3e75c104 F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 -F src/os_unix.c ce491421b3a54b63094a155eeac668a3bc8e5b86a5a58551d906e5b5affb443f +F src/os_unix.c 9e02186f0dcde77d3ec27d4409664eb692c4e8ea0257e30c0b7f25969fa69fec F src/os_win.c eb03c6d52f893bcd7fdd4c6006674c13c1b5e49543fec98d605201af2997171c F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 0b6bd5442733b2e08d0673de6cdafe3e7ab0b5715e4844ac836ab346b1d9ed89 @@ -1707,7 +1707,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 2df6bbf1b8ca881c8a465d6624de66fde4c5975ccae6b2f2dda392b137f577de -R e03e92857318b640fae3aae1181693fa +P 88258770adead70fa101c74e266a37bb9aaffac0ba738a4b345617feb8c46477 +R 2e4bdb0c045376ff7154d29ff239bf61 U drh -Z 8b8a951294db4a887bc90d59e44c7680 +Z 37b9f8fa92af1d03e78cb9f20faaf7e7 diff --git a/manifest.uuid b/manifest.uuid index d969c3165e..f3e38fcfac 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -88258770adead70fa101c74e266a37bb9aaffac0ba738a4b345617feb8c46477 \ No newline at end of file +38f654dc117bb11d61407327884b5c913cfaf920547bebc4ce5a4cf983e5c20e \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index b24c6861d3..cb2ae33d0a 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -468,7 +468,11 @@ static struct unix_syscall { #endif #define osFchown ((int(*)(int,uid_t,gid_t))aSyscall[20].pCurrent) +#if defined(HAVE_FCHOWN) { "geteuid", (sqlite3_syscall_ptr)geteuid, 0 }, +#else + { "geteuid", (sqlite3_syscall_ptr)0, 0 }, +#endif #define osGeteuid ((uid_t(*)(void))aSyscall[21].pCurrent) #if !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 From 62856465f2350ccba3aa95798e6af543e466f521 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 19 Feb 2018 17:03:23 +0000 Subject: [PATCH 076/107] In the printf() library, measure width and precision in characters rather than bytes if the "!" (alternate-form-2) flag is present on a %s or %z substitution. FossilOrigin-Name: ca31c6630422fca70e626dd38aae96296bd8535d491ca52391624a5e7e663636 --- manifest | 15 +++++++++------ manifest.uuid | 2 +- src/printf.c | 24 +++++++++++++++++++++--- 3 files changed, 31 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index ea203fe2f3..51334ef416 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Only\stry\sto\suse\sthe\sgeteuid()\sinterface\son\sunix\sif\sHAVE_FCHOWN\sis\sdefined.\nThis\sfixes\sthe\sbuild\sfor\svxWorks,\swe\sare\stold. -D 2018-02-19T16:34:31.891 +C In\sthe\sprintf()\slibrary,\smeasure\swidth\sand\sprecision\sin\scharacters\srather\nthan\sbytes\sif\sthe\s"!"\s(alternate-form-2)\sflag\sis\spresent\son\sa\s%s\sor\s%z\nsubstitution. +D 2018-02-19T17:03:23.575 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -484,7 +484,7 @@ F src/pcache1.c 716975564c15eb6679e97f734cec1bfd6c16ac3d4010f05f1f8e509fc7d19880 F src/pragma.c bea56df3ae0637768c0da4fbbb8f2492f780980d95000034a105ff291bf7ca69 F src/pragma.h bb83728944b42f6d409c77f5838a8edbdb0fe83046c5496ffc9602b40340a324 F src/prepare.c 259f4e7960c47082c9653f3d5f0c294abd68bb9c3aab86de7630700cba1c20fb -F src/printf.c 9506b4b96e59c0467047155f09015750cb2878aeda3d39e5610c1192ddc3c41c +F src/printf.c a94897345b3228d97c6aa06297890b7c62bc47acc86e34d3634d125426d78695 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac @@ -1707,7 +1707,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 88258770adead70fa101c74e266a37bb9aaffac0ba738a4b345617feb8c46477 -R 2e4bdb0c045376ff7154d29ff239bf61 +P 38f654dc117bb11d61407327884b5c913cfaf920547bebc4ce5a4cf983e5c20e +R 07cd50bae7c4080dd411c728ce9cf3a0 +T *branch * printf-enhancement +T *sym-printf-enhancement * +T -sym-trunk * U drh -Z 37b9f8fa92af1d03e78cb9f20faaf7e7 +Z 9d6ee38fe0b61a32cc5a7309de027572 diff --git a/manifest.uuid b/manifest.uuid index f3e38fcfac..29c0e9958e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -38f654dc117bb11d61407327884b5c913cfaf920547bebc4ce5a4cf983e5c20e \ No newline at end of file +ca31c6630422fca70e626dd38aae96296bd8535d491ca52391624a5e7e663636 \ No newline at end of file diff --git a/src/printf.c b/src/printf.c index ca8d26e4f9..f6980c90db 100644 --- a/src/printf.c +++ b/src/printf.c @@ -654,10 +654,25 @@ void sqlite3VXPrintf( zExtra = bufpt; } if( precision>=0 ){ - for(length=0; length 0 && z[0] ){ + SQLITE_SKIP_UTF8(z); + } + length = (int)(z - (unsigned char*)bufpt); + }else{ + for(length=0; length0 ){ + /* Adjust width to account for extra bytes in UTF-8 characters */ + int ii = length - 1; + while( ii>=0 ) if( (bufpt[ii--] & 0xc0)==0x80 ) width++; + } break; case etSQLESCAPE: /* Escape ' characters */ case etSQLESCAPE2: /* Escape ' and enclose in '...' */ @@ -700,7 +715,7 @@ void sqlite3VXPrintf( if( needQuote ) bufpt[j++] = q; bufpt[j] = 0; length = j; - /* The precision in %q and %Q means how many input characters to + /* The precision in %q and %Q means how many input bytes to ** consume, not the length of the output... ** if( precision>=0 && precision0 ){ From 57e3ba7623c1d819399f779e83372fea1727b9bf Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 19 Feb 2018 18:03:10 +0000 Subject: [PATCH 077/107] Make the alternate-form-2 flag ("!") change the meaning of width and precision from bytes to characters for the %q, %Q, and %w extensions of printf(). FossilOrigin-Name: 391540acbea34eb88c75b1152b458d8936b3fdfff47633b9ec897775b015ba4c --- manifest | 15 ++++++--------- manifest.uuid | 2 +- src/printf.c | 20 +++++++++++++------- 3 files changed, 20 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index 51334ef416..867a07179e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\sprintf()\slibrary,\smeasure\swidth\sand\sprecision\sin\scharacters\srather\nthan\sbytes\sif\sthe\s"!"\s(alternate-form-2)\sflag\sis\spresent\son\sa\s%s\sor\s%z\nsubstitution. -D 2018-02-19T17:03:23.575 +C Make\sthe\salternate-form-2\sflag\s("!")\schange\sthe\smeaning\sof\swidth\sand\sprecision\nfrom\sbytes\sto\scharacters\sfor\sthe\s%q,\s%Q,\sand\s%w\sextensions\sof\sprintf(). +D 2018-02-19T18:03:10.959 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -484,7 +484,7 @@ F src/pcache1.c 716975564c15eb6679e97f734cec1bfd6c16ac3d4010f05f1f8e509fc7d19880 F src/pragma.c bea56df3ae0637768c0da4fbbb8f2492f780980d95000034a105ff291bf7ca69 F src/pragma.h bb83728944b42f6d409c77f5838a8edbdb0fe83046c5496ffc9602b40340a324 F src/prepare.c 259f4e7960c47082c9653f3d5f0c294abd68bb9c3aab86de7630700cba1c20fb -F src/printf.c a94897345b3228d97c6aa06297890b7c62bc47acc86e34d3634d125426d78695 +F src/printf.c ab91b63394bf07a430af0c644f8e42ca136c015e2533544f585dab8b0a408b84 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac @@ -1707,10 +1707,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 38f654dc117bb11d61407327884b5c913cfaf920547bebc4ce5a4cf983e5c20e -R 07cd50bae7c4080dd411c728ce9cf3a0 -T *branch * printf-enhancement -T *sym-printf-enhancement * -T -sym-trunk * +P ca31c6630422fca70e626dd38aae96296bd8535d491ca52391624a5e7e663636 +R d4aa6a8346f9339a7533b828b4137a72 U drh -Z 9d6ee38fe0b61a32cc5a7309de027572 +Z bc6f530d4e962e8540ac43cd25813033 diff --git a/manifest.uuid b/manifest.uuid index 29c0e9958e..50989e81be 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ca31c6630422fca70e626dd38aae96296bd8535d491ca52391624a5e7e663636 \ No newline at end of file +391540acbea34eb88c75b1152b458d8936b3fdfff47633b9ec897775b015ba4c \ No newline at end of file diff --git a/src/printf.c b/src/printf.c index f6980c90db..ff948ff5a4 100644 --- a/src/printf.c +++ b/src/printf.c @@ -668,15 +668,16 @@ void sqlite3VXPrintf( }else{ length = 0x7fffffff & (int)strlen(bufpt); } + adjust_width_for_utf8: if( flag_altform2 && width>0 ){ /* Adjust width to account for extra bytes in UTF-8 characters */ int ii = length - 1; while( ii>=0 ) if( (bufpt[ii--] & 0xc0)==0x80 ) width++; } break; - case etSQLESCAPE: /* Escape ' characters */ - case etSQLESCAPE2: /* Escape ' and enclose in '...' */ - case etSQLESCAPE3: { /* Escape " characters */ + case etSQLESCAPE: /* %q: Escape ' characters */ + case etSQLESCAPE2: /* %Q: Escape ' and enclose in '...' */ + case etSQLESCAPE3: { /* %w: Escape " characters */ int i, j, k, n, isnull; int needQuote; char ch; @@ -690,9 +691,17 @@ void sqlite3VXPrintf( } isnull = escarg==0; if( isnull ) escarg = (xtype==etSQLESCAPE2 ? "NULL" : "(NULL)"); + /* For %q, %Q, and %w, the precision is the number of byte (or + ** characters if the ! flags is present) to use from the input. + ** Because of the extra quoting characters inserted, the number + ** of output characters may be larger than the precision. + */ k = precision; for(i=n=0; k!=0 && (ch=escarg[i])!=0; i++, k--){ if( ch==q ) n++; + if( flag_altform2 && (ch&0xc0)==0xc0 ){ + while( (escarg[i+1]&0xc0)==0x80 ){ i++; } + } } needQuote = !isnull && xtype==etSQLESCAPE2; n += i + 3; @@ -715,10 +724,7 @@ void sqlite3VXPrintf( if( needQuote ) bufpt[j++] = q; bufpt[j] = 0; length = j; - /* The precision in %q and %Q means how many input bytes to - ** consume, not the length of the output... - ** if( precision>=0 && precision Date: Mon, 19 Feb 2018 18:56:52 +0000 Subject: [PATCH 078/107] Fix the %c substitution in printf() so that it works with unicode characters. FossilOrigin-Name: c35be1d9e9132fbadc74d207c1088e5a710a151c0835c38a63bf3c8e6fd711a0 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/printf.c | 38 +++++++++++++++++++++++++++++++++----- 3 files changed, 40 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 867a07179e..151cc25b6f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\sthe\salternate-form-2\sflag\s("!")\schange\sthe\smeaning\sof\swidth\sand\sprecision\nfrom\sbytes\sto\scharacters\sfor\sthe\s%q,\s%Q,\sand\s%w\sextensions\sof\sprintf(). -D 2018-02-19T18:03:10.959 +C Fix\sthe\s%c\ssubstitution\sin\sprintf()\sso\sthat\sit\sworks\swith\sunicode\scharacters. +D 2018-02-19T18:56:52.191 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -484,7 +484,7 @@ F src/pcache1.c 716975564c15eb6679e97f734cec1bfd6c16ac3d4010f05f1f8e509fc7d19880 F src/pragma.c bea56df3ae0637768c0da4fbbb8f2492f780980d95000034a105ff291bf7ca69 F src/pragma.h bb83728944b42f6d409c77f5838a8edbdb0fe83046c5496ffc9602b40340a324 F src/prepare.c 259f4e7960c47082c9653f3d5f0c294abd68bb9c3aab86de7630700cba1c20fb -F src/printf.c ab91b63394bf07a430af0c644f8e42ca136c015e2533544f585dab8b0a408b84 +F src/printf.c 721a26a112e895ec7c7388fab9e8a1eb1c3fe55493cd14a19eb26ffd8ea9798e F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac @@ -1707,7 +1707,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ca31c6630422fca70e626dd38aae96296bd8535d491ca52391624a5e7e663636 -R d4aa6a8346f9339a7533b828b4137a72 +P 391540acbea34eb88c75b1152b458d8936b3fdfff47633b9ec897775b015ba4c +R 71fe5339e526e4b9a6fb2ef9bc7b6d4e U drh -Z bc6f530d4e962e8540ac43cd25813033 +Z 27f720e6a7a3028447e586595bfbe60c diff --git a/manifest.uuid b/manifest.uuid index 50989e81be..34e844ad2e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -391540acbea34eb88c75b1152b458d8936b3fdfff47633b9ec897775b015ba4c \ No newline at end of file +c35be1d9e9132fbadc74d207c1088e5a710a151c0835c38a63bf3c8e6fd711a0 \ No newline at end of file diff --git a/src/printf.c b/src/printf.c index ff948ff5a4..ada582ebdf 100644 --- a/src/printf.c +++ b/src/printf.c @@ -624,9 +624,37 @@ void sqlite3VXPrintf( case etCHARX: if( bArgList ){ bufpt = getTextArg(pArgList); - c = bufpt ? bufpt[0] : 0; + length = 0; + if( bufpt ){ + buf[0] = c = *(bufpt++); + length = 1; + if( (c&0xc0)==0xc0 ){ + while( length<4 && (bufpt[0]&0xc0)==0x80 ){ + buf[length++] = *(bufpt++); + } + } + } }else{ - c = va_arg(ap,int); + unsigned int ch = va_arg(ap,unsigned int); + if( ch<0x00080 ){ + buf[0] = ch & 0xff; + length = 1; + }else if( ch<0x00800 ){ + buf[0] = 0xc0 + (u8)((ch>>6)&0x1f); + buf[1] = 0x80 + (u8)(ch & 0x3f); + length = 2; + }else if( ch<0x10000 ){ + buf[0] = 0xe0 + (u8)((ch>>12)&0x0f); + buf[1] = 0x80 + (u8)((ch>>6) & 0x3f); + buf[2] = 0x80 + (u8)(ch & 0x3f); + length = 3; + }else{ + buf[0] = 0xf0 + (u8)((ch>>18) & 0x07); + buf[1] = 0x80 + (u8)((ch>>12) & 0x3f); + buf[2] = 0x80 + (u8)((ch>>6) & 0x3f); + buf[3] = 0x80 + (u8)(ch & 0x3f); + length = 4; + } } if( precision>1 ){ width -= precision-1; @@ -634,10 +662,10 @@ void sqlite3VXPrintf( sqlite3AppendChar(pAccum, width-1, ' '); width = 0; } - sqlite3AppendChar(pAccum, precision-1, c); + while( precision-- > 1 ){ + sqlite3StrAccumAppend(pAccum, buf, length); + } } - length = 1; - buf[0] = c; bufpt = buf; break; case etSTRING: From cf7c8370d19b10b23e6b2360137490a3eddddc02 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 19 Feb 2018 20:23:20 +0000 Subject: [PATCH 079/107] Test cases and a bug fix on the new unicode handling in %c. FossilOrigin-Name: e41d64e95b9b6fec5dc329553822925e7d648a41912b420cfa1ba198736e6bab --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/printf.c | 3 ++- test/printf2.test | 25 +++++++++++++++++++++++++ 4 files changed, 35 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 151cc25b6f..e688a8efc4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\s%c\ssubstitution\sin\sprintf()\sso\sthat\sit\sworks\swith\sunicode\scharacters. -D 2018-02-19T18:56:52.191 +C Test\scases\sand\sa\sbug\sfix\son\sthe\snew\sunicode\shandling\sin\s%c. +D 2018-02-19T20:23:20.986 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -484,7 +484,7 @@ F src/pcache1.c 716975564c15eb6679e97f734cec1bfd6c16ac3d4010f05f1f8e509fc7d19880 F src/pragma.c bea56df3ae0637768c0da4fbbb8f2492f780980d95000034a105ff291bf7ca69 F src/pragma.h bb83728944b42f6d409c77f5838a8edbdb0fe83046c5496ffc9602b40340a324 F src/prepare.c 259f4e7960c47082c9653f3d5f0c294abd68bb9c3aab86de7630700cba1c20fb -F src/printf.c 721a26a112e895ec7c7388fab9e8a1eb1c3fe55493cd14a19eb26ffd8ea9798e +F src/printf.c 1c8986a3ab20f0cdc7bdf1b7f6ea7b9857acdd34eba5c82e0be70fed825d2f9f F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac @@ -1141,7 +1141,7 @@ F test/pragma4.test 3046501bee2f652dc2a4f9c87781e2741361d6864439c8381aba6c3b774b F test/pragma5.test 824ce6ced5d6b7ec71abe37fc6005ff836fe39d638273dc5192b39864b9ee983 F test/pragmafault.test 275edaf3161771d37de60e5c2b412627ac94cef11739236bec12ed1258b240f8 F test/printf.test b3ff34e73d59124140eaf89f7672e21bc2ca5fcc -F test/printf2.test 9e6db85f81c63f2367c34a9d7db384088bd374ad +F test/printf2.test e2bb746fdc8bbf45bbd495ba6f3a8b2256a47428a333274ae01b30bfeb938ca1 F test/progress.test ebab27f670bd0d4eb9d20d49cef96e68141d92fb F test/ptrchng.test ef1aa72d6cf35a2bbd0869a649b744e9d84977fc F test/pushdown.test 5e72c51c5e33253ed639ccee1e01ce62d62b6eee5ca893cd82334e4ee7b1d7fc @@ -1707,7 +1707,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 391540acbea34eb88c75b1152b458d8936b3fdfff47633b9ec897775b015ba4c -R 71fe5339e526e4b9a6fb2ef9bc7b6d4e +P c35be1d9e9132fbadc74d207c1088e5a710a151c0835c38a63bf3c8e6fd711a0 +R fe2c99c67a288621be87056c5aacba15 U drh -Z 27f720e6a7a3028447e586595bfbe60c +Z fd62d4a92d36a20a423e3f08ea4c3a69 diff --git a/manifest.uuid b/manifest.uuid index 34e844ad2e..6a39fe7287 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c35be1d9e9132fbadc74d207c1088e5a710a151c0835c38a63bf3c8e6fd711a0 \ No newline at end of file +e41d64e95b9b6fec5dc329553822925e7d648a41912b420cfa1ba198736e6bab \ No newline at end of file diff --git a/src/printf.c b/src/printf.c index ada582ebdf..725caa2d5e 100644 --- a/src/printf.c +++ b/src/printf.c @@ -667,7 +667,8 @@ void sqlite3VXPrintf( } } bufpt = buf; - break; + flag_altform2 = 1; + goto adjust_width_for_utf8; case etSTRING: case etDYNSTRING: if( bArgList ){ diff --git a/test/printf2.test b/test/printf2.test index d30966d167..c73727d005 100644 --- a/test/printf2.test +++ b/test/printf2.test @@ -148,6 +148,31 @@ do_execsql_test printf2-4.10 { SELECT printf('|%,d|%,d|',1234567890,-1234567890); } {|1,234,567,890|-1,234,567,890|} +# 2018-02-19. Unicode characters with %c +do_execsql_test printf2-5.100 { + SELECT printf('(%8c)',char(11106)); +} {{( ⭢)}} +do_execsql_test printf2-5.101 { + SELECT printf('(%-8c)',char(11106)); +} {{(⭢ )}} +do_execsql_test printf2-5.102 { + SELECT printf('(%5.3c)',char(1492)); +} {{( ההה)}} +do_execsql_test printf2-5.103 { + SELECT printf('(%-5.3c)',char(1492)); +} {{(ההה )}} +do_execsql_test printf2-5.104 { + SELECT printf('(%3.3c)',char(1492)); +} {{(ההה)}} +do_execsql_test printf2-5.105 { + SELECT printf('(%-3.3c)',char(1492)); +} {{(ההה)}} +do_execsql_test printf2-5.104 { + SELECT printf('(%2c)',char(1513)); +} {{( ש)}} +do_execsql_test printf2-5.106 { + SELECT printf('(%-2c)',char(1513)); +} {{(ש )}} finish_test From 57f29b7bfbc0bc69f2c2f0b73e22a5aa0664d13c Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 19 Feb 2018 20:41:01 +0000 Subject: [PATCH 080/107] More test cases for unicode handling in printf. FossilOrigin-Name: ac188211119bfd77f43e4aba8acab733a56aac90b4c8a11f7ef8e6388582438f --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/printf2.test | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 39 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index e688a8efc4..a521ef217b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Test\scases\sand\sa\sbug\sfix\son\sthe\snew\sunicode\shandling\sin\s%c. -D 2018-02-19T20:23:20.986 +C More\stest\scases\sfor\sunicode\shandling\sin\sprintf. +D 2018-02-19T20:41:01.120 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -1141,7 +1141,7 @@ F test/pragma4.test 3046501bee2f652dc2a4f9c87781e2741361d6864439c8381aba6c3b774b F test/pragma5.test 824ce6ced5d6b7ec71abe37fc6005ff836fe39d638273dc5192b39864b9ee983 F test/pragmafault.test 275edaf3161771d37de60e5c2b412627ac94cef11739236bec12ed1258b240f8 F test/printf.test b3ff34e73d59124140eaf89f7672e21bc2ca5fcc -F test/printf2.test e2bb746fdc8bbf45bbd495ba6f3a8b2256a47428a333274ae01b30bfeb938ca1 +F test/printf2.test 30b5dd0b4b992dc5626496846ecce17ff592cacbcb11c3e589f3ac4d7e129dae F test/progress.test ebab27f670bd0d4eb9d20d49cef96e68141d92fb F test/ptrchng.test ef1aa72d6cf35a2bbd0869a649b744e9d84977fc F test/pushdown.test 5e72c51c5e33253ed639ccee1e01ce62d62b6eee5ca893cd82334e4ee7b1d7fc @@ -1707,7 +1707,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c35be1d9e9132fbadc74d207c1088e5a710a151c0835c38a63bf3c8e6fd711a0 -R fe2c99c67a288621be87056c5aacba15 +P e41d64e95b9b6fec5dc329553822925e7d648a41912b420cfa1ba198736e6bab +R c809a7387d4329cd8d06217ca9198d46 U drh -Z fd62d4a92d36a20a423e3f08ea4c3a69 +Z c256c2f346298486ba50767fa31b18cb diff --git a/manifest.uuid b/manifest.uuid index 6a39fe7287..4a4024e523 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e41d64e95b9b6fec5dc329553822925e7d648a41912b420cfa1ba198736e6bab \ No newline at end of file +ac188211119bfd77f43e4aba8acab733a56aac90b4c8a11f7ef8e6388582438f \ No newline at end of file diff --git a/test/printf2.test b/test/printf2.test index c73727d005..998038f88e 100644 --- a/test/printf2.test +++ b/test/printf2.test @@ -174,5 +174,37 @@ do_execsql_test printf2-5.106 { SELECT printf('(%-2c)',char(1513)); } {{(ש )}} +# 2018-02-19. Unicode characters with the "!" flag in %s and friends. +do_execsql_test printf2-6.100 { + SELECT printf('(%!.3s)','הנה מה־טוב ומה־נעים שבת אחים גם־יחד'); +} {(הנה)} +do_execsql_test printf2-6.101 { + SELECT printf('(%.6s)','הנה מה־טוב ומה־נעים שבת אחים גם־יחד'); +} {(הנה)} +do_execsql_test printf2-6.102 { + SELECT printf('(%!5.3s)','הנה מה־טוב ומה־נעים שבת אחים גם־יחד'); +} {{( הנה)}} +do_execsql_test printf2-6.103 { + SELECT printf('(%8.6s)','הנה מה־טוב ומה־נעים שבת אחים גם־יחד'); +} {{( הנה)}} +do_execsql_test printf2-6.104 { + SELECT printf('(%!-5.3s)','הנה מה־טוב ומה־נעים שבת אחים גם־יחד'); +} {{(הנה )}} +do_execsql_test printf2-6.105 { + SELECT printf('(%-8.6s)','הנה מה־טוב ומה־נעים שבת אחים גם־יחד'); +} {{(הנה )}} +do_execsql_test printf2-6.106 { + SELECT printf('(%!.3Q)','הנה מה־טוב ומה־נעים שבת אחים גם־יחד'); +} {('הנה')} +do_execsql_test printf2-6.107 { + SELECT printf('(%.6Q)','הנה מה־טוב ומה־נעים שבת אחים גם־יחד'); +} {('הנה')} +do_execsql_test printf2-6.108 { + SELECT printf('(%!7.3Q)','הנה מה־טוב ומה־נעים שבת אחים גם־יחד'); +} {{( 'הנה')}} +do_execsql_test printf2-6.109 { + SELECT printf('(%10.6Q)','הנה מה־טוב ומה־נעים שבת אחים גם־יחד'); +} {{( 'הנה')}} + finish_test From a15a7c352ce0aad7143e589e32a4d9e60f5ce9e2 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 19 Feb 2018 21:58:16 +0000 Subject: [PATCH 081/107] Printing a value of 0 using %c terminates the string. FossilOrigin-Name: 255612f0a131f2f522cbca3cc5a1edcf7e38938abca25ba421e7e38a422db9c9 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/printf.c | 5 +++-- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index a521ef217b..5fae45a4a2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C More\stest\scases\sfor\sunicode\shandling\sin\sprintf. -D 2018-02-19T20:41:01.120 +C Printing\sa\svalue\sof\s0\susing\s%c\sterminates\sthe\sstring. +D 2018-02-19T21:58:16.694 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -484,7 +484,7 @@ F src/pcache1.c 716975564c15eb6679e97f734cec1bfd6c16ac3d4010f05f1f8e509fc7d19880 F src/pragma.c bea56df3ae0637768c0da4fbbb8f2492f780980d95000034a105ff291bf7ca69 F src/pragma.h bb83728944b42f6d409c77f5838a8edbdb0fe83046c5496ffc9602b40340a324 F src/prepare.c 259f4e7960c47082c9653f3d5f0c294abd68bb9c3aab86de7630700cba1c20fb -F src/printf.c 1c8986a3ab20f0cdc7bdf1b7f6ea7b9857acdd34eba5c82e0be70fed825d2f9f +F src/printf.c 07244a9e1755247c9543db551acd2f3cce461b28d13e9f3ab87053c35fd62717 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac @@ -1707,7 +1707,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e41d64e95b9b6fec5dc329553822925e7d648a41912b420cfa1ba198736e6bab -R c809a7387d4329cd8d06217ca9198d46 +P ac188211119bfd77f43e4aba8acab733a56aac90b4c8a11f7ef8e6388582438f +R 508c3db0e2913d65a43c01bb9b80f5ab U drh -Z c256c2f346298486ba50767fa31b18cb +Z 626afeea462f05f2075e81e36cf30b70 diff --git a/manifest.uuid b/manifest.uuid index 4a4024e523..3b60577eae 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ac188211119bfd77f43e4aba8acab733a56aac90b4c8a11f7ef8e6388582438f \ No newline at end of file +255612f0a131f2f522cbca3cc5a1edcf7e38938abca25ba421e7e38a422db9c9 \ No newline at end of file diff --git a/src/printf.c b/src/printf.c index 725caa2d5e..11d5b82828 100644 --- a/src/printf.c +++ b/src/printf.c @@ -624,15 +624,16 @@ void sqlite3VXPrintf( case etCHARX: if( bArgList ){ bufpt = getTextArg(pArgList); - length = 0; + length = 1; if( bufpt ){ buf[0] = c = *(bufpt++); - length = 1; if( (c&0xc0)==0xc0 ){ while( length<4 && (bufpt[0]&0xc0)==0x80 ){ buf[length++] = *(bufpt++); } } + }else{ + buf[0] = 0; } }else{ unsigned int ch = va_arg(ap,unsigned int); From b0b6f8783c8a80fbf68e38a3230182edeacef306 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 20 Feb 2018 13:46:20 +0000 Subject: [PATCH 082/107] Remove a lot of the text describing extended format options from the documentation on sqlite3_mprintf() and friends, since that information is now covered by the separate printf.html document. Provide links to that other document. No changes to code. FossilOrigin-Name: 99eec556f065ad19548e48d1f4ae0a3767b4e82e1c83fa2365062e3c5e0071fb --- manifest | 13 ++++----- manifest.uuid | 2 +- src/sqlite.h.in | 78 +++++-------------------------------------------- 3 files changed, 14 insertions(+), 79 deletions(-) diff --git a/manifest b/manifest index 2e00962c43..9788e54316 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\sthe\sstring\sformatter\s(used\sby\sprintf())\sso\sthat\sthe\swidth\sand\nprecision\sof\sstring\ssubstitution\soperators\srefer\sto\scharacters\sinstead\sof\nbytes\swhen\sthe\salternate-form-2\sflag\s("!")\sis\sused.\s\sAlso\sfix\sthe\s%c\nsubstition\sto\salways\swork\swithin\sunicode,\sregardless\sof\sthe\s\nalternate-form-2\sflag. -D 2018-02-19T22:46:19.020 +C Remove\sa\slot\sof\sthe\stext\sdescribing\sextended\sformat\soptions\sfrom\sthe\ndocumentation\son\ssqlite3_mprintf()\sand\sfriends,\ssince\sthat\sinformation\sis\nnow\scovered\sby\sthe\sseparate\sprintf.html\sdocument.\s\sProvide\slinks\sto\sthat\nother\sdocument.\s\sNo\schanges\sto\scode. +D 2018-02-20T13:46:20.902 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -490,7 +490,7 @@ F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c f02352ac5cbb6fad3804add825743b924cfb6c742ba2e8666d726828a9985d73 F src/shell.c.in c1b14bc0bc1fe73a97dfaad55ca09f5d6e44f6019977d94e7e3d1861616035e1 -F src/sqlite.h.in 51f9acf52c80113d793ddd38b3940ad6895d97b4752503b19291fb8fcbf54c5e +F src/sqlite.h.in 235e942dd8c01b414c2996828b4d0d2500faf8850f8b24ae17d31c172f519e69 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 83a3c4ce93d650bedfd1aa558cb85a516bd6d094445ee989740827d0d944368d F src/sqliteInt.h fd8943e9d6b8bd04b3a22ef806b843c2c06f022db3f37f1436ac3a9b04e5b8a1 @@ -1707,8 +1707,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 38f654dc117bb11d61407327884b5c913cfaf920547bebc4ce5a4cf983e5c20e 255612f0a131f2f522cbca3cc5a1edcf7e38938abca25ba421e7e38a422db9c9 -R 508c3db0e2913d65a43c01bb9b80f5ab -T +closed 255612f0a131f2f522cbca3cc5a1edcf7e38938abca25ba421e7e38a422db9c9 +P c883c4d33f4cd722982aca17eeb355d19eeed793991d923b5494d173b1d0c73a +R 2031e9cc56213989aa9a3f7b2af81256 U drh -Z 1886f73e61ed30038df9856ac6df181f +Z 242cc6a18e31e977a6ec01c768dc69ec diff --git a/manifest.uuid b/manifest.uuid index 25a24484fc..736ace92ca 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c883c4d33f4cd722982aca17eeb355d19eeed793991d923b5494d173b1d0c73a \ No newline at end of file +99eec556f065ad19548e48d1f4ae0a3767b4e82e1c83fa2365062e3c5e0071fb \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 1a0164150b..a4344935da 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -2496,16 +2496,16 @@ void sqlite3_free_table(char **result); ** ** These routines are work-alikes of the "printf()" family of functions ** from the standard C library. -** These routines understand most of the common K&R formatting options, -** plus some additional non-standard formats, detailed below. -** Note that some of the more obscure formatting options from recent -** C-library standards are omitted from this implementation. +** These routines understand most of the common formatting options from +** the standard library printf() +** plus some additional non-standard formats ([%q], [%Q], [%w], and [%z]). +** See the [built-in printf()] documentation for details. ** ** ^The sqlite3_mprintf() and sqlite3_vmprintf() routines write their -** results into memory obtained from [sqlite3_malloc()]. +** results into memory obtained from [sqlite3_malloc64()]. ** The strings returned by these two routines should be ** released by [sqlite3_free()]. ^Both routines return a -** NULL pointer if [sqlite3_malloc()] is unable to allocate enough +** NULL pointer if [sqlite3_malloc64()] is unable to allocate enough ** memory to hold the resulting string. ** ** ^(The sqlite3_snprintf() routine is similar to "snprintf()" from @@ -2529,71 +2529,7 @@ void sqlite3_free_table(char **result); ** ** ^The sqlite3_vsnprintf() routine is a varargs version of sqlite3_snprintf(). ** -** These routines all implement some additional formatting -** options that are useful for constructing SQL statements. -** All of the usual printf() formatting options apply. In addition, there -** is are "%q", "%Q", "%w" and "%z" options. -** -** ^(The %q option works like %s in that it substitutes a nul-terminated -** string from the argument list. But %q also doubles every '\'' character. -** %q is designed for use inside a string literal.)^ By doubling each '\'' -** character it escapes that character and allows it to be inserted into -** the string. -** -** For example, assume the string variable zText contains text as follows: -** -** 
-**  char *zText = "It's a happy day!";
-**
-** -** One can use this text in an SQL statement as follows: -** -** 
-**  char *zSQL = sqlite3_mprintf("INSERT INTO table VALUES('%q')", zText);
-**  sqlite3_exec(db, zSQL, 0, 0, 0);
-**  sqlite3_free(zSQL);
-**
-** -** Because the %q format string is used, the '\'' character in zText -** is escaped and the SQL generated is as follows: -** -** 
-**  INSERT INTO table1 VALUES('It''s a happy day!')
-**
-** -** This is correct. Had we used %s instead of %q, the generated SQL -** would have looked like this: -** -** 
-**  INSERT INTO table1 VALUES('It's a happy day!');
-**
-** -** This second example is an SQL syntax error. As a general rule you should -** always use %q instead of %s when inserting text into a string literal. -** -** ^(The %Q option works like %q except it also adds single quotes around -** the outside of the total string. Additionally, if the parameter in the -** argument list is a NULL pointer, %Q substitutes the text "NULL" (without -** single quotes).)^ So, for example, one could say: -** -** 
-**  char *zSQL = sqlite3_mprintf("INSERT INTO table VALUES(%Q)", zText);
-**  sqlite3_exec(db, zSQL, 0, 0, 0);
-**  sqlite3_free(zSQL);
-**
-** -** The code above will render a correct SQL statement in the zSQL -** variable even if the zText variable is a NULL pointer. -** -** ^(The "%w" formatting option is like "%q" except that it expects to -** be contained within double-quotes instead of single quotes, and it -** escapes the double-quote character instead of the single-quote -** character.)^ The "%w" formatting option is intended for safely inserting -** table and column names into a constructed SQL statement. -** -** ^(The "%z" formatting option works like "%s" but with the -** addition that after the string has been read and copied into -** the result, [sqlite3_free()] is called on the input string.)^ +** See also: [built-in printf()], [printf() SQL function] */ char *sqlite3_mprintf(const char*,...); char *sqlite3_vmprintf(const char*, va_list); From cc398969e058b006bf87e35700b366977fc8682c Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 20 Feb 2018 15:23:37 +0000 Subject: [PATCH 083/107] Optimize calls to sqlite3_mprintf("%z...") so that they attempt to append text onto the end of the existing memory allocation rather than reallocating and copying. FossilOrigin-Name: 4bc8a48e644562f6e6192f4c6fc4a70f6bb59f8126ed6c6dc876bedf65d74cda --- Makefile.in | 1 + Makefile.msc | 2 +- main.mk | 1 + manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/printf.c | 17 +++++++++++++++++ 6 files changed, 30 insertions(+), 11 deletions(-) diff --git a/Makefile.in b/Makefile.in index 7bc413e843..1766ca580e 100644 --- a/Makefile.in +++ b/Makefile.in @@ -591,6 +591,7 @@ SHELL_OPT += -DSQLITE_INTROSPECTION_PRAGMAS FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1 FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ FUZZCHECK_OPT += -DSQLITE_MAX_MEMORY=50000000 +FUZZCHECK_OPT += -DSQLITE_PRINTF_PRECISION_LIMIT=1000 FUZZCHECK_SRC = $(TOP)/test/fuzzcheck.c $(TOP)/test/ossfuzz.c DBFUZZ_OPT = diff --git a/Makefile.msc b/Makefile.msc index 875aeedeab..af053cd615 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1615,7 +1615,7 @@ SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_RTREE # MPTESTER_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS5 FUZZERSHELL_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 -FUZZCHECK_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ -DSQLITE_MAX_MEMORY=50000000 +FUZZCHECK_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ -DSQLITE_MAX_MEMORY=50000000 -DSQLITE_PRINTF_PRECISION_LIMIT=1000 FUZZCHECK_SRC = $(TOP)\test\fuzzcheck.c $(TOP)\test\ossfuzz.c OSSSHELL_SRC = $(TOP)\test\ossshell.c $(TOP)\test\ossfuzz.c DBFUZZ_COMPILE_OPTS = -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION diff --git a/main.mk b/main.mk index 2eddfc6f27..c5a0aa6cd3 100644 --- a/main.mk +++ b/main.mk @@ -519,6 +519,7 @@ SHELL_OPT += -DSQLITE_INTROSPECTION_PRAGMAS FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1 FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_MEMSYS5 FUZZCHECK_OPT += -DSQLITE_MAX_MEMORY=50000000 +FUZZCHECK_OPT += -DSQLITE_PRINTF_PRECISION_LIMIT=1000 DBFUZZ_OPT = KV_OPT = -DSQLITE_THREADSAFE=0 -DSQLITE_DIRECT_OVERFLOW_READ ST_OPT = -DSQLITE_THREADSAFE=0 diff --git a/manifest b/manifest index 9788e54316..76a5ca6866 100644 --- a/manifest +++ b/manifest @@ -1,10 +1,10 @@ -C Remove\sa\slot\sof\sthe\stext\sdescribing\sextended\sformat\soptions\sfrom\sthe\ndocumentation\son\ssqlite3_mprintf()\sand\sfriends,\ssince\sthat\sinformation\sis\nnow\scovered\sby\sthe\sseparate\sprintf.html\sdocument.\s\sProvide\slinks\sto\sthat\nother\sdocument.\s\sNo\schanges\sto\scode. -D 2018-02-20T13:46:20.902 +C Optimize\scalls\sto\ssqlite3_mprintf("%z...")\sso\sthat\sthey\sattempt\sto\sappend\ntext\sonto\sthe\send\sof\sthe\sexisting\smemory\sallocation\srather\sthan\sreallocating\nand\scopying. +D 2018-02-20T15:23:37.534 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea -F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea +F Makefile.in a2d2fb8d17c39ab5ec52beb27850b903949080848236923f436156b72a958737 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 -F Makefile.msc b5e9eb7212c5a61d4800172c710376d1158fd5eea2edc0be8e05c0c372e767e4 +F Makefile.msc b53ebf2538324c914e4308a6cd44610eb1ae50bdbd17d7ee65f680918e54da88 F README.md d748f58e3ab0fe0307fb4ae0942b415d93dcc4288756e366cc9e7cf8260c093f F VERSION cdf91ac446255ecf3d8f6d8c3ee40d64123235ae5b3cef29d344e61b45ec3759 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -411,7 +411,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk c8c473bd91d553acab3fb0608ddb69fc769c7bcf6d9e258800504bfda86c792b +F main.mk 04969462bfd32c9f08d4a6d40622e8c43c8c5ecfb2ee52ffb5737c5eca4b0c03 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 @@ -484,7 +484,7 @@ F src/pcache1.c 716975564c15eb6679e97f734cec1bfd6c16ac3d4010f05f1f8e509fc7d19880 F src/pragma.c bea56df3ae0637768c0da4fbbb8f2492f780980d95000034a105ff291bf7ca69 F src/pragma.h bb83728944b42f6d409c77f5838a8edbdb0fe83046c5496ffc9602b40340a324 F src/prepare.c 259f4e7960c47082c9653f3d5f0c294abd68bb9c3aab86de7630700cba1c20fb -F src/printf.c 07244a9e1755247c9543db551acd2f3cce461b28d13e9f3ab87053c35fd62717 +F src/printf.c d3b7844ddeb11fbbdd38dd84d09c9c1ac171d21fb038473c3aa97981201cc660 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac @@ -1707,7 +1707,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c883c4d33f4cd722982aca17eeb355d19eeed793991d923b5494d173b1d0c73a -R 2031e9cc56213989aa9a3f7b2af81256 +P 99eec556f065ad19548e48d1f4ae0a3767b4e82e1c83fa2365062e3c5e0071fb +R 7c8988fb832914bacef8f6a6a302789a U drh -Z 242cc6a18e31e977a6ec01c768dc69ec +Z 0178b35e80c45b1a39d6c8cc991625e8 diff --git a/manifest.uuid b/manifest.uuid index 736ace92ca..9279bc5dbb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -99eec556f065ad19548e48d1f4ae0a3767b4e82e1c83fa2365062e3c5e0071fb \ No newline at end of file +4bc8a48e644562f6e6192f4c6fc4a70f6bb59f8126ed6c6dc876bedf65d74cda \ No newline at end of file diff --git a/src/printf.c b/src/printf.c index 11d5b82828..fcbd7fdbe0 100644 --- a/src/printf.c +++ b/src/printf.c @@ -206,6 +206,11 @@ void sqlite3VXPrintf( PrintfArguments *pArgList = 0; /* Arguments for SQLITE_PRINTF_SQLFUNC */ char buf[etBUFSIZE]; /* Conversion buffer */ + /* pAccum never starts out with an empty buffer that was obtained from + ** malloc(). This precondition is required by the mprintf("%z...") + ** optimization. */ + assert( pAccum->nChar>0 || (pAccum->printfFlags&SQLITE_PRINTF_MALLOCED)==0 ); + bufpt = 0; if( (pAccum->printfFlags & SQLITE_PRINTF_SQLFUNC)!=0 ){ pArgList = va_arg(ap, PrintfArguments*); @@ -681,6 +686,18 @@ void sqlite3VXPrintf( if( bufpt==0 ){ bufpt = ""; }else if( xtype==etDYNSTRING ){ + if( pAccum->nChar==0 && pAccum->mxAlloc && width==0 && precision<0 ){ + /* Special optimization for sqlite3_mprintf("%z..."): + ** Extend an existing memory allocation rather than creating + ** a new one. */ + assert( (pAccum->printfFlags&SQLITE_PRINTF_MALLOCED)==0 ); + pAccum->zText = bufpt; + pAccum->nAlloc = sqlite3DbMallocSize(pAccum->db, bufpt); + pAccum->nChar = 0x7fffffff & (int)strlen(bufpt); + pAccum->printfFlags |= SQLITE_PRINTF_MALLOCED; + length = 0; + break; + } zExtra = bufpt; } if( precision>=0 ){ From 3993678d3d5b27dbeb3d9298c71cbda17f74d699 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 20 Feb 2018 16:39:11 +0000 Subject: [PATCH 084/107] Remove a superfluous "#if 1". No logic changes. FossilOrigin-Name: caaffd32001ff44c31f2d6dbe921f9d294f869c2c4a81e943fdf3f43f8d9313c --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/attach.c | 2 -- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 76a5ca6866..26878a1bec 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Optimize\scalls\sto\ssqlite3_mprintf("%z...")\sso\sthat\sthey\sattempt\sto\sappend\ntext\sonto\sthe\send\sof\sthe\sexisting\smemory\sallocation\srather\sthan\sreallocating\nand\scopying. -D 2018-02-20T15:23:37.534 +C Remove\sa\ssuperfluous\s"#if\s1".\s\sNo\slogic\schanges. +D 2018-02-20T16:39:11.853 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in a2d2fb8d17c39ab5ec52beb27850b903949080848236923f436156b72a958737 @@ -425,7 +425,7 @@ F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c cf7a8af45cb0ace672f47a1b29ab24092a9e8cd8d945a9974e3b5d925f548594 F src/analyze.c 6b42e36a5dcc2703a771f2411bd5e99524bd62c7ecde209bb88dfb04c72f046e -F src/attach.c e1c0d72dd0747da84aa1352f8729906b1d8d2a162ba7f6c475980be202b37c46 +F src/attach.c 79cb6b365d79b96b15dd429aa3e4c3b99335039e059baca8df28f1708fbff261 F src/auth.c 6277d63837357549fe14e723490d6dc1a38768d71c795c5eb5c0f8a99f918f73 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33 @@ -1707,7 +1707,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 99eec556f065ad19548e48d1f4ae0a3767b4e82e1c83fa2365062e3c5e0071fb -R 7c8988fb832914bacef8f6a6a302789a +P 4bc8a48e644562f6e6192f4c6fc4a70f6bb59f8126ed6c6dc876bedf65d74cda +R b395f52b369116e5af9544f274df7b1d U drh -Z 0178b35e80c45b1a39d6c8cc991625e8 +Z 0f63b7ed04628440113fb474f826bbce diff --git a/manifest.uuid b/manifest.uuid index 9279bc5dbb..b912daee1d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4bc8a48e644562f6e6192f4c6fc4a70f6bb59f8126ed6c6dc876bedf65d74cda \ No newline at end of file +caaffd32001ff44c31f2d6dbe921f9d294f869c2c4a81e943fdf3f43f8d9313c \ No newline at end of file diff --git a/src/attach.c b/src/attach.c index 67fc4afa82..f3d68553b6 100644 --- a/src/attach.c +++ b/src/attach.c @@ -504,7 +504,6 @@ int sqlite3FixSelect( if( sqlite3FixExpr(pFix, pSelect->pLimit) ){ return 1; } -#if 1 if( pSelect->pWith ){ int i; for(i=0; ipWith->nCte; i++){ @@ -513,7 +512,6 @@ int sqlite3FixSelect( } } } -#endif pSelect = pSelect->pPrior; } return 0; From 7436f1e491c7df4a438a1b9d73f32243362224a0 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 20 Feb 2018 17:02:31 +0000 Subject: [PATCH 085/107] In the speed-check.sh script, make WAL mode the default. Add the --legacy option to do DELETE mode. Continue to use DELETE mode for long-term performance graphs since WAL mode was not always available, but use WAL mode for day-to-day performance measurements. FossilOrigin-Name: cd81613850c0458b4b5b2460b9b38ce7ff6e557f7add9615058bb96f5a344ccd --- manifest | 12 ++++++------ manifest.uuid | 2 +- tool/speed-check.sh | 9 ++++++++- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 26878a1bec..d34c77f35d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sa\ssuperfluous\s"#if\s1".\s\sNo\slogic\schanges. -D 2018-02-20T16:39:11.853 +C In\sthe\sspeed-check.sh\sscript,\smake\sWAL\smode\sthe\sdefault.\s\sAdd\sthe\s--legacy\noption\sto\sdo\sDELETE\smode.\s\sContinue\sto\suse\sDELETE\smode\sfor\slong-term\sperformance\ngraphs\ssince\sWAL\smode\swas\snot\salways\savailable,\sbut\suse\sWAL\smode\sfor\s\nday-to-day\sperformance\smeasurements. +D 2018-02-20T17:02:31.731 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in a2d2fb8d17c39ab5ec52beb27850b903949080848236923f436156b72a958737 @@ -1665,7 +1665,7 @@ F tool/showstat4.c 0682ebea7abf4d3657f53c4a243f2e7eab48eab344ed36a94bb75dcd19a5c F tool/showwal.c ad9d768f96ca6199ad3a8c9562d679680bd032dd01204ea3e5ea6fb931d81847 F tool/soak1.tcl 8d407956e1a45b485a8e072470a3e629a27037fe F tool/spaceanal.tcl 4bfd19aad7eb3ce0372ef0255f58035e0bba4ff5e9acfd763a10c6fb365c8dec -F tool/speed-check.sh c6239c71ba6951ac5d093bbea55aadf019246c8e7a0c079422e62312a6ab1247 +F tool/speed-check.sh 4ff9b095cf1a7643f0264e7fb7d23f0b12b7cce587a9de315877c378e90eeaf4 F tool/speedtest.tcl 06c76698485ccf597b9e7dbb1ac70706eb873355 F tool/speedtest16.c ecb6542862151c3e6509bbc00509b234562ae81e F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff @@ -1707,7 +1707,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4bc8a48e644562f6e6192f4c6fc4a70f6bb59f8126ed6c6dc876bedf65d74cda -R b395f52b369116e5af9544f274df7b1d +P caaffd32001ff44c31f2d6dbe921f9d294f869c2c4a81e943fdf3f43f8d9313c +R fbade0c89f4079ed24b0fd1c0a4dd6f0 U drh -Z 0f63b7ed04628440113fb474f826bbce +Z 870fe7a9ea537077e2a70e18f17526cf diff --git a/manifest.uuid b/manifest.uuid index b912daee1d..d2359b7ccc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -caaffd32001ff44c31f2d6dbe921f9d294f869c2c4a81e943fdf3f43f8d9313c \ No newline at end of file +cd81613850c0458b4b5b2460b9b38ce7ff6e557f7add9615058bb96f5a344ccd \ No newline at end of file diff --git a/tool/speed-check.sh b/tool/speed-check.sh index cc31e6aa0a..fc05ac98ef 100644 --- a/tool/speed-check.sh +++ b/tool/speed-check.sh @@ -40,6 +40,7 @@ BASELINE="trunk" doExplain=0 doCachegrind=1 doVdbeProfile=0 +doWal=1 while test "$1" != ""; do case $1 in --reprepare) @@ -63,8 +64,11 @@ while test "$1" != ""; do --temp) SPEEDTEST_OPTS="$SPEEDTEST_OPTS --temp 6" ;; + --legacy) + doWal=0 + ;; --wal) - SPEEDTEST_OPTS="$SPEEDTEST_OPTS --journal wal" + doWal=1 ;; --size) shift; SIZE=$1 @@ -134,6 +138,9 @@ while test "$1" != ""; do esac shift done +if test $doWal -eq 1; then + SPEEDTEST_OPTS="$SPEEDTEST_OPTS --journal wal" +fi SPEEDTEST_OPTS="$SPEEDTEST_OPTS --size $SIZE" echo "NAME = $NAME" | tee summary-$NAME.txt echo "SPEEDTEST_OPTS = $SPEEDTEST_OPTS" | tee -a summary-$NAME.txt From 2e178d7321265e21fcec226345267d57a61560ca Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 20 Feb 2018 22:20:57 +0000 Subject: [PATCH 086/107] Make the walIndexPage() routine about 3x faster by factoring out the seldom used reallocation logic into a separate subroutine. FossilOrigin-Name: e2b107141cd97bd4ab240748a9ce43fc2ec950ea74610697a4a7a3d7a6441e6b --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/wal.c | 16 +++++++++++++++- 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index d34c77f35d..2255c10023 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\sspeed-check.sh\sscript,\smake\sWAL\smode\sthe\sdefault.\s\sAdd\sthe\s--legacy\noption\sto\sdo\sDELETE\smode.\s\sContinue\sto\suse\sDELETE\smode\sfor\slong-term\sperformance\ngraphs\ssince\sWAL\smode\swas\snot\salways\savailable,\sbut\suse\sWAL\smode\sfor\s\nday-to-day\sperformance\smeasurements. -D 2018-02-20T17:02:31.731 +C Make\sthe\swalIndexPage()\sroutine\sabout\s3x\sfaster\sby\sfactoring\sout\sthe\sseldom\nused\sreallocation\slogic\sinto\sa\sseparate\ssubroutine. +D 2018-02-20T22:20:57.397 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in a2d2fb8d17c39ab5ec52beb27850b903949080848236923f436156b72a958737 @@ -568,7 +568,7 @@ F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c 5a3f464edd64596f601683ed321d12e6fd93c5fb9afdfb3653d6ffd0fee9c48f +F src/wal.c fff06582a9611cd472fe67ba2b37da1ba052b48a41e3ee60ad55e87faf724c75 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a F src/walker.c da987a20d40145c0a03c07d8fefcb2ed363becc7680d0500d9c79915591f5b1f F src/where.c 7cae47e813393d70c6d327fdf000fcb30f76b1b0b5a5b52ff6402e0c658de32c @@ -1707,7 +1707,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P caaffd32001ff44c31f2d6dbe921f9d294f869c2c4a81e943fdf3f43f8d9313c -R fbade0c89f4079ed24b0fd1c0a4dd6f0 +P cd81613850c0458b4b5b2460b9b38ce7ff6e557f7add9615058bb96f5a344ccd +R 1787f3579805a4f44923446da8d3d679 U drh -Z 870fe7a9ea537077e2a70e18f17526cf +Z 192c58e6de32e2b3cf41a7ef956f2494 diff --git a/manifest.uuid b/manifest.uuid index d2359b7ccc..83c33c8bfe 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cd81613850c0458b4b5b2460b9b38ce7ff6e557f7add9615058bb96f5a344ccd \ No newline at end of file +e2b107141cd97bd4ab240748a9ce43fc2ec950ea74610697a4a7a3d7a6441e6b \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 7b49f3dab4..99832dff3d 100644 --- a/src/wal.c +++ b/src/wal.c @@ -554,7 +554,11 @@ struct WalIterator { ** page and SQLITE_OK is returned. If an error (an OOM or VFS error) occurs, ** then an SQLite error code is returned and *ppPage is set to 0. */ -static int walIndexPage(Wal *pWal, int iPage, volatile u32 **ppPage){ +static SQLITE_NOINLINE int walIndexPageRealloc( + Wal *pWal, /* The WAL context */ + int iPage, /* The page we seek */ + volatile u32 **ppPage /* Write the page pointer here */ +){ int rc = SQLITE_OK; /* Enlarge the pWal->apWiData[] array if required */ @@ -596,6 +600,16 @@ static int walIndexPage(Wal *pWal, int iPage, volatile u32 **ppPage){ assert( iPage==0 || *ppPage || rc!=SQLITE_OK ); return rc; } +static int walIndexPage( + Wal *pWal, /* The WAL context */ + int iPage, /* The page we seek */ + volatile u32 **ppPage /* Write the page pointer here */ +){ + if( pWal->nWiData<=iPage || (*ppPage = pWal->apWiData[iPage])==0 ){ + return walIndexPageRealloc(pWal, iPage, ppPage); + } + return SQLITE_OK; +} /* ** Return a pointer to the WalCkptInfo structure in the wal-index. From 4f4e49a98b8d3a28384e4a14df23ae4b98e63868 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 21 Feb 2018 00:48:02 +0000 Subject: [PATCH 087/107] Remove an unnecessary memset(). FossilOrigin-Name: e7b7f5884a78630c6ba6a217c36247397e62f1928d4b480df929efcfc4a01893 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_unix.c | 2 -- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 2255c10023..671a63c223 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\sthe\swalIndexPage()\sroutine\sabout\s3x\sfaster\sby\sfactoring\sout\sthe\sseldom\nused\sreallocation\slogic\sinto\sa\sseparate\ssubroutine. -D 2018-02-20T22:20:57.397 +C Remove\san\sunnecessary\smemset(). +D 2018-02-21T00:48:02.760 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in a2d2fb8d17c39ab5ec52beb27850b903949080848236923f436156b72a958737 @@ -472,7 +472,7 @@ F src/os.c 22d31db3ca5a96a408fbf1ceeaaebcaf64c87024d2ff9fe1cf2ddbec3e75c104 F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 -F src/os_unix.c 9e02186f0dcde77d3ec27d4409664eb692c4e8ea0257e30c0b7f25969fa69fec +F src/os_unix.c 928586cbde31890538854f1657463392d63b2d7794f577774573ae53ec283d7b F src/os_win.c eb03c6d52f893bcd7fdd4c6006674c13c1b5e49543fec98d605201af2997171c F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 0b6bd5442733b2e08d0673de6cdafe3e7ab0b5715e4844ac836ab346b1d9ed89 @@ -1707,7 +1707,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P cd81613850c0458b4b5b2460b9b38ce7ff6e557f7add9615058bb96f5a344ccd -R 1787f3579805a4f44923446da8d3d679 +P e2b107141cd97bd4ab240748a9ce43fc2ec950ea74610697a4a7a3d7a6441e6b +R 59bb924e6fde113e9c73dbca193c7d8f U drh -Z 192c58e6de32e2b3cf41a7ef956f2494 +Z 888a0e04ef92ada18bb625445f9ec36e diff --git a/manifest.uuid b/manifest.uuid index 83c33c8bfe..de2a013349 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e2b107141cd97bd4ab240748a9ce43fc2ec950ea74610697a4a7a3d7a6441e6b \ No newline at end of file +e7b7f5884a78630c6ba6a217c36247397e62f1928d4b480df929efcfc4a01893 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index cb2ae33d0a..5a1d396ce2 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4180,12 +4180,10 @@ static int unixShmSystemLock( if( pShmNode->h>=0 ){ /* Initialize the locking parameters */ - memset(&f, 0, sizeof(f)); f.l_type = lockType; f.l_whence = SEEK_SET; f.l_start = ofst; f.l_len = n; - rc = osFcntl(pShmNode->h, F_SETLK, &f); rc = (rc!=(-1)) ? SQLITE_OK : SQLITE_BUSY; } From 8d3e15eec45466b1c89117af5f8d8a0bb6216512 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 21 Feb 2018 01:05:37 +0000 Subject: [PATCH 088/107] Small performance improvement in sqltie3WalFindFrame(). FossilOrigin-Name: 52013cad0e6ce2d694f25e2984a76d438cef724f0e07c8bb9d3dce8a3caf4350 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/wal.c | 3 ++- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 671a63c223..1939578313 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\san\sunnecessary\smemset(). -D 2018-02-21T00:48:02.760 +C Small\sperformance\simprovement\sin\ssqltie3WalFindFrame(). +D 2018-02-21T01:05:37.077 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in a2d2fb8d17c39ab5ec52beb27850b903949080848236923f436156b72a958737 @@ -568,7 +568,7 @@ F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c fff06582a9611cd472fe67ba2b37da1ba052b48a41e3ee60ad55e87faf724c75 +F src/wal.c 634908f6bc2b5147c1a4218381393fc5f19cfc16bef2d9fa8029c2a7f31b73ae F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a F src/walker.c da987a20d40145c0a03c07d8fefcb2ed363becc7680d0500d9c79915591f5b1f F src/where.c 7cae47e813393d70c6d327fdf000fcb30f76b1b0b5a5b52ff6402e0c658de32c @@ -1707,7 +1707,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e2b107141cd97bd4ab240748a9ce43fc2ec950ea74610697a4a7a3d7a6441e6b -R 59bb924e6fde113e9c73dbca193c7d8f +P e7b7f5884a78630c6ba6a217c36247397e62f1928d4b480df929efcfc4a01893 +R cec4fbff2931f49a6ea05bd82279c171 U drh -Z 888a0e04ef92ada18bb625445f9ec36e +Z ba266545389d418145bdf93b79a10d36 diff --git a/manifest.uuid b/manifest.uuid index de2a013349..bd3bf9c056 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e7b7f5884a78630c6ba6a217c36247397e62f1928d4b480df929efcfc4a01893 \ No newline at end of file +52013cad0e6ce2d694f25e2984a76d438cef724f0e07c8bb9d3dce8a3caf4350 \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 99832dff3d..72bb600f5c 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2882,7 +2882,7 @@ int sqlite3WalFindFrame( ** table after the current read-transaction had started. */ iMinHash = walFramePage(pWal->minFrame); - for(iHash=walFramePage(iLast); iHash>=iMinHash && iRead==0; iHash--){ + for(iHash=walFramePage(iLast); iHash>=iMinHash; iHash--){ volatile ht_slot *aHash; /* Pointer to hash table */ volatile u32 *aPgno; /* Pointer to array of page numbers */ u32 iZero; /* Frame number corresponding to aPgno[0] */ @@ -2905,6 +2905,7 @@ int sqlite3WalFindFrame( return SQLITE_CORRUPT_BKPT; } } + if( iRead ) break; } #ifdef SQLITE_ENABLE_EXPENSIVE_ASSERT From c0ec2f77627556ea88fe2b4f9ffaf74b33ad9091 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 21 Feb 2018 01:48:22 +0000 Subject: [PATCH 089/107] Remove an unreachable branch in the walIndexPage() logic. FossilOrigin-Name: 4e61a9731f5949dcf05c541106b56be16add04a7730613481fc8519d3e4f493b --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/wal.c | 29 ++++++++++++++--------------- 3 files changed, 21 insertions(+), 22 deletions(-) diff --git a/manifest b/manifest index 1939578313..80d8dc6b2b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Small\sperformance\simprovement\sin\ssqltie3WalFindFrame(). -D 2018-02-21T01:05:37.077 +C Remove\san\sunreachable\sbranch\sin\sthe\swalIndexPage()\slogic. +D 2018-02-21T01:48:22.912 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in a2d2fb8d17c39ab5ec52beb27850b903949080848236923f436156b72a958737 @@ -568,7 +568,7 @@ F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2 F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c 634908f6bc2b5147c1a4218381393fc5f19cfc16bef2d9fa8029c2a7f31b73ae +F src/wal.c 0ba9af2589b67c4bc60c773c7d2080491099037af21e73545f932bb29bf70b65 F src/wal.h 8de5d2d3de0956d6f6cb48c83a4012d5f227b8fe940f3a349a4b7e85ebcb492a F src/walker.c da987a20d40145c0a03c07d8fefcb2ed363becc7680d0500d9c79915591f5b1f F src/where.c 7cae47e813393d70c6d327fdf000fcb30f76b1b0b5a5b52ff6402e0c658de32c @@ -1707,7 +1707,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e7b7f5884a78630c6ba6a217c36247397e62f1928d4b480df929efcfc4a01893 -R cec4fbff2931f49a6ea05bd82279c171 +P 52013cad0e6ce2d694f25e2984a76d438cef724f0e07c8bb9d3dce8a3caf4350 +R 85b71d55df961ca31201d9b641eb44c9 U drh -Z ba266545389d418145bdf93b79a10d36 +Z e9ca7c00b9c8d598dec87551eaa9cc3d diff --git a/manifest.uuid b/manifest.uuid index bd3bf9c056..9ccc6e494d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -52013cad0e6ce2d694f25e2984a76d438cef724f0e07c8bb9d3dce8a3caf4350 \ No newline at end of file +4e61a9731f5949dcf05c541106b56be16add04a7730613481fc8519d3e4f493b \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 72bb600f5c..d56946e309 100644 --- a/src/wal.c +++ b/src/wal.c @@ -577,21 +577,20 @@ static SQLITE_NOINLINE int walIndexPageRealloc( } /* Request a pointer to the required page from the VFS */ - if( pWal->apWiData[iPage]==0 ){ - if( pWal->exclusiveMode==WAL_HEAPMEMORY_MODE ){ - pWal->apWiData[iPage] = (u32 volatile *)sqlite3MallocZero(WALINDEX_PGSZ); - if( !pWal->apWiData[iPage] ) rc = SQLITE_NOMEM_BKPT; - }else{ - rc = sqlite3OsShmMap(pWal->pDbFd, iPage, WALINDEX_PGSZ, - pWal->writeLock, (void volatile **)&pWal->apWiData[iPage] - ); - assert( pWal->apWiData[iPage]!=0 || rc!=SQLITE_OK || pWal->writeLock==0 ); - testcase( pWal->apWiData[iPage]==0 && rc==SQLITE_OK ); - if( (rc&0xff)==SQLITE_READONLY ){ - pWal->readOnly |= WAL_SHM_RDONLY; - if( rc==SQLITE_READONLY ){ - rc = SQLITE_OK; - } + assert( pWal->apWiData[iPage]==0 ); + if( pWal->exclusiveMode==WAL_HEAPMEMORY_MODE ){ + pWal->apWiData[iPage] = (u32 volatile *)sqlite3MallocZero(WALINDEX_PGSZ); + if( !pWal->apWiData[iPage] ) rc = SQLITE_NOMEM_BKPT; + }else{ + rc = sqlite3OsShmMap(pWal->pDbFd, iPage, WALINDEX_PGSZ, + pWal->writeLock, (void volatile **)&pWal->apWiData[iPage] + ); + assert( pWal->apWiData[iPage]!=0 || rc!=SQLITE_OK || pWal->writeLock==0 ); + testcase( pWal->apWiData[iPage]==0 && rc==SQLITE_OK ); + if( (rc&0xff)==SQLITE_READONLY ){ + pWal->readOnly |= WAL_SHM_RDONLY; + if( rc==SQLITE_READONLY ){ + rc = SQLITE_OK; } } } From d6ec7a6ab3e9a8ea2df0cf3375a4e2334311a3eb Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 22 Feb 2018 18:28:19 +0000 Subject: [PATCH 090/107] Updates to the README.md file. Mention the ability to use SQLite-archives for download. FossilOrigin-Name: ab9a7ae4eec0aaa5e0b472998f6de7212055d93c163f0b66da60b956bcc45add --- README.md | 16 +++++++++------- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index e5fa2acb67..50954b6cde 100644 --- a/README.md +++ b/README.md @@ -14,15 +14,17 @@ SQLite sources are managed using the [Fossil](https://www.fossil-scm.org/), a distributed version control system that was specifically designed to support SQLite development. If you do not want to use Fossil, you can download tarballs or ZIP -archives as follows: +archives or [SQLite archives](https://sqlite.org/sqlar) as follows: - * Lastest trunk check-in: - or - . + * Lastest trunk check-in as + [Tarball](https://www.sqlite.org/src/tarball/sqlite.tar.gz), + [ZIP-archive](https://www.sqlite.org/src/zip/sqlite.zip), or + [SQLite-archive](https://www.sqlite.org/src/sqlar/sqlite.sqlar). - * Latest release: - or - . + * Latest release as + [Tarball](https://www.sqlite.org/src/tarball/sqlite.tar.gz?r=release), + [ZIP-archive](https://www.sqlite.org/src/zip/sqlite.zip?r=release), or + [SQLite-archive](https://www.sqlite.org/src/sqlar/sqlite.sqlar?r=release). * For other check-ins, substitute an appropriate branch name or tag or hash prefix for "release" in the URLs of the previous diff --git a/manifest b/manifest index 80d8dc6b2b..e3dde34935 100644 --- a/manifest +++ b/manifest @@ -1,11 +1,11 @@ -C Remove\san\sunreachable\sbranch\sin\sthe\swalIndexPage()\slogic. -D 2018-02-21T01:48:22.912 +C Updates\sto\sthe\sREADME.md\sfile.\s\sMention\sthe\sability\sto\suse\sSQLite-archives\sfor\ndownload. +D 2018-02-22T18:28:19.974 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in a2d2fb8d17c39ab5ec52beb27850b903949080848236923f436156b72a958737 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b53ebf2538324c914e4308a6cd44610eb1ae50bdbd17d7ee65f680918e54da88 -F README.md d748f58e3ab0fe0307fb4ae0942b415d93dcc4288756e366cc9e7cf8260c093f +F README.md 7c0991bd30eab5f377cd1508a619fc7ccd8014897da31e57f60d9675c0779abd F VERSION cdf91ac446255ecf3d8f6d8c3ee40d64123235ae5b3cef29d344e61b45ec3759 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2 @@ -1707,7 +1707,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 52013cad0e6ce2d694f25e2984a76d438cef724f0e07c8bb9d3dce8a3caf4350 -R 85b71d55df961ca31201d9b641eb44c9 +P 4e61a9731f5949dcf05c541106b56be16add04a7730613481fc8519d3e4f493b +R aa9874414cd1a4e6c530b39e5a39ec09 U drh -Z e9ca7c00b9c8d598dec87551eaa9cc3d +Z 856f42b132f15d59c7f2cba2fefa2460 diff --git a/manifest.uuid b/manifest.uuid index 9ccc6e494d..db728d9dc2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4e61a9731f5949dcf05c541106b56be16add04a7730613481fc8519d3e4f493b \ No newline at end of file +ab9a7ae4eec0aaa5e0b472998f6de7212055d93c163f0b66da60b956bcc45add \ No newline at end of file From 4f46b58a2d9483b921132977c818c1b6951030a0 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 22 Feb 2018 18:30:53 +0000 Subject: [PATCH 091/107] For the "SQLite archive" link in the README.md file, point to the CLI .archive command documentation, not the original "sqlar" website. FossilOrigin-Name: 4290faf5f7223ebe35b8ed5c4f0ba2444c1764501cabc09a79000f9c1d0890c7 --- README.md | 2 +- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 50954b6cde..a92e5ae24e 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ SQLite sources are managed using the [Fossil](https://www.fossil-scm.org/), a distributed version control system that was specifically designed to support SQLite development. If you do not want to use Fossil, you can download tarballs or ZIP -archives or [SQLite archives](https://sqlite.org/sqlar) as follows: +archives or [SQLite archives](https://sqlite.org/cli.html#sqlar) as follows: * Lastest trunk check-in as [Tarball](https://www.sqlite.org/src/tarball/sqlite.tar.gz), diff --git a/manifest b/manifest index e3dde34935..f9ea1ef022 100644 --- a/manifest +++ b/manifest @@ -1,11 +1,11 @@ -C Updates\sto\sthe\sREADME.md\sfile.\s\sMention\sthe\sability\sto\suse\sSQLite-archives\sfor\ndownload. -D 2018-02-22T18:28:19.974 +C For\sthe\s"SQLite\sarchive"\slink\sin\sthe\sREADME.md\sfile,\spoint\sto\sthe\nCLI\s.archive\scommand\sdocumentation,\snot\sthe\soriginal\s"sqlar"\swebsite. +D 2018-02-22T18:30:53.011 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in a2d2fb8d17c39ab5ec52beb27850b903949080848236923f436156b72a958737 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b53ebf2538324c914e4308a6cd44610eb1ae50bdbd17d7ee65f680918e54da88 -F README.md 7c0991bd30eab5f377cd1508a619fc7ccd8014897da31e57f60d9675c0779abd +F README.md 1d5342ebda97420f114283e604e5fe99b0da939d63b76d492eabbaae23488276 F VERSION cdf91ac446255ecf3d8f6d8c3ee40d64123235ae5b3cef29d344e61b45ec3759 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2 @@ -1707,7 +1707,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4e61a9731f5949dcf05c541106b56be16add04a7730613481fc8519d3e4f493b -R aa9874414cd1a4e6c530b39e5a39ec09 +P ab9a7ae4eec0aaa5e0b472998f6de7212055d93c163f0b66da60b956bcc45add +R 3cb70dc7ae645278be920e32ff6e511f U drh -Z 856f42b132f15d59c7f2cba2fefa2460 +Z c1c7fcd61cd44109ceb9fbe2b0c63c1e diff --git a/manifest.uuid b/manifest.uuid index db728d9dc2..4c38faf3fa 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ab9a7ae4eec0aaa5e0b472998f6de7212055d93c163f0b66da60b956bcc45add \ No newline at end of file +4290faf5f7223ebe35b8ed5c4f0ba2444c1764501cabc09a79000f9c1d0890c7 \ No newline at end of file From fda0867873cc3cf5823416a10eb10c69da761af6 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Fri, 23 Feb 2018 13:38:20 +0000 Subject: [PATCH 092/107] Enable the JSON1 extension when compiling the test fixture with MSVC. FossilOrigin-Name: e9e9f27b3d496531905ad7459cf60366ad3798f507266134dd6388cdec50fdc4 --- Makefile.msc | 1 + manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/Makefile.msc b/Makefile.msc index af053cd615..1b2bcb55ac 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -2252,6 +2252,7 @@ TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_SERIES_CONSTRAINT_VERIFY=1 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_DEFAULT_PAGE_SIZE=1024 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_DBPAGE_VTAB +TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_JSON1 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) $(TEST_CCONV_OPTS) TESTFIXTURE_SRC0 = $(TESTEXT) $(TESTSRC2) diff --git a/manifest b/manifest index f9ea1ef022..8c403db8d8 100644 --- a/manifest +++ b/manifest @@ -1,10 +1,10 @@ -C For\sthe\s"SQLite\sarchive"\slink\sin\sthe\sREADME.md\sfile,\spoint\sto\sthe\nCLI\s.archive\scommand\sdocumentation,\snot\sthe\soriginal\s"sqlar"\swebsite. -D 2018-02-22T18:30:53.011 +C Enable\sthe\sJSON1\sextension\swhen\scompiling\sthe\stest\sfixture\swith\sMSVC. +D 2018-02-23T13:38:20.020 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in a2d2fb8d17c39ab5ec52beb27850b903949080848236923f436156b72a958737 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 -F Makefile.msc b53ebf2538324c914e4308a6cd44610eb1ae50bdbd17d7ee65f680918e54da88 +F Makefile.msc bf19d3a0eb849bd3b114653b0e455aa5b2799a96f413287a5866013db0e47f30 F README.md 1d5342ebda97420f114283e604e5fe99b0da939d63b76d492eabbaae23488276 F VERSION cdf91ac446255ecf3d8f6d8c3ee40d64123235ae5b3cef29d344e61b45ec3759 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -1707,7 +1707,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ab9a7ae4eec0aaa5e0b472998f6de7212055d93c163f0b66da60b956bcc45add -R 3cb70dc7ae645278be920e32ff6e511f -U drh -Z c1c7fcd61cd44109ceb9fbe2b0c63c1e +P 4290faf5f7223ebe35b8ed5c4f0ba2444c1764501cabc09a79000f9c1d0890c7 +R 3d31ca3626fb7de7ae6ca98028d90678 +U mistachkin +Z 415294341d4b98b1263aa22d9cb61bd0 diff --git a/manifest.uuid b/manifest.uuid index 4c38faf3fa..db4761886f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4290faf5f7223ebe35b8ed5c4f0ba2444c1764501cabc09a79000f9c1d0890c7 \ No newline at end of file +e9e9f27b3d496531905ad7459cf60366ad3798f507266134dd6388cdec50fdc4 \ No newline at end of file From 5facffbc70893cea166235f87ac3c2ae7ee6fbe1 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Fri, 23 Feb 2018 13:38:54 +0000 Subject: [PATCH 093/107] Fix harmless compiler warnings in the zipfile extension seen with MSVC. FossilOrigin-Name: 15c587cf6ffc920dc37f67cacb4f0db6a603fb998a22a639f755387910516414 --- ext/misc/zipfile.c | 14 +++++++------- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/ext/misc/zipfile.c b/ext/misc/zipfile.c index f88b7e22d6..a400ae30e1 100644 --- a/ext/misc/zipfile.c +++ b/ext/misc/zipfile.c @@ -737,7 +737,7 @@ static void zipfileMtimeToDos(ZipfileCDS *pCds, u32 mUnixTime){ int hr, min, sec; A = (int)((JD - 1867216.25)/36524.25); - A = JD + 1 + A - (A/4); + A = (int)(JD + 1 + A - (A/4)); B = A + 1524; C = (int)((B - 122.1)/365.25); D = (36525*(C&32767))/100; @@ -1623,7 +1623,7 @@ static int zipfileUpdate( pNew->cds.szUncompressed = (u32)sz; pNew->cds.iExternalAttr = (mode<<16); pNew->cds.iOffset = (u32)pTab->szCurrent; - pNew->cds.nFile = nPath; + pNew->cds.nFile = (u16)nPath; pNew->mUnixTime = (u32)mTime; rc = zipfileAppendEntry(pTab, pNew, pData, nData); zipfileAddEntry(pTab, pOld, pNew); @@ -1926,7 +1926,7 @@ void zipfileStep(sqlite3_context *pCtx, int nVal, sqlite3_value **apVal){ /* Inspect the 'method' parameter. This must be either 0 (store), 8 (use ** deflate compression) or NULL (choose automatically). */ if( pMethod && SQLITE_NULL!=sqlite3_value_type(pMethod) ){ - iMethod = sqlite3_value_int64(pMethod); + iMethod = (int)sqlite3_value_int64(pMethod); if( iMethod!=0 && iMethod!=8 ){ zErr = sqlite3_mprintf("illegal method value: %d", iMethod); rc = SQLITE_ERROR; @@ -1993,14 +1993,14 @@ void zipfileStep(sqlite3_context *pCtx, int nVal, sqlite3_value **apVal){ e.cds.iVersionMadeBy = ZIPFILE_NEWENTRY_MADEBY; e.cds.iVersionExtract = ZIPFILE_NEWENTRY_REQUIRED; e.cds.flags = ZIPFILE_NEWENTRY_FLAGS; - e.cds.iCompression = iMethod; + e.cds.iCompression = (u16)iMethod; zipfileMtimeToDos(&e.cds, (u32)e.mUnixTime); e.cds.crc32 = iCrc32; e.cds.szCompressed = nData; e.cds.szUncompressed = szUncompressed; e.cds.iExternalAttr = (mode<<16); e.cds.iOffset = p->body.n; - e.cds.nFile = nName; + e.cds.nFile = (u16)nName; e.cds.zFile = zName; /* Append the LFH to the body of the new archive */ @@ -2047,8 +2047,8 @@ void zipfileFinal(sqlite3_context *pCtx){ if( p==0 ) return; if( p->nEntry>0 ){ memset(&eocd, 0, sizeof(eocd)); - eocd.nEntry = p->nEntry; - eocd.nEntryTotal = p->nEntry; + eocd.nEntry = (u16)p->nEntry; + eocd.nEntryTotal = (u16)p->nEntry; eocd.nSize = p->cds.n; eocd.iOffset = p->body.n; diff --git a/manifest b/manifest index 8c403db8d8..a1b9650113 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enable\sthe\sJSON1\sextension\swhen\scompiling\sthe\stest\sfixture\swith\sMSVC. -D 2018-02-23T13:38:20.020 +C Fix\sharmless\scompiler\swarnings\sin\sthe\szipfile\sextension\sseen\swith\sMSVC. +D 2018-02-23T13:38:54.742 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in a2d2fb8d17c39ab5ec52beb27850b903949080848236923f436156b72a958737 @@ -304,7 +304,7 @@ F ext/misc/vfsstat.c bf10ef0bc51e1ad6756629e1edb142f7a8db1178 F ext/misc/vtablog.c 31d0d8f4406795679dcd3a67917c213d3a2a5fb3ea5de35f6e773491ed7e13c9 F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 -F ext/misc/zipfile.c 3c3e21d0351a44c34c67ca7e833ba04e0de34421b4c9359aa764919ee36b37ef +F ext/misc/zipfile.c bc88dbf9aa26c24595d158c1398208786ef34e7aee19ea29e8f0110c752fca03 F ext/misc/zorder.c b0ff58fa643afa1d846786d51ea8d5c4b6b35aa0254ab5a82617db92f3adda64 F ext/rbu/rbu.c ea7d1b7eb44c123a2a619332e19fe5313500705c4a58aaa1887905c0d83ffc2e F ext/rbu/rbu1.test 43836fac8c7179a358eaf38a8a1ef3d6e6285842 @@ -1707,7 +1707,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4290faf5f7223ebe35b8ed5c4f0ba2444c1764501cabc09a79000f9c1d0890c7 -R 3d31ca3626fb7de7ae6ca98028d90678 +P e9e9f27b3d496531905ad7459cf60366ad3798f507266134dd6388cdec50fdc4 +R 654cd747b1d6661b630741ec0eabab23 U mistachkin -Z 415294341d4b98b1263aa22d9cb61bd0 +Z f6be006b93932e4aec25faabbec16d8c diff --git a/manifest.uuid b/manifest.uuid index db4761886f..3218ef0aa6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e9e9f27b3d496531905ad7459cf60366ad3798f507266134dd6388cdec50fdc4 \ No newline at end of file +15c587cf6ffc920dc37f67cacb4f0db6a603fb998a22a639f755387910516414 \ No newline at end of file From 007c843b0f9b832c21eac4274b361173544c9bb7 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 26 Feb 2018 03:20:18 +0000 Subject: [PATCH 094/107] Experimental implementation of IS TRUE and IS FALSE operators. All TRUE and FALSE to act like constants if the names do not resolve to a column name. FossilOrigin-Name: 40314bc999af08ab10e654241208842b4bb95b19858d11249444372250ea4160 --- manifest | 23 +++++++++++++---------- manifest.uuid | 2 +- src/expr.c | 25 +++++++++++++++++++++++++ src/resolve.c | 36 +++++++++++++++++++++++++++++++----- src/vdbe.c | 6 +++--- tool/addopcodes.tcl | 3 +++ 6 files changed, 76 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index a1b9650113..3417c2a6b1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarnings\sin\sthe\szipfile\sextension\sseen\swith\sMSVC. -D 2018-02-23T13:38:54.742 +C Experimental\simplementation\sof\sIS\sTRUE\sand\sIS\sFALSE\soperators.\s\sAll\sTRUE\sand\nFALSE\sto\sact\slike\sconstants\sif\sthe\snames\sdo\snot\sresolve\sto\sa\scolumn\sname. +D 2018-02-26T03:20:18.624 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in a2d2fb8d17c39ab5ec52beb27850b903949080848236923f436156b72a958737 @@ -441,7 +441,7 @@ F src/date.c ebe1dc7c8a347117bb02570f1a931c62dd78f4a2b1b516f4837d45b7d6426957 F src/dbpage.c 8db4c97f630e7d83f884ea75caf1ffd0988c160e9d530194d93721c80821e0f6 F src/dbstat.c 7a4ba8518b6369ef3600c49cf9c918ad979acba610b2aebef1b656d649b96720 F src/delete.c 20c8788451dc737a967c87ea53ad43544d617f5b57d32ccce8bd52a0daf9e89b -F src/expr.c 9e06de431c09f144438aa6895ea4d4290fa3c6875bfcc3ba331012ca78deadf0 +F src/expr.c 56bfb9937e14c8daf3e94d23ee9c516f35573afcbf973dc2fae2d60405e18862 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c d617daf66b5515e2b42c1405b2b4984c30ca50fb705ab164271a9bf66c69e331 F src/func.c 94f42cba2cc1c34aeaa441022ba0170ec3fec4bba54db4e0ded085c6dc0fdc51 @@ -486,7 +486,7 @@ F src/pragma.h bb83728944b42f6d409c77f5838a8edbdb0fe83046c5496ffc9602b40340a324 F src/prepare.c 259f4e7960c47082c9653f3d5f0c294abd68bb9c3aab86de7630700cba1c20fb F src/printf.c d3b7844ddeb11fbbdd38dd84d09c9c1ac171d21fb038473c3aa97981201cc660 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 -F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730 +F src/resolve.c e3555b156d60407c23692c1e99c05cdf8257f1b7530684dce85247012e58bd8a F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c f02352ac5cbb6fad3804add825743b924cfb6c742ba2e8666d726828a9985d73 F src/shell.c.in c1b14bc0bc1fe73a97dfaad55ca09f5d6e44f6019977d94e7e3d1861616035e1 @@ -557,7 +557,7 @@ F src/update.c a90a32ffc0100265b0693dbbdbe490756447af181f5ea2c138cce515b08c8795 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5 F src/util.c d9eb0a6c4aae1b00a7369eadd7ca0bbe946cb4c953b6751aa20d357c2f482157 F src/vacuum.c 762ee9bbf8733d87d8cd06f58d950e881982e416f8c767334a40ffd341b6bff5 -F src/vdbe.c 72f5a4b95be486170104779aeef000423acabf55ac0c661afdae889be72c8922 +F src/vdbe.c d221ab77e0997d410004d46cb373f3b848d994b64b439dc69db9d7d43b255397 F src/vdbe.h 134beb7a12a6213c00eba58febaede33447cc4441bc568a0d9c144b33fc3720a F src/vdbeInt.h 8d7d07f13cb3c4cbca91e22ba4a1920e542dda7c5d9299920432a0b3d5b009f5 F src/vdbeapi.c fea41171884a4de119f8b10ab514c788674eeeb7f27218bb6d008e1310bfd07f @@ -1612,7 +1612,7 @@ F test/zipfilefault.test 050be76778d2ec83566a542800fc3967f65ce719671a819016ada8b F tool/GetFile.cs a15e08acb5dd7539b75ba23501581d7c2b462cb5 F tool/GetTclKit.bat 8995df40c4209808b31f24de0b58f90930239a234f7591e3675d45bfbb990c5d F tool/Replace.cs 02c67258801c2fb5f63231e0ac0f220b4b36ba91 -F tool/addopcodes.tcl 7181c041d495e3f26acc36d15c86923ed722285f9015f017f41a3efdb9a0dab4 +F tool/addopcodes.tcl 92514b0db662868f1ec803ea95d8f5cf31213ba022fdfd5b7d289518d2e1eecf F tool/build-all-msvc.bat c12328d06c45fec8baada5949e3d5af54bf8c887 x F tool/build-shell.sh 950f47c6174f1eea171319438b93ba67ff5bf367 F tool/cg_anno.tcl f95b0006c52cf7f0496b506343415b6ee3cdcdd3 x @@ -1707,7 +1707,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e9e9f27b3d496531905ad7459cf60366ad3798f507266134dd6388cdec50fdc4 -R 654cd747b1d6661b630741ec0eabab23 -U mistachkin -Z f6be006b93932e4aec25faabbec16d8c +P 15c587cf6ffc920dc37f67cacb4f0db6a603fb998a22a639f755387910516414 +R b02e960cb9fe764e3ce324741c78d420 +T *branch * is-true-operator +T *sym-is-true-operator * +T -sym-trunk * +U drh +Z 2f211764040206a38839d885e0d3c8ec diff --git a/manifest.uuid b/manifest.uuid index 3218ef0aa6..8ab9156881 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -15c587cf6ffc920dc37f67cacb4f0db6a603fb998a22a639f755387910516414 \ No newline at end of file +40314bc999af08ab10e654241208842b4bb95b19858d11249444372250ea4160 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 1b87734923..5079035ca1 100644 --- a/src/expr.c +++ b/src/expr.c @@ -3543,6 +3543,14 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ codeInteger(pParse, pExpr, 0, target); return target; } + case TK_TRUE: { + sqlite3VdbeAddOp2(v, OP_Integer, 1, target); + return target; + } + case TK_FALSE: { + sqlite3VdbeAddOp2(v, OP_Integer, 0, target); + return target; + } #ifndef SQLITE_OMIT_FLOATING_POINT case TK_FLOAT: { assert( !ExprHasProperty(pExpr, EP_IntValue) ); @@ -3698,6 +3706,13 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ sqlite3VdbeAddOp2(v, op, r1, inReg); break; } + case TK_ISTRUE: { + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + testcase( regFree1==0 ); + sqlite3VdbeAddOp2(v, OP_Not, r1, inReg); + sqlite3VdbeAddOp2(v, OP_Not, inReg, inReg); + break; + } case TK_ISNULL: case TK_NOTNULL: { int addr; @@ -4473,6 +4488,11 @@ void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ sqlite3ExprIfFalse(pParse, pExpr->pLeft, dest, jumpIfNull); break; } + case TK_ISTRUE: { + testcase( jumpIfNull==0 ); + sqlite3ExprIfTrue(pParse, pExpr->pLeft, dest, jumpIfNull); + break; + } case TK_IS: case TK_ISNOT: testcase( op==TK_IS ); @@ -4627,6 +4647,11 @@ void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ sqlite3ExprIfTrue(pParse, pExpr->pLeft, dest, jumpIfNull); break; } + case TK_ISTRUE: { + testcase( jumpIfNull==0 ); + sqlite3ExprIfFalse(pParse, pExpr->pLeft, dest, jumpIfNull); + break; + } case TK_IS: case TK_ISNOT: testcase( pExpr->op==TK_IS ); diff --git a/src/resolve.c b/src/resolve.c index f735fffa02..fb00788dd0 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -431,10 +431,22 @@ static int lookupName( ** Because no reference was made to outer contexts, the pNC->nRef ** fields are not changed in any context. */ - if( cnt==0 && zTab==0 && ExprHasProperty(pExpr,EP_DblQuoted) ){ - pExpr->op = TK_STRING; - pExpr->pTab = 0; - return WRC_Prune; + if( cnt==0 && zTab==0 ){ + if( ExprHasProperty(pExpr,EP_DblQuoted) ){ + pExpr->op = TK_STRING; + pExpr->pTab = 0; + return WRC_Prune; + } + if( sqlite3StrICmp(zCol, "true")==0 ){ + pExpr->op = TK_TRUE; + pExpr->pTab = 0; + return WRC_Prune; + } + if( sqlite3StrICmp(zCol, "false")==0 ){ + pExpr->op = TK_FALSE; + pExpr->pTab = 0; + return WRC_Prune; + } } /* @@ -783,6 +795,21 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ notValid(pParse, pNC, "parameters", NC_IsCheck|NC_PartIdx|NC_IdxExpr); break; } + case TK_IS: + /* Handle special cases of "x IS TRUE" and "x IS FALSE". The first + ** is transformed into "+x" and the second into "NOT x". */ + if( pExpr->pRight->op==TK_ID ){ + int rc = resolveExprStep(pWalker, pExpr->pRight); + if( rc==WRC_Abort ) return WRC_Abort; + if( pExpr->pRight->op==TK_TRUE ){ + pExpr->op = TK_ISTRUE; + return WRC_Continue; + }else if( pExpr->pRight->op==TK_FALSE ){ + pExpr->op = TK_NOT; + return WRC_Continue; + } + } + /* Fall thru */ case TK_BETWEEN: case TK_EQ: case TK_NE: @@ -790,7 +817,6 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ case TK_LE: case TK_GT: case TK_GE: - case TK_IS: case TK_ISNOT: { int nLeft, nRight; if( pParse->db->mallocFailed ) break; diff --git a/src/vdbe.c b/src/vdbe.c index ebf221d02f..7115c8cf7d 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2211,10 +2211,10 @@ case OP_Or: { /* same as TK_OR, in1, in2, out3 */ case OP_Not: { /* same as TK_NOT, in1, out2 */ pIn1 = &aMem[pOp->p1]; pOut = &aMem[pOp->p2]; - sqlite3VdbeMemSetNull(pOut); if( (pIn1->flags & MEM_Null)==0 ){ - pOut->flags = MEM_Int; - pOut->u.i = !sqlite3VdbeIntValue(pIn1); + sqlite3VdbeMemSetInt64(pOut, !sqlite3VdbeIntValue(pIn1)); + }else{ + sqlite3VdbeMemSetNull(pOut); } break; } diff --git a/tool/addopcodes.tcl b/tool/addopcodes.tcl index dfd7fac88c..7059d71363 100644 --- a/tool/addopcodes.tcl +++ b/tool/addopcodes.tcl @@ -22,6 +22,8 @@ close $in # ILLEGAL *must* be the last two token codes and they must be in that order. # set extras { + TRUE + FALSE ISNOT FUNCTION COLUMN @@ -29,6 +31,7 @@ set extras { AGG_COLUMN UMINUS UPLUS + ISTRUE REGISTER VECTOR SELECT_COLUMN From 1fcfa7240f9ab22d11f12d92378c64e89fbf36fb Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 26 Feb 2018 15:27:31 +0000 Subject: [PATCH 095/107] Always interpret non-zero floating-point values as true even if their integer part is zero. Fix for ticket [36fae083b450e3af857a459e20]. FossilOrigin-Name: a983fa857048c748112c2f805f13aadeae43b43b3d27fa55433a3699b51c9a0e --- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- src/vdbe.c | 41 +++++++++++++---------------------------- src/vdbeInt.h | 1 + src/vdbemem.c | 10 ++++++++++ test/expr.test | 22 ++++++++++++++++++++++ 6 files changed, 57 insertions(+), 39 deletions(-) diff --git a/manifest b/manifest index a1b9650113..007a40eb4c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarnings\sin\sthe\szipfile\sextension\sseen\swith\sMSVC. -D 2018-02-23T13:38:54.742 +C Always\sinterpret\snon-zero\sfloating-point\svalues\sas\strue\seven\sif\stheir\ninteger\spart\sis\szero.\s\sFix\sfor\sticket\s[36fae083b450e3af857a459e20]. +D 2018-02-26T15:27:31.204 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in a2d2fb8d17c39ab5ec52beb27850b903949080848236923f436156b72a958737 @@ -557,13 +557,13 @@ F src/update.c a90a32ffc0100265b0693dbbdbe490756447af181f5ea2c138cce515b08c8795 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5 F src/util.c d9eb0a6c4aae1b00a7369eadd7ca0bbe946cb4c953b6751aa20d357c2f482157 F src/vacuum.c 762ee9bbf8733d87d8cd06f58d950e881982e416f8c767334a40ffd341b6bff5 -F src/vdbe.c 72f5a4b95be486170104779aeef000423acabf55ac0c661afdae889be72c8922 +F src/vdbe.c bf51a88d09f3a3f22fec7e81fc5a1a6b72a7bc9a1863c8ab48d4983916e25fbd F src/vdbe.h 134beb7a12a6213c00eba58febaede33447cc4441bc568a0d9c144b33fc3720a -F src/vdbeInt.h 8d7d07f13cb3c4cbca91e22ba4a1920e542dda7c5d9299920432a0b3d5b009f5 +F src/vdbeInt.h 95f7adfdc5c8f1353321f55a6c5ec00a90877e3b85af5159e393afb41ff54110 F src/vdbeapi.c fea41171884a4de119f8b10ab514c788674eeeb7f27218bb6d008e1310bfd07f F src/vdbeaux.c 2756ac68ac259c416554100598fc291870063288cd7e1af22847f57b3e130e56 F src/vdbeblob.c f5c70f973ea3a9e915d1693278a5f890dc78594300cf4d54e64f2b0917c94191 -F src/vdbemem.c 21c1d00f0b901dda0a2eccf56bf0d86648cae1bde37de0606ae4e7cdb0ff6d3d +F src/vdbemem.c 414e28d3a7e2a8bee2bb247de115dcbc68e3cbac284d5862d077002f7a93bce1 F src/vdbesort.c 731a09e5cb9e96b70c394c1b7cf3860fbe84acca7682e178615eb941a3a0ef2f F src/vdbetrace.c 48e11ebe040c6b41d146abed2602e3d00d621d7ebe4eb29b0a0f1617fd3c2f6c F src/vtab.c 0e4885495172e1bdf54b12cce23b395ac74ef5729031f15e1bc1e3e6b360ed1a @@ -792,7 +792,7 @@ F test/exclusive.test 1206b87e192497d78c7f35552e86a9d05421498da300fb1cce5ca5351c F test/exclusive2.test 984090e8e9d1b331d2e8111daf6e5d61dda0bef7 F test/exec.test e949714dc127eaa5ecc7d723efec1ec27118fdd7 F test/exists.test 79a75323c78f02bbe9c251ea502a092f9ef63dac -F test/expr.test 66a2c9ac34f74f036faa4092f5402c7d3162fc93 +F test/expr.test 4d34609c823fb0b856a6f71ffd75a2d03e7ccc366f8d182ae901b4467abb9caa F test/extension01.test 00d13cec817f331a687a243e0e5a2d87b0e358c9 F test/extraquick.test cb254400bd42bfb777ff675356aabf3287978f79 F test/fallocate.test 07416bd593a116d5893cb244f45a94d5c6fe030561df3bd972e6135f8106e509 @@ -1707,7 +1707,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e9e9f27b3d496531905ad7459cf60366ad3798f507266134dd6388cdec50fdc4 -R 654cd747b1d6661b630741ec0eabab23 -U mistachkin -Z f6be006b93932e4aec25faabbec16d8c +P 15c587cf6ffc920dc37f67cacb4f0db6a603fb998a22a639f755387910516414 +R ff7cf472c64b844ed90baa6da5e6de1d +U drh +Z 4ad98123770e35c50780573ab58f9c3a diff --git a/manifest.uuid b/manifest.uuid index 3218ef0aa6..48e20093ff 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -15c587cf6ffc920dc37f67cacb4f0db6a603fb998a22a639f755387910516414 \ No newline at end of file +a983fa857048c748112c2f805f13aadeae43b43b3d27fa55433a3699b51c9a0e \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index ebf221d02f..b4c96b498f 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2172,18 +2172,8 @@ case OP_Or: { /* same as TK_OR, in1, in2, out3 */ int v1; /* Left operand: 0==FALSE, 1==TRUE, 2==UNKNOWN or NULL */ int v2; /* Right operand: 0==FALSE, 1==TRUE, 2==UNKNOWN or NULL */ - pIn1 = &aMem[pOp->p1]; - if( pIn1->flags & MEM_Null ){ - v1 = 2; - }else{ - v1 = sqlite3VdbeIntValue(pIn1)!=0; - } - pIn2 = &aMem[pOp->p2]; - if( pIn2->flags & MEM_Null ){ - v2 = 2; - }else{ - v2 = sqlite3VdbeIntValue(pIn2)!=0; - } + v1 = sqlite3VdbeBooleanValue(&aMem[pOp->p1], 2); + v2 = sqlite3VdbeBooleanValue(&aMem[pOp->p2], 2); if( pOp->opcode==OP_And ){ static const unsigned char and_logic[] = { 0, 0, 0, 0, 1, 2, 0, 2, 2 }; v1 = and_logic[v1*3+v2]; @@ -2214,7 +2204,7 @@ case OP_Not: { /* same as TK_NOT, in1, out2 */ sqlite3VdbeMemSetNull(pOut); if( (pIn1->flags & MEM_Null)==0 ){ pOut->flags = MEM_Int; - pOut->u.i = !sqlite3VdbeIntValue(pIn1); + pOut->u.i = !sqlite3VdbeBooleanValue(pIn1, 0); } break; } @@ -2281,30 +2271,25 @@ case OP_Once: { /* jump */ ** is considered true if it is numeric and non-zero. If the value ** in P1 is NULL then take the jump if and only if P3 is non-zero. */ +case OP_If: { /* jump, in1 */ + int c; + c = sqlite3VdbeBooleanValue(&aMem[pOp->p1], pOp->p3); + VdbeBranchTaken(c!=0, 2); + if( c ) goto jump_to_p2; + break; +} + /* Opcode: IfNot P1 P2 P3 * * ** ** Jump to P2 if the value in register P1 is False. The value ** is considered false if it has a numeric value of zero. If the value ** in P1 is NULL then take the jump if and only if P3 is non-zero. */ -case OP_If: /* jump, in1 */ case OP_IfNot: { /* jump, in1 */ int c; - pIn1 = &aMem[pOp->p1]; - if( pIn1->flags & MEM_Null ){ - c = pOp->p3; - }else{ - if( pIn1->flags & MEM_Int ){ - c = pIn1->u.i!=0; - }else{ - c = sqlite3VdbeRealValue(pIn1)!=0.0; - } - if( pOp->opcode==OP_IfNot ) c = !c; - } + c = !sqlite3VdbeBooleanValue(&aMem[pOp->p1], !pOp->p3); VdbeBranchTaken(c!=0, 2); - if( c ){ - goto jump_to_p2; - } + if( c ) goto jump_to_p2; break; } diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 1e4f615ba2..44f901abf7 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -486,6 +486,7 @@ int sqlite3VdbeMemStringify(Mem*, u8, u8); i64 sqlite3VdbeIntValue(Mem*); int sqlite3VdbeMemIntegerify(Mem*); double sqlite3VdbeRealValue(Mem*); +int sqlite3VdbeBooleanValue(Mem*, int ifNull); void sqlite3VdbeIntegerAffinity(Mem*); int sqlite3VdbeMemRealify(Mem*); int sqlite3VdbeMemNumerify(Mem*); diff --git a/src/vdbemem.c b/src/vdbemem.c index c02370c738..8df3c0d53b 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -572,6 +572,16 @@ double sqlite3VdbeRealValue(Mem *pMem){ } } +/* +** Return 1 if pMem represents true, and return 0 if pMem represents false. +** Return the value ifNull if pMem is NULL. +*/ +int sqlite3VdbeBooleanValue(Mem *pMem, int ifNull){ + if( pMem->flags & MEM_Int ) return pMem->u.i!=0; + if( pMem->flags & MEM_Null ) return ifNull; + return sqlite3VdbeRealValue(pMem)!=0.0; +} + /* ** The MEM structure is already a MEM_Real. Try to also make it a ** MEM_Int if we can. diff --git a/test/expr.test b/test/expr.test index 7a6d477259..edef3486af 100644 --- a/test/expr.test +++ b/test/expr.test @@ -977,6 +977,28 @@ do_execsql_test expr-13.9 { SELECT '' <= ""; } {1} +# 2018-02-26. Ticket https://www.sqlite.org/src/tktview/36fae083b450e3af85 +# +do_execsql_test expr-14.1 { + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(x); + INSERT INTO t1 VALUES(0),(1),(NULL),(0.5),('1x'),('0x'); + SELECT count(*) FROM t1 + WHERE (x OR (8==9)) != (CASE WHEN x THEN 1 ELSE 0 END); +} {0} +do_execsql_test expr-14.2 { + SELECT count(*) FROM t1 + WHERE (x OR (8==9)) != (NOT NOT x); +} {0} +do_execsql_test expr-14.3 { + SELECT sum(NOT x) FROM t1 + WHERE x +} {0} +do_execsql_test expr-14.4 { + SELECT sum(CASE WHEN x THEN 0 ELSE 1 END) FROM t1 + WHERE x +} {0} + finish_test From f7e867c347d274f3feb08452afa4053048c7cb04 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Mon, 26 Feb 2018 16:49:20 +0000 Subject: [PATCH 096/107] Fix heap issue in the shell on Win32 caused by allocator mismatch. FossilOrigin-Name: 7b3a75519aa72a6b49a125ee84946ba2c0722dd6644ee0c80ba567eb85daf5c9 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/shell.c.in | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 007a40eb4c..b60224db6e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Always\sinterpret\snon-zero\sfloating-point\svalues\sas\strue\seven\sif\stheir\ninteger\spart\sis\szero.\s\sFix\sfor\sticket\s[36fae083b450e3af857a459e20]. -D 2018-02-26T15:27:31.204 +C Fix\sheap\sissue\sin\sthe\sshell\son\sWin32\scaused\sby\sallocator\smismatch. +D 2018-02-26T16:49:20.522 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in a2d2fb8d17c39ab5ec52beb27850b903949080848236923f436156b72a958737 @@ -489,7 +489,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c f02352ac5cbb6fad3804add825743b924cfb6c742ba2e8666d726828a9985d73 -F src/shell.c.in c1b14bc0bc1fe73a97dfaad55ca09f5d6e44f6019977d94e7e3d1861616035e1 +F src/shell.c.in 47dac081e15544f4d4373a837215391459971b312c8bed15903e7e9045c3e652 F src/sqlite.h.in 235e942dd8c01b414c2996828b4d0d2500faf8850f8b24ae17d31c172f519e69 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 83a3c4ce93d650bedfd1aa558cb85a516bd6d094445ee989740827d0d944368d @@ -1707,7 +1707,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 15c587cf6ffc920dc37f67cacb4f0db6a603fb998a22a639f755387910516414 -R ff7cf472c64b844ed90baa6da5e6de1d -U drh -Z 4ad98123770e35c50780573ab58f9c3a +P a983fa857048c748112c2f805f13aadeae43b43b3d27fa55433a3699b51c9a0e +R 1433684d91ba65f3b7614e9fa9c71d0a +U mistachkin +Z b329981237c7dadda32eba65e163ae21 diff --git a/manifest.uuid b/manifest.uuid index 48e20093ff..f5717e276e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a983fa857048c748112c2f805f13aadeae43b43b3d27fa55433a3699b51c9a0e \ No newline at end of file +7b3a75519aa72a6b49a125ee84946ba2c0722dd6644ee0c80ba567eb85daf5c9 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index c074b5ca7b..121a6c3506 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -8490,8 +8490,8 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ data.doXdgOpen = 0; clearTempFile(&data); #if !SQLITE_SHELL_IS_UTF8 - for(i=0; i Date: Mon, 26 Feb 2018 17:03:03 +0000 Subject: [PATCH 097/107] More tests pursuant to ticket [36fae083b450e3af857a459e20]. FossilOrigin-Name: c26f236da61aef7c7a8fe38f1c7f33eea894c8fae38de6a085d4cce933a70502 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/expr.test | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 42 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index b60224db6e..983bc3f72d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sheap\sissue\sin\sthe\sshell\son\sWin32\scaused\sby\sallocator\smismatch. -D 2018-02-26T16:49:20.522 +C More\stests\spursuant\sto\sticket\s[36fae083b450e3af857a459e20]. +D 2018-02-26T17:03:03.328 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in a2d2fb8d17c39ab5ec52beb27850b903949080848236923f436156b72a958737 @@ -792,7 +792,7 @@ F test/exclusive.test 1206b87e192497d78c7f35552e86a9d05421498da300fb1cce5ca5351c F test/exclusive2.test 984090e8e9d1b331d2e8111daf6e5d61dda0bef7 F test/exec.test e949714dc127eaa5ecc7d723efec1ec27118fdd7 F test/exists.test 79a75323c78f02bbe9c251ea502a092f9ef63dac -F test/expr.test 4d34609c823fb0b856a6f71ffd75a2d03e7ccc366f8d182ae901b4467abb9caa +F test/expr.test 7cb55e80aeb41d65fec968c08212505123063fea60bdc355d764d747670e9eea F test/extension01.test 00d13cec817f331a687a243e0e5a2d87b0e358c9 F test/extraquick.test cb254400bd42bfb777ff675356aabf3287978f79 F test/fallocate.test 07416bd593a116d5893cb244f45a94d5c6fe030561df3bd972e6135f8106e509 @@ -1707,7 +1707,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a983fa857048c748112c2f805f13aadeae43b43b3d27fa55433a3699b51c9a0e -R 1433684d91ba65f3b7614e9fa9c71d0a +P 7b3a75519aa72a6b49a125ee84946ba2c0722dd6644ee0c80ba567eb85daf5c9 +R 54e3679053cfb34f03d5fc5a51c25e95 U mistachkin -Z b329981237c7dadda32eba65e163ae21 +Z 67d82893997d32bf8b3a9fa69332f798 diff --git a/manifest.uuid b/manifest.uuid index f5717e276e..5e91c27a63 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7b3a75519aa72a6b49a125ee84946ba2c0722dd6644ee0c80ba567eb85daf5c9 \ No newline at end of file +c26f236da61aef7c7a8fe38f1c7f33eea894c8fae38de6a085d4cce933a70502 \ No newline at end of file diff --git a/test/expr.test b/test/expr.test index edef3486af..3cdc9180e8 100644 --- a/test/expr.test +++ b/test/expr.test @@ -1000,5 +1000,40 @@ do_execsql_test expr-14.4 { } {0} +foreach {tn val} [list 1 NaN 2 -NaN 3 NaN0 4 -NaN0 5 Inf 6 -Inf] { + do_execsql_test expr-15.$tn.1 { + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(x); + INSERT INTO t1 VALUES(0),(1),(NULL),(0.5),('1x'),('0x'); + } + + do_test expr-15.$tn.2 { + set ::STMT [sqlite3_prepare db "INSERT INTO t1 VALUES(?)" -1 TAIL] + sqlite3_bind_double $::STMT 1 $val + sqlite3_step $::STMT + sqlite3_reset $::STMT + sqlite3_finalize $::STMT + } {SQLITE_OK} + + do_execsql_test expr-15.$tn.3 { + SELECT count(*) FROM t1 + WHERE (x OR (8==9)) != (CASE WHEN x THEN 1 ELSE 0 END); + } {0} + + do_execsql_test expr-15.$tn.4 { + SELECT count(*) FROM t1 + WHERE (x OR (8==9)) != (NOT NOT x); + } {0} + + do_execsql_test expr-15.$tn.5 { + SELECT sum(NOT x) FROM t1 + WHERE x + } {0} + + do_execsql_test expr-15.$tn.6 { + SELECT sum(CASE WHEN x THEN 0 ELSE 1 END) FROM t1 + WHERE x + } {0} +} finish_test From 8abed7b90769a6f63ef2681cfd4ae92d617b883f Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 26 Feb 2018 18:49:05 +0000 Subject: [PATCH 098/107] Refactor for correct NULL handling in the IS TRUE, IS FALSE, IS NOT TRUE, and IS NOT FALSE operators. FossilOrigin-Name: cf2abd59be9971a55bd3d6c5df374c6aaa23bf81819482b42f01ee2484dcd739 --- manifest | 21 ++++++------ manifest.uuid | 2 +- src/expr.c | 53 +++++++++++++++++++++-------- src/resolve.c | 34 +++++++++---------- src/sqliteInt.h | 3 +- src/vdbe.c | 28 ++++++++++++++++ test/istrue.test | 81 +++++++++++++++++++++++++++++++++++++++++++++ tool/addopcodes.tcl | 5 ++- 8 files changed, 181 insertions(+), 46 deletions(-) create mode 100644 test/istrue.test diff --git a/manifest b/manifest index afbeea2798..d23a52f63f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\sfix\sfor\sdetermining\struth\sof\sfloating\spoint\svalues\sfrom\strunk. -D 2018-02-26T15:31:39.919 +C Refactor\sfor\scorrect\sNULL\shandling\sin\sthe\sIS\sTRUE,\sIS\sFALSE,\sIS\sNOT\sTRUE,\nand\sIS\sNOT\sFALSE\soperators. +D 2018-02-26T18:49:05.668 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in a2d2fb8d17c39ab5ec52beb27850b903949080848236923f436156b72a958737 @@ -441,7 +441,7 @@ F src/date.c ebe1dc7c8a347117bb02570f1a931c62dd78f4a2b1b516f4837d45b7d6426957 F src/dbpage.c 8db4c97f630e7d83f884ea75caf1ffd0988c160e9d530194d93721c80821e0f6 F src/dbstat.c 7a4ba8518b6369ef3600c49cf9c918ad979acba610b2aebef1b656d649b96720 F src/delete.c 20c8788451dc737a967c87ea53ad43544d617f5b57d32ccce8bd52a0daf9e89b -F src/expr.c 56bfb9937e14c8daf3e94d23ee9c516f35573afcbf973dc2fae2d60405e18862 +F src/expr.c 4344e70a158c0092e0c14c564d352546259297996cba250f1d6d9dbf830a415d F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c d617daf66b5515e2b42c1405b2b4984c30ca50fb705ab164271a9bf66c69e331 F src/func.c 94f42cba2cc1c34aeaa441022ba0170ec3fec4bba54db4e0ded085c6dc0fdc51 @@ -486,14 +486,14 @@ F src/pragma.h bb83728944b42f6d409c77f5838a8edbdb0fe83046c5496ffc9602b40340a324 F src/prepare.c 259f4e7960c47082c9653f3d5f0c294abd68bb9c3aab86de7630700cba1c20fb F src/printf.c d3b7844ddeb11fbbdd38dd84d09c9c1ac171d21fb038473c3aa97981201cc660 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 -F src/resolve.c e3555b156d60407c23692c1e99c05cdf8257f1b7530684dce85247012e58bd8a +F src/resolve.c 7e6a9702fe3bbf556b1105497ba7a51c24bf606bcc2a7c329122b31e04149005 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c f02352ac5cbb6fad3804add825743b924cfb6c742ba2e8666d726828a9985d73 F src/shell.c.in c1b14bc0bc1fe73a97dfaad55ca09f5d6e44f6019977d94e7e3d1861616035e1 F src/sqlite.h.in 235e942dd8c01b414c2996828b4d0d2500faf8850f8b24ae17d31c172f519e69 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 83a3c4ce93d650bedfd1aa558cb85a516bd6d094445ee989740827d0d944368d -F src/sqliteInt.h fd8943e9d6b8bd04b3a22ef806b843c2c06f022db3f37f1436ac3a9b04e5b8a1 +F src/sqliteInt.h d38064626689d9a832f17cfc0feef250e34a9f6f2589805e8cef5e85c38c18fa F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -557,7 +557,7 @@ F src/update.c a90a32ffc0100265b0693dbbdbe490756447af181f5ea2c138cce515b08c8795 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5 F src/util.c d9eb0a6c4aae1b00a7369eadd7ca0bbe946cb4c953b6751aa20d357c2f482157 F src/vacuum.c 762ee9bbf8733d87d8cd06f58d950e881982e416f8c767334a40ffd341b6bff5 -F src/vdbe.c 2d1338273ff6d6430c0d7b88afbd8f4683df0d23980ab1e638a6b490e9766e80 +F src/vdbe.c 39675332be215b4672f8800b9355e190476ff5fe35fe71f2c9944a3f9dce17b7 F src/vdbe.h 134beb7a12a6213c00eba58febaede33447cc4441bc568a0d9c144b33fc3720a F src/vdbeInt.h 95f7adfdc5c8f1353321f55a6c5ec00a90877e3b85af5159e393afb41ff54110 F src/vdbeapi.c fea41171884a4de119f8b10ab514c788674eeeb7f27218bb6d008e1310bfd07f @@ -1001,6 +1001,7 @@ F test/ioerr3.test d3cec5e1a11ad6d27527d0d38573fbff14c71bdd F test/ioerr4.test f130fe9e71008577b342b8874d52984bd04ede2c F test/ioerr5.test 2edfa4fb0f896f733071303b42224df8bedd9da4 F test/ioerr6.test a395a6ab144b26a9e3e21059a1ab6a7149cca65b +F test/istrue.test e95d4366a705232d81781545f314457f4cc407c8a53b5eac96703f220a74303a F test/join.test 442c462eea85cf065d70a663c626b780a95af6e11585d909bb63b87598afe678 F test/join2.test f1b5ba94ba70a791df2d99e61adcebf62edacad1eca57881c97eb3e2ca718595 F test/join3.test 6f0c774ff1ba0489e6c88a3e77b9d3528fb4fda0 @@ -1612,7 +1613,7 @@ F test/zipfilefault.test 050be76778d2ec83566a542800fc3967f65ce719671a819016ada8b F tool/GetFile.cs a15e08acb5dd7539b75ba23501581d7c2b462cb5 F tool/GetTclKit.bat 8995df40c4209808b31f24de0b58f90930239a234f7591e3675d45bfbb990c5d F tool/Replace.cs 02c67258801c2fb5f63231e0ac0f220b4b36ba91 -F tool/addopcodes.tcl 92514b0db662868f1ec803ea95d8f5cf31213ba022fdfd5b7d289518d2e1eecf +F tool/addopcodes.tcl 0288d5b26b9b35f4cb5affb76eec63f1dfce117bbc2020066708069ef60b86ff F tool/build-all-msvc.bat c12328d06c45fec8baada5949e3d5af54bf8c887 x F tool/build-shell.sh 950f47c6174f1eea171319438b93ba67ff5bf367 F tool/cg_anno.tcl f95b0006c52cf7f0496b506343415b6ee3cdcdd3 x @@ -1707,7 +1708,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 40314bc999af08ab10e654241208842b4bb95b19858d11249444372250ea4160 a983fa857048c748112c2f805f13aadeae43b43b3d27fa55433a3699b51c9a0e -R 38dbfe1b01398cfde35ddf8702196d69 +P 003dc140536d0dd4384252ae1b82827b5d8b2b399520f99549b9266a482009fb +R e29768d7d377928f9be17fdcc6f50028 U drh -Z bda364b06b801f317fc0151c8a36021e +Z 99658c0596100aeaf925626dd549f8e1 diff --git a/manifest.uuid b/manifest.uuid index 4d9aad0fd3..d9fe0c4992 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -003dc140536d0dd4384252ae1b82827b5d8b2b399520f99549b9266a482009fb \ No newline at end of file +cf2abd59be9971a55bd3d6c5df374c6aaa23bf81819482b42f01ee2484dcd739 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 5079035ca1..f32a03a6d7 100644 --- a/src/expr.c +++ b/src/expr.c @@ -3543,12 +3543,8 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ codeInteger(pParse, pExpr, 0, target); return target; } - case TK_TRUE: { - sqlite3VdbeAddOp2(v, OP_Integer, 1, target); - return target; - } - case TK_FALSE: { - sqlite3VdbeAddOp2(v, OP_Integer, 0, target); + case TK_TRUEFALSE: { + sqlite3VdbeAddOp2(v, OP_Integer, pExpr->iTable, target); return target; } #ifndef SQLITE_OMIT_FLOATING_POINT @@ -3706,11 +3702,14 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ sqlite3VdbeAddOp2(v, op, r1, inReg); break; } - case TK_ISTRUE: { + case TK_TRUTH: { + assert( pExpr->pRight->op==TK_TRUEFALSE ); + assert( pExpr->pRight->iTable==0 || pExpr->pRight->iTable==1 ); + assert( pExpr->op2==TK_IS || pExpr->op2==TK_ISNOT ); r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); testcase( regFree1==0 ); - sqlite3VdbeAddOp2(v, OP_Not, r1, inReg); - sqlite3VdbeAddOp2(v, OP_Not, inReg, inReg); + sqlite3VdbeAddOp4Int(v, OP_IsTrue, r1, inReg, !pExpr->pRight->iTable, + pExpr->pRight->iTable ^ (pExpr->op2==TK_IS)); break; } case TK_ISNULL: @@ -4488,9 +4487,21 @@ void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ sqlite3ExprIfFalse(pParse, pExpr->pLeft, dest, jumpIfNull); break; } - case TK_ISTRUE: { + case TK_TRUTH: { + int isNot; testcase( jumpIfNull==0 ); - sqlite3ExprIfTrue(pParse, pExpr->pLeft, dest, jumpIfNull); + assert( pExpr->pRight->op==TK_TRUEFALSE ); + assert( pExpr->pRight->iTable==0 || pExpr->pRight->iTable==1 ); + testcase( pExpr->pRight->iTable==0 ); + assert( pExpr->op2==TK_IS || pExpr->op2==TK_ISNOT ); + isNot = pExpr->op2==TK_ISNOT; + if( pExpr->pRight->iTable ^ isNot ){ + sqlite3ExprIfTrue(pParse, pExpr->pLeft, dest, + isNot ? SQLITE_JUMPIFNULL : 0); + }else{ + sqlite3ExprIfFalse(pParse, pExpr->pLeft, dest, + isNot ? SQLITE_JUMPIFNULL : 0); + } break; } case TK_IS: @@ -4647,9 +4658,25 @@ void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ sqlite3ExprIfTrue(pParse, pExpr->pLeft, dest, jumpIfNull); break; } - case TK_ISTRUE: { + case TK_TRUTH: { testcase( jumpIfNull==0 ); - sqlite3ExprIfFalse(pParse, pExpr->pLeft, dest, jumpIfNull); + int isNot; + testcase( jumpIfNull==0 ); + assert( pExpr->pRight->op==TK_TRUEFALSE ); + assert( pExpr->pRight->iTable==0 || pExpr->pRight->iTable==1 ); + testcase( pExpr->pRight->iTable==0 ); + assert( pExpr->op2==TK_IS || pExpr->op2==TK_ISNOT ); + isNot = pExpr->op2==TK_ISNOT; + if( pExpr->pRight->iTable ^ isNot ){ + /* IS TRUE and IS NOT FALSE */ + sqlite3ExprIfFalse(pParse, pExpr->pLeft, dest, + isNot ? 0 : SQLITE_JUMPIFNULL); + + }else{ + /* IS FALSE and IS NOT TRUE */ + sqlite3ExprIfTrue(pParse, pExpr->pLeft, dest, + isNot ? 0: SQLITE_JUMPIFNULL); + } break; } case TK_IS: diff --git a/src/resolve.c b/src/resolve.c index fb00788dd0..c6b086c3ad 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -437,13 +437,9 @@ static int lookupName( pExpr->pTab = 0; return WRC_Prune; } - if( sqlite3StrICmp(zCol, "true")==0 ){ - pExpr->op = TK_TRUE; - pExpr->pTab = 0; - return WRC_Prune; - } - if( sqlite3StrICmp(zCol, "false")==0 ){ - pExpr->op = TK_FALSE; + if( sqlite3StrICmp(zCol, "true")==0 || sqlite3StrICmp(zCol, "false")==0 ){ + pExpr->op = TK_TRUEFALSE; + pExpr->iTable = zCol[4]==0; pExpr->pTab = 0; return WRC_Prune; } @@ -796,28 +792,30 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ break; } case TK_IS: - /* Handle special cases of "x IS TRUE" and "x IS FALSE". The first - ** is transformed into "+x" and the second into "NOT x". */ - if( pExpr->pRight->op==TK_ID ){ - int rc = resolveExprStep(pWalker, pExpr->pRight); + case TK_ISNOT: { + Expr *pRight; + assert( !ExprHasProperty(pExpr, EP_Reduced) ); + /* Handle special cases of "x IS TRUE", "x IS FALSE", "x IS NOT TRUE", + ** and "x IS NOT FALSE". */ + if( (pRight = pExpr->pRight)->op==TK_ID ){ + int rc = resolveExprStep(pWalker, pRight); if( rc==WRC_Abort ) return WRC_Abort; - if( pExpr->pRight->op==TK_TRUE ){ - pExpr->op = TK_ISTRUE; - return WRC_Continue; - }else if( pExpr->pRight->op==TK_FALSE ){ - pExpr->op = TK_NOT; + if( pRight->op==TK_TRUEFALSE ){ + assert( pRight->iTable==0 || pRight->iTable==1 ); + pExpr->op2 = pExpr->op; + pExpr->op = TK_TRUTH; return WRC_Continue; } } /* Fall thru */ + } case TK_BETWEEN: case TK_EQ: case TK_NE: case TK_LT: case TK_LE: case TK_GT: - case TK_GE: - case TK_ISNOT: { + case TK_GE: { int nLeft, nRight; if( pParse->db->mallocFailed ) break; assert( pExpr->pLeft!=0 ); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index f4c82cdb68..6da57c25f6 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2387,7 +2387,8 @@ struct Expr { ** TK_REGISTER: register number ** TK_TRIGGER: 1 -> new, 0 -> old ** EP_Unlikely: 134217728 times likelihood - ** TK_SELECT: 1st register of result vector */ + ** TK_SELECT: 1st register of result vector + ** TK_TRUEFALSE: 1 for true, 0 for false */ ynVar iColumn; /* TK_COLUMN: column index. -1 for rowid. ** TK_VARIABLE: variable number (always >= 1). ** TK_SELECT_COLUMN: column of the result vector */ diff --git a/src/vdbe.c b/src/vdbe.c index 61fc55e4de..cdc16292b8 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2191,6 +2191,34 @@ case OP_Or: { /* same as TK_OR, in1, in2, out3 */ break; } +/* Opcode: IsTrue P1 P2 P3 P4 * +** Synopsis: r[P2] = coalesce(r[P1]==TRUE,P3) ^ P4 +** +** This opcode implements the IS TRUE, IS FALSE, IS NOT TRUE, and +** IS NOT FALSE operators. +** +** Interpret the value in register P1 as a boolean value. Store the that +** boolean (a 0 or 1) in register P2. Or if the value in register P1 is +** NULL, then the P3 is stored in register P2. Invert the answer if P4 +** is 1. +** +** The logic is summarized like this: +** +**
    +**
  • P3==0, P4==0 → r[P2] = r[P1] IS TRUE +**
  • P3==1, P4==1 → r[P2] = r[P1] IS FALSE +**
  • P3==0, P4==1 → r[P2] = r[P1] IS NOT TRUE +**
  • P3==1, P4==0 → r[P2] = r[P1] IS NOT FALSE +**
+*/ +case OP_IsTrue: { /* in1, out2 */ + assert( pOp->p4type==P4_INT32 ); + assert( pOp->p4.i==0 || pOp->p4.i==1 ); + sqlite3VdbeMemSetInt64(&aMem[pOp->p2], + sqlite3VdbeBooleanValue(&aMem[pOp->p1], pOp->p3) ^ pOp->p4.i); + break; +} + /* Opcode: Not P1 P2 * * * ** Synopsis: r[P2]= !r[P1] ** diff --git a/test/istrue.test b/test/istrue.test new file mode 100644 index 0000000000..cb6bfc52dc --- /dev/null +++ b/test/istrue.test @@ -0,0 +1,81 @@ +# 2018-02-26 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing expressions of the form +# +# x IS TRUE +# x IS FALSE +# x IS NOT TRUE +# x IS NOT FALSE +# +# Tests are also included for the use of TRUE and FALSE as +# literal values. + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +do_execsql_test istrue-100 { + CREATE TABLE t1(x INTEGER PRIMARY KEY, y BOOLEAN); + INSERT INTO t1 VALUES(1, true),(2, false),(3, null); + SELECT x FROM t1 WHERE y IS TRUE; +} {1} +do_execsql_test istrue-110 { + SELECT x FROM t1 WHERE y IS FALSE; +} {2} +do_execsql_test istrue-120 { + SELECT x FROM t1 WHERE y IS NULL; +} {3} +do_execsql_test istrue-130 { + SELECT x FROM t1 WHERE y IS NOT TRUE; +} {2 3} +do_execsql_test istrue-140 { + SELECT x FROM t1 WHERE y IS NOT FALSE; +} {1 3} +do_execsql_test istrue-150 { + SELECT x FROM t1 WHERE y IS NOT NULL; +} {1 2} +unset -nocomplain X +set X 9 +do_execsql_test istrue-160 { + SELECT x FROM t1 WHERE y IS TRUE OR (8==$X) +} {1} +do_execsql_test istrue-170 { + SELECT x FROM t1 WHERE y IS FALSE OR (8==$X) +} {2} +do_execsql_test istrue-180 { + SELECT x FROM t1 WHERE y IS NULL OR (8==$X); +} {3} +do_execsql_test istrue-190 { + SELECT x FROM t1 WHERE y IS NOT TRUE OR (8==$X); +} {2 3} +do_execsql_test istrue-200 { + SELECT x FROM t1 WHERE y IS NOT FALSE OR (8==$X); +} {1 3} +do_execsql_test istrue-210 { + SELECT x FROM t1 WHERE y IS NOT NULL OR (8==$X); +} {1 2} + +do_execsql_test istrue-300 { + SELECT x, + y IS TRUE, y IS FALSE, y is NULL, + y IS NOT TRUE, y IS NOT FALSE, y IS NOT NULL, '|' + FROM t1 ORDER BY x; +} {1 1 0 0 0 1 1 | 2 0 1 0 1 0 1 | 3 0 0 1 1 1 0 |} + +do_execsql_test istrue-400 { + SELECT x FROM t1 WHERE true; +} {1 2 3} +do_execsql_test istrue-410 { + SELECT x FROM t1 WHERE false; +} {} + + +finish_test diff --git a/tool/addopcodes.tcl b/tool/addopcodes.tcl index 7059d71363..070779029b 100644 --- a/tool/addopcodes.tcl +++ b/tool/addopcodes.tcl @@ -22,8 +22,7 @@ close $in # ILLEGAL *must* be the last two token codes and they must be in that order. # set extras { - TRUE - FALSE + TRUEFALSE ISNOT FUNCTION COLUMN @@ -31,7 +30,7 @@ set extras { AGG_COLUMN UMINUS UPLUS - ISTRUE + TRUTH REGISTER VECTOR SELECT_COLUMN From 3432821192f3c0ddcb5588f401daf661060ea124 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 26 Feb 2018 19:03:25 +0000 Subject: [PATCH 099/107] Enhance TreeView so that it can display the new IS TRUE expression trees. FossilOrigin-Name: 7e38305e6f71c3353df3d11d099e255eefbacceafd7ed2ccaba257fe2e7b8c38 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/treeview.c | 18 ++++++++++++++++++ 3 files changed, 25 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index d23a52f63f..7117845e2a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Refactor\sfor\scorrect\sNULL\shandling\sin\sthe\sIS\sTRUE,\sIS\sFALSE,\sIS\sNOT\sTRUE,\nand\sIS\sNOT\sFALSE\soperators. -D 2018-02-26T18:49:05.668 +C Enhance\sTreeView\sso\sthat\sit\scan\sdisplay\sthe\snew\sIS\sTRUE\sexpression\strees. +D 2018-02-26T19:03:25.495 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in a2d2fb8d17c39ab5ec52beb27850b903949080848236923f436156b72a958737 @@ -551,7 +551,7 @@ F src/test_windirent.h 90dfbe95442c9762357fe128dc7ae3dc199d006de93eb33ba3972e0a9 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c F src/tokenize.c 5b0c661a85f783d35b9883830736eeb63be4aefc4f6b7d9cd081d48782c041e2 -F src/treeview.c eae35972ff44f67064de2eaf35f04afe94e7aea3271a8b3bcebb3f954880fec3 +F src/treeview.c bdc8ca85b6f9b56b10381801b656f0ec0f4a86a3f5006cd3f9faa24fc15ea443 F src/trigger.c a34539c69433276d37b0da9a89c117726ff2d292c0902895af1f393a983cd3a1 F src/update.c a90a32ffc0100265b0693dbbdbe490756447af181f5ea2c138cce515b08c8795 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5 @@ -1708,7 +1708,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 003dc140536d0dd4384252ae1b82827b5d8b2b399520f99549b9266a482009fb -R e29768d7d377928f9be17fdcc6f50028 +P cf2abd59be9971a55bd3d6c5df374c6aaa23bf81819482b42f01ee2484dcd739 +R 6f6e40c4ec384ce48a27d1e80052a614 U drh -Z 99658c0596100aeaf925626dd549f8e1 +Z deaee9f4cadd8927720fc9b75136faa8 diff --git a/manifest.uuid b/manifest.uuid index d9fe0c4992..46f8b1b4b2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cf2abd59be9971a55bd3d6c5df374c6aaa23bf81819482b42f01ee2484dcd739 \ No newline at end of file +7e38305e6f71c3353df3d11d099e255eefbacceafd7ed2ccaba257fe2e7b8c38 \ No newline at end of file diff --git a/src/treeview.c b/src/treeview.c index 6dd386ffe6..ea91710cfd 100644 --- a/src/treeview.c +++ b/src/treeview.c @@ -292,6 +292,10 @@ void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 moreToFollow){ sqlite3TreeViewLine(pView,"NULL"); break; } + case TK_TRUEFALSE: { + sqlite3TreeViewLine(pView, pExpr->iTable ? "TRUE":"FALSE"); + break; + } #ifndef SQLITE_OMIT_BLOB_LITERAL case TK_BLOB: { sqlite3TreeViewLine(pView,"%s", pExpr->u.zToken); @@ -348,6 +352,20 @@ void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 moreToFollow){ case TK_ISNULL: zUniOp = "ISNULL"; break; case TK_NOTNULL: zUniOp = "NOTNULL"; break; + case TK_TRUTH: { + assert( pExpr->op2==TK_IS || pExpr->op2==TK_ISNOT ); + assert( pExpr->pRight ); + assert( pExpr->pRight->op==TK_TRUEFALSE ); + assert( pExpr->pRight->iTable==0 || pExpr->pRight->iTable==1 ); + switch( (pExpr->op2==TK_ISNOT)*2 + pExpr->pRight->iTable ){ + case 0: zUniOp = "IS-FALSE"; break; + case 1: zUniOp = "IS-TRUE"; break; + case 2: zUniOp = "IS-NOT-FALSE"; break; + case 3: zUniOp = "IS-NOT-TRUE"; break; + } + break; + } + case TK_SPAN: { sqlite3TreeViewLine(pView, "SPAN %Q", pExpr->u.zToken); sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); From 171d16bb2187b41636980b809d04d923e2811523 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 26 Feb 2018 20:15:54 +0000 Subject: [PATCH 100/107] Get the "DEFAULT true" and "DEFAULT false" phrases working correctly in CREATE TABLE. FossilOrigin-Name: 8002f87d96b3f885cd208e7d204907a33ba60c4057ce2338b71e2de41215b0e5 --- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- src/expr.c | 28 +++++++++++++++++++++++++++- src/parse.y | 2 ++ src/resolve.c | 6 ++---- src/sqliteInt.h | 1 + test/istrue.test | 11 +++++++++++ 7 files changed, 54 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index 7117845e2a..39f7ed6fcb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\sTreeView\sso\sthat\sit\scan\sdisplay\sthe\snew\sIS\sTRUE\sexpression\strees. -D 2018-02-26T19:03:25.495 +C Get\sthe\s"DEFAULT\strue"\sand\s"DEFAULT\sfalse"\sphrases\sworking\scorrectly\sin\nCREATE\sTABLE. +D 2018-02-26T20:15:54.167 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in a2d2fb8d17c39ab5ec52beb27850b903949080848236923f436156b72a958737 @@ -441,7 +441,7 @@ F src/date.c ebe1dc7c8a347117bb02570f1a931c62dd78f4a2b1b516f4837d45b7d6426957 F src/dbpage.c 8db4c97f630e7d83f884ea75caf1ffd0988c160e9d530194d93721c80821e0f6 F src/dbstat.c 7a4ba8518b6369ef3600c49cf9c918ad979acba610b2aebef1b656d649b96720 F src/delete.c 20c8788451dc737a967c87ea53ad43544d617f5b57d32ccce8bd52a0daf9e89b -F src/expr.c 4344e70a158c0092e0c14c564d352546259297996cba250f1d6d9dbf830a415d +F src/expr.c 2662b7cb76943b5865fe5e62f21615e7f86928b98f0b908b339f84a91ace8f8e F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c d617daf66b5515e2b42c1405b2b4984c30ca50fb705ab164271a9bf66c69e331 F src/func.c 94f42cba2cc1c34aeaa441022ba0170ec3fec4bba54db4e0ded085c6dc0fdc51 @@ -477,7 +477,7 @@ F src/os_win.c eb03c6d52f893bcd7fdd4c6006674c13c1b5e49543fec98d605201af2997171c F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 0b6bd5442733b2e08d0673de6cdafe3e7ab0b5715e4844ac836ab346b1d9ed89 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a -F src/parse.y 4e750e1b261ff9f1d0b6b5d40a829c66d691899f48953fde839d8b52d41aa148 +F src/parse.y 6319911fb7ab37907fb1ea14ddfea429b2c42707276eaa388b6c402891dfc091 F src/pcache.c 7ae91a4557a43d77d449accbfdc68846e6516f8e2eda46e8bbe4536fb669b201 F src/pcache.h 072f94d29281cffd99e46c1539849f248c4b56ae7684c1f36626797fee375170 F src/pcache1.c 716975564c15eb6679e97f734cec1bfd6c16ac3d4010f05f1f8e509fc7d19880 @@ -486,14 +486,14 @@ F src/pragma.h bb83728944b42f6d409c77f5838a8edbdb0fe83046c5496ffc9602b40340a324 F src/prepare.c 259f4e7960c47082c9653f3d5f0c294abd68bb9c3aab86de7630700cba1c20fb F src/printf.c d3b7844ddeb11fbbdd38dd84d09c9c1ac171d21fb038473c3aa97981201cc660 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 -F src/resolve.c 7e6a9702fe3bbf556b1105497ba7a51c24bf606bcc2a7c329122b31e04149005 +F src/resolve.c 6f8a45063c693019734887b5f95e860137960fae324ef8b6db36f06e8c53240e F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c f02352ac5cbb6fad3804add825743b924cfb6c742ba2e8666d726828a9985d73 F src/shell.c.in c1b14bc0bc1fe73a97dfaad55ca09f5d6e44f6019977d94e7e3d1861616035e1 F src/sqlite.h.in 235e942dd8c01b414c2996828b4d0d2500faf8850f8b24ae17d31c172f519e69 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 83a3c4ce93d650bedfd1aa558cb85a516bd6d094445ee989740827d0d944368d -F src/sqliteInt.h d38064626689d9a832f17cfc0feef250e34a9f6f2589805e8cef5e85c38c18fa +F src/sqliteInt.h a6856d3d1aefd594b2c8c6245c6fbcba3d691e38555e22910ee6e7a839bc3897 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -1001,7 +1001,7 @@ F test/ioerr3.test d3cec5e1a11ad6d27527d0d38573fbff14c71bdd F test/ioerr4.test f130fe9e71008577b342b8874d52984bd04ede2c F test/ioerr5.test 2edfa4fb0f896f733071303b42224df8bedd9da4 F test/ioerr6.test a395a6ab144b26a9e3e21059a1ab6a7149cca65b -F test/istrue.test e95d4366a705232d81781545f314457f4cc407c8a53b5eac96703f220a74303a +F test/istrue.test c15523b11ea315e7c2855e12f39ddd9df8852d23b8caafceb53a5f5dca907c91 F test/join.test 442c462eea85cf065d70a663c626b780a95af6e11585d909bb63b87598afe678 F test/join2.test f1b5ba94ba70a791df2d99e61adcebf62edacad1eca57881c97eb3e2ca718595 F test/join3.test 6f0c774ff1ba0489e6c88a3e77b9d3528fb4fda0 @@ -1708,7 +1708,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P cf2abd59be9971a55bd3d6c5df374c6aaa23bf81819482b42f01ee2484dcd739 -R 6f6e40c4ec384ce48a27d1e80052a614 +P 7e38305e6f71c3353df3d11d099e255eefbacceafd7ed2ccaba257fe2e7b8c38 +R 8bbc8431995f4fe7e9fb21f05431fedd U drh -Z deaee9f4cadd8927720fc9b75136faa8 +Z 0df480b7d34c4b4166a378f5cc8b7dd8 diff --git a/manifest.uuid b/manifest.uuid index 46f8b1b4b2..4d5c7fa522 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7e38305e6f71c3353df3d11d099e255eefbacceafd7ed2ccaba257fe2e7b8c38 \ No newline at end of file +8002f87d96b3f885cd208e7d204907a33ba60c4057ce2338b71e2de41215b0e5 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index f32a03a6d7..75d8db7e42 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1123,7 +1123,7 @@ static int dupedExprStructSize(Expr *p, int flags){ assert( flags==EXPRDUP_REDUCE || flags==0 ); /* Only one flag value allowed */ assert( EXPR_FULLSIZE<=0xfff ); assert( (0xfff & (EP_Reduced|EP_TokenOnly))==0 ); - if( 0==flags || p->op==TK_SELECT_COLUMN ){ + if( 0==flags || p->op==TK_SELECT_COLUMN || p->op==TK_TRUEFALSE ){ nSize = EXPR_FULLSIZE; }else{ assert( !ExprHasProperty(p, EP_TokenOnly|EP_Reduced) ); @@ -1732,6 +1732,26 @@ int sqlite3SelectWalkFail(Walker *pWalker, Select *NotUsed){ return WRC_Abort; } +/* +** If the input expression is an ID with the name "true" or "false" +** then convert it into an appropriate TK_TRUEFALSE term. Return true +** if a conversion occurred, and false if the expression is unaltered. +*/ +int sqlite3ExprIdToTrueFalse(Expr *pExpr){ + assert( pExpr->op==TK_ID || pExpr->op==TK_STRING ); + if( sqlite3StrICmp(pExpr->u.zToken, "true")==0 + || sqlite3StrICmp(pExpr->u.zToken, "false")==0 + ){ + pExpr->op = TK_TRUEFALSE; + pExpr->iTable = pExpr->u.zToken[4]==0; + pExpr->pTab = 0; + ExprSetProperty(pExpr, EP_NoReduce); + return 1; + } + return 0; +} + + /* ** These routines are Walker callbacks used to check expressions to ** see if they are "constant" for some definition of constant. The @@ -1779,6 +1799,12 @@ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){ return WRC_Abort; } case TK_ID: + /* Convert "true" or "false" in a DEFAULT clause into the + ** appropriate TK_TRUEFALSE operator */ + if( pWalker->eCode>=4 && sqlite3ExprIdToTrueFalse(pExpr) ){ + return WRC_Prune; + } + /* Fall thru */ case TK_COLUMN: case TK_AGG_FUNCTION: case TK_AGG_COLUMN: diff --git a/src/parse.y b/src/parse.y index d9cf1cb87d..ff2e74af01 100644 --- a/src/parse.y +++ b/src/parse.y @@ -313,6 +313,8 @@ ccons ::= DEFAULT MINUS(A) term(X) scanpt(Z). { } ccons ::= DEFAULT scanpt id(X). { Expr *p = tokenExpr(pParse, TK_STRING, X); + sqlite3ExprIdToTrueFalse(p); + testcase( p->op==TK_TRUEFALSE && p->iTable==0 ); sqlite3AddDefaultValue(pParse,p,X.z,X.z+X.n); } diff --git a/src/resolve.c b/src/resolve.c index c6b086c3ad..33dd4dcb7b 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -432,15 +432,13 @@ static int lookupName( ** fields are not changed in any context. */ if( cnt==0 && zTab==0 ){ + assert( pExpr->op==TK_ID ); if( ExprHasProperty(pExpr,EP_DblQuoted) ){ pExpr->op = TK_STRING; pExpr->pTab = 0; return WRC_Prune; } - if( sqlite3StrICmp(zCol, "true")==0 || sqlite3StrICmp(zCol, "false")==0 ){ - pExpr->op = TK_TRUEFALSE; - pExpr->iTable = zCol[4]==0; - pExpr->pTab = 0; + if( sqlite3ExprIdToTrueFalse(pExpr) ){ return WRC_Prune; } } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 6da57c25f6..09413de004 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3840,6 +3840,7 @@ void sqlite3EndTransaction(Parse*,int); void sqlite3Savepoint(Parse*, int, Token*); void sqlite3CloseSavepoints(sqlite3 *); void sqlite3LeaveMutexAndCloseZombie(sqlite3*); +int sqlite3ExprIdToTrueFalse(Expr*); int sqlite3ExprIsConstant(Expr*); int sqlite3ExprIsConstantNotJoin(Expr*); int sqlite3ExprIsConstantOrFunction(Expr*, u8); diff --git a/test/istrue.test b/test/istrue.test index cb6bfc52dc..5ffc7f5f4f 100644 --- a/test/istrue.test +++ b/test/istrue.test @@ -77,5 +77,16 @@ do_execsql_test istrue-410 { SELECT x FROM t1 WHERE false; } {} +do_execsql_test istrue-500 { + CREATE TABLE t2( + a INTEGER PRIMARY KEY, + b BOOLEAN DEFAULT true, + c BOOLEAN DEFAULT(true), + d BOOLEAN DEFAULT false, + e BOOLEAN DEFAULT(false) + ); + INSERT INTO t2 DEFAULT VALUES; + SELECT * FROM t2; +} {1 1 1 0 0} finish_test From 43c4ac8bfecba96e4db6d075c5fa743a4fe25ea5 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 26 Feb 2018 21:26:27 +0000 Subject: [PATCH 101/107] Code simplifications. New test cases. FossilOrigin-Name: 57508518ef9d003d259ba98dcc32e5104aca26731a7161808741fe10bc0830d0 --- manifest | 22 +++++++++++----------- manifest.uuid | 2 +- src/expr.c | 47 +++++++++++++++++++++++++++-------------------- src/parse.y | 2 +- src/resolve.c | 1 - src/sqliteInt.h | 4 ++-- src/treeview.c | 16 ++++++++-------- test/istrue.test | 34 ++++++++++++++++++++++++++++++++++ 8 files changed, 84 insertions(+), 44 deletions(-) diff --git a/manifest b/manifest index 39f7ed6fcb..96e1edd95d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Get\sthe\s"DEFAULT\strue"\sand\s"DEFAULT\sfalse"\sphrases\sworking\scorrectly\sin\nCREATE\sTABLE. -D 2018-02-26T20:15:54.167 +C Code\ssimplifications.\s\sNew\stest\scases. +D 2018-02-26T21:26:27.326 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in a2d2fb8d17c39ab5ec52beb27850b903949080848236923f436156b72a958737 @@ -441,7 +441,7 @@ F src/date.c ebe1dc7c8a347117bb02570f1a931c62dd78f4a2b1b516f4837d45b7d6426957 F src/dbpage.c 8db4c97f630e7d83f884ea75caf1ffd0988c160e9d530194d93721c80821e0f6 F src/dbstat.c 7a4ba8518b6369ef3600c49cf9c918ad979acba610b2aebef1b656d649b96720 F src/delete.c 20c8788451dc737a967c87ea53ad43544d617f5b57d32ccce8bd52a0daf9e89b -F src/expr.c 2662b7cb76943b5865fe5e62f21615e7f86928b98f0b908b339f84a91ace8f8e +F src/expr.c ee29e9c2af181dd8fe4476f7e983aa13aa1634d00dd955a90ffbeb4732d7bc39 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c d617daf66b5515e2b42c1405b2b4984c30ca50fb705ab164271a9bf66c69e331 F src/func.c 94f42cba2cc1c34aeaa441022ba0170ec3fec4bba54db4e0ded085c6dc0fdc51 @@ -477,7 +477,7 @@ F src/os_win.c eb03c6d52f893bcd7fdd4c6006674c13c1b5e49543fec98d605201af2997171c F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 0b6bd5442733b2e08d0673de6cdafe3e7ab0b5715e4844ac836ab346b1d9ed89 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a -F src/parse.y 6319911fb7ab37907fb1ea14ddfea429b2c42707276eaa388b6c402891dfc091 +F src/parse.y b875c74b72497f3cf23d9c6227ec0c859824ac9a5b23380cc1ed8adcff4857a0 F src/pcache.c 7ae91a4557a43d77d449accbfdc68846e6516f8e2eda46e8bbe4536fb669b201 F src/pcache.h 072f94d29281cffd99e46c1539849f248c4b56ae7684c1f36626797fee375170 F src/pcache1.c 716975564c15eb6679e97f734cec1bfd6c16ac3d4010f05f1f8e509fc7d19880 @@ -486,14 +486,14 @@ F src/pragma.h bb83728944b42f6d409c77f5838a8edbdb0fe83046c5496ffc9602b40340a324 F src/prepare.c 259f4e7960c47082c9653f3d5f0c294abd68bb9c3aab86de7630700cba1c20fb F src/printf.c d3b7844ddeb11fbbdd38dd84d09c9c1ac171d21fb038473c3aa97981201cc660 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 -F src/resolve.c 6f8a45063c693019734887b5f95e860137960fae324ef8b6db36f06e8c53240e +F src/resolve.c 66c73fcb7719b8ff0e841b58338f13604ff3e2b50a723f9b8f383595735262f6 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c f02352ac5cbb6fad3804add825743b924cfb6c742ba2e8666d726828a9985d73 F src/shell.c.in c1b14bc0bc1fe73a97dfaad55ca09f5d6e44f6019977d94e7e3d1861616035e1 F src/sqlite.h.in 235e942dd8c01b414c2996828b4d0d2500faf8850f8b24ae17d31c172f519e69 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 83a3c4ce93d650bedfd1aa558cb85a516bd6d094445ee989740827d0d944368d -F src/sqliteInt.h a6856d3d1aefd594b2c8c6245c6fbcba3d691e38555e22910ee6e7a839bc3897 +F src/sqliteInt.h ce6d2c08df3216a69d3f7fb740e8ddb7ef94169277776778e7cfce05ab245d10 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -551,7 +551,7 @@ F src/test_windirent.h 90dfbe95442c9762357fe128dc7ae3dc199d006de93eb33ba3972e0a9 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c F src/tokenize.c 5b0c661a85f783d35b9883830736eeb63be4aefc4f6b7d9cd081d48782c041e2 -F src/treeview.c bdc8ca85b6f9b56b10381801b656f0ec0f4a86a3f5006cd3f9faa24fc15ea443 +F src/treeview.c b2470f160658181f5d1d7672144d5e7a2635a398a2f062842b7b4e0c766c4158 F src/trigger.c a34539c69433276d37b0da9a89c117726ff2d292c0902895af1f393a983cd3a1 F src/update.c a90a32ffc0100265b0693dbbdbe490756447af181f5ea2c138cce515b08c8795 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5 @@ -1001,7 +1001,7 @@ F test/ioerr3.test d3cec5e1a11ad6d27527d0d38573fbff14c71bdd F test/ioerr4.test f130fe9e71008577b342b8874d52984bd04ede2c F test/ioerr5.test 2edfa4fb0f896f733071303b42224df8bedd9da4 F test/ioerr6.test a395a6ab144b26a9e3e21059a1ab6a7149cca65b -F test/istrue.test c15523b11ea315e7c2855e12f39ddd9df8852d23b8caafceb53a5f5dca907c91 +F test/istrue.test 2d89873d72248b7a0ef7b5457120100cce75a19181bcf4b30e90a3f96a5c5558 F test/join.test 442c462eea85cf065d70a663c626b780a95af6e11585d909bb63b87598afe678 F test/join2.test f1b5ba94ba70a791df2d99e61adcebf62edacad1eca57881c97eb3e2ca718595 F test/join3.test 6f0c774ff1ba0489e6c88a3e77b9d3528fb4fda0 @@ -1708,7 +1708,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7e38305e6f71c3353df3d11d099e255eefbacceafd7ed2ccaba257fe2e7b8c38 -R 8bbc8431995f4fe7e9fb21f05431fedd +P 8002f87d96b3f885cd208e7d204907a33ba60c4057ce2338b71e2de41215b0e5 +R 5cfd707edda06e1acf2e2d72bf60da27 U drh -Z 0df480b7d34c4b4166a378f5cc8b7dd8 +Z d672c3ecc0520c79d23db3920b6bc66e diff --git a/manifest.uuid b/manifest.uuid index 4d5c7fa522..16a67a9163 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8002f87d96b3f885cd208e7d204907a33ba60c4057ce2338b71e2de41215b0e5 \ No newline at end of file +57508518ef9d003d259ba98dcc32e5104aca26731a7161808741fe10bc0830d0 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 75d8db7e42..fae2c11d9c 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1123,7 +1123,7 @@ static int dupedExprStructSize(Expr *p, int flags){ assert( flags==EXPRDUP_REDUCE || flags==0 ); /* Only one flag value allowed */ assert( EXPR_FULLSIZE<=0xfff ); assert( (0xfff & (EP_Reduced|EP_TokenOnly))==0 ); - if( 0==flags || p->op==TK_SELECT_COLUMN || p->op==TK_TRUEFALSE ){ + if( 0==flags || p->op==TK_SELECT_COLUMN ){ nSize = EXPR_FULLSIZE; }else{ assert( !ExprHasProperty(p, EP_TokenOnly|EP_Reduced) ); @@ -1743,14 +1743,22 @@ int sqlite3ExprIdToTrueFalse(Expr *pExpr){ || sqlite3StrICmp(pExpr->u.zToken, "false")==0 ){ pExpr->op = TK_TRUEFALSE; - pExpr->iTable = pExpr->u.zToken[4]==0; - pExpr->pTab = 0; - ExprSetProperty(pExpr, EP_NoReduce); return 1; } return 0; } +/* +** The argument is one of a TK_TRUEFALSE term. Return 1 if it is TRUE +** and 0 if it is FALSE. +*/ +int sqlite3ExprTruthOperand(const Expr *pExpr){ + assert( pExpr->op==TK_TRUEFALSE ); + assert( sqlite3StrICmp(pExpr->u.zToken,"true")==0 + || sqlite3StrICmp(pExpr->u.zToken,"false")==0 ); + return pExpr->u.zToken[4]==0; +} + /* ** These routines are Walker callbacks used to check expressions to @@ -3570,7 +3578,7 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ return target; } case TK_TRUEFALSE: { - sqlite3VdbeAddOp2(v, OP_Integer, pExpr->iTable, target); + sqlite3VdbeAddOp2(v, OP_Integer, sqlite3ExprTruthOperand(pExpr), target); return target; } #ifndef SQLITE_OMIT_FLOATING_POINT @@ -3729,13 +3737,12 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ break; } case TK_TRUTH: { - assert( pExpr->pRight->op==TK_TRUEFALSE ); - assert( pExpr->pRight->iTable==0 || pExpr->pRight->iTable==1 ); - assert( pExpr->op2==TK_IS || pExpr->op2==TK_ISNOT ); + int isTrue; r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); testcase( regFree1==0 ); - sqlite3VdbeAddOp4Int(v, OP_IsTrue, r1, inReg, !pExpr->pRight->iTable, - pExpr->pRight->iTable ^ (pExpr->op2==TK_IS)); + isTrue = sqlite3ExprTruthOperand(pExpr->pRight); + sqlite3VdbeAddOp4Int(v, OP_IsTrue, r1, inReg, !isTrue, + isTrue ^ (pExpr->op2==TK_IS)); break; } case TK_ISNULL: @@ -4515,13 +4522,13 @@ void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ } case TK_TRUTH: { int isNot; + int isTrue; testcase( jumpIfNull==0 ); - assert( pExpr->pRight->op==TK_TRUEFALSE ); - assert( pExpr->pRight->iTable==0 || pExpr->pRight->iTable==1 ); - testcase( pExpr->pRight->iTable==0 ); - assert( pExpr->op2==TK_IS || pExpr->op2==TK_ISNOT ); isNot = pExpr->op2==TK_ISNOT; - if( pExpr->pRight->iTable ^ isNot ){ + isTrue = sqlite3ExprTruthOperand(pExpr->pRight); + testcase( isTrue && isNot ); + testcase( isTrue && !isNot ); + if( isTrue ^ isNot ){ sqlite3ExprIfTrue(pParse, pExpr->pLeft, dest, isNot ? SQLITE_JUMPIFNULL : 0); }else{ @@ -4685,15 +4692,15 @@ void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ break; } case TK_TRUTH: { - testcase( jumpIfNull==0 ); int isNot; + int isTrue; testcase( jumpIfNull==0 ); - assert( pExpr->pRight->op==TK_TRUEFALSE ); - assert( pExpr->pRight->iTable==0 || pExpr->pRight->iTable==1 ); - testcase( pExpr->pRight->iTable==0 ); assert( pExpr->op2==TK_IS || pExpr->op2==TK_ISNOT ); isNot = pExpr->op2==TK_ISNOT; - if( pExpr->pRight->iTable ^ isNot ){ + isTrue = sqlite3ExprTruthOperand(pExpr->pRight); + testcase( isTrue && isNot ); + testcase( isTrue && !isNot ); + if( isTrue ^ isNot ){ /* IS TRUE and IS NOT FALSE */ sqlite3ExprIfFalse(pParse, pExpr->pLeft, dest, isNot ? 0 : SQLITE_JUMPIFNULL); diff --git a/src/parse.y b/src/parse.y index ff2e74af01..81e4fdf1e0 100644 --- a/src/parse.y +++ b/src/parse.y @@ -314,7 +314,7 @@ ccons ::= DEFAULT MINUS(A) term(X) scanpt(Z). { ccons ::= DEFAULT scanpt id(X). { Expr *p = tokenExpr(pParse, TK_STRING, X); sqlite3ExprIdToTrueFalse(p); - testcase( p->op==TK_TRUEFALSE && p->iTable==0 ); + testcase( p->op==TK_TRUEFALSE && sqlite3ExprTruthOperand(p) ); sqlite3AddDefaultValue(pParse,p,X.z,X.z+X.n); } diff --git a/src/resolve.c b/src/resolve.c index 33dd4dcb7b..7ae49bde57 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -799,7 +799,6 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ int rc = resolveExprStep(pWalker, pRight); if( rc==WRC_Abort ) return WRC_Abort; if( pRight->op==TK_TRUEFALSE ){ - assert( pRight->iTable==0 || pRight->iTable==1 ); pExpr->op2 = pExpr->op; pExpr->op = TK_TRUTH; return WRC_Continue; diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 09413de004..4dd58d3370 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2387,8 +2387,7 @@ struct Expr { ** TK_REGISTER: register number ** TK_TRIGGER: 1 -> new, 0 -> old ** EP_Unlikely: 134217728 times likelihood - ** TK_SELECT: 1st register of result vector - ** TK_TRUEFALSE: 1 for true, 0 for false */ + ** TK_SELECT: 1st register of result vector */ ynVar iColumn; /* TK_COLUMN: column index. -1 for rowid. ** TK_VARIABLE: variable number (always >= 1). ** TK_SELECT_COLUMN: column of the result vector */ @@ -3841,6 +3840,7 @@ void sqlite3Savepoint(Parse*, int, Token*); void sqlite3CloseSavepoints(sqlite3 *); void sqlite3LeaveMutexAndCloseZombie(sqlite3*); int sqlite3ExprIdToTrueFalse(Expr*); +int sqlite3ExprTruthOperand(const Expr*); int sqlite3ExprIsConstant(Expr*); int sqlite3ExprIsConstantNotJoin(Expr*); int sqlite3ExprIsConstantOrFunction(Expr*, u8); diff --git a/src/treeview.c b/src/treeview.c index ea91710cfd..cfe818268f 100644 --- a/src/treeview.c +++ b/src/treeview.c @@ -293,7 +293,8 @@ void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 moreToFollow){ break; } case TK_TRUEFALSE: { - sqlite3TreeViewLine(pView, pExpr->iTable ? "TRUE":"FALSE"); + sqlite3TreeViewLine(pView, + sqlite3ExprTruthOperand(pExpr) ? "TRUE" : "FALSE"); break; } #ifndef SQLITE_OMIT_BLOB_LITERAL @@ -353,16 +354,15 @@ void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 moreToFollow){ case TK_NOTNULL: zUniOp = "NOTNULL"; break; case TK_TRUTH: { + int x; + const char *azOp[] = { + "IS-FALSE", "IS-TRUE", "IS-NOT-FALSE", "IS-NOT-TRUE" + }; assert( pExpr->op2==TK_IS || pExpr->op2==TK_ISNOT ); assert( pExpr->pRight ); assert( pExpr->pRight->op==TK_TRUEFALSE ); - assert( pExpr->pRight->iTable==0 || pExpr->pRight->iTable==1 ); - switch( (pExpr->op2==TK_ISNOT)*2 + pExpr->pRight->iTable ){ - case 0: zUniOp = "IS-FALSE"; break; - case 1: zUniOp = "IS-TRUE"; break; - case 2: zUniOp = "IS-NOT-FALSE"; break; - case 3: zUniOp = "IS-NOT-TRUE"; break; - } + x = (pExpr->op2==TK_ISNOT)*2 + sqlite3ExprTruthOperand(pExpr->pRight); + zUniOp = azOp[x]; break; } diff --git a/test/istrue.test b/test/istrue.test index 5ffc7f5f4f..250f1f9d5d 100644 --- a/test/istrue.test +++ b/test/istrue.test @@ -88,5 +88,39 @@ do_execsql_test istrue-500 { INSERT INTO t2 DEFAULT VALUES; SELECT * FROM t2; } {1 1 1 0 0} +do_execsql_test istrue-510 { + DROP TABLE t2; + CREATE TABLE t2( + a INTEGER PRIMARY KEY, + b BOOLEAN DEFAULT(not true), + c BOOLEAN DEFAULT(not false) + ); + INSERT INTO t2(a) VALUES(99); + SELECT * FROM t2; +} {99 0 1} +do_execsql_test istrue-520 { + DROP TABLE t2; + CREATE TABLE t2( + a INTEGER PRIMARY KEY, + b BOOLEAN CHECK(b IS TRUE), + c BOOLEAN CHECK(c IS FALSE), + d BOOLEAN CHECK(d IS NOT TRUE), + e BOOLEAN CHECK(e IS NOT FALSE) + ); + INSERT INTO t2 VALUES(1,true,false,null,null); + SELECT * FROM t2; +} {1 1 0 {} {}} +do_catchsql_test istrue-521 { + INSERT INTO t2 VALUES(2,false,false,null,null); +} {1 {CHECK constraint failed: t2}} +do_catchsql_test istrue-522 { + INSERT INTO t2 VALUES(2,true,true,null,null); +} {1 {CHECK constraint failed: t2}} +do_catchsql_test istrue-523 { + INSERT INTO t2 VALUES(2,true,false,true,null); +} {1 {CHECK constraint failed: t2}} +do_catchsql_test istrue-524 { + INSERT INTO t2 VALUES(2,true,false,null,false); +} {1 {CHECK constraint failed: t2}} finish_test From e39ef31cd68218eaf39bf02cdb365b936158baef Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 27 Feb 2018 00:58:13 +0000 Subject: [PATCH 102/107] Remove an unnecessary decision. FossilOrigin-Name: adcb4665490404e8da2de128dd5df9aeed8e08893649dedb6788ebb5d957b400 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/expr.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 96e1edd95d..7f3497c990 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Code\ssimplifications.\s\sNew\stest\scases. -D 2018-02-26T21:26:27.326 +C Remove\san\sunnecessary\sdecision. +D 2018-02-27T00:58:13.219 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in a2d2fb8d17c39ab5ec52beb27850b903949080848236923f436156b72a958737 @@ -441,7 +441,7 @@ F src/date.c ebe1dc7c8a347117bb02570f1a931c62dd78f4a2b1b516f4837d45b7d6426957 F src/dbpage.c 8db4c97f630e7d83f884ea75caf1ffd0988c160e9d530194d93721c80821e0f6 F src/dbstat.c 7a4ba8518b6369ef3600c49cf9c918ad979acba610b2aebef1b656d649b96720 F src/delete.c 20c8788451dc737a967c87ea53ad43544d617f5b57d32ccce8bd52a0daf9e89b -F src/expr.c ee29e9c2af181dd8fe4476f7e983aa13aa1634d00dd955a90ffbeb4732d7bc39 +F src/expr.c 03975fff227cd24b17e86b5bcc64e0fb9deecccd1b50d23c5f1f8a6852b99064 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c d617daf66b5515e2b42c1405b2b4984c30ca50fb705ab164271a9bf66c69e331 F src/func.c 94f42cba2cc1c34aeaa441022ba0170ec3fec4bba54db4e0ded085c6dc0fdc51 @@ -1708,7 +1708,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8002f87d96b3f885cd208e7d204907a33ba60c4057ce2338b71e2de41215b0e5 -R 5cfd707edda06e1acf2e2d72bf60da27 +P 57508518ef9d003d259ba98dcc32e5104aca26731a7161808741fe10bc0830d0 +R 8ef9da63ad321cc3596e92a715293098 U drh -Z d672c3ecc0520c79d23db3920b6bc66e +Z 8fed3cfabeba6ea8fc4880d89a25745b diff --git a/manifest.uuid b/manifest.uuid index 16a67a9163..622a0019fa 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -57508518ef9d003d259ba98dcc32e5104aca26731a7161808741fe10bc0830d0 \ No newline at end of file +adcb4665490404e8da2de128dd5df9aeed8e08893649dedb6788ebb5d957b400 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index fae2c11d9c..e1959761f4 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1809,7 +1809,7 @@ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){ case TK_ID: /* Convert "true" or "false" in a DEFAULT clause into the ** appropriate TK_TRUEFALSE operator */ - if( pWalker->eCode>=4 && sqlite3ExprIdToTrueFalse(pExpr) ){ + if( sqlite3ExprIdToTrueFalse(pExpr) ){ return WRC_Prune; } /* Fall thru */ From 96acafbea7bc8b7c688f0a39b98bcd0ed00dc35c Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 27 Feb 2018 14:49:25 +0000 Subject: [PATCH 103/107] Clean up comments and variable names prior to merge. FossilOrigin-Name: 6445519e91c4f98b4a9a45d5091d733c31497ebf0eb23a76edce3091f626035d --- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- src/expr.c | 40 +++++++++++++++++++++------------------- src/parse.y | 2 +- src/sqliteInt.h | 2 +- src/treeview.c | 4 ++-- src/vdbe.c | 11 ++++++----- 7 files changed, 42 insertions(+), 39 deletions(-) diff --git a/manifest b/manifest index 7f3497c990..bdef730d7d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\san\sunnecessary\sdecision. -D 2018-02-27T00:58:13.219 +C Clean\sup\scomments\sand\svariable\snames\sprior\sto\smerge. +D 2018-02-27T14:49:25.328 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in a2d2fb8d17c39ab5ec52beb27850b903949080848236923f436156b72a958737 @@ -441,7 +441,7 @@ F src/date.c ebe1dc7c8a347117bb02570f1a931c62dd78f4a2b1b516f4837d45b7d6426957 F src/dbpage.c 8db4c97f630e7d83f884ea75caf1ffd0988c160e9d530194d93721c80821e0f6 F src/dbstat.c 7a4ba8518b6369ef3600c49cf9c918ad979acba610b2aebef1b656d649b96720 F src/delete.c 20c8788451dc737a967c87ea53ad43544d617f5b57d32ccce8bd52a0daf9e89b -F src/expr.c 03975fff227cd24b17e86b5bcc64e0fb9deecccd1b50d23c5f1f8a6852b99064 +F src/expr.c 904f85e5f5fa232488cbb662fcf3306ae203728751468670fb9840a9b1886abc F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c d617daf66b5515e2b42c1405b2b4984c30ca50fb705ab164271a9bf66c69e331 F src/func.c 94f42cba2cc1c34aeaa441022ba0170ec3fec4bba54db4e0ded085c6dc0fdc51 @@ -477,7 +477,7 @@ F src/os_win.c eb03c6d52f893bcd7fdd4c6006674c13c1b5e49543fec98d605201af2997171c F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 0b6bd5442733b2e08d0673de6cdafe3e7ab0b5715e4844ac836ab346b1d9ed89 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a -F src/parse.y b875c74b72497f3cf23d9c6227ec0c859824ac9a5b23380cc1ed8adcff4857a0 +F src/parse.y 97b31a676ec1cb049970883a317efcd07ca8c7daded950c8de49199cdfdb1dfa F src/pcache.c 7ae91a4557a43d77d449accbfdc68846e6516f8e2eda46e8bbe4536fb669b201 F src/pcache.h 072f94d29281cffd99e46c1539849f248c4b56ae7684c1f36626797fee375170 F src/pcache1.c 716975564c15eb6679e97f734cec1bfd6c16ac3d4010f05f1f8e509fc7d19880 @@ -493,7 +493,7 @@ F src/shell.c.in c1b14bc0bc1fe73a97dfaad55ca09f5d6e44f6019977d94e7e3d1861616035e F src/sqlite.h.in 235e942dd8c01b414c2996828b4d0d2500faf8850f8b24ae17d31c172f519e69 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 83a3c4ce93d650bedfd1aa558cb85a516bd6d094445ee989740827d0d944368d -F src/sqliteInt.h ce6d2c08df3216a69d3f7fb740e8ddb7ef94169277776778e7cfce05ab245d10 +F src/sqliteInt.h 6c321e404f97117c9b676ac1a8591a22d095bf5ab730624becea3ad93067c89c F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -551,13 +551,13 @@ F src/test_windirent.h 90dfbe95442c9762357fe128dc7ae3dc199d006de93eb33ba3972e0a9 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c F src/tokenize.c 5b0c661a85f783d35b9883830736eeb63be4aefc4f6b7d9cd081d48782c041e2 -F src/treeview.c b2470f160658181f5d1d7672144d5e7a2635a398a2f062842b7b4e0c766c4158 +F src/treeview.c e0d62678314abf0e1e6d09ea405f4de3f404a17b4641c2169c304b5edf509320 F src/trigger.c a34539c69433276d37b0da9a89c117726ff2d292c0902895af1f393a983cd3a1 F src/update.c a90a32ffc0100265b0693dbbdbe490756447af181f5ea2c138cce515b08c8795 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5 F src/util.c d9eb0a6c4aae1b00a7369eadd7ca0bbe946cb4c953b6751aa20d357c2f482157 F src/vacuum.c 762ee9bbf8733d87d8cd06f58d950e881982e416f8c767334a40ffd341b6bff5 -F src/vdbe.c 39675332be215b4672f8800b9355e190476ff5fe35fe71f2c9944a3f9dce17b7 +F src/vdbe.c 99a0f275c83c01332b9af6179f2ee1e5d24ad81f90f17f8364397f8a3d20fb40 F src/vdbe.h 134beb7a12a6213c00eba58febaede33447cc4441bc568a0d9c144b33fc3720a F src/vdbeInt.h 95f7adfdc5c8f1353321f55a6c5ec00a90877e3b85af5159e393afb41ff54110 F src/vdbeapi.c fea41171884a4de119f8b10ab514c788674eeeb7f27218bb6d008e1310bfd07f @@ -1708,7 +1708,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 57508518ef9d003d259ba98dcc32e5104aca26731a7161808741fe10bc0830d0 -R 8ef9da63ad321cc3596e92a715293098 +P adcb4665490404e8da2de128dd5df9aeed8e08893649dedb6788ebb5d957b400 +R 691db3d03f207528a0777702d3649318 U drh -Z 8fed3cfabeba6ea8fc4880d89a25745b +Z 3708f8d55564f3165287f25ec6417cc9 diff --git a/manifest.uuid b/manifest.uuid index 622a0019fa..d25d53f33e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -adcb4665490404e8da2de128dd5df9aeed8e08893649dedb6788ebb5d957b400 \ No newline at end of file +6445519e91c4f98b4a9a45d5091d733c31497ebf0eb23a76edce3091f626035d \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index e1959761f4..d2fad0ffc3 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1734,8 +1734,8 @@ int sqlite3SelectWalkFail(Walker *pWalker, Select *NotUsed){ /* ** If the input expression is an ID with the name "true" or "false" -** then convert it into an appropriate TK_TRUEFALSE term. Return true -** if a conversion occurred, and false if the expression is unaltered. +** then convert it into an TK_TRUEFALSE term. Return non-zero if +** the conversion happened, and zero if the expression is unaltered. */ int sqlite3ExprIdToTrueFalse(Expr *pExpr){ assert( pExpr->op==TK_ID || pExpr->op==TK_STRING ); @@ -1749,10 +1749,10 @@ int sqlite3ExprIdToTrueFalse(Expr *pExpr){ } /* -** The argument is one of a TK_TRUEFALSE term. Return 1 if it is TRUE +** The argument must be a TK_TRUEFALSE Expr node. Return 1 if it is TRUE ** and 0 if it is FALSE. */ -int sqlite3ExprTruthOperand(const Expr *pExpr){ +int sqlite3ExprTruthValue(const Expr *pExpr){ assert( pExpr->op==TK_TRUEFALSE ); assert( sqlite3StrICmp(pExpr->u.zToken,"true")==0 || sqlite3StrICmp(pExpr->u.zToken,"false")==0 ); @@ -3578,7 +3578,7 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ return target; } case TK_TRUEFALSE: { - sqlite3VdbeAddOp2(v, OP_Integer, sqlite3ExprTruthOperand(pExpr), target); + sqlite3VdbeAddOp2(v, OP_Integer, sqlite3ExprTruthValue(pExpr), target); return target; } #ifndef SQLITE_OMIT_FLOATING_POINT @@ -3737,12 +3737,15 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ break; } case TK_TRUTH: { - int isTrue; + int isTrue; /* IS TRUE or IS NOT TRUE */ + int bNormal; /* IS TRUE or IS FALSE */ r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); testcase( regFree1==0 ); - isTrue = sqlite3ExprTruthOperand(pExpr->pRight); - sqlite3VdbeAddOp4Int(v, OP_IsTrue, r1, inReg, !isTrue, - isTrue ^ (pExpr->op2==TK_IS)); + isTrue = sqlite3ExprTruthValue(pExpr->pRight); + bNormal = pExpr->op2==TK_IS; + testcase( isTrue && bNormal); + testcase( !isTrue && bNormal); + sqlite3VdbeAddOp4Int(v, OP_IsTrue, r1, inReg, !isTrue, isTrue ^ bNormal); break; } case TK_ISNULL: @@ -4521,13 +4524,13 @@ void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ break; } case TK_TRUTH: { - int isNot; - int isTrue; + int isNot; /* IS NOT TRUE or IS NOT FALSE */ + int isTrue; /* IS TRUE or IS NOT TRUE */ testcase( jumpIfNull==0 ); isNot = pExpr->op2==TK_ISNOT; - isTrue = sqlite3ExprTruthOperand(pExpr->pRight); + isTrue = sqlite3ExprTruthValue(pExpr->pRight); testcase( isTrue && isNot ); - testcase( isTrue && !isNot ); + testcase( !isTrue && isNot ); if( isTrue ^ isNot ){ sqlite3ExprIfTrue(pParse, pExpr->pLeft, dest, isNot ? SQLITE_JUMPIFNULL : 0); @@ -4692,14 +4695,13 @@ void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ break; } case TK_TRUTH: { - int isNot; - int isTrue; + int isNot; /* IS NOT TRUE or IS NOT FALSE */ + int isTrue; /* IS TRUE or IS NOT TRUE */ testcase( jumpIfNull==0 ); - assert( pExpr->op2==TK_IS || pExpr->op2==TK_ISNOT ); isNot = pExpr->op2==TK_ISNOT; - isTrue = sqlite3ExprTruthOperand(pExpr->pRight); + isTrue = sqlite3ExprTruthValue(pExpr->pRight); testcase( isTrue && isNot ); - testcase( isTrue && !isNot ); + testcase( !isTrue && isNot ); if( isTrue ^ isNot ){ /* IS TRUE and IS NOT FALSE */ sqlite3ExprIfFalse(pParse, pExpr->pLeft, dest, @@ -4708,7 +4710,7 @@ void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ }else{ /* IS FALSE and IS NOT TRUE */ sqlite3ExprIfTrue(pParse, pExpr->pLeft, dest, - isNot ? 0: SQLITE_JUMPIFNULL); + isNot ? 0 : SQLITE_JUMPIFNULL); } break; } diff --git a/src/parse.y b/src/parse.y index 81e4fdf1e0..49cc1cb0d4 100644 --- a/src/parse.y +++ b/src/parse.y @@ -314,7 +314,7 @@ ccons ::= DEFAULT MINUS(A) term(X) scanpt(Z). { ccons ::= DEFAULT scanpt id(X). { Expr *p = tokenExpr(pParse, TK_STRING, X); sqlite3ExprIdToTrueFalse(p); - testcase( p->op==TK_TRUEFALSE && sqlite3ExprTruthOperand(p) ); + testcase( p->op==TK_TRUEFALSE && sqlite3ExprTruthValue(p) ); sqlite3AddDefaultValue(pParse,p,X.z,X.z+X.n); } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 4dd58d3370..eba52b6770 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3840,7 +3840,7 @@ void sqlite3Savepoint(Parse*, int, Token*); void sqlite3CloseSavepoints(sqlite3 *); void sqlite3LeaveMutexAndCloseZombie(sqlite3*); int sqlite3ExprIdToTrueFalse(Expr*); -int sqlite3ExprTruthOperand(const Expr*); +int sqlite3ExprTruthValue(const Expr*); int sqlite3ExprIsConstant(Expr*); int sqlite3ExprIsConstantNotJoin(Expr*); int sqlite3ExprIsConstantOrFunction(Expr*, u8); diff --git a/src/treeview.c b/src/treeview.c index cfe818268f..e33fe6e227 100644 --- a/src/treeview.c +++ b/src/treeview.c @@ -294,7 +294,7 @@ void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 moreToFollow){ } case TK_TRUEFALSE: { sqlite3TreeViewLine(pView, - sqlite3ExprTruthOperand(pExpr) ? "TRUE" : "FALSE"); + sqlite3ExprTruthValue(pExpr) ? "TRUE" : "FALSE"); break; } #ifndef SQLITE_OMIT_BLOB_LITERAL @@ -361,7 +361,7 @@ void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 moreToFollow){ assert( pExpr->op2==TK_IS || pExpr->op2==TK_ISNOT ); assert( pExpr->pRight ); assert( pExpr->pRight->op==TK_TRUEFALSE ); - x = (pExpr->op2==TK_ISNOT)*2 + sqlite3ExprTruthOperand(pExpr->pRight); + x = (pExpr->op2==TK_ISNOT)*2 + sqlite3ExprTruthValue(pExpr->pRight); zUniOp = azOp[x]; break; } diff --git a/src/vdbe.c b/src/vdbe.c index cdc16292b8..10715a8232 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2197,7 +2197,7 @@ case OP_Or: { /* same as TK_OR, in1, in2, out3 */ ** This opcode implements the IS TRUE, IS FALSE, IS NOT TRUE, and ** IS NOT FALSE operators. ** -** Interpret the value in register P1 as a boolean value. Store the that +** Interpret the value in register P1 as a boolean value. Store that ** boolean (a 0 or 1) in register P2. Or if the value in register P1 is ** NULL, then the P3 is stored in register P2. Invert the answer if P4 ** is 1. @@ -2205,15 +2205,16 @@ case OP_Or: { /* same as TK_OR, in1, in2, out3 */ ** The logic is summarized like this: ** **
    -**
  • P3==0, P4==0 → r[P2] = r[P1] IS TRUE -**
  • P3==1, P4==1 → r[P2] = r[P1] IS FALSE -**
  • P3==0, P4==1 → r[P2] = r[P1] IS NOT TRUE -**
  • P3==1, P4==0 → r[P2] = r[P1] IS NOT FALSE +**
  • If P3==0 and P4==0 then r[P2] := r[P1] IS TRUE +**
  • If P3==1 and P4==1 then r[P2] := r[P1] IS FALSE +**
  • If P3==0 and P4==1 then r[P2] := r[P1] IS NOT TRUE +**
  • If P3==1 and P4==0 then r[P2] := r[P1] IS NOT FALSE **
*/ case OP_IsTrue: { /* in1, out2 */ assert( pOp->p4type==P4_INT32 ); assert( pOp->p4.i==0 || pOp->p4.i==1 ); + assert( pOp->p3==0 || pOp->p3==1 ); sqlite3VdbeMemSetInt64(&aMem[pOp->p2], sqlite3VdbeBooleanValue(&aMem[pOp->p1], pOp->p3) ^ pOp->p4.i); break; From d7fd899eeb8d29b0bc59727a2e137d4c341100cb Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 28 Feb 2018 04:30:55 +0000 Subject: [PATCH 104/107] Fix a potential NULL pointer deref following OOM in the new IS TRUE logic. FossilOrigin-Name: 48775ec693c21927ff26448acc614dd67d56240fa263499da2a9fe405c546c32 --- manifest | 13 ++++++------- manifest.uuid | 2 +- src/parse.y | 6 ++++-- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 50048f9068..be35bd4d18 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\ssupport\sfor\sTRUE\sand\sFALSE\skeywords\sand\sfor\soperators\sIS\sTRUE,\sIS\sFALSE,\nIS\sNOT\sTRUE,\sand\sIS\sNOT\sFALSE.\s\sIf\sthere\sis\sare\scolumns\snamed\sTRUE\sor\sFALSE,\nthen\sthe\skeywords\sresolve\sto\sthe\scolumn\snames,\sfor\scompatibility.\s\sThe\nbehavior\sof\sthe\s"DEFAULT\strue"\sphrase\sis\schanged\sto\smean\swhat\sit\ssays,\nrather\sthan\sbeing\san\salias\sfor\s"DEFAULT\s'true'". -D 2018-02-27T15:40:59.030 +C Fix\sa\spotential\sNULL\spointer\sderef\sfollowing\sOOM\sin\sthe\snew\sIS\sTRUE\nlogic. +D 2018-02-28T04:30:55.210 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in a2d2fb8d17c39ab5ec52beb27850b903949080848236923f436156b72a958737 @@ -477,7 +477,7 @@ F src/os_win.c eb03c6d52f893bcd7fdd4c6006674c13c1b5e49543fec98d605201af2997171c F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 0b6bd5442733b2e08d0673de6cdafe3e7ab0b5715e4844ac836ab346b1d9ed89 F src/pager.h 581698f2177e8bd4008fe4760898ce20b6133d1df22139b9101b5155f900df7a -F src/parse.y 97b31a676ec1cb049970883a317efcd07ca8c7daded950c8de49199cdfdb1dfa +F src/parse.y 3be4b2b5e33ef7dab3128b765898ee786fc6cd698a7f053f756a68f2f238f0aa F src/pcache.c 7ae91a4557a43d77d449accbfdc68846e6516f8e2eda46e8bbe4536fb669b201 F src/pcache.h 072f94d29281cffd99e46c1539849f248c4b56ae7684c1f36626797fee375170 F src/pcache1.c 716975564c15eb6679e97f734cec1bfd6c16ac3d4010f05f1f8e509fc7d19880 @@ -1708,8 +1708,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c26f236da61aef7c7a8fe38f1c7f33eea894c8fae38de6a085d4cce933a70502 6445519e91c4f98b4a9a45d5091d733c31497ebf0eb23a76edce3091f626035d -R 74016598c506c5fb19fa13d7ec93e943 -T +closed 6445519e91c4f98b4a9a45d5091d733c31497ebf0eb23a76edce3091f626035d +P 9a7f02c50e6c8a3dc3addf4e51b7e04bd31670d6ce58d26d55273c43b5ecc8b0 +R 8cff6bf49e22739f377b0c86155b1f52 U drh -Z 4b60eba7ac8a552c151e343e027ad980 +Z 08a4a2d4426a2c0bef4340ae5149b6ad diff --git a/manifest.uuid b/manifest.uuid index 50dab302b5..8f7e16b28e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9a7f02c50e6c8a3dc3addf4e51b7e04bd31670d6ce58d26d55273c43b5ecc8b0 \ No newline at end of file +48775ec693c21927ff26448acc614dd67d56240fa263499da2a9fe405c546c32 \ No newline at end of file diff --git a/src/parse.y b/src/parse.y index 49cc1cb0d4..9c41484ab2 100644 --- a/src/parse.y +++ b/src/parse.y @@ -313,8 +313,10 @@ ccons ::= DEFAULT MINUS(A) term(X) scanpt(Z). { } ccons ::= DEFAULT scanpt id(X). { Expr *p = tokenExpr(pParse, TK_STRING, X); - sqlite3ExprIdToTrueFalse(p); - testcase( p->op==TK_TRUEFALSE && sqlite3ExprTruthValue(p) ); + if( p ){ + sqlite3ExprIdToTrueFalse(p); + testcase( p->op==TK_TRUEFALSE && sqlite3ExprTruthValue(p) ); + } sqlite3AddDefaultValue(pParse,p,X.z,X.z+X.n); } From 6a8a629e6bf8886e1f5b43f4cdcdd850f96b3bd3 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 28 Feb 2018 21:50:00 +0000 Subject: [PATCH 105/107] Use <pre> around code snippets in the documentation for sessions interfaces. This is a documentation change only with no changes to code. FossilOrigin-Name: c949b915e893e917315ce21092d4c4bbd3e1b88d5326928f71dcc2f18f300702 --- ext/session/sqlite3session.h | 4 ++++ manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/ext/session/sqlite3session.h b/ext/session/sqlite3session.h index 9f33855df0..f9e371fb9a 100644 --- a/ext/session/sqlite3session.h +++ b/ext/session/sqlite3session.h @@ -678,6 +678,7 @@ int sqlite3changeset_fk_conflicts( ** to that error is returned by this function. Otherwise, SQLITE_OK is ** returned. This is to allow the following pattern (pseudo-code): ** +** 
 **   sqlite3changeset_start();
 **   while( SQLITE_ROW==sqlite3changeset_next() ){
 **     // Do something with change.
@@ -686,6 +687,7 @@ int sqlite3changeset_fk_conflicts(
 **   if( rc!=SQLITE_OK ){
 **     // An error has occurred 
 **   }
+**
*/ int sqlite3changeset_finalize(sqlite3_changeset_iter *pIter); @@ -733,6 +735,7 @@ int sqlite3changeset_invert( ** sqlite3_changegroup object. Calling it produces similar results as the ** following code fragment: ** +** 
 **   sqlite3_changegroup *pGrp;
 **   rc = sqlite3_changegroup_new(&pGrp);
 **   if( rc==SQLITE_OK ) rc = sqlite3changegroup_add(pGrp, nA, pA);
@@ -743,6 +746,7 @@ int sqlite3changeset_invert(
 **     *ppOut = 0;
 **     *pnOut = 0;
 **   }
+**
** ** Refer to the sqlite3_changegroup documentation below for details. */ diff --git a/manifest b/manifest index be35bd4d18..20de1e13a0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\spotential\sNULL\spointer\sderef\sfollowing\sOOM\sin\sthe\snew\sIS\sTRUE\nlogic. -D 2018-02-28T04:30:55.210 +C Use\s<pre>\saround\scode\ssnippets\sin\sthe\sdocumentation\sfor\ssessions\ninterfaces.\s\sThis\sis\sa\sdocumentation\schange\sonly\swith\sno\schanges\sto\scode. +D 2018-02-28T21:50:00.053 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in a2d2fb8d17c39ab5ec52beb27850b903949080848236923f436156b72a958737 @@ -403,7 +403,7 @@ F ext/session/sessionfault2.test 04aa0bc9aa70ea43d8de82c4f648db4de1e990b0 F ext/session/sessionstat1.test 41cd97c2e48619a41cdf8ae749e1b25f34719de638689221aa43971be693bf4e F ext/session/sessionwor.test 2f3744236dc8b170a695b7d8ddc8c743c7e79fdc F ext/session/sqlite3session.c a5b7aed647abe7e366254b755597fce3f2719d82c98990cb0e1e07a3d203fe2b -F ext/session/sqlite3session.h 01774161cbd328fe3d496323655b9cc142317ff1fb1ae15c1232075ea240e3a4 +F ext/session/sqlite3session.h 357b889d5d1be268bd39bce9df8d28fb4afc88c35b5ebf62dad6aeb725d3f37e F ext/session/test_session.c eb0bd6c1ea791c1d66ee4ef94c16500dad936386 F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 @@ -1708,7 +1708,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9a7f02c50e6c8a3dc3addf4e51b7e04bd31670d6ce58d26d55273c43b5ecc8b0 -R 8cff6bf49e22739f377b0c86155b1f52 +P 48775ec693c21927ff26448acc614dd67d56240fa263499da2a9fe405c546c32 +R 534465c4041022fe3e1312b87af4cd46 U drh -Z 08a4a2d4426a2c0bef4340ae5149b6ad +Z d5651a3a39c99623b3bbaa536731261b diff --git a/manifest.uuid b/manifest.uuid index 8f7e16b28e..33f071d785 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -48775ec693c21927ff26448acc614dd67d56240fa263499da2a9fe405c546c32 \ No newline at end of file +c949b915e893e917315ce21092d4c4bbd3e1b88d5326928f71dcc2f18f300702 \ No newline at end of file From bda30ce4d82fa2f26309028e93dbacf2e7612818 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 28 Feb 2018 22:21:29 +0000 Subject: [PATCH 106/107] Enhance the sessions documentation to show the methods of the various objects. FossilOrigin-Name: e01177754ad6d9e2d38adddddd2e2e212094dac1154bda5fcee61ca8b678ae0f --- ext/session/sqlite3session.h | 33 ++++++++++++++++++++++++++++++++- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 39 insertions(+), 8 deletions(-) diff --git a/ext/session/sqlite3session.h b/ext/session/sqlite3session.h index f9e371fb9a..81b1ed06d6 100644 --- a/ext/session/sqlite3session.h +++ b/ext/session/sqlite3session.h @@ -13,16 +13,23 @@ extern "C" { /* ** CAPI3REF: Session Object Handle +** +** An instance of this object is a [session] that can be used to +** record changes to a database. */ typedef struct sqlite3_session sqlite3_session; /* ** CAPI3REF: Changeset Iterator Handle +** +** An instance of this object is as as a cursor for iterating +** over the elements of a [changeset] or [patchset]. */ typedef struct sqlite3_changeset_iter sqlite3_changeset_iter; /* ** CAPI3REF: Create A New Session Object +** CONSTRUCTOR: sqlite3_session ** ** Create a new session object attached to database handle db. If successful, ** a pointer to the new object is written to *ppSession and SQLITE_OK is @@ -59,6 +66,7 @@ int sqlite3session_create( /* ** CAPI3REF: Delete A Session Object +** DESTRUCTOR: sqlite3_session ** ** Delete a session object previously allocated using ** [sqlite3session_create()]. Once a session object has been deleted, the @@ -74,6 +82,7 @@ void sqlite3session_delete(sqlite3_session *pSession); /* ** CAPI3REF: Enable Or Disable A Session Object +** METHOD: sqlite3_session ** ** Enable or disable the recording of changes by a session object. When ** enabled, a session object records changes made to the database. When @@ -93,6 +102,7 @@ int sqlite3session_enable(sqlite3_session *pSession, int bEnable); /* ** CAPI3REF: Set Or Clear the Indirect Change Flag +** METHOD: sqlite3_session ** ** Each change recorded by a session object is marked as either direct or ** indirect. A change is marked as indirect if either: @@ -122,6 +132,7 @@ int sqlite3session_indirect(sqlite3_session *pSession, int bIndirect); /* ** CAPI3REF: Attach A Table To A Session Object +** METHOD: sqlite3_session ** ** If argument zTab is not NULL, then it is the name of a table to attach ** to the session object passed as the first argument. All subsequent changes @@ -184,6 +195,7 @@ int sqlite3session_attach( /* ** CAPI3REF: Set a table filter on a Session Object. +** METHOD: sqlite3_session ** ** The second argument (xFilter) is the "filter callback". For changes to rows ** in tables that are not attached to the Session object, the filter is called @@ -202,6 +214,7 @@ void sqlite3session_table_filter( /* ** CAPI3REF: Generate A Changeset From A Session Object +** METHOD: sqlite3_session ** ** Obtain a changeset containing changes to the tables attached to the ** session object passed as the first argument. If successful, @@ -311,7 +324,8 @@ int sqlite3session_changeset( ); /* -** CAPI3REF: Load The Difference Between Tables Into A Session +** CAPI3REF: Load The Difference Between Tables Into A Session +** METHOD: sqlite3_session ** ** If it is not already attached to the session object passed as the first ** argument, this function attaches table zTbl in the same manner as the @@ -376,6 +390,7 @@ int sqlite3session_diff( /* ** CAPI3REF: Generate A Patchset From A Session Object +** METHOD: sqlite3_session ** ** The differences between a patchset and a changeset are that: ** @@ -427,6 +442,7 @@ int sqlite3session_isempty(sqlite3_session *pSession); /* ** CAPI3REF: Create An Iterator To Traverse A Changeset +** CONSTRUCTOR: sqlite3_changeset_iter ** ** Create an iterator used to iterate through the contents of a changeset. ** If successful, *pp is set to point to the iterator handle and SQLITE_OK @@ -467,6 +483,7 @@ int sqlite3changeset_start( /* ** CAPI3REF: Advance A Changeset Iterator +** METHOD: sqlite3_changeset_iter ** ** This function may only be used with iterators created by function ** [sqlite3changeset_start()]. If it is called on an iterator passed to @@ -491,6 +508,7 @@ int sqlite3changeset_next(sqlite3_changeset_iter *pIter); /* ** CAPI3REF: Obtain The Current Operation From A Changeset Iterator +** METHOD: sqlite3_changeset_iter ** ** The pIter argument passed to this function may either be an iterator ** passed to a conflict-handler by [sqlite3changeset_apply()], or an iterator @@ -525,6 +543,7 @@ int sqlite3changeset_op( /* ** CAPI3REF: Obtain The Primary Key Definition Of A Table +** METHOD: sqlite3_changeset_iter ** ** For each modified table, a changeset includes the following: ** @@ -556,6 +575,7 @@ int sqlite3changeset_pk( /* ** CAPI3REF: Obtain old.* Values From A Changeset Iterator +** METHOD: sqlite3_changeset_iter ** ** The pIter argument passed to this function may either be an iterator ** passed to a conflict-handler by [sqlite3changeset_apply()], or an iterator @@ -586,6 +606,7 @@ int sqlite3changeset_old( /* ** CAPI3REF: Obtain new.* Values From A Changeset Iterator +** METHOD: sqlite3_changeset_iter ** ** The pIter argument passed to this function may either be an iterator ** passed to a conflict-handler by [sqlite3changeset_apply()], or an iterator @@ -619,6 +640,7 @@ int sqlite3changeset_new( /* ** CAPI3REF: Obtain Conflicting Row Values From A Changeset Iterator +** METHOD: sqlite3_changeset_iter ** ** This function should only be used with iterator objects passed to a ** conflict-handler callback by [sqlite3changeset_apply()] with either @@ -646,6 +668,7 @@ int sqlite3changeset_conflict( /* ** CAPI3REF: Determine The Number Of Foreign Key Constraint Violations +** METHOD: sqlite3_changeset_iter ** ** This function may only be called with an iterator passed to an ** SQLITE_CHANGESET_FOREIGN_KEY conflict handler callback. In this case @@ -662,6 +685,7 @@ int sqlite3changeset_fk_conflicts( /* ** CAPI3REF: Finalize A Changeset Iterator +** METHOD: sqlite3_changeset_iter ** ** This function is used to finalize an iterator allocated with ** [sqlite3changeset_start()]. @@ -762,11 +786,15 @@ int sqlite3changeset_concat( /* ** CAPI3REF: Changegroup Handle +** +** A changegroup is an object used to combine two or more +** [changesets] or [patchsets] */ typedef struct sqlite3_changegroup sqlite3_changegroup; /* ** CAPI3REF: Create A New Changegroup Object +** CONSTRUCTOR: sqlite3_changegroup ** ** An sqlite3_changegroup object is used to combine two or more changesets ** (or patchsets) into a single changeset (or patchset). A single changegroup @@ -804,6 +832,7 @@ int sqlite3changegroup_new(sqlite3_changegroup **pp); /* ** CAPI3REF: Add A Changeset To A Changegroup +** METHOD: sqlite3_changegroup ** ** Add all changes within the changeset (or patchset) in buffer pData (size ** nData bytes) to the changegroup. @@ -881,6 +910,7 @@ int sqlite3changegroup_add(sqlite3_changegroup*, int nData, void *pData); /* ** CAPI3REF: Obtain A Composite Changeset From A Changegroup +** METHOD: sqlite3_changegroup ** ** Obtain a buffer containing a changeset (or patchset) representing the ** current contents of the changegroup. If the inputs to the changegroup @@ -911,6 +941,7 @@ int sqlite3changegroup_output( /* ** CAPI3REF: Delete A Changegroup Object +** DESTRUCTOR: sqlite3_changegroup */ void sqlite3changegroup_delete(sqlite3_changegroup*); diff --git a/manifest b/manifest index 20de1e13a0..07d15b424e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Use\s<pre>\saround\scode\ssnippets\sin\sthe\sdocumentation\sfor\ssessions\ninterfaces.\s\sThis\sis\sa\sdocumentation\schange\sonly\swith\sno\schanges\sto\scode. -D 2018-02-28T21:50:00.053 +C Enhance\sthe\ssessions\sdocumentation\sto\sshow\sthe\smethods\sof\sthe\svarious\sobjects. +D 2018-02-28T22:21:29.549 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in a2d2fb8d17c39ab5ec52beb27850b903949080848236923f436156b72a958737 @@ -403,7 +403,7 @@ F ext/session/sessionfault2.test 04aa0bc9aa70ea43d8de82c4f648db4de1e990b0 F ext/session/sessionstat1.test 41cd97c2e48619a41cdf8ae749e1b25f34719de638689221aa43971be693bf4e F ext/session/sessionwor.test 2f3744236dc8b170a695b7d8ddc8c743c7e79fdc F ext/session/sqlite3session.c a5b7aed647abe7e366254b755597fce3f2719d82c98990cb0e1e07a3d203fe2b -F ext/session/sqlite3session.h 357b889d5d1be268bd39bce9df8d28fb4afc88c35b5ebf62dad6aeb725d3f37e +F ext/session/sqlite3session.h b4de978c24a48a0d9b3b92ddfb749f4b07461766325ee950f5ecb8384c10606f F ext/session/test_session.c eb0bd6c1ea791c1d66ee4ef94c16500dad936386 F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 @@ -1708,7 +1708,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 48775ec693c21927ff26448acc614dd67d56240fa263499da2a9fe405c546c32 -R 534465c4041022fe3e1312b87af4cd46 +P c949b915e893e917315ce21092d4c4bbd3e1b88d5326928f71dcc2f18f300702 +R 6f69334171b7b3454d79aa33bdfe9b61 U drh -Z d5651a3a39c99623b3bbaa536731261b +Z 67038639e12c985c1bd40cee53ce76b7 diff --git a/manifest.uuid b/manifest.uuid index 33f071d785..36563f126d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c949b915e893e917315ce21092d4c4bbd3e1b88d5326928f71dcc2f18f300702 \ No newline at end of file +e01177754ad6d9e2d38adddddd2e2e212094dac1154bda5fcee61ca8b678ae0f \ No newline at end of file From dd8a4af8e77795f8d0982db57e3e959b7ba108ec Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 1 Mar 2018 12:05:51 +0000 Subject: [PATCH 107/107] Fix some crashes in the sqlite3changeset_apply() function that could be caused by corrupt changeset blobs. FossilOrigin-Name: 745a9a7fef0f28a57ea3f44899058993f6ecdedda52c81a09a4a9ce09c9004d6 --- ext/session/session4.test | 25 ++++++++++++++++++++ ext/session/sqlite3session.c | 46 ++++++++++++++++++++++++++---------- manifest | 16 ++++++------- manifest.uuid | 2 +- 4 files changed, 67 insertions(+), 22 deletions(-) diff --git a/ext/session/session4.test b/ext/session/session4.test index 120a230b57..d07c57ffd6 100644 --- a/ext/session/session4.test +++ b/ext/session/session4.test @@ -64,4 +64,29 @@ do_test 1.3 { list [catch { sqlite3changeset_apply db $x xConflict } msg] $msg } {1 SQLITE_CORRUPT} +#------------------------------------------------------------------------- +reset_db +do_execsql_test 2.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY,b,c,d); + CREATE TABLE t2(e TEXT PRIMARY KEY NOT NULL,f,g); + CREATE TABLE t3(w REAL PRIMARY KEY NOT NULL,x,y); + CREATE TABLE t4(z PRIMARY KEY) WITHOUT ROWID; +} + +foreach {tn blob} { + 1 {54010174340012000000} + 2 {54fefe8bcb0012000300} + 3 {5480809280808001017434001200fb} + 4 {50af9c939c9c9cb09c9c6400b09c9c6400} + 5 {12000300} + 6 {09847304} + 7 {5401017434001208} +} { +if {$tn==7} breakpoint + do_test 2.$tn { + set changeset [binary decode hex $blob] + list [catch { sqlite3changeset_apply db $changeset xConflict } msg] $msg + } {1 SQLITE_CORRUPT} +} + finish_test diff --git a/ext/session/sqlite3session.c b/ext/session/sqlite3session.c index bdecdd1031..2a130c7e9e 100644 --- a/ext/session/sqlite3session.c +++ b/ext/session/sqlite3session.c @@ -2718,15 +2718,18 @@ static int sessionReadRecord( for(i=0; iiNext>=pIn->nData ){ + rc = SQLITE_CORRUPT; + }else{ + rc = sessionInputBuffer(pIn, 9); + } if( rc==SQLITE_OK ){ eType = pIn->aData[pIn->iNext++]; - } - - assert( apOut[i]==0 ); - if( eType ){ - apOut[i] = sqlite3ValueNew(0); - if( !apOut[i] ) rc = SQLITE_NOMEM; + assert( apOut[i]==0 ); + if( eType ){ + apOut[i] = sqlite3ValueNew(0); + if( !apOut[i] ) rc = SQLITE_NOMEM; + } } if( rc==SQLITE_OK ){ @@ -2857,11 +2860,15 @@ static int sessionChangesetReadTblhdr(sqlite3_changeset_iter *p){ int nByte; int nVarint; nVarint = sessionVarintGet(&p->in.aData[p->in.iNext], &p->nCol); - nCopy -= nVarint; - p->in.iNext += nVarint; - nByte = p->nCol * sizeof(sqlite3_value*) * 2 + nCopy; - p->tblhdr.nBuf = 0; - sessionBufferGrow(&p->tblhdr, nByte, &rc); + if( p->nCol>0 ){ + nCopy -= nVarint; + p->in.iNext += nVarint; + nByte = p->nCol * sizeof(sqlite3_value*) * 2 + nCopy; + p->tblhdr.nBuf = 0; + sessionBufferGrow(&p->tblhdr, nByte, &rc); + }else{ + rc = SQLITE_CORRUPT; + } } if( rc==SQLITE_OK ){ @@ -2939,6 +2946,13 @@ static int sessionChangesetNext( op = p->in.aData[p->in.iNext++]; } + if( p->zTab==0 ){ + /* The first record in the changeset is not a table header. Must be a + ** corrupt changeset. */ + assert( p->in.iNext==1 ); + return (p->rc = SQLITE_CORRUPT_BKPT); + } + p->op = op; p->bIndirect = p->in.aData[p->in.iNext++]; if( p->op!=SQLITE_UPDATE && p->op!=SQLITE_DELETE && p->op!=SQLITE_INSERT ){ @@ -3708,7 +3722,13 @@ static int sessionBindRow( if( !abPK || abPK[i] ){ sqlite3_value *pVal; (void)xValue(pIter, i, &pVal); - rc = sessionBindValue(pStmt, i+1, pVal); + if( pVal==0 ){ + /* The value in the changeset was "undefined". This indicates a + ** corrupt changeset blob. */ + rc = SQLITE_CORRUPT; + }else{ + rc = sessionBindValue(pStmt, i+1, pVal); + } } } return rc; diff --git a/manifest b/manifest index 07d15b424e..f09706c7be 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\sthe\ssessions\sdocumentation\sto\sshow\sthe\smethods\sof\sthe\svarious\sobjects. -D 2018-02-28T22:21:29.549 +C Fix\ssome\scrashes\sin\sthe\ssqlite3changeset_apply()\sfunction\sthat\scould\sbe\scaused\nby\scorrupt\schangeset\sblobs. +D 2018-03-01T12:05:51.293 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in a2d2fb8d17c39ab5ec52beb27850b903949080848236923f436156b72a958737 @@ -381,7 +381,7 @@ F ext/session/changeset.c 4ccbaa4531944c24584bf6a61ba3a39c62b6267a F ext/session/session1.test 736d7ff178662f0b717c37f46531b84a5ce0210ccb0c4edf629c55dbcbbc3ea1 F ext/session/session2.test 284de45abae4cc1082bc52012ee81521d5ac58e0 F ext/session/session3.test ce9ce3dfa489473987f899e9f6a0f2db9bde3479 -F ext/session/session4.test 457b02bdc349eb01151e54de014df77abd3c08c8 +F ext/session/session4.test 488539cee1d2510b415236fac2727575f4110e7609a500eb21c8d9e517dfff38 F ext/session/session5.test 716bc6fafd625ce60dfa62ae128971628c1a1169 F ext/session/session6.test 443789bc2fca12e4f7075cf692c60b8a2bea1a26 F ext/session/session8.test 8e194b3f655d861ca36de5d4de53f702751bab3b @@ -402,7 +402,7 @@ F ext/session/sessionfault.test da273f2712b6411e85e71465a1733b8501dbf6f7 F ext/session/sessionfault2.test 04aa0bc9aa70ea43d8de82c4f648db4de1e990b0 F ext/session/sessionstat1.test 41cd97c2e48619a41cdf8ae749e1b25f34719de638689221aa43971be693bf4e F ext/session/sessionwor.test 2f3744236dc8b170a695b7d8ddc8c743c7e79fdc -F ext/session/sqlite3session.c a5b7aed647abe7e366254b755597fce3f2719d82c98990cb0e1e07a3d203fe2b +F ext/session/sqlite3session.c bd8e52f8b4adef1d886564655030d5a7675baf59d52542c8f7eda99971048e13 F ext/session/sqlite3session.h b4de978c24a48a0d9b3b92ddfb749f4b07461766325ee950f5ecb8384c10606f F ext/session/test_session.c eb0bd6c1ea791c1d66ee4ef94c16500dad936386 F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 @@ -1708,7 +1708,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c949b915e893e917315ce21092d4c4bbd3e1b88d5326928f71dcc2f18f300702 -R 6f69334171b7b3454d79aa33bdfe9b61 -U drh -Z 67038639e12c985c1bd40cee53ce76b7 +P e01177754ad6d9e2d38adddddd2e2e212094dac1154bda5fcee61ca8b678ae0f +R 72900d8c116195630f7d9bfcd1153ca1 +U dan +Z e1e8f32ef49baada43e1fbc29b2fefd6 diff --git a/manifest.uuid b/manifest.uuid index 36563f126d..bad79d7fbb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e01177754ad6d9e2d38adddddd2e2e212094dac1154bda5fcee61ca8b678ae0f \ No newline at end of file +745a9a7fef0f28a57ea3f44899058993f6ecdedda52c81a09a4a9ce09c9004d6 \ No newline at end of file