From 99f363b35ed536b3f9ed6f14ec1d4915e732c122 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 25 Jun 2020 20:28:13 +0000 Subject: [PATCH 1/2] Enhance the --verify option to speedtest1.c so that it computes and displays a hash of the result from all SQL queries, for verification purposes. FossilOrigin-Name: 60d1e46c8c8a3c853034fd79f204bcb5d50d1c366eb246849c333a2d0abc2648 --- manifest | 15 ++++--- manifest.uuid | 2 +- test/speedtest1.c | 112 +++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 121 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 56768b3a54..1dcf574f4d 100644 --- a/manifest +++ b/manifest @@ -1,6 +1,6 @@ B 7a876209a678a34c198b54ceef9e3c041f128a14dc73357f6a57cadadaa6cf7b -C Add\sthe\sieee754_mantissa()\sand\sieee754_exponent()\sfunctions\sto\sthe\siee754\nextension.\s\sBuild\sthe\sieee754\sextension\sinto\sthe\sCLI. -D 2020-06-24T15:06:29.041 +C Enhance\sthe\s--verify\soption\sto\sspeedtest1.c\sso\sthat\sit\scomputes\sand\sdisplays\na\shash\sof\sthe\sresult\sfrom\sall\sSQL\squeries,\sfor\sverification\spurposes. +D 2020-06-25T20:28:13.760 F Makefile.in 19374a5db06c3199ec1bab71ab74a103d8abf21053c05e9389255dc58083f806 F Makefile.msc 48f5a3fc32672c09ad73795749f6253e406a31526935fbbffd8f021108d54574 F autoconf/Makefile.am a8d1d24affe52ebf8d7ddcf91aa973fa0316618ab95bb68c87cabf8faf527dc8 @@ -12,11 +12,14 @@ F src/build.c ba1bbe563a3dc02d5fed20537603181e5289c13ea30ae5e775f552e7557adbfa F src/shell.c.in d663152487d4bfddea0f6d21ebc2ed51575d22657a02c6828afd344bbd4651af F src/test1.c fe56c4bcaa2685ca9aa25d817a0ee9345e189aff4a5a71a3d8ba946c7776feb8 F test/decimal.test 12739a01bdba4c4d79f95b323e6b67b9fad1ab6ffb56116bd2b9c81a5b19e1d9 -F test/speedtest1.c ea201573f9b27542ea1e74a68e74f121e0eb04c89e67039f40ed68f1b833339f +F test/speedtest1.c 62a22866f2f1bf3bfd81ded4473314d5508209e7758e122746ace56c138a0f66 F tool/mkautoconfamal.sh f62353eb6c06ab264da027fd4507d09914433dbdcab9cb011cdc18016f1ab3b8 F tool/mksqlite3c.tcl f4ef476510eca4124c874a72029f1e01bc54a896b1724e8f9eef0d8bfae0e84c F tool/mksqlite3h.tcl 1f5e4a1dbbbc43c83cc6e74fe32c6c620502240b66c7c0f33a51378e78fc4edf -P 838817b680f02b3845d6d56f85d5d36fa5ae7453afef7a1a5a24624255f2dc3e -R 6e286f8b0cf2dcbbb10b38a65d995df9 +P db2f0836b64cd2e119684f1cf75fa3b19a84ca6aca1a239f7e2b9298016e2c95 +R 612c27b420c2073a1256c7d89a8eeaa4 +T *branch * speedtest-hash +T *sym-speedtest-hash * +T -sym-trunk * U drh -Z 549c83ceeab0b10ef9bb12772423a56c +Z 5bfc672238a7ff5aba5dfa178b2aded4 diff --git a/manifest.uuid b/manifest.uuid index 766ebfb8a3..d888b0868d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -db2f0836b64cd2e119684f1cf75fa3b19a84ca6aca1a239f7e2b9298016e2c95 \ No newline at end of file +60d1e46c8c8a3c853034fd79f204bcb5d50d1c366eb246849c333a2d0abc2648 \ No newline at end of file diff --git a/test/speedtest1.c b/test/speedtest1.c index ddd2c35911..6703b38752 100644 --- a/test/speedtest1.c +++ b/test/speedtest1.c @@ -41,7 +41,6 @@ static const char zHelp[] = " --without-rowid Use WITHOUT ROWID where appropriate\n" ; - #include "sqlite3.h" #include #include @@ -61,6 +60,87 @@ static const char zHelp[] = # define sqlite3_int64 sqlite_int64 #endif +typedef sqlite3_uint64 u64; + +#ifndef SPEEDTEST_OMIT_HASH +/**************************************************************************** +** Hash algorithm used to verify that compilation is not miscompiled +** in such a was as to generate an incorrect result. +*/ +/* +** State structure for a Hash hash in progress +*/ +typedef struct HashContext HashContext; +struct HashContext { + unsigned char isInit; /* True if initialized */ + unsigned char i, j; /* State variables */ + unsigned char s[256]; /* State variables */ + unsigned char r[32]; /* Result */ +}; + +/* +** Initialize a new hash. iSize determines the size of the hash +** in bits and should be one of 224, 256, 384, or 512. Or iSize +** can be zero to use the default hash size of 256 bits. +*/ +static void HashInit(HashContext *p){ + unsigned int k; + p->i = 0; + p->j = 0; + for(k=0; k<256; k++) p->s[k] = k; +} + +/* +** Make consecutive calls to the HashUpdate function to add new content +** to the hash +*/ +static void HashUpdate( + HashContext *p, + const unsigned char *aData, + unsigned int nData +){ + unsigned char t; + unsigned char i = p->i; + unsigned char j = p->j; + unsigned int k; + for(k=0; ks[i] + aData[k]; + t = p->s[j]; + p->s[j] = p->s[i]; + p->s[i] = t; + i++; + } + p->i = i; + p->j = j; +} + +/* +** After all content has been added, invoke HashFinal() to compute +** the final hash. The function returns a pointer to the binary +** hash value. +*/ +static unsigned char *HashFinal(HashContext *p){ + unsigned int k; + unsigned char t, i, j; + i = p->i; + j = p->j; + for(k=0; k<32; k++){ + i++; + t = p->s[i]; + j += t; + p->s[i] = p->s[j]; + p->s[j] = t; + t += p->s[i]; + p->r[k] = p->s[t]; + } + return p->r; +} + +/* End of the Hash hashing logic +*****************************************************************************/ +#endif /* SPEEDTEST_OMIT_HASH */ + + /* All global state is held in this structure */ static struct Global { sqlite3 *db; /* The open database connection */ @@ -80,8 +160,12 @@ static struct Global { const char *zNN; /* Might be NOT NULL */ const char *zPK; /* Might be UNIQUE or PRIMARY KEY */ unsigned int x, y; /* Pseudo-random number generator state */ + u64 nResByte; /* Total number of result bytes */ int nResult; /* Size of the current result */ char zResult[3000]; /* Text of the current result */ +#ifndef SPEEDTEST_OMIT_HASH + HashContext hash; /* Hash of all output */ +#endif } g; /* Return " TEMP" or "", as appropriate for creating a table. @@ -91,6 +175,8 @@ static const char *isTemp(int N){ } + + /* Print an error message and exit */ static void fatal_error(const char *zMsg, ...){ va_list ap; @@ -324,6 +410,21 @@ void speedtest1_final(void){ printf(" TOTAL%.*s %4d.%03ds\n", NAMEWIDTH-5, zDots, (int)(g.iTotal/1000), (int)(g.iTotal%1000)); } + if( g.bVerify ){ +#ifndef SPEEDTEST_OMIT_HASH + int i; + unsigned char *aHash = HashFinal(&g.hash); +#endif + printf("SQL Output Verification:\n"); + printf(" size: %llu\n", g.nResByte); +#ifndef SPEEDTEST_OMIT_HASH + printf(" hash: "); + for(i=0; i<32; i++){ + printf("%02x", aHash[i]); + } + printf("\n"); +#endif + } } /* Print an SQL statement to standard output */ @@ -434,6 +535,12 @@ void speedtest1_run(void){ const char *z = (const char*)sqlite3_column_text(g.pStmt, i); if( z==0 ) z = "nil"; len = (int)strlen(z); +#ifndef SPEEDTEST_OMIT_HASH + if( g.bVerify ){ + HashUpdate(&g.hash, (unsigned char*)z, len); + g.nResByte += len; + } +#endif if( g.nResult+len0 ) g.zResult[g.nResult++] = ' '; memcpy(g.zResult + g.nResult, z, len+1); @@ -2140,6 +2247,9 @@ int main(int argc, char **argv){ zEncoding = "utf16be"; }else if( strcmp(z,"verify")==0 ){ g.bVerify = 1; +#ifndef SPEEDTEST_OMIT_HASH + HashInit(&g.hash); +#endif }else if( strcmp(z,"without-rowid")==0 ){ g.zWR = "WITHOUT ROWID"; g.zPK = "PRIMARY KEY"; From 3c99dbb18bdd46548c61cee1114bb011bdb4bbfb Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 26 Jun 2020 14:05:58 +0000 Subject: [PATCH 2/2] Add --verify to speed-check.sh and add --memdb and --output to speedtest1.c. Other improvements to speedtest1.c. FossilOrigin-Name: 89a11120ab2ce13f8a539cb05a9d0628a1f83b4790910b2023c21d60aabc43ee --- manifest | 16 ++-- manifest.uuid | 2 +- test/speedtest1.c | 208 ++++++++++++++++++++++++++------------------ tool/speed-check.sh | 4 + 4 files changed, 136 insertions(+), 94 deletions(-) diff --git a/manifest b/manifest index 1dcf574f4d..6e062f2132 100644 --- a/manifest +++ b/manifest @@ -1,6 +1,6 @@ B 7a876209a678a34c198b54ceef9e3c041f128a14dc73357f6a57cadadaa6cf7b -C Enhance\sthe\s--verify\soption\sto\sspeedtest1.c\sso\sthat\sit\scomputes\sand\sdisplays\na\shash\sof\sthe\sresult\sfrom\sall\sSQL\squeries,\sfor\sverification\spurposes. -D 2020-06-25T20:28:13.760 +C Add\s--verify\sto\sspeed-check.sh\sand\sadd\s--memdb\sand\s--output\sto\sspeedtest1.c.\nOther\simprovements\sto\sspeedtest1.c. +D 2020-06-26T14:05:58.668 F Makefile.in 19374a5db06c3199ec1bab71ab74a103d8abf21053c05e9389255dc58083f806 F Makefile.msc 48f5a3fc32672c09ad73795749f6253e406a31526935fbbffd8f021108d54574 F autoconf/Makefile.am a8d1d24affe52ebf8d7ddcf91aa973fa0316618ab95bb68c87cabf8faf527dc8 @@ -12,14 +12,12 @@ F src/build.c ba1bbe563a3dc02d5fed20537603181e5289c13ea30ae5e775f552e7557adbfa F src/shell.c.in d663152487d4bfddea0f6d21ebc2ed51575d22657a02c6828afd344bbd4651af F src/test1.c fe56c4bcaa2685ca9aa25d817a0ee9345e189aff4a5a71a3d8ba946c7776feb8 F test/decimal.test 12739a01bdba4c4d79f95b323e6b67b9fad1ab6ffb56116bd2b9c81a5b19e1d9 -F test/speedtest1.c 62a22866f2f1bf3bfd81ded4473314d5508209e7758e122746ace56c138a0f66 +F test/speedtest1.c 18c3a29fba651788a3dc423277d4aa1245860f03fa115f5cad1bebdfbf92bc02 F tool/mkautoconfamal.sh f62353eb6c06ab264da027fd4507d09914433dbdcab9cb011cdc18016f1ab3b8 F tool/mksqlite3c.tcl f4ef476510eca4124c874a72029f1e01bc54a896b1724e8f9eef0d8bfae0e84c F tool/mksqlite3h.tcl 1f5e4a1dbbbc43c83cc6e74fe32c6c620502240b66c7c0f33a51378e78fc4edf -P db2f0836b64cd2e119684f1cf75fa3b19a84ca6aca1a239f7e2b9298016e2c95 -R 612c27b420c2073a1256c7d89a8eeaa4 -T *branch * speedtest-hash -T *sym-speedtest-hash * -T -sym-trunk * +F tool/speed-check.sh 615cbdf50f1409ef3bbf9f682e396df80f49d97ed93ed3e61c8e91fae6afde58 +P 60d1e46c8c8a3c853034fd79f204bcb5d50d1c366eb246849c333a2d0abc2648 +R d502ce56ec4f409f68ad0bf60c478eac U drh -Z 5bfc672238a7ff5aba5dfa178b2aded4 +Z 9ade6808e0b03cd3693ddfa12386f5e0 diff --git a/manifest.uuid b/manifest.uuid index d888b0868d..226b741c4e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -60d1e46c8c8a3c853034fd79f204bcb5d50d1c366eb246849c333a2d0abc2648 \ No newline at end of file +89a11120ab2ce13f8a539cb05a9d0628a1f83b4790910b2023c21d60aabc43ee \ No newline at end of file diff --git a/test/speedtest1.c b/test/speedtest1.c index 6703b38752..fb63244444 100644 --- a/test/speedtest1.c +++ b/test/speedtest1.c @@ -7,7 +7,7 @@ static const char zHelp[] = "Usage: %s [--options] DATABASE\n" "Options:\n" " --autovacuum Enable AUTOVACUUM mode\n" - " --cachesize N Set the cache size to N\n" + " --cachesize N Set the cache size to N\n" " --exclusive Enable locking_mode=EXCLUSIVE\n" " --explain Like --sqlonly but with added EXPLAIN keywords\n" " --heap SZ MIN Memory allocator uses SZ bytes & min allocation MIN\n" @@ -15,11 +15,13 @@ static const char zHelp[] = " --journal M Set the journal_mode to M\n" " --key KEY Set the encryption key to KEY\n" " --lookaside N SZ Configure lookaside for N slots of SZ bytes each\n" + " --memdb Use an in-memory database\n" " --mmap SZ MMAP the first SZ bytes of the database file\n" " --multithread Set multithreaded mode\n" " --nomemstat Disable memory statistics\n" " --nosync Set PRAGMA synchronous=OFF\n" " --notnull Add NOT NULL constraints to table columns\n" + " --output FILE Store SQL output in FILE\n" " --pagesize N Set the page size to N\n" " --pcache N SZ Configure N pages of pagecache each of size SZ bytes\n" " --primarykey Use PRIMARY KEY instead of UNIQUE where appropriate\n" @@ -62,11 +64,6 @@ static const char zHelp[] = typedef sqlite3_uint64 u64; -#ifndef SPEEDTEST_OMIT_HASH -/**************************************************************************** -** Hash algorithm used to verify that compilation is not miscompiled -** in such a was as to generate an incorrect result. -*/ /* ** State structure for a Hash hash in progress */ @@ -78,68 +75,6 @@ struct HashContext { unsigned char r[32]; /* Result */ }; -/* -** Initialize a new hash. iSize determines the size of the hash -** in bits and should be one of 224, 256, 384, or 512. Or iSize -** can be zero to use the default hash size of 256 bits. -*/ -static void HashInit(HashContext *p){ - unsigned int k; - p->i = 0; - p->j = 0; - for(k=0; k<256; k++) p->s[k] = k; -} - -/* -** Make consecutive calls to the HashUpdate function to add new content -** to the hash -*/ -static void HashUpdate( - HashContext *p, - const unsigned char *aData, - unsigned int nData -){ - unsigned char t; - unsigned char i = p->i; - unsigned char j = p->j; - unsigned int k; - for(k=0; ks[i] + aData[k]; - t = p->s[j]; - p->s[j] = p->s[i]; - p->s[i] = t; - i++; - } - p->i = i; - p->j = j; -} - -/* -** After all content has been added, invoke HashFinal() to compute -** the final hash. The function returns a pointer to the binary -** hash value. -*/ -static unsigned char *HashFinal(HashContext *p){ - unsigned int k; - unsigned char t, i, j; - i = p->i; - j = p->j; - for(k=0; k<32; k++){ - i++; - t = p->s[i]; - j += t; - p->s[i] = p->s[j]; - p->s[j] = t; - t += p->s[i]; - p->r[k] = p->s[t]; - } - return p->r; -} - -/* End of the Hash hashing logic -*****************************************************************************/ -#endif /* SPEEDTEST_OMIT_HASH */ - /* All global state is held in this structure */ static struct Global { @@ -164,6 +99,7 @@ static struct Global { int nResult; /* Size of the current result */ char zResult[3000]; /* Text of the current result */ #ifndef SPEEDTEST_OMIT_HASH + FILE *hashFile; /* Store all hash results in this file */ HashContext hash; /* Hash of all output */ #endif } g; @@ -174,9 +110,6 @@ static const char *isTemp(int N){ return g.eTemp>=N ? " TEMP" : ""; } - - - /* Print an error message and exit */ static void fatal_error(const char *zMsg, ...){ va_list ap; @@ -186,6 +119,72 @@ static void fatal_error(const char *zMsg, ...){ exit(1); } +#ifndef SPEEDTEST_OMIT_HASH +/**************************************************************************** +** Hash algorithm used to verify that compilation is not miscompiled +** in such a was as to generate an incorrect result. +*/ + +/* +** Initialize a new hash. iSize determines the size of the hash +** in bits and should be one of 224, 256, 384, or 512. Or iSize +** can be zero to use the default hash size of 256 bits. +*/ +static void HashInit(void){ + unsigned int k; + g.hash.i = 0; + g.hash.j = 0; + for(k=0; k<256; k++) g.hash.s[k] = k; +} + +/* +** Make consecutive calls to the HashUpdate function to add new content +** to the hash +*/ +static void HashUpdate( + const unsigned char *aData, + unsigned int nData +){ + unsigned char t; + unsigned char i = g.hash.i; + unsigned char j = g.hash.j; + unsigned int k; + if( g.hashFile ) fwrite(aData, 1, nData, g.hashFile); + for(k=0; k=3006000 }else if( strcmp(z,"multithread")==0 ){ sqlite3_config(SQLITE_CONFIG_MULTITHREAD); @@ -2194,6 +2212,22 @@ int main(int argc, char **argv){ noSync = 1; }else if( strcmp(z,"notnull")==0 ){ g.zNN = "NOT NULL"; + }else if( strcmp(z,"output")==0 ){ +#ifdef SPEEDTEST_OMIT_HASH + fatal_error("The --output option is not supported with" + " -DSPEEDTEST_OMIT_HASH\n"); +#else + if( i>=argc-1 ) fatal_error("missing argument on %s\n", argv[i]); + i++; + if( strcmp(argv[i],"-")==0 ){ + g.hashFile = stdout; + }else{ + g.hashFile = fopen(argv[i], "wb"); + if( g.hashFile==0 ){ + fatal_error("cannot open \"%s\" for writing\n", argv[i]); + } + } +#endif }else if( strcmp(z,"pagesize")==0 ){ if( i>=argc-1 ) fatal_error("missing argument on %s\n", argv[i]); pageSize = integerValue(argv[++i]); @@ -2248,7 +2282,7 @@ int main(int argc, char **argv){ }else if( strcmp(z,"verify")==0 ){ g.bVerify = 1; #ifndef SPEEDTEST_OMIT_HASH - HashInit(&g.hash); + HashInit(); #endif }else if( strcmp(z,"without-rowid")==0 ){ g.zWR = "WITHOUT ROWID"; @@ -2291,13 +2325,13 @@ int main(int argc, char **argv){ sqlite3_initialize(); /* Open the database and the input file */ - if( sqlite3_open(zDbName, &g.db) ){ + if( sqlite3_open(memDb ? ":memory:" : zDbName, &g.db) ){ fatal_error("Cannot open database file: %s\n", zDbName); } #if SQLITE_VERSION_NUMBER>=3006001 if( nLook>0 && szLook>0 ){ pLook = malloc( nLook*szLook ); - rc = sqlite3_db_config(g.db, SQLITE_DBCONFIG_LOOKASIDE, pLook, szLook,nLook); + rc = sqlite3_db_config(g.db, SQLITE_DBCONFIG_LOOKASIDE,pLook,szLook,nLook); if( rc ) fatal_error("lookaside configuration failed: %d\n", rc); } #endif @@ -2307,6 +2341,9 @@ int main(int argc, char **argv){ #ifndef SQLITE_OMIT_DEPRECATED if( doTrace ) sqlite3_trace(g.db, traceCallback, 0); #endif + if( memDb>0 ){ + speedtest1_exec("PRAGMA temp_store=memory"); + } if( mmapSize>0 ){ speedtest1_exec("PRAGMA mmap_size=%d", mmapSize); } @@ -2346,6 +2383,9 @@ int main(int argc, char **argv){ }else{ zTSet = ""; } + if( g.iTotal>0 || zComma!=0 ){ + printf(" Begin testset \"%s\"\n", zThisTest); + } if( strcmp(zThisTest,"main")==0 ){ testset_main(); }else if( strcmp(zThisTest,"debug1")==0 ){ diff --git a/tool/speed-check.sh b/tool/speed-check.sh index 414e4b4482..1be7c9e5d3 100644 --- a/tool/speed-check.sh +++ b/tool/speed-check.sh @@ -79,6 +79,10 @@ while test "$1" != ""; do ;; --legacy) doWal=0 + CC_OPTS="$CC_OPTS -DSPEEDTEST_OMIT_HASH" + ;; + --verify) + SPEEDTEST_OPTS="$SPEEDTEST_OPTS --verify" ;; --wal) doWal=1