From 8cbadb0211a9476b6acca6f9b3be7c524a978694 Mon Sep 17 00:00:00 2001 From: danielk1977 Date: Thu, 3 May 2007 16:31:26 +0000 Subject: [PATCH] Test cases and minor bugfixes for incremental blob APIs. (CVS 3907) FossilOrigin-Name: e12c522383bd40af375a52d2e68612c4dc7fd4db --- manifest | 22 ++++---- manifest.uuid | 2 +- src/sqlite.h.in | 61 ++++++++++++++++++++-- src/tclsqlite.c | 28 ++++++---- src/vdbeblob.c | 24 +++++---- test/incrblob.test | 125 ++++++++++++++++++++++++++++++++++++++++++--- test/tkt2332.test | 5 +- test/types3.test | 4 +- 8 files changed, 225 insertions(+), 46 deletions(-) diff --git a/manifest b/manifest index 85f064ef32..4d2482e94d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sbug\swhere\saccessPayload()\swas\scalling\sPagerWrite()\son\sthe\swrong\spage\shandle.\sTicket\s#2332.\s(CVS\s3906) -D 2007-05-03T13:11:32 +C Test\scases\sand\sminor\sbugfixes\sfor\sincremental\sblob\sAPIs.\s(CVS\s3907) +D 2007-05-03T16:31:26 F Makefile.in 8cab54f7c9f5af8f22fd97ddf1ecfd1e1860de62 F Makefile.linux-gcc 2d8574d1ba75f129aba2019f0b959db380a90935 F README 9c4e2d6706bdcc3efdd773ce752a8cdab4f90028 @@ -97,11 +97,11 @@ F src/random.c 6119474a6f6917f708c1dee25b9a8e519a620e88 F src/select.c 3c8f3bc7fd823abb8af30ec89ba6bcc515923fa1 F src/server.c 087b92a39d883e3fa113cae259d64e4c7438bc96 F src/shell.c 3ae4654560e91220a95738a73d135d91d937cda1 -F src/sqlite.h.in 1e053c58fd4df28c38ffdca2443b16d5f76f6f1e +F src/sqlite.h.in a666300976897eced975b448f722a722b362c6b1 F src/sqlite3ext.h 7d0d363ea7327e817ef0dfe1b7eee1f171b72890 F src/sqliteInt.h 0b14d0eae083aafca0562d2261a404e5e5abc5f0 F src/table.c 6d0da66dde26ee75614ed8f584a1996467088d06 -F src/tclsqlite.c 23082fa8affdf3ae73937ca0755754fc562674bc +F src/tclsqlite.c dde509871614d17f8ab5f3b4bc496b0af07280c7 F src/test1.c 29a39fdde51f4612082ecf3f5af54dac93766f87 F src/test2.c 24458b17ab2f3c90cbc1c8446bd7ffe69be62f88 F src/test3.c 946ea9d1a8c928656e3c70f0a2fcb8e733a15e86 @@ -130,7 +130,7 @@ F src/vdbe.h 0025259af1939fb264a545816c69e4b5b8d52691 F src/vdbeInt.h cb02cbbceddf3b40d49012e9f41576f17bcbec97 F src/vdbeapi.c 37d793559390bec8a00c556f651f21b5f9e589af F src/vdbeaux.c 8c7f22e22d1ea578971f5a3fcd3a56a6882ced64 -F src/vdbeblob.c 0e070ded61b5db6ac55085d542bda5c2ee9e1a5f +F src/vdbeblob.c 74fe0c7fc149a80715be7e3a33ed0e545d5e33e1 F src/vdbefifo.c 3ca8049c561d5d67cbcb94dc909ae9bb68c0bf8f F src/vdbemem.c ba98f8572ec4609846b368fa7580db178022f1bb F src/vtab.c 89a0d5f39c1beba65a77fdb4d507b831fc5e6baf @@ -242,7 +242,7 @@ F test/fts2n.test a70357e72742681eaebfdbe9007b87ff3b771638 F test/func.test 6727c7729472ae52b5acd86e802f89aa350ba50f F test/hook.test 7e7645fd9a033f79cce8fdff151e32715e7ec50a F test/in.test 369cb2aa1eab02296b4ec470732fe8c131260b1d -F test/incrblob.test cca13ec6c3a163c86821dd49a81bb5b519850e79 +F test/incrblob.test 9f6f5c23716d6c9386d1011cff732399900750df F test/incrvacuum.test 2173bc075c7b3b96ccf228d737dd4f5c29500dc4 F test/incrvacuum_ioerr.test 0ebc382bcc2036ec58cf49cc5ffada45f75d907b F test/index.test e65df12bed94b2903ee89987115e1578687e9266 @@ -360,7 +360,7 @@ F test/tkt2192.test 480d0e017ddb01a46ee20809427370f343bb3c03 F test/tkt2213.test 8cf7c446e1fcd0627fffe7fc19046eb24ac7333b F test/tkt2251.test 3f0549213386ed911715665a908ff2bb7a871002 F test/tkt2285.test c618085f0c13ec3347e607f83c34ada0721b4bfa -F test/tkt2332.test a7d678dc146ca7d2dae7bcc19f4659a389225b36 +F test/tkt2332.test 1623a64e0dfd5cf6d02e095d49ed3af1010da7c9 F test/trace.test 75ffc1b992c780d054748a656e3e7fd674f18567 F test/trans.test 3fe1b9e03b523482eee2b869858c5c1eca7b218b F test/trigger1.test b361161cf20614024cc1e52ea0bdec250776b2ae @@ -373,7 +373,7 @@ F test/trigger7.test 0afa870be2ce1b132cdb85b17a4a4ef45aa8cece F test/trigger8.test 3a09275aa2214fdff56f731b1e775d8dfee4408a F test/types.test 98e7a631bddf0806204358b452b02d0e319318a6 F test/types2.test 3555aacf8ed8dc883356e59efc314707e6247a84 -F test/types3.test ea0ddf793ad5cd17c3b029dd8f48473059f873b6 +F test/types3.test c08b2b960064be30f5237cd2cf4680f32ba190be F test/unique.test 0253c4227a5dc533e312202ce21ecfad18058d18 F test/update.test 7669ca789d62c258b678e8aa7a22a57eac10f2cf F test/utf16.test 20e2d9ba0d57e952a18b1ac8deab9ad49e082893 @@ -473,7 +473,7 @@ F www/tclsqlite.tcl bb0d1357328a42b1993d78573e587c6dcbc964b9 F www/vdbe.tcl 87a31ace769f20d3627a64fa1fade7fed47b90d0 F www/version3.tcl 890248cf7b70e60c383b0e84d77d5132b3ead42b F www/whentouse.tcl fc46eae081251c3c181bd79c5faef8195d7991a5 -P 678d672b73cc7b7f563c15daee3831cb5bbd890e -R 3fe50c067312d08e6c011ae37b9a3ab6 +P cf9eeba7be64ad29cddd320832db10c799fb6e8e +R 8a6f6ed85fe488d5b2d00df7c522a2c5 U danielk1977 -Z 99cbb46fec6c9867e749c0e87c86be78 +Z 6a4151dc8ab67d985ea9dda4365f5921 diff --git a/manifest.uuid b/manifest.uuid index 5335083151..f6165e2adc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cf9eeba7be64ad29cddd320832db10c799fb6e8e \ No newline at end of file +e12c522383bd40af375a52d2e68612c4dc7fd4db \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 17a0a14d5b..4af11e26ca 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -12,7 +12,7 @@ ** This header file defines the interface that the SQLite library ** presents to client programs. ** -** @(#) $Id: sqlite.h.in,v 1.203 2007/05/02 01:34:31 drh Exp $ +** @(#) $Id: sqlite.h.in,v 1.204 2007/05/03 16:31:26 danielk1977 Exp $ */ #ifndef _SQLITE3_H_ #define _SQLITE3_H_ @@ -1873,8 +1873,29 @@ int sqlite3_overload_function(sqlite3*, const char *zFuncName, int nArg); ****** EXPERIMENTAL - subject to change without notice ************** */ +/* +** An instance of the following opaque structure is used to +** represent an open blob handle. +*/ typedef struct sqlite3_blob sqlite3_blob; +/* +** Open a handle to the blob located in row iRow,, column zColumn, +** table zTable in database zDb. i.e. the same blob that would +** be selected by: +** +** "SELECT zColumn FROM zDb.zTable WHERE rowid = iRow; +** +** If the flags parameter is non-zero, the blob is opened for +** read and write access. If it is zero, the blob is opened for read +** access. +** +** On success, SQLITE_OK is returned and the new blob-handle is +** written to *ppBlob. Otherwise an error code is returned and +** any value written to *ppBlob should not be used by the caller. +** This function sets the database-handle error code and message +** accessible via sqlite3_errcode() and sqlite3_errmsg(). +*/ int sqlite3_blob_open( sqlite3*, const char *zDb, @@ -1885,12 +1906,46 @@ int sqlite3_blob_open( sqlite3_blob **ppBlob ); +/* +** Close an open blob handle. +*/ int sqlite3_blob_close(sqlite3_blob *); -int sqlite3_blob_read(sqlite3_blob *, void *z, int n, int iOffset); -int sqlite3_blob_write(sqlite3_blob *, const void *z, int n, int iOffset); +/* +** Return the size in bytes of the blob accessible via the open +** blob-handle passed as an argument. +*/ int sqlite3_blob_bytes(sqlite3_blob *); +/* +** This function is used to read data from an open blob-handle into +** a caller supplied buffer. n bytes of data are copied into buffer +** z from the open blob, starting at offset iOffset. +** +** On success, SQLITE_OK is returned. Otherwise, an SQLite error +** code. +*/ +int sqlite3_blob_read(sqlite3_blob *, void *z, int n, int iOffset); + +/* +** This function is used to write data from an open blob-handle into +** a user supplied buffer. n bytes of data are copied from the buffer +** pointed to by z into the open blob, starting at offset iOffset. +** +** If the blob-handle passed as the first argument was not opened for +** writing (the flags parameter to sqlite3_blob_open was zero), this +** function returns SQLITE_READONLY. +** +** This function may only modify the contents of the blob, it is +** not possible to increase the size of a blob using this API. If +** offset iOffset is less than n bytes from the end of the blob, +** SQLITE_ERROR is returned and no data is written. +** +** On success, SQLITE_OK is returned. Otherwise, an SQLite error +** code. If an error occurs, this function sets the +*/ +int sqlite3_blob_write(sqlite3_blob *, const void *z, int n, int iOffset); + /* ** Undo the hack that converts floating point types to integer for ** builds on processors without floating point support. diff --git a/src/tclsqlite.c b/src/tclsqlite.c index 70481a75ab..1a2567e8cb 100644 --- a/src/tclsqlite.c +++ b/src/tclsqlite.c @@ -12,7 +12,7 @@ ** A TCL Interface to SQLite. Append this file to sqlite3.c and ** compile the whole thing to build a TCL-enabled version of SQLite. ** -** $Id: tclsqlite.c,v 1.181 2007/05/02 13:16:31 danielk1977 Exp $ +** $Id: tclsqlite.c,v 1.182 2007/05/03 16:31:26 danielk1977 Exp $ */ #include "tcl.h" #include @@ -299,18 +299,20 @@ static int createIncrblobChannel( const char *zDb, const char *zTable, const char *zColumn, - sqlite_int64 iRow + sqlite_int64 iRow, + int isReadonly ){ IncrblobChannel *p; + sqlite3 *db = pDb->db; sqlite3_blob *pBlob; int rc; - int flags = TCL_READABLE|TCL_WRITABLE; + int flags = TCL_READABLE|(isReadonly ? 0 : TCL_WRITABLE); /* This variable is used to name the channels: "incrblob_[incr count]" */ static int count = 0; char zChannel[64]; - rc = sqlite3_blob_open(pDb->db, zDb, zTable, zColumn, iRow, 1, &pBlob); + rc = sqlite3_blob_open(db, zDb, zTable, zColumn, iRow, !isReadonly, &pBlob); if( rc!=SQLITE_OK ){ Tcl_SetResult(interp, (char *)sqlite3_errmsg(pDb->db), TCL_VOLATILE); return TCL_ERROR; @@ -1849,20 +1851,26 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){ } /* - ** $db incrblob ?DB? TABLE COLUMN ROWID + ** $db incrblob ?-readonly? ?DB? TABLE COLUMN ROWID */ case DB_INCRBLOB: { + int isReadonly = 0; const char *zDb = "main"; const char *zTable; const char *zColumn; sqlite_int64 iRow; - if( objc!=5 && objc!=6 ){ - Tcl_WrongNumArgs(interp, 2, objv, "?DB? TABLE ROWID"); + /* Check for the -readonly option */ + if( objc>3 && strcmp(Tcl_GetString(objv[2]), "-readonly")==0 ){ + isReadonly = 1; + } + + if( objc!=(5+isReadonly) && objc!=(6+isReadonly) ){ + Tcl_WrongNumArgs(interp, 2, objv, "?-readonly? ?DB? TABLE COLUMN ROWID"); return TCL_ERROR; } - if( objc==6 ){ + if( objc==(6+isReadonly) ){ zDb = Tcl_GetString(objv[2]); } zTable = Tcl_GetString(objv[objc-3]); @@ -1870,7 +1878,9 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){ rc = Tcl_GetWideIntFromObj(interp, objv[objc-1], &iRow); if( rc==TCL_OK ){ - rc = createIncrblobChannel(interp, pDb, zDb, zTable, zColumn, iRow); + rc = createIncrblobChannel( + interp, pDb, zDb, zTable, zColumn, iRow, isReadonly + ); } break; } diff --git a/src/vdbeblob.c b/src/vdbeblob.c index 7c55a635b9..b095a5e814 100644 --- a/src/vdbeblob.c +++ b/src/vdbeblob.c @@ -10,7 +10,7 @@ ** ************************************************************************* ** -** $Id: vdbeblob.c,v 1.3 2007/05/03 11:43:33 danielk1977 Exp $ +** $Id: vdbeblob.c,v 1.4 2007/05/03 16:31:26 danielk1977 Exp $ */ #include "sqliteInt.h" @@ -43,7 +43,6 @@ int sqlite3_blob_open( int flags, /* True -> read/write access, false -> read-only */ sqlite3_blob **ppBlob ){ - int rc = SQLITE_OK; int nAttempt = 0; int iCol; /* Index of zColumn in row-record */ @@ -83,6 +82,8 @@ int sqlite3_blob_open( }; Vdbe *v = 0; + int rc = SQLITE_OK; + char zErr[128] = {0}; do { Parse sParse; @@ -98,9 +99,12 @@ int sqlite3_blob_open( pTab = sqlite3LocateTable(&sParse, zTable, zDb); if( !pTab ){ - sqlite3Error(db, sParse.rc, "%s", sParse.zErrMsg); + if( sParse.zErrMsg ){ + sqlite3_snprintf(sizeof(zErr), zErr, "%s", sParse.zErrMsg); + zErr[sizeof(zErr)-1] = '\0'; + } sqliteFree(sParse.zErrMsg); - rc = sParse.rc; + rc = SQLITE_ERROR; sqlite3SafetyOff(db); goto blob_open_out; } @@ -112,8 +116,7 @@ int sqlite3_blob_open( } } if( iCol==pTab->nCol ){ - sqlite3Error(db, SQLITE_ERROR, "no such column: %s", zColumn); - sqliteFree(sParse.zErrMsg); + sprintf(zErr, "no such column: \"%s\"", zColumn); rc = SQLITE_ERROR; sqlite3SafetyOff(db); goto blob_open_out; @@ -162,6 +165,7 @@ int sqlite3_blob_open( if( rc!=SQLITE_ROW ){ nAttempt++; rc = sqlite3_finalize((sqlite3_stmt *)v); + sprintf(zErr, "no such rowid: %lld", iRow); v = 0; } } while( nAttempt<5 && rc==SQLITE_SCHEMA ); @@ -191,17 +195,15 @@ int sqlite3_blob_open( pBlob->nByte = sqlite3VdbeSerialTypeLen(type); *ppBlob = (sqlite3_blob *)pBlob; rc = SQLITE_OK; - }else{ - if( rc==SQLITE_DONE ){ - rc = SQLITE_ERROR; - } + }else if( rc==SQLITE_OK ){ + rc = SQLITE_ERROR; } blob_open_out: if( rc!=SQLITE_OK || sqlite3MallocFailed() ){ sqlite3_finalize((sqlite3_stmt *)v); } - sqlite3Error(db, rc, ""); + sqlite3Error(db, rc, zErr); return sqlite3ApiExit(db, rc); } diff --git a/test/incrblob.test b/test/incrblob.test index e482f23333..96038c6c6e 100644 --- a/test/incrblob.test +++ b/test/incrblob.test @@ -9,7 +9,7 @@ # #*********************************************************************** # -# $Id: incrblob.test,v 1.4 2007/05/03 13:11:32 danielk1977 Exp $ +# $Id: incrblob.test,v 1.5 2007/05/03 16:31:26 danielk1977 Exp $ # set testdir [file dirname $argv0] @@ -79,18 +79,25 @@ do_test incrblob-1.3.10 { #------------------------------------------------------------------------ -# incrblob-2.*: Test that the following operations use ptrmap pages: +# incrblob-2.*: +# +# Test that the following operations use ptrmap pages to reduce +# unnecessary reads: # # * Reading near the end of a blob, -# * Writing near the end of a blob (TODO), -# * SELECT a column value that is located on an overflow page (TODO). -# +# * Writing near the end of a blob, and +# * SELECT a column value that is located on an overflow page. # proc nRead {db} { set bt [btree_from_db $db] array set stats [btree_pager_stats $bt] return $stats(read) } +proc nWrite {db} { + set bt [btree_from_db $db] + array set stats [btree_pager_stats $bt] + return $stats(write) +} foreach AutoVacuumMode [list 0 1] { @@ -104,9 +111,9 @@ foreach AutoVacuumMode [list 0 1] { set ::str [string repeat abcdefghij 2900] execsql { BEGIN; - CREATE TABLE blobs(k PRIMARY KEY, v BLOB); + CREATE TABLE blobs(k PRIMARY KEY, v BLOB, i INTEGER); DELETE FROM blobs; - INSERT INTO blobs VALUES('one', $::str || randstr(500,500)); + INSERT INTO blobs VALUES('one', $::str || randstr(500,500), 45); COMMIT; } expr [file size test.db]/1024 @@ -136,9 +143,111 @@ foreach AutoVacuumMode [list 0 1] { nRead db } [expr $AutoVacuumMode ? 4 : 30] - do_test incrblob-2.$AutoVacuumMode.3 { + do_test incrblob-2.$AutoVacuumMode.4 { string range [db one {SELECT v FROM blobs}] end-19 end } $::fragment + + do_test incrblob-2.$AutoVacuumMode.5 { + # Open and close the db to make sure the page cache is empty. + db close + sqlite3 db test.db + + # Write the second-to-last 20 bytes of the blob via a blob handle. + # + set ::blob [db incrblob blobs v 1] + seek $::blob -40 end + puts -nonewline $::blob "1234567890abcdefghij" + flush $::blob + + # If the database is not in auto-vacuum mode, the whole of + # the overflow-chain must be scanned. In auto-vacuum mode, + # sqlite uses the ptrmap pages to avoid reading the other pages. + # + nRead db + } [expr $AutoVacuumMode ? 4 : 30] + + # Pages 1 (the write-counter) and 32 (the blob data) were written. + do_test incrblob-2.$AutoVacuumMode.6 { + close $::blob + nWrite db + } 2 + + do_test incrblob-2.$AutoVacuumMode.7 { + string range [db one {SELECT v FROM blobs}] end-39 end-20 + } "1234567890abcdefghij" + + do_test incrblob-2.$AutoVacuumMode.8 { + # Open and close the db to make sure the page cache is empty. + db close + sqlite3 db test.db + + execsql { SELECT i FROM blobs } + } {45} + + do_test incrblob-2.$AutoVacuumMode.9 { + nRead db + } [expr $AutoVacuumMode ? 4 : 30] } +#------------------------------------------------------------------------ +# incrblob-3.*: +# +# Test the outcome of trying to write to a read-only blob handle. +# +# TODO: The following test only tests the tcl interface, not the +# underlying sqlite3 interface. Need to find some other method +# to call sqlite3_blob_write() on a readonly handle... +# +do_test incrblob-3.1 { + set ::blob [db incrblob -readonly blobs v 1] + seek $::blob -40 end + read $::blob 20 +} "1234567890abcdefghij" +do_test incrblob-3.2 { + seek $::blob 0 + set rc [catch { + puts -nonewline $::blob "helloworld" + } msg] + list $rc $msg +} "1 {channel \"$::blob\" wasn't opened for writing}" + +#------------------------------------------------------------------------ +# incrblob-4.*: +# +# Try a couple of error conditions: +# +# 4.1 - Attempt to open a row that does not exist. +# 4.2 - Attempt to open a column that does not exist. +# 4.3 - Attempt to open a table that does not exist. +# 4.4 - Attempt to open a database that does not exist. +# +do_test incrblob-4.1 { + set rc [catch { + set ::blob [db incrblob blobs v 2] + } msg ] + list $rc $msg +} {1 {no such rowid: 2}} + +do_test incrblob-4.2 { + set rc [catch { + set ::blob [db incrblob blobs blue 1] + } msg ] + list $rc $msg +} {1 {no such column: "blue"}} + +do_test incrblob-4.3 { + set rc [catch { + set ::blob [db incrblob nosuchtable blue 1] + } msg ] + list $rc $msg +} {1 {no such table: main.nosuchtable}} + +do_test incrblob-4.4 { + set rc [catch { + set ::blob [db incrblob nosuchdb blobs v 1] + } msg ] + list $rc $msg +} {1 {no such table: nosuchdb.blobs}} + finish_test + diff --git a/test/tkt2332.test b/test/tkt2332.test index 5da3e7bd4f..e57bafa9c5 100644 --- a/test/tkt2332.test +++ b/test/tkt2332.test @@ -9,7 +9,7 @@ # #*********************************************************************** # -# $Id: tkt2332.test,v 1.1 2007/05/03 13:11:32 danielk1977 Exp $ +# $Id: tkt2332.test,v 1.2 2007/05/03 16:31:26 danielk1977 Exp $ # set testdir [file dirname $argv0] @@ -56,4 +56,7 @@ foreach Len [list 10000 100000 1000000] { incr ::iKey } +# Free memory: +unset ::blobstr + finish_test diff --git a/test/types3.test b/test/types3.test index 2319ba93d6..78bbca50d9 100644 --- a/test/types3.test +++ b/test/types3.test @@ -12,7 +12,7 @@ # of this file is testing the interaction of SQLite manifest types # with Tcl dual-representations. # -# $Id: types3.test,v 1.5 2006/01/17 09:35:03 danielk1977 Exp $ +# $Id: types3.test,v 1.6 2007/05/03 16:31:26 danielk1977 Exp $ # set testdir [file dirname $argv0] @@ -27,7 +27,7 @@ do_test types3-1.1 { # A variable with an integer representation comes in as INTEGER do_test types3-1.2 { - set V [expr {1+2}] + set V [expr {int(1+2)}] concat [tcl_variable_type V] [execsql {SELECT typeof(:V)}] } {int integer} do_test types3-1.3 {