From e1cd98740f70d75812628f9658cf948b420f10f6 Mon Sep 17 00:00:00 2001 From: danielk1977 Date: Sat, 22 May 2004 10:33:04 +0000 Subject: [PATCH] Add a couple of tests for UTF-16 databases. (CVS 1438) FossilOrigin-Name: d7551df8c32c4981d94eb57cd2a999592e91f50f --- manifest | 17 ++++--- manifest.uuid | 2 +- src/sqlite.h.in | 16 +++++- src/test1.c | 79 +++++++++++++++++++++++++++-- src/vdbe.c | 97 ++++++++++++++++++------------------ test/enc2.test | 129 ++++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 274 insertions(+), 66 deletions(-) create mode 100644 test/enc2.test diff --git a/manifest b/manifest index 21a5f921a0..3ba973ebfc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Use\sthe\snew\sform\sof\sthe\ssqlite3_open()\sAPI\severywhere.\s(CVS\s1437) -D 2004-05-22T09:21:21 +C Add\sa\scouple\sof\stests\sfor\sUTF-16\sdatabases.\s(CVS\s1438) +D 2004-05-22T10:33:04 F Makefile.in ab7b0d5118e2da97bac66be8684a1034e3500f5a F Makefile.linux-gcc b86a99c493a5bfb402d1d9178dcdc4bd4b32f906 F README f1de682fbbd94899d50aca13d387d1b3fd3be2dd @@ -49,11 +49,11 @@ F src/printf.c ef750e8e2398ca7e8b58be991075f08c6a7f0e53 F src/random.c eff68e3f257e05e81eae6c4d50a51eb88beb4ff3 F src/select.c 7d77a8bed7eeac23216d42fc1be006fb4352fcdc F src/shell.c 657623c2a3df126538d41842c2146cadbd52b154 -F src/sqlite.h.in 56eb3abf26b35ad58bd0d7615673065572e53253 +F src/sqlite.h.in 9a4c374f4030cde181593166d71bf376274ca45c F src/sqliteInt.h 4b45892cb082f4883efb58c5e13328c42cbc7642 F src/table.c af14284fa36c8d41f6829e3f2819dce07d3e2de2 F src/tclsqlite.c f241854328ee2b06006efded270d84799159f760 -F src/test1.c 4ed0aee9bd363068b1bf6a8ad58f5c94172a5db1 +F src/test1.c b5f2f9f9d866c8a586b8d47c5999d2cbefaac686 F src/test2.c 6195a1ca2c8d0d2d93644e86da3289b403486872 F src/test3.c 5e4a6d596f982f6f47a5f9f75ede9b4a3b739968 F src/test4.c b9947c319a5c023c10c1e953e6610abd571c2283 @@ -64,7 +64,7 @@ F src/update.c 1a5e9182596f3ea8c7a141e308a3d2a7e5689fee F src/utf.c 537e1c98cddc623628d44497ec02c2246cf66dea F src/util.c 5cbeb452da09cfc7248de9948c15b14d840723f7 F src/vacuum.c 8734f89742f246abd91dbd3e087fc153bddbfbad -F src/vdbe.c 3ce76c80887e5a5a0a850b2cb7ea1e353c113e36 +F src/vdbe.c a21826df111a46e772cb37b73a6b2726c5c93883 F src/vdbe.h 391d5642a83af686f35c228fcd36cb4456d68f44 F src/vdbeInt.h f40e8048d644c8389cda16f46479376f763d56e6 F src/vdbeaux.c d3dfb6d40eb1fdf2626896e8b13fe7b50134ff12 @@ -89,6 +89,7 @@ F test/crashtest1.c 09c1c7d728ccf4feb9e481671e29dda5669bbcc2 F test/date.test aed5030482ebc02bd8d386c6c86a29f694ab068d F test/delete.test 92256384f1801760180ded129f7427884cf28886 F test/enc.test a55481d45ff493804e8d88357feb4642fc50a6b2 +F test/enc2.test c72d5d74f2f6731c9e7f00f20caab0aeba65027e F test/expr.test 8b62f3fcac64fbd5c3d43d7a7984245743dcbe65 F test/fkey1.test d65c824459916249bee501532d6154ddab0b5db7 F test/format3.test 149cc166c97923fa60def047e90dd3fb32bba916 @@ -196,7 +197,7 @@ F www/sqlite.tcl 3c83b08cf9f18aa2d69453ff441a36c40e431604 F www/tclsqlite.tcl b9271d44dcf147a93c98f8ecf28c927307abd6da F www/vdbe.tcl 9b9095d4495f37697fd1935d10e14c6015e80aa1 F www/whentouse.tcl a8335bce47cc2fddb07f19052cb0cb4d9129a8e4 -P 802d65affcafffda33e2ff1cbd4e4869dc3814df -R 8b27932509e64c7f6375a72db4ed8e0b +P b449217318ade3196757bef8aaf7302634f0f9b6 +R 6f88cc157d8cb8bd7f75314a4406aae5 U danielk1977 -Z bed47f028169fc74361bb2f3dd3e6944 +Z d9c4233e4b530d40467b54b803289542 diff --git a/manifest.uuid b/manifest.uuid index 0a1db66df1..37e82fde35 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b449217318ade3196757bef8aaf7302634f0f9b6 \ No newline at end of file +d7551df8c32c4981d94eb57cd2a999592e91f50f \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 785f7c99bc..ebc976ed32 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -12,7 +12,7 @@ ** This header file defines the interface that the SQLite library ** presents to client programs. ** -** @(#) $Id: sqlite.h.in,v 1.69 2004/05/22 09:21:21 danielk1977 Exp $ +** @(#) $Id: sqlite.h.in,v 1.70 2004/05/22 10:33:04 danielk1977 Exp $ */ #ifndef _SQLITE_H_ #define _SQLITE_H_ @@ -1291,7 +1291,7 @@ const void *sqlite3_column_data16(sqlite3_stmt*,int); /* ** The first parameter is a compiled SQL statement for which the most ** recent call to sqlite3_step() has returned SQLITE_ROW. This routine -** retrieves the length of the data in bytse returned by the +** retrieves the length of the data in bytes returned by the ** sqlite3_column_data() routine for the same second parameter value. ** ** If sqlite3_column_data() returns a UTF-8 string, then the length @@ -1300,6 +1300,18 @@ const void *sqlite3_column_data16(sqlite3_stmt*,int); */ int sqlite3_column_bytes(sqlite3_stmt*,int); +/* +** The first parameter is a compiled SQL statement for which the most +** recent call to sqlite3_step() has returned SQLITE_ROW. This routine +** retrieves the length of the data in bytes returned by the +** sqlite3_column_data() routine for the same second parameter value. +** +** If sqlite3_column_data() returns a UTF-16 string, then the length +** returned by this function includes the nul terminator character (two +** bytes) at the end of the UTF-16 string. +*/ +int sqlite3_column_bytes16(sqlite3_stmt *, int); + /* ** The first parameter is a compiled SQL statement for which the most ** recent call to sqlite3_step() has returned SQLITE_ROW. This routine diff --git a/src/test1.c b/src/test1.c index 5288a9cc38..2a208124fc 100644 --- a/src/test1.c +++ b/src/test1.c @@ -13,7 +13,7 @@ ** is not included in the SQLite library. It is used for automated ** testing of the SQLite library. ** -** $Id: test1.c,v 1.46 2004/05/22 09:21:21 danielk1977 Exp $ +** $Id: test1.c,v 1.47 2004/05/22 10:33:04 danielk1977 Exp $ */ #include "sqliteInt.h" #include "tcl.h" @@ -1416,7 +1416,7 @@ static int test_step_new( sqlite3_stmt *pStmt; int rc; - if( objc!=3 ){ + if( objc!=2 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", Tcl_GetString(objv[0]), " STMT", 0); return TCL_ERROR; @@ -1425,7 +1425,74 @@ static int test_step_new( if( getStmtPointer(interp, Tcl_GetString(objv[1]), &pStmt) ) return TCL_ERROR; rc = sqlite3_step_new(pStmt); - if( rc!=SQLITE_OK ) return TCL_ERROR; + if( rc!=SQLITE_DONE && rc!=SQLITE_ROW ) return TCL_ERROR; + return TCL_OK; +} + +/* +** Usage: sqlite3_column_data STMT column +** +** Advance the statement to the next row. +*/ +static int test_column_data( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + sqlite3_stmt *pStmt; + int col; + Tcl_Obj *pRet; + + if( objc!=3 ){ + Tcl_AppendResult(interp, "wrong # args: should be \"", + Tcl_GetString(objv[0]), " STMT column", 0); + return TCL_ERROR; + } + + if( getStmtPointer(interp, Tcl_GetString(objv[1]), &pStmt) ) return TCL_ERROR; + if( Tcl_GetIntFromObj(interp, objv[2], &col) ) return TCL_ERROR; + + if( SQLITE3_BLOB==sqlite3_column_type(pStmt, col) ){ + int len = sqlite3_column_bytes(pStmt, col); + pRet = Tcl_NewByteArrayObj(sqlite3_column_data(pStmt, col), len); + }else{ + pRet = Tcl_NewStringObj(sqlite3_column_data(pStmt, col), -1); + } + Tcl_SetObjResult(interp, pRet); + + return TCL_OK; +} + +/* +** Usage: sqlite3_column_data16 STMT column +** +** Advance the statement to the next row. +*/ +static int test_column_data16( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + sqlite3_stmt *pStmt; + int col; + Tcl_Obj *pRet; + int len; + + if( objc!=3 ){ + Tcl_AppendResult(interp, "wrong # args: should be \"", + Tcl_GetString(objv[0]), " STMT column", 0); + return TCL_ERROR; + } + + if( getStmtPointer(interp, Tcl_GetString(objv[1]), &pStmt) ) return TCL_ERROR; + if( Tcl_GetIntFromObj(interp, objv[2], &col) ) return TCL_ERROR; + + len = sqlite3_column_bytes16(pStmt, col); + pRet = Tcl_NewByteArrayObj(sqlite3_column_data16(pStmt, col), len); + Tcl_SetObjResult(interp, pRet); + return TCL_OK; } @@ -1526,9 +1593,11 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ { "sqlite3_prepare16", (Tcl_ObjCmdProc*)test_prepare16 }, { "sqlite3_open", (Tcl_ObjCmdProc*)test_open }, { "sqlite3_open16", (Tcl_ObjCmdProc*)test_open16 }, - { "sqlite3_finalize", (Tcl_ObjCmdProc*)test_finalize }, - { "sqlite3_reset", (Tcl_ObjCmdProc*)test_reset }, + { "sqlite3_finalize", (Tcl_ObjCmdProc*)test_finalize }, + { "sqlite3_reset", (Tcl_ObjCmdProc*)test_reset }, { "sqlite3_step", (Tcl_ObjCmdProc*)test_step_new }, + { "sqlite3_column_data", (Tcl_ObjCmdProc*)test_column_data }, + { "sqlite3_column_data16", (Tcl_ObjCmdProc*)test_column_data16 }, { "add_reverse_collating_func", (Tcl_ObjCmdProc*)reverse_collfunc }, }; int i; diff --git a/src/vdbe.c b/src/vdbe.c index b4e429f185..1b675f9281 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -43,7 +43,7 @@ ** in this file for details. If in doubt, do not deviate from existing ** commenting and indentation practices when changing or adding code. ** -** $Id: vdbe.c,v 1.317 2004/05/22 07:27:46 danielk1977 Exp $ +** $Id: vdbe.c,v 1.318 2004/05/22 10:33:04 danielk1977 Exp $ */ #include "sqliteInt.h" #include "os.h" @@ -69,6 +69,7 @@ int sqlite3_search_count = 0; */ int sqlite3_interrupt_count = 0; +#if 0 /* ** NulTermify ** Stringify @@ -94,51 +95,7 @@ typedef struct MemRecord MemRecord; static int Recordify(Mem *pMem){ return 0; } - -#define NulTermify(P) if(((P)->flags & MEM_Str)==0){hardStringify(P);} \ - else if(((P)->flags & MEM_Term)==0){hardNulTermify(P);} -static int hardNulTermify(Mem *pStack){ - int flags = pStack->flags; - - assert( !(flags&MEM_Term) && (flags&MEM_Str) ); - assert( flags&(MEM_Utf8|MEM_Utf16le|MEM_Utf16be) ); - - if( flags&MEM_Utf8 ){ - /* If the string is already dynamically allocated, use sqliteRealloc() - ** to allocate extra space for the terminator. - */ - if( flags&MEM_Dyn ){ - pStack->z = sqliteRealloc(pStack->z, pStack->n+1); - if( !pStack->z ){ - return 1; - } - } - - if( flags&(MEM_Static|MEM_Ephem|MEM_Short) ){ - if( pStack->n+1zShort, pStack->z, pStack->n); - pStack->flags = MEM_Short|MEM_Str|MEM_Utf8|MEM_Term; - } - }else{ - char *z = sqliteMalloc(pStack->n+1); - if( !z ){ - return 1; - } - memcpy(z, pStack->z, pStack->n); - pStack->z = z; - pStack->flags = MEM_Dyn|MEM_Str|MEM_Utf8|MEM_Term; - } - } - - pStack->z[pStack->n] = '\0'; - pStack->n++; - }else{ - assert(0); - } - - return 0; -} +#endif /* ** Convert the given stack entity into a string if it isn't one @@ -532,12 +489,13 @@ const unsigned char *sqlite3_column_data(sqlite3_stmt *pStmt, int i){ } /* -** Return the number of bytes of data that will be returned by the -** equivalent sqlite3_column_data() call. +** Return the value of the 'i'th column of the current row of the currently +** executing statement pStmt. */ -int sqlite3_column_bytes(sqlite3_stmt *pStmt, int i){ - Vdbe *pVm = (Vdbe *)pStmt; +const void *sqlite3_column_data16(sqlite3_stmt *pStmt, int i){ int vals; + Vdbe *pVm = (Vdbe *)pStmt; + Mem *pVal; vals = sqlite3_value_count(pStmt); if( i>=vals || i<0 ){ @@ -545,7 +503,46 @@ int sqlite3_column_bytes(sqlite3_stmt *pStmt, int i){ return 0; } + pVal = &pVm->pTos[(1-vals)+i]; + if( pVal->flags&MEM_Null ){ + return 0; + } + + if( !(pVal->flags&MEM_Blob) ){ + Stringify(pVal); + if( SQLITE3_BIGENDIAN ){ + SetEncoding(pVal, MEM_Utf16be|MEM_Term); + }else{ + SetEncoding(pVal, MEM_Utf16le|MEM_Term); + } + } + + return pVal->z; +} + +/* +** Return the number of bytes of data that will be returned by the +** equivalent sqlite3_column_data() call. +*/ +int sqlite3_column_bytes(sqlite3_stmt *pStmt, int i){ + Vdbe *pVm = (Vdbe *)pStmt; + if( sqlite3_column_data(pStmt, i) ){ + int vals = sqlite3_value_count(pStmt); + return pVm->pTos[(1-vals)+i].n; + } + return 0; +} + +/* +** Return the number of bytes of data that will be returned by the +** equivalent sqlite3_column_data16() call. +*/ +int sqlite3_column_bytes16(sqlite3_stmt *pStmt, int i){ + Vdbe *pVm = (Vdbe *)pStmt; + + if( sqlite3_column_data16(pStmt, i) ){ + int vals = sqlite3_value_count(pStmt); return pVm->pTos[(1-vals)+i].n; } return 0; diff --git a/test/enc2.test b/test/enc2.test new file mode 100644 index 0000000000..9a6955ac54 --- /dev/null +++ b/test/enc2.test @@ -0,0 +1,129 @@ +# 2002 May 24 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The focus of +# this file is testing the SQLite routines used for converting between the +# various suported unicode encodings (UTF-8, UTF-16, UTF-16le and +# UTF-16be). +# +# $Id: enc2.test,v 1.1 2004/05/22 10:33:04 danielk1977 Exp $ + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +db close + +# Return the UTF-8 representation of the supplied UTF-16 string $str. +proc utf8 {str} { + # If $str ends in two 0x00 0x00 bytes, knock these off before + # converting to UTF-8 using TCL. + binary scan $str \c* vals + if {[lindex $vals end]==0 && [lindex $vals end-1]==0} { + set str [binary format \c* [lrange $vals 0 end-2]] + } + + set r [encoding convertfrom unicode $str] + return $r +} + +# +# This proc contains all the tests in this file. It is run +# three times. Each time the file 'test.db' contains a database +# with the following contents: +set dbcontents { + CREATE TABLE t1(a PRIMARY KEY, b, c); + INSERT INTO t1 VALUES('one', 'I', 1); +} +# This proc tests that we can open and manipulate the test.db +# database, and that it is possible to retreive values in +# various text encodings. +# +proc run_test_script {t} { + +# Open the database and pull out a (the) row. +do_test $t.1 { + set DB [sqlite db test.db] + execsql {SELECT * FROM t1} +} {one I 1} + +# Insert some data +do_test $t.2 { + execsql {INSERT INTO t1 VALUES('two', 'II', 2);} + execsql {SELECT * FROM t1} +} {one I 1 two II 2} + +# Insert some data using the COPY command. +do_test $t.3 { + set f [open data.txt w] + puts $f "three\tIII\t3" + puts $f "four\tIV\t4" + puts $f "five\tV\t5" + close $f + execsql {COPY t1 FROM 'data.txt'} + execsql {SELECT * FROM t1} +} {one I 1 two II 2 three III 3 four IV 4 five V 5} + +# Use the index +do_test $t.4 { + execsql { + SELECT * FROM t1 WHERE a = 'one'; + } +} {one I 1} +do_test $t.5 { + execsql { + SELECT * FROM t1 WHERE a = 'four'; + } +} {four IV 4} +do_test $t.6 { + execsql { + SELECT * FROM t1 WHERE a IN ('one', 'two'); + } +} {one I 1 two II 2} + +# Now check that we can retrieve data in both UTF-16 and UTF-8 +do_test $t.7 { + set STMT [sqlite3_prepare $DB "SELECT a FROM t1 WHERE c>3;" -1 TAIL] + sqlite3_step $STMT + sqlite3_column_data $STMT 0 +} {four} + +do_test $t.8 { + sqlite3_step $STMT + utf8 [sqlite3_column_data16 $STMT 0] +} {five} + +do_test $t.9 { + sqlite3_finalize $STMT +} {} + +do_test $t.99 { + db close +} {} + +} + +# The three unicode encodings understood by SQLite. +set encodings [list -utf8 -utf16be -utf16le] + +set i 1 +foreach enc $encodings { + file delete -force test.db + sqlite db test.db $enc + execsql $dbcontents + db close + run_test_script enc2-$i + incr i +} + +finish_test + + + +