From ac15e2d7ccfb706d020d4a42fc4d809f6ca1cca8 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 14 Dec 2017 19:15:07 +0000 Subject: [PATCH] Have the writefile() function optionally set the modification-time of the files it writes or creates. And many small fixes to the new code on this branch. FossilOrigin-Name: 7b51269caebe1492885fe9b965892f49a3f8bdb1d666b0203d594c30f9e83938 --- ext/misc/fileio.c | 136 ++++++++++++++++++++++++++++++++++++---------- manifest | 16 +++--- manifest.uuid | 2 +- src/shell.c.in | 91 ++++++++++++++++--------------- test/shell8.test | 15 +++++ 5 files changed, 177 insertions(+), 83 deletions(-) diff --git a/ext/misc/fileio.c b/ext/misc/fileio.c index 2320173997..7dbac4043f 100644 --- a/ext/misc/fileio.c +++ b/ext/misc/fileio.c @@ -11,21 +11,67 @@ ****************************************************************************** ** ** This SQLite extension implements SQL functions readfile() and -** writefile(). +** writefile(), and eponymous virtual type "fsdir". ** -** Also, an eponymous virtual table type "fsdir". Used as follows: +** WRITEFILE(FILE, DATA [, MODE [, MTIME]]): ** -** SELECT * FROM fsdir($dirname); +** If neither of the optional arguments is present, then this UDF +** function writes blob DATA to file FILE. If successful, the number +** of bytes written is returned. If an error occurs, NULL is returned. ** -** Returns one row for each entry in the directory $dirname. No row is -** returned for "." or "..". Row columns are as follows: +** If the first option argument - MODE - is present, then it must +** be passed an integer value that corresponds to a POSIX mode +** value (file type + permissions, as returned in the stat.st_mode +** field by the stat() system call). Three types of files may +** be written/created: ** -** name: Name of directory entry. -** mode: Value of stat.st_mode for directory entry. -** mtime: Value of stat.st_mtime for directory entry. -** data: For a regular file, a blob containing the file data. For a -** symlink, a text value containing the text of the link. For a -** directory, NULL. +** regular files: (mode & 0170000)==0100000 +** symbolic links: (mode & 0170000)==0120000 +** directories: (mode & 0170000)==0040000 +** +** For a directory, the DATA is ignored. For a symbolic link, it is +** interpreted as text and used as the target of the link. For a +** regular file, it is interpreted as a blob and written into the +** named file. Regardless of the type of file, its permissions are +** set to (mode & 0777) before returning. +** +** If the optional MTIME argument is present, then it is interpreted +** as an integer - the number of seconds since the unix epoch. The +** modification-time of the target file is set to this value before +** returning. +** +** If three or more arguments are passed to this function and an +** error is encountered, an exception is raised. +** +** READFILE(FILE): +** +** Read and return the contents of file FILE (type blob) from disk. +** +** FSDIR: +** +** Used as follows: +** +** SELECT * FROM fsdir($path [, $dir]); +** +** Parameter $path is an absolute or relative pathname. If the file that it +** refers to does not exist, it is an error. If the path refers to a regular +** file or symbolic link, it returns a single row. Or, if the path refers +** to a directory, it returns one row for the directory, and one row for each +** file within the hierarchy rooted at $path. +** +** Each row has the following columns: +** +** name: Path to file or directory (text value). +** mode: Value of stat.st_mode for directory entry (an integer). +** mtime: Value of stat.st_mtime for directory entry (an integer). +** data: For a regular file, a blob containing the file data. For a +** symlink, a text value containing the text of the link. For a +** directory, NULL. +** +** If a non-NULL value is specified for the optional $dir parameter and +** $path is a relative path, then $path is interpreted relative to $dir. +** And the paths returned in the "name" column of the table are also +** relative to directory $dir. */ #include "sqlite3ext.h" SQLITE_EXTENSION_INIT1 @@ -45,6 +91,10 @@ SQLITE_EXTENSION_INIT1 #define FSDIR_SCHEMA "(name,mode,mtime,data,path HIDDEN,dir HIDDEN)" +/* +** Set the result stored by context ctx to a blob containing the +** contents of file zName. +*/ static void readFileContents(sqlite3_context *ctx, const char *zName){ FILE *in; long nIn; @@ -81,6 +131,10 @@ static void readfileFunc( readFileContents(context, zName); } +/* +** Set the error message contained in context ctx to the results of +** vprintf(zFmt, ...). +*/ static void ctxErrorMsg(sqlite3_context *ctx, const char *zFmt, ...){ char *zMsg = 0; va_list ap; @@ -138,11 +192,16 @@ static int makeDirectory( return rc; } +/* +** This function does the work for the writefile() UDF. Refer to +** header comments at the top of this file for details. +*/ static int writeFile( - sqlite3_context *pCtx, - const char *zFile, - mode_t mode, - sqlite3_value *pData + sqlite3_context *pCtx, /* Context to return bytes written in */ + const char *zFile, /* File to write */ + sqlite3_value *pData, /* Data to write */ + mode_t mode, /* MODE parameter passed to writefile() */ + sqlite3_int64 mtime /* MTIME parameter (or -1 to not set time) */ ){ if( S_ISLNK(mode) ){ const char *zTo = (const char*)sqlite3_value_text(pData); @@ -178,22 +237,30 @@ static int writeFile( } } fclose(out); - if( rc==0 && chmod(zFile, mode & 0777) ){ + if( rc==0 && mode && chmod(zFile, mode & 0777) ){ rc = 1; } if( rc ) return 2; sqlite3_result_int64(pCtx, nWrite); } } + + if( mtime>=0 ){ + struct timespec times[2]; + times[0].tv_nsec = times[1].tv_nsec = 0; + times[0].tv_sec = time(0); + times[1].tv_sec = mtime; + if( utimensat(AT_FDCWD, zFile, times, AT_SYMLINK_NOFOLLOW) ){ + return 1; + } + } + return 0; } /* -** Implementation of the "writefile(W,X[,Y]])" SQL function. -** -** The argument X is written into file W. The number of bytes written is -** returned. Or NULL is returned if something goes wrong, such as being unable -** to open file X for writing. +** Implementation of the "writefile(W,X[,Y[,Z]]])" SQL function. +** Refer to header comments at the top of this file for details. */ static void writefileFunc( sqlite3_context *context, @@ -203,8 +270,9 @@ static void writefileFunc( const char *zFile; mode_t mode = 0; int res; + sqlite3_int64 mtime = -1; - if( argc<2 || argc>3 ){ + if( argc<2 || argc>4 ){ sqlite3_result_error(context, "wrong number of arguments to function writefile()", -1 ); @@ -214,18 +282,20 @@ static void writefileFunc( zFile = (const char*)sqlite3_value_text(argv[0]); if( zFile==0 ) return; if( argc>=3 ){ - sqlite3_result_int(context, 0); mode = sqlite3_value_int(argv[2]); } + if( argc==4 ){ + mtime = sqlite3_value_int64(argv[3]); + } - res = writeFile(context, zFile, mode, argv[1]); + res = writeFile(context, zFile, argv[1], mode, mtime); if( res==1 && errno==ENOENT ){ if( makeDirectory(zFile, mode)==SQLITE_OK ){ - res = writeFile(context, zFile, mode, argv[1]); + res = writeFile(context, zFile, argv[1], mode, mtime); } } - if( res!=0 ){ + if( argc>2 && res!=0 ){ if( S_ISLNK(mode) ){ ctxErrorMsg(context, "failed to create symlink: %s", zFile); }else if( S_ISDIR(mode) ){ @@ -313,6 +383,10 @@ static int fsdirOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ return SQLITE_OK; } +/* +** Reset a cursor back to the state it was in when first returned +** by fsdirOpen(). +*/ static void fsdirResetCursor(fsdir_cursor *pCur){ int i; for(i=0; i<=pCur->iLvl; i++){ @@ -341,6 +415,10 @@ static int fsdirClose(sqlite3_vtab_cursor *cur){ return SQLITE_OK; } +/* +** Set the error message for the virtual table associated with cursor +** pCur to the results of vprintf(zFmt, ...). +*/ static void fsdirSetErrmsg(fsdir_cursor *pCur, const char *zFmt, ...){ va_list ap; va_start(ap, zFmt); @@ -586,6 +664,9 @@ static int fsdirBestIndex( return SQLITE_OK; } +/* +** Register the "fsdir" virtual table. +*/ static int fsdirRegister(sqlite3 *db){ static sqlite3_module fsdirModule = { 0, /* iVersion */ @@ -611,9 +692,6 @@ static int fsdirRegister(sqlite3 *db){ }; int rc = sqlite3_create_module(db, "fsdir", &fsdirModule, 0); - if( rc==SQLITE_OK ){ - rc = sqlite3_create_module(db, "fsentry", &fsdirModule, (void*)1); - } return rc; } #else /* SQLITE_OMIT_VIRTUALTABLE */ diff --git a/manifest b/manifest index dda85f8287..2ee47c023e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improve\serror\sand\susage\smessages\soutput\sby\sthe\sshell\s".ar"\scommand. -D 2017-12-14T15:40:42.931 +C Have\sthe\swritefile()\sfunction\soptionally\sset\sthe\smodification-time\sof\sthe\nfiles\sit\swrites\sor\screates.\sAnd\smany\ssmall\sfixes\sto\sthe\snew\scode\son\sthis\nbranch. +D 2017-12-14T19:15:07.381 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8 @@ -270,7 +270,7 @@ F ext/misc/compress.c 122faa92d25033d6c3f07c39231de074ab3d2e83 F ext/misc/csv.c 1a009b93650732e22334edc92459c4630b9fa703397cbb3c8ca279921a36ca11 F ext/misc/dbdump.c 3509fa6b8932d04e932d6b6b827b6a82ca362781b8e8f3c77336f416793e215e F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2 -F ext/misc/fileio.c 29b7fc94752fff6245cf4a81455f98cf6778ec1102ca7e67bf693d41a7db4307 +F ext/misc/fileio.c 014152d4133e7b29eab8eb39d0c640659c23a6d23d882b4778f487ae7d1a457b F ext/misc/fuzzer.c 7c64b8197bb77b7d64eff7cac7848870235d4c25 F ext/misc/ieee754.c f190d0cc5182529acb15babd177781be1ac1718c F ext/misc/json1.c dbe086615b9546c156bf32b9378fc09383b58bd17513b866cfd24c1e15281984 @@ -475,7 +475,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c 17e220191860a64a18c084141e1a8b7309e166a6f2d42c02021af27ea080d157 -F src/shell.c.in 4bdd2efe722005180365698f2a3de51e22ae1e9bb61c868006bc8f2c5e02eb98 +F src/shell.c.in 074b2129559a0aa712a367317f7e7daf4740925ec2c123b529800628eb10dc73 F src/sqlite.h.in 364515dd186285f3c01f5cab42e7db7edc47c70e87b6a25de389a2e6b8c413fd F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34 @@ -1215,7 +1215,7 @@ F test/shell4.test 89ad573879a745974ff2df20ff97c5d6ffffbd5d F test/shell5.test 23939a4c51f0421330ea61dbd3c74f9c215f5f8d3d1a94846da6ffc777a35458 F test/shell6.test 1ceb51b2678c472ba6cf1e5da96679ce8347889fe2c3bf93a0e0fa73f00b00d3 F test/shell7.test 115132f66d0463417f408562cc2cf534f6bbc6d83a6d50f0072a9eb171bae97f -F test/shell8.test 0f7dfc5b33bde7143df8e37cbb4ae6ccc7e91f87232dc8e5e02be03117cdebb8 +F test/shell8.test 96f35965fe84d633fb2338696f5cbc1bcf6bdbdd79677244bc617a8452851dc7 F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3 F test/show_speedtest1_rtree.tcl 32e6c5f073d7426148a6936a0408f4b5b169aba5 F test/shrink.test 1b4330b1fd9e818c04726d45cb28db73087535ce @@ -1683,7 +1683,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 803156cba8b056a1cb8d1bb186a57454afe72341abe7de1dfe529234c3415cd2 -R 69895ee1194373ab4ba11578d798fe26 +P b9d2d5d97291bf3d1392232e3705cca89dc7b918db2b08067b2b013ea39320e0 +R 975f9981c88e18e3ddfa1fe4bb6e7fae U dan -Z 5ea750b0cf239f9442948a3eda934166 +Z 1e84576de6ef44efce8abf1d868c1119 diff --git a/manifest.uuid b/manifest.uuid index ce1ae1dcb3..31073ba115 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b9d2d5d97291bf3d1392232e3705cca89dc7b918db2b08067b2b013ea39320e0 \ No newline at end of file +7b51269caebe1492885fe9b965892f49a3f8bdb1d666b0203d594c30f9e83938 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 2a0f9fde9e..8b16082add 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -4092,23 +4092,25 @@ static void shellPrepare( } } -static void shellPrepare2( +static void shellPreparePrintf( sqlite3 *db, int *pRc, - const char *zSql, - const char *zTail, - sqlite3_stmt **ppStmt + sqlite3_stmt **ppStmt, + const char *zFmt, + ... ){ - if( *pRc==SQLITE_OK && zTail ){ - char *z = sqlite3_mprintf("%s %s", zSql, zTail); + *ppStmt = 0; + if( *pRc==SQLITE_OK ){ + va_list ap; + char *z; + va_start(ap, zFmt); + z = sqlite3_vmprintf(zFmt, ap); if( z==0 ){ *pRc = SQLITE_NOMEM; }else{ shellPrepare(db, pRc, z, ppStmt); sqlite3_free(z); } - }else{ - shellPrepare(db, pRc, zSql, ppStmt); } } @@ -4429,23 +4431,27 @@ static int arCheckEntries(sqlite3 *db, ArCommand *pAr){ static void arWhereClause( int *pRc, ArCommand *pAr, - char **pzWhere /* OUT: New WHERE clause (or NULL) */ + char **pzWhere /* OUT: New WHERE clause */ ){ char *zWhere = 0; if( *pRc==SQLITE_OK ){ - int i; - const char *zSep = "WHERE "; - for(i=0; inArg; i++){ - const char *z = pAr->azArg[i]; - zWhere = sqlite3_mprintf( - "%z%s name = '%q' OR name BETWEEN '%q/' AND '%q0'", - zWhere, zSep, z, z, z - ); - if( zWhere==0 ){ - *pRc = SQLITE_NOMEM; - break; + if( pAr->nArg==0 ){ + zWhere = sqlite3_mprintf("1"); + }else{ + int i; + const char *zSep = ""; + for(i=0; inArg; i++){ + const char *z = pAr->azArg[i]; + zWhere = sqlite3_mprintf( + "%z%s name = '%q' OR name BETWEEN '%q/' AND '%q0'", + zWhere, zSep, z, z, z + ); + if( zWhere==0 ){ + *pRc = SQLITE_NOMEM; + break; + } + zSep = " OR "; } - zSep = " OR "; } } *pzWhere = zWhere; @@ -4455,7 +4461,7 @@ static void arWhereClause( ** Implementation of .ar "lisT" command. */ static int arListCommand(ShellState *p, sqlite3 *db, ArCommand *pAr){ - const char *zSql = "SELECT name FROM sqlar"; + const char *zSql = "SELECT name FROM sqlar WHERE %s"; char *zWhere = 0; sqlite3_stmt *pSql = 0; int rc; @@ -4463,7 +4469,7 @@ static int arListCommand(ShellState *p, sqlite3 *db, ArCommand *pAr){ rc = arCheckEntries(db, pAr); arWhereClause(&rc, pAr, &zWhere); - shellPrepare2(db, &rc, zSql, zWhere, &pSql); + shellPreparePrintf(db, &rc, &pSql, zSql, zWhere); while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){ raw_printf(p->out, "%s\n", sqlite3_column_text(pSql, 0)); } @@ -4482,14 +4488,14 @@ static int arExtractCommand(ShellState *p, sqlite3 *db, ArCommand *pAr){ "ELSE" " data " "END, " - "mode) FROM sqlar"; - const char *zSql2 = "SELECT :1 || name, mtime FROM sqlar"; + "mode, mtime) FROM sqlar WHERE (%s) AND (data IS NULL OR :2 = 0)"; struct timespec times[2]; sqlite3_stmt *pSql = 0; int rc = SQLITE_OK; char *zDir = 0; char *zWhere = 0; + int i; /* If arguments are specified, check that they actually exist within ** the archive before proceeding. And formulate a WHERE clause to @@ -4509,31 +4515,26 @@ static int arExtractCommand(ShellState *p, sqlite3 *db, ArCommand *pAr){ memset(times, 0, sizeof(times)); times[0].tv_sec = time(0); - shellPrepare2(db, &rc, zSql1, zWhere, &pSql); + shellPreparePrintf(db, &rc, &pSql, zSql1, zWhere); if( rc==SQLITE_OK ){ sqlite3_bind_text(pSql, 1, zDir, -1, SQLITE_STATIC); - } - while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){ - if( pAr->bVerbose ){ - raw_printf(p->out, "%s\n", sqlite3_column_text(pSql, 0)); - } - } - shellFinalize(&rc, pSql); - shellPrepare2(db, &rc, zSql2, zWhere, &pSql); - if( rc==SQLITE_OK ){ - sqlite3_bind_text(pSql, 1, zDir, -1, SQLITE_STATIC); - } - while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){ - const char *zPath = (const char*)sqlite3_column_text(pSql, 0); - times[1].tv_sec = (time_t)sqlite3_column_int64(pSql, 1); - if( utimensat(AT_FDCWD, zPath, times, AT_SYMLINK_NOFOLLOW) ){ - raw_printf(stderr, "failed to set timestamp for %s\n", zPath); - rc = SQLITE_ERROR; - break; + /* Run the SELECT statement twice. The first time, writefile() is called + ** for all archive members that should be extracted. The second time, + ** only for the directories. This is because the timestamps for + ** extracted directories must be reset after they are populated (as + ** populating them changes the timestamp). */ + for(i=0; i<2; i++){ + sqlite3_bind_int(pSql, 2, i); + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){ + if( i==0 && pAr->bVerbose ){ + raw_printf(p->out, "%s\n", sqlite3_column_text(pSql, 0)); + } + } + shellReset(&rc, pSql); } + shellFinalize(&rc, pSql); } - shellFinalize(&rc, pSql); sqlite3_free(zDir); sqlite3_free(zWhere); diff --git a/test/shell8.test b/test/shell8.test index 07065b56bd..14980a84a5 100644 --- a/test/shell8.test +++ b/test/shell8.test @@ -139,6 +139,21 @@ foreach {tn tcl} { catchcmd ":memory:" $x3 dir_to_list ar3 } $expected + + # This is a repeat of test 1.$tn.1, except that there is a 2 second + # pause between creating the archive and extracting its contents. + # This is to test that timestamps are set correctly. + # + # Because it is slow, only do this for $tn==1. + if {$tn==1} { + do_test 1.$tn.1 { + catchcmd test_ar.db $c1 + file delete -force ar1 + after 2000 + catchcmd test_ar.db $x1 + dir_to_list ar1 + } $expected + } } finish_test