From fd0245d771542a60f17fb7aa5c1ab13990d0a92f Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 7 Dec 2017 15:44:29 +0000 Subject: [PATCH] Begin adding support for the sqlar archive format to shell.c. There is no "extract" command so far, only "create". FossilOrigin-Name: c9827a01a6e107f38f85c2b2c1c7a599e443067b106217e965b6936441ca619d --- ext/misc/fileio.c | 403 ++++++++++++++++++++++++++++++++++++++++++++-- manifest | 16 +- manifest.uuid | 2 +- src/shell.c.in | 215 +++++++++++++++++++++++++ 4 files changed, 611 insertions(+), 25 deletions(-) diff --git a/ext/misc/fileio.c b/ext/misc/fileio.c index 2c00ad971d..475f3713ce 100644 --- a/ext/misc/fileio.c +++ b/ext/misc/fileio.c @@ -12,11 +12,46 @@ ** ** This SQLite extension implements SQL functions readfile() and ** writefile(). +** +** Also, an eponymous virtual table type "fsdir". Used as follows: +** +** SELECT * FROM fsdir($dirname); +** +** Returns one row for each entry in the directory $dirname. No row is +** returned for "." or "..". Row columns are as follows: +** +** name: Name of directory entry. +** mode: Value of stat.st_mode for directory entry. +** mtime: Value of stat.st_mtime for directory entry. +** data: For a regular file, a blob containing the file data. For a +** symlink, a text value containing the text of the link. For a +** directory, NULL. */ #include "sqlite3ext.h" SQLITE_EXTENSION_INIT1 #include +#define FSDIR_SCHEMA "CREATE TABLE x(name,mode,mtime,data,dir HIDDEN)" + +static void readFileContents(sqlite3_context *ctx, const char *zName){ + FILE *in; + long nIn; + void *pBuf; + + in = fopen(zName, "rb"); + if( in==0 ) return; + fseek(in, 0, SEEK_END); + nIn = ftell(in); + rewind(in); + pBuf = sqlite3_malloc( nIn ); + if( pBuf && 1==fread(pBuf, nIn, 1, in) ){ + sqlite3_result_blob(ctx, pBuf, nIn, sqlite3_free); + }else{ + sqlite3_free(pBuf); + } + fclose(in); +} + /* ** Implementation of the "readfile(X)" SQL function. The entire content ** of the file named X is read and returned as a BLOB. NULL is returned @@ -28,25 +63,10 @@ static void readfileFunc( sqlite3_value **argv ){ const char *zName; - FILE *in; - long nIn; - void *pBuf; - (void)(argc); /* Unused parameter */ zName = (const char*)sqlite3_value_text(argv[0]); if( zName==0 ) return; - in = fopen(zName, "rb"); - if( in==0 ) return; - fseek(in, 0, SEEK_END); - nIn = ftell(in); - rewind(in); - pBuf = sqlite3_malloc( nIn ); - if( pBuf && 1==fread(pBuf, nIn, 1, in) ){ - sqlite3_result_blob(context, pBuf, nIn, sqlite3_free); - }else{ - sqlite3_free(pBuf); - } - fclose(in); + readFileContents(context, zName); } /* @@ -80,6 +100,354 @@ static void writefileFunc( sqlite3_result_int64(context, rc); } +#ifndef SQLITE_OMIT_VIRTUALTABLE + +#include +#include +#include +#include + +/* +*/ +typedef struct fsdir_cursor fsdir_cursor; +struct fsdir_cursor { + sqlite3_vtab_cursor base; /* Base class - must be first */ + int eType; /* One of FSDIR_DIR or FSDIR_ENTRY */ + DIR *pDir; /* From opendir() */ + struct stat sStat; /* Current lstat() results */ + char *zDir; /* Directory to read */ + int nDir; /* Value of strlen(zDir) */ + char *zPath; /* Path to current entry */ + int bEof; + sqlite3_int64 iRowid; /* Current rowid */ +}; + +typedef struct fsdir_tab fsdir_tab; +struct fsdir_tab { + sqlite3_vtab base; /* Base class - must be first */ + int eType; /* One of FSDIR_DIR or FSDIR_ENTRY */ +}; + +#define FSDIR_DIR 0 +#define FSDIR_ENTRY 1 + +/* +** Construct a new fsdir virtual table object. +*/ +static int fsdirConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + fsdir_tab *pNew = 0; + int rc; + + rc = sqlite3_declare_vtab(db, FSDIR_SCHEMA); + if( rc==SQLITE_OK ){ + pNew = (fsdir_tab*)sqlite3_malloc( sizeof(*pNew) ); + if( pNew==0 ) return SQLITE_NOMEM; + memset(pNew, 0, sizeof(*pNew)); + pNew->eType = (pAux==0 ? FSDIR_DIR : FSDIR_ENTRY); + } + *ppVtab = (sqlite3_vtab*)pNew; + return rc; +} + +/* +** This method is the destructor for fsdir vtab objects. +*/ +static int fsdirDisconnect(sqlite3_vtab *pVtab){ + sqlite3_free(pVtab); + return SQLITE_OK; +} + +/* +** Constructor for a new fsdir_cursor object. +*/ +static int fsdirOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ + fsdir_cursor *pCur; + pCur = sqlite3_malloc( sizeof(*pCur) ); + if( pCur==0 ) return SQLITE_NOMEM; + memset(pCur, 0, sizeof(*pCur)); + pCur->eType = ((fsdir_tab*)p)->eType; + *ppCursor = &pCur->base; + return SQLITE_OK; +} + +/* +** Destructor for an fsdir_cursor. +*/ +static int fsdirClose(sqlite3_vtab_cursor *cur){ + fsdir_cursor *pCur = (fsdir_cursor*)cur; + if( pCur->pDir ) closedir(pCur->pDir); + sqlite3_free(pCur->zDir); + sqlite3_free(pCur->zPath); + sqlite3_free(pCur); + return SQLITE_OK; +} + +/* +** Advance an fsdir_cursor to its next row of output. +*/ +static int fsdirNext(sqlite3_vtab_cursor *cur){ + fsdir_cursor *pCur = (fsdir_cursor*)cur; + struct dirent *pEntry; + + if( pCur->eType==FSDIR_ENTRY ){ + pCur->bEof = 1; + return SQLITE_OK; + } + + sqlite3_free(pCur->zPath); + pCur->zPath = 0; + + while( 1 ){ + pEntry = readdir(pCur->pDir); + if( pEntry ){ + if( strcmp(pEntry->d_name, ".") + && strcmp(pEntry->d_name, "..") + ){ + pCur->zPath = sqlite3_mprintf("%s/%s", pCur->zDir, pEntry->d_name); + if( pCur->zPath==0 ) return SQLITE_NOMEM; + lstat(pCur->zPath, &pCur->sStat); + break; + } + }else{ + pCur->bEof = 1; + break; + } + } + + pCur->iRowid++; + return SQLITE_OK; +} + +/* +** Return values of columns for the row at which the series_cursor +** is currently pointing. +*/ +static int fsdirColumn( + sqlite3_vtab_cursor *cur, /* The cursor */ + sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ + int i /* Which column to return */ +){ + fsdir_cursor *pCur = (fsdir_cursor*)cur; + switch( i ){ + case 0: { /* name */ + const char *zName; + if( pCur->eType==FSDIR_DIR ){ + zName = &pCur->zPath[pCur->nDir+1]; + }else{ + zName = pCur->zPath; + } + sqlite3_result_text(ctx, zName, -1, SQLITE_TRANSIENT); + break; + } + + case 1: /* mode */ + sqlite3_result_int64(ctx, pCur->sStat.st_mode); + break; + + case 2: /* mode */ + sqlite3_result_int64(ctx, pCur->sStat.st_mtime); + break; + + case 3: { + mode_t m = pCur->sStat.st_mode; + if( S_ISDIR(m) ){ + sqlite3_result_null(ctx); + }else if( S_ISLNK(m) ){ + char aStatic[64]; + char *aBuf = aStatic; + int nBuf = 64; + int n; + + while( 1 ){ + n = readlink(pCur->zPath, aBuf, nBuf); + if( nzPath); + } + } + } + return SQLITE_OK; +} + +/* +** Return the rowid for the current row. In this implementation, the +** first row returned is assigned rowid value 1, and each subsequent +** row a value 1 more than that of the previous. +*/ +static int fsdirRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + fsdir_cursor *pCur = (fsdir_cursor*)cur; + *pRowid = pCur->iRowid; + return SQLITE_OK; +} + +/* +** Return TRUE if the cursor has been moved off of the last +** row of output. +*/ +static int fsdirEof(sqlite3_vtab_cursor *cur){ + fsdir_cursor *pCur = (fsdir_cursor*)cur; + return pCur->bEof; +} + +static void fsdirSetErrmsg(fsdir_cursor *pCur, const char *zFmt, ...){ + va_list ap; + va_start(ap, zFmt); + pCur->base.pVtab->zErrMsg = sqlite3_vmprintf(zFmt, ap); + va_end(ap); +} + +/* +** xFilter callback. +*/ +static int fsdirFilter( + sqlite3_vtab_cursor *cur, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + const char *zDir = 0; + fsdir_cursor *pCur = (fsdir_cursor*)cur; + + sqlite3_free(pCur->zDir); + pCur->iRowid = 0; + pCur->zDir = 0; + pCur->bEof = 0; + if( pCur->pDir ){ + closedir(pCur->pDir); + pCur->pDir = 0; + } + + if( idxNum==0 ){ + fsdirSetErrmsg(pCur, "table function fsdir requires an argument"); + return SQLITE_ERROR; + } + + assert( argc==1 ); + zDir = (const char*)sqlite3_value_text(argv[0]); + if( zDir==0 ){ + fsdirSetErrmsg(pCur, "table function fsdir requires a non-NULL argument"); + return SQLITE_ERROR; + } + + pCur->zDir = sqlite3_mprintf("%s", zDir); + if( pCur->zDir==0 ){ + return SQLITE_NOMEM; + } + + if( pCur->eType==FSDIR_ENTRY ){ + int rc = lstat(pCur->zDir, &pCur->sStat); + if( rc ){ + fsdirSetErrmsg(pCur, "cannot stat file: %s", pCur->zDir); + }else{ + pCur->zPath = sqlite3_mprintf("%s", pCur->zDir); + if( pCur->zPath==0 ) return SQLITE_NOMEM; + } + return SQLITE_OK; + }else{ + pCur->nDir = strlen(pCur->zDir); + pCur->pDir = opendir(zDir); + if( pCur->pDir==0 ){ + fsdirSetErrmsg(pCur, "error in opendir(\"%s\")", zDir); + return SQLITE_ERROR; + } + + return fsdirNext(cur); + } +} + +/* +** SQLite will invoke this method one or more times while planning a query +** that uses the generate_series virtual table. This routine needs to create +** a query plan for each invocation and compute an estimated cost for that +** plan. +** +** In this implementation idxNum is used to represent the +** query plan. idxStr is unused. +** +** The query plan is represented by bits in idxNum: +** +** (1) start = $value -- constraint exists +** (2) stop = $value -- constraint exists +** (4) step = $value -- constraint exists +** (8) output in descending order +*/ +static int fsdirBestIndex( + sqlite3_vtab *tab, + sqlite3_index_info *pIdxInfo +){ + int i; /* Loop over constraints */ + + const struct sqlite3_index_constraint *pConstraint; + pConstraint = pIdxInfo->aConstraint; + for(i=0; inConstraint; i++, pConstraint++){ + if( pConstraint->usable==0 ) continue; + if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; + if( pConstraint->iColumn!=4 ) continue; + break; + } + + if( inConstraint ){ + pIdxInfo->aConstraintUsage[i].omit = 1; + pIdxInfo->aConstraintUsage[i].argvIndex = 1; + pIdxInfo->idxNum = 1; + pIdxInfo->estimatedCost = 10.0; + }else{ + pIdxInfo->idxNum = 0; + pIdxInfo->estimatedCost = (double)(((sqlite3_int64)1) << 50); + } + + return SQLITE_OK; +} + +static int fsdirRegister(sqlite3 *db){ + static sqlite3_module fsdirModule = { + 0, /* iVersion */ + 0, /* xCreate */ + fsdirConnect, /* xConnect */ + fsdirBestIndex, /* xBestIndex */ + fsdirDisconnect, /* xDisconnect */ + 0, /* xDestroy */ + fsdirOpen, /* xOpen - open a cursor */ + fsdirClose, /* xClose - close a cursor */ + fsdirFilter, /* xFilter - configure scan constraints */ + fsdirNext, /* xNext - advance a cursor */ + fsdirEof, /* xEof - check for end of scan */ + fsdirColumn, /* xColumn - read data */ + fsdirRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ + }; + + int rc = sqlite3_create_module(db, "fsdir", &fsdirModule, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_module(db, "fsentry", &fsdirModule, (void*)1); + } + return rc; +} +#else /* SQLITE_OMIT_VIRTUALTABLE */ +# define fsdirRegister(x) SQLITE_OK +#endif #ifdef _WIN32 __declspec(dllexport) @@ -98,5 +466,8 @@ int sqlite3_fileio_init( rc = sqlite3_create_function(db, "writefile", 2, SQLITE_UTF8, 0, writefileFunc, 0, 0); } + if( rc==SQLITE_OK ){ + rc = fsdirRegister(db); + } return rc; } diff --git a/manifest b/manifest index 3e24dc86b3..dc93367667 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C For\sMSVC,\ssimplify\sdefault\slocations\sfor\sTcl\sand\sICU\sby\susing\sdirectories\sinside\s'compat'. -D 2017-12-05T19:07:30.651 +C Begin\sadding\ssupport\sfor\sthe\ssqlar\sarchive\sformat\sto\sshell.c.\sThere\sis\sno\n"extract"\scommand\sso\sfar,\sonly\s"create". +D 2017-12-07T15:44:29.604 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8 @@ -269,7 +269,7 @@ F ext/misc/compress.c 122faa92d25033d6c3f07c39231de074ab3d2e83 F ext/misc/csv.c 1a009b93650732e22334edc92459c4630b9fa703397cbb3c8ca279921a36ca11 F ext/misc/dbdump.c 3509fa6b8932d04e932d6b6b827b6a82ca362781b8e8f3c77336f416793e215e F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2 -F ext/misc/fileio.c b1aa06c0f1dac277695d4529e5e976c65ab5678dcbb53a0304deaa8adc44b332 +F ext/misc/fileio.c bd2dd9bd22a509f972a4658be18bbfef80aec3cbc3e18948c5e8c5e29ece9939 F ext/misc/fuzzer.c 7c64b8197bb77b7d64eff7cac7848870235d4c25 F ext/misc/ieee754.c f190d0cc5182529acb15babd177781be1ac1718c F ext/misc/json1.c dbe086615b9546c156bf32b9378fc09383b58bd17513b866cfd24c1e15281984 @@ -474,7 +474,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c 17e220191860a64a18c084141e1a8b7309e166a6f2d42c02021af27ea080d157 -F src/shell.c.in ab727c09b4c87c0c1db32d2fe0a910c0a8e468a0209233328753f5526d6c6c73 +F src/shell.c.in 56c4c091c74af2c7858f2d8af962caa632889596435b17bffbc308058fb305cc F src/sqlite.h.in 8fd97993d48b50b9bade38c52f12d175942c9497c960905610c7b03a3e4b5818 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34 @@ -1681,7 +1681,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e1838cee3847301ef491467dc75d9c4e1e3b12599596c058bdb14319a52fd8a0 -R a792968384e617f4a37df83169716b24 -U mistachkin -Z 06008f15f266edb3571035a6d698db2a +P 8155b5ac850327ea76aba2adf624132f3e05024c973afd218b12f186fc7630e8 +R 1ffe38726fc0bf4dcb6cd1bf88405c9b +U dan +Z f98960131c8a51264c03e5b1a40bd1b7 diff --git a/manifest.uuid b/manifest.uuid index b50deeb9bc..3b745e1dd5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8155b5ac850327ea76aba2adf624132f3e05024c973afd218b12f186fc7630e8 \ No newline at end of file +c9827a01a6e107f38f85c2b2c1c7a599e443067b106217e965b6936441ca619d \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index edd75b078c..9015a4330a 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -4074,6 +4074,214 @@ static int lintDotCommand( return SQLITE_ERROR; } +static void shellPrepare( + ShellState *p, + int *pRc, + const char *zSql, + sqlite3_stmt **ppStmt +){ + *ppStmt = 0; + if( *pRc==SQLITE_OK ){ + int rc = sqlite3_prepare_v2(p->db, zSql, -1, ppStmt, 0); + if( rc!=SQLITE_OK ){ + raw_printf(stderr, "sql error: %s (%d)\n", + sqlite3_errmsg(p->db), sqlite3_errcode(p->db) + ); + *pRc = rc; + } + } +} + +static void shellFinalize( + int *pRc, + sqlite3_stmt *pStmt +){ + int rc = sqlite3_finalize(pStmt); + if( *pRc==SQLITE_OK ) *pRc = rc; +} + +static void shellReset( + int *pRc, + sqlite3_stmt *pStmt +){ + int rc = sqlite3_reset(pStmt); + if( *pRc==SQLITE_OK ) *pRc = rc; +} + +/* +** Implementation of .ar "eXtract" command. +*/ +static int arExtractCommand(ShellState *p, int bVerbose){ + const char *zSql = + "SELECT name, mode, mtime, sz, data FROM sqlar"; + sqlite3_stmt *pSql = 0; + int rc = SQLITE_OK; + + shellPrepare(p, &rc, zSql, &pSql); + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){ + } + + shellFinalize(&rc, pSql); + return rc; +} + +/* +** Implementation of .ar "Create" command. +** +** Create the "sqlar" table in the database if it does not already exist. +** Then add each file in the azFile[] array to the archive. Directories +** are added recursively. If argument bVerbose is non-zero, a message is +** printed on stdout for each file archived. +*/ +static int arCreateCommand( + ShellState *p, /* Shell state pointer */ + char **azFile, /* Array of files to add to archive */ + int nFile, /* Number of entries in azFile[] */ + int bVerbose /* True to be verbose on stdout */ +){ + const char *zSql = + "WITH f(n, m, t, d) AS (" + " SELECT name, mode, mtime, data FROM fsentry(:1) UNION ALL " + " SELECT n || '/' || name, mode, mtime, data " + " FROM f, fsdir(n) WHERE (m&?)" + ") SELECT * FROM f"; + + const char *zSqlar = + "CREATE TABLE IF NOT EXISTS sqlar(" + "name TEXT PRIMARY KEY, -- name of the file\n" + "mode INT, -- access permissions\n" + "mtime INT, -- last modification time\n" + "sz INT, -- original file size\n" + "data BLOB -- compressed content\n" + ")"; + + const char *zInsert = "REPLACE INTO sqlar VALUES(?, ?, ?, ?, ?)"; + + sqlite3_stmt *pStmt = 0; /* Directory traverser */ + sqlite3_stmt *pInsert = 0; /* Compilation of zInsert */ + int i; /* For iterating through azFile[] */ + int rc; /* Return code */ + + Bytef *aCompress = 0; /* Compression buffer */ + int nCompress = 0; /* Size of compression buffer */ + + rc = sqlite3_exec(p->db, "SAVEPOINT ar;", 0, 0, 0); + if( rc!=SQLITE_OK ) return rc; + + rc = sqlite3_exec(p->db, zSqlar, 0, 0, 0); + shellPrepare(p, &rc, zInsert, &pInsert); + shellPrepare(p, &rc, zSql, &pStmt); + + for(i=0; inCompress ){ + Bytef *aNew = sqlite3_realloc(aCompress, nReq); + if( aNew==0 ){ + rc = SQLITE_NOMEM; + }else{ + aCompress = aNew; + nCompress = nReq; + } + } + + if( Z_OK!=compress(aCompress, &nReq, pData, sz) ){ + rc = SQLITE_ERROR; + } + if( nReqdb, "ROLLBACK TO ar; RELEASE ar;", 0, 0, 0); + }else{ + rc = sqlite3_exec(p->db, "RELEASE ar;", 0, 0, 0); + } + shellFinalize(&rc, pStmt); + shellFinalize(&rc, pInsert); + sqlite3_free(aCompress); + return rc; +} + +/* +** Implementation of ".ar" dot command. +*/ +static int arDotCommand( + ShellState *pState, /* Current shell tool state */ + char **azArg, /* Array of arguments passed to dot command */ + int nArg /* Number of entries in azArg[] */ +){ + int bVerbose = 0; + char cmd = 0; + int i; + int n1; + if( nArg<=1 ) goto usage; + + n1 = strlen(azArg[1]); + for(i=0; i=3 && strncmp(azArg[0], "backup", n)==0) || (c=='s' && n>=3 && strncmp(azArg[0], "save", n)==0) ){