Enhance virtual table "fsdir" in ext/misc/fileio.c. Add support for "-C" to

the shell command's ".ar c" command.

FossilOrigin-Name: 0394889afed2479773af594e2d9659cf58b8959004ebcdeaff8e08e5dae684ef

ext/misc/fileio.c

@@ -42,7 +42,7 @@ SQLITE_EXTENSION_INIT1
 #include <utime.h>
 
 
-#define FSDIR_SCHEMA "CREATE TABLE x(name,mode,mtime,data,dir HIDDEN)"
+#define FSDIR_SCHEMA "(name,mode,mtime,data,path HIDDEN,dir HIDDEN)"
 
 static void readFileContents(sqlite3_context *ctx, const char *zName){
   FILE *in;
@@ -166,29 +166,36 @@ static void writefileFunc(
 #ifndef SQLITE_OMIT_VIRTUALTABLE
 
 /* 
+** Cursor type for recursively iterating through a directory structure.
 */
 typedef struct fsdir_cursor fsdir_cursor;
+typedef struct FsdirLevel FsdirLevel;
+
+struct FsdirLevel {
+  DIR *pDir;                 /* From opendir() */
+  char *zDir;                /* Name of directory (nul-terminated) */
+};
+
 struct fsdir_cursor {
   sqlite3_vtab_cursor base;  /* Base class - must be first */
-  int eType;                 /* One of FSDIR_DIR or FSDIR_ENTRY */
+
-  DIR *pDir;                 /* From opendir() */
+  int nLvl;                  /* Number of entries in aLvl[] array */
+  int iLvl;                  /* Index of current entry */
+  FsdirLevel *aLvl;          /* Hierarchy of directories being traversed */
+
+  const char *zBase;
+  int nBase;
+
   struct stat sStat;         /* Current lstat() results */
-  char *zDir;                /* Directory to read */
-  int nDir;                  /* Value of strlen(zDir) */
   char *zPath;               /* Path to current entry */
-  int bEof;
   sqlite3_int64 iRowid;      /* Current rowid */
 };
 
 typedef struct fsdir_tab fsdir_tab;
 struct fsdir_tab {
   sqlite3_vtab base;         /* Base class - must be first */
-  int eType;                 /* One of FSDIR_DIR or FSDIR_ENTRY */
 };
 
-#define FSDIR_DIR   0
-#define FSDIR_ENTRY 1
-
 /*
 ** Construct a new fsdir virtual table object.
 */
@@ -202,12 +209,11 @@ static int fsdirConnect(
   fsdir_tab *pNew = 0;
   int rc;
 
-  rc = sqlite3_declare_vtab(db, FSDIR_SCHEMA);
+  rc = sqlite3_declare_vtab(db, "CREATE TABLE x" FSDIR_SCHEMA);
   if( rc==SQLITE_OK ){
     pNew = (fsdir_tab*)sqlite3_malloc( sizeof(*pNew) );
     if( pNew==0 ) return SQLITE_NOMEM;
     memset(pNew, 0, sizeof(*pNew));
-    pNew->eType = (pAux==0 ? FSDIR_DIR : FSDIR_ENTRY);
   }
   *ppVtab = (sqlite3_vtab*)pNew;
   return rc;
@@ -229,56 +235,107 @@ static int fsdirOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){
   pCur = sqlite3_malloc( sizeof(*pCur) );
   if( pCur==0 ) return SQLITE_NOMEM;
   memset(pCur, 0, sizeof(*pCur));
-  pCur->eType = ((fsdir_tab*)p)->eType;
+  pCur->iLvl = -1;
   *ppCursor = &pCur->base;
   return SQLITE_OK;
 }
 
+static void fsdirResetCursor(fsdir_cursor *pCur){
+  int i;
+  for(i=0; i<=pCur->iLvl; i++){
+    FsdirLevel *pLvl = &pCur->aLvl[i];
+    if( pLvl->pDir ) closedir(pLvl->pDir);
+    sqlite3_free(pLvl->zDir);
+  }
+  sqlite3_free(pCur->zPath);
+  pCur->aLvl = 0;
+  pCur->zPath = 0;
+  pCur->zBase = 0;
+  pCur->nBase = 0;
+  pCur->iLvl = -1;
+  pCur->iRowid = 1;
+}
+
 /*
 ** Destructor for an fsdir_cursor.
 */
 static int fsdirClose(sqlite3_vtab_cursor *cur){
   fsdir_cursor *pCur = (fsdir_cursor*)cur;
-  if( pCur->pDir ) closedir(pCur->pDir);
+
-  sqlite3_free(pCur->zDir);
+  fsdirResetCursor(pCur);
-  sqlite3_free(pCur->zPath);
+  sqlite3_free(pCur->aLvl);
   sqlite3_free(pCur);
   return SQLITE_OK;
 }
 
+static void fsdirSetErrmsg(fsdir_cursor *pCur, const char *zFmt, ...){
+  va_list ap;
+  va_start(ap, zFmt);
+  pCur->base.pVtab->zErrMsg = sqlite3_vmprintf(zFmt, ap);
+  va_end(ap);
+}
+
+
 /*
 ** Advance an fsdir_cursor to its next row of output.
 */
 static int fsdirNext(sqlite3_vtab_cursor *cur){
   fsdir_cursor *pCur = (fsdir_cursor*)cur;
-  struct dirent *pEntry;
+  mode_t m = pCur->sStat.st_mode;
 
-  if( pCur->eType==FSDIR_ENTRY ){
+  pCur->iRowid++;
-    pCur->bEof = 1;
+  if( S_ISDIR(m) ){
-    return SQLITE_OK;
+    /* Descend into this directory */
-  }
+    int iNew = pCur->iLvl + 1;
-
+    FsdirLevel *pLvl;
-  sqlite3_free(pCur->zPath);
+    if( iNew>=pCur->nLvl ){
-  pCur->zPath = 0;
+      int nNew = iNew+1;
-
+      int nByte = nNew*sizeof(FsdirLevel);
-  while( 1 ){
+      FsdirLevel *aNew = (FsdirLevel*)sqlite3_realloc(pCur->aLvl, nByte);
-    pEntry = readdir(pCur->pDir);
+      if( aNew==0 ) return SQLITE_NOMEM;
-    if( pEntry ){
+      memset(&aNew[pCur->nLvl], 0, sizeof(FsdirLevel)*(nNew-pCur->nLvl));
-      if( strcmp(pEntry->d_name, ".") 
+      pCur->aLvl = aNew;
-       && strcmp(pEntry->d_name, "..") 
+      pCur->nLvl = nNew;
-      ){
+    }
-        pCur->zPath = sqlite3_mprintf("%s/%s", pCur->zDir, pEntry->d_name);
+    pCur->iLvl = iNew;
-        if( pCur->zPath==0 ) return SQLITE_NOMEM;
+    pLvl = &pCur->aLvl[iNew];
-        lstat(pCur->zPath, &pCur->sStat);
+    
-        break;
+    pLvl->zDir = pCur->zPath;
-      }
+    pCur->zPath = 0;
-    }else{
+    pLvl->pDir = opendir(pLvl->zDir);
-      pCur->bEof = 1;
+    if( pLvl->pDir==0 ){
-      break;
+      fsdirSetErrmsg(pCur, "cannot read directory: %s", pCur->zPath);
+      return SQLITE_ERROR;
     }
   }
 
-  pCur->iRowid++;
+  while( pCur->iLvl>=0 ){
+    FsdirLevel *pLvl = &pCur->aLvl[pCur->iLvl];
+    struct dirent *pEntry = readdir(pLvl->pDir);
+    if( pEntry ){
+      if( pEntry->d_name[0]=='.' ){
+       if( pEntry->d_name[1]=='.' && pEntry->d_name[2]=='\0' ) continue;
+       if( pEntry->d_name[1]=='\0' ) continue;
+      }
+      sqlite3_free(pCur->zPath);
+      pCur->zPath = sqlite3_mprintf("%s/%s", pLvl->zDir, pEntry->d_name);
+      if( pCur->zPath==0 ) return SQLITE_NOMEM;
+      if( lstat(pCur->zPath, &pCur->sStat) ){
+        fsdirSetErrmsg(pCur, "cannot stat file: %s", pCur->zPath);
+        return SQLITE_ERROR;
+      }
+      return SQLITE_OK;
+    }
+    closedir(pLvl->pDir);
+    sqlite3_free(pLvl->zDir);
+    pLvl->pDir = 0;
+    pLvl->zDir = 0;
+    pCur->iLvl--;
+  }
+
+  /* EOF */
+  sqlite3_free(pCur->zPath);
+  pCur->zPath = 0;
   return SQLITE_OK;
 }
 
@@ -294,13 +351,7 @@ static int fsdirColumn(
   fsdir_cursor *pCur = (fsdir_cursor*)cur;
   switch( i ){
     case 0: { /* name */
-      const char *zName;
+      sqlite3_result_text(ctx, &pCur->zPath[pCur->nBase], -1, SQLITE_TRANSIENT);
-      if( pCur->eType==FSDIR_DIR ){
-        zName = &pCur->zPath[pCur->nDir+1];
-      }else{
-        zName = pCur->zPath;
-      }
-      sqlite3_result_text(ctx, zName, -1, SQLITE_TRANSIENT);
       break;
     }
 
@@ -308,11 +359,11 @@ static int fsdirColumn(
       sqlite3_result_int64(ctx, pCur->sStat.st_mode);
       break;
 
-    case 2: /* mode */
+    case 2: /* mtime */
       sqlite3_result_int64(ctx, pCur->sStat.st_mtime);
       break;
 
-    case 3: {
+    case 3: { /* data */
       mode_t m = pCur->sStat.st_mode;
       if( S_ISDIR(m) ){
         sqlite3_result_null(ctx);
@@ -361,14 +412,7 @@ static int fsdirRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
 */
 static int fsdirEof(sqlite3_vtab_cursor *cur){
   fsdir_cursor *pCur = (fsdir_cursor*)cur;
-  return pCur->bEof;
+  return (pCur->zPath==0);
-}
-
-static void fsdirSetErrmsg(fsdir_cursor *pCur, const char *zFmt, ...){
-  va_list ap;
-  va_start(ap, zFmt);
-  pCur->base.pVtab->zErrMsg = sqlite3_vmprintf(zFmt, ap);
-  va_end(ap);
 }
 
 /*
@@ -382,51 +426,38 @@ static int fsdirFilter(
   const char *zDir = 0;
   fsdir_cursor *pCur = (fsdir_cursor*)cur;
 
-  sqlite3_free(pCur->zDir);
+  fsdirResetCursor(pCur);
-  pCur->iRowid = 0;
-  pCur->zDir = 0;
-  pCur->bEof = 0;
-  if( pCur->pDir ){
-    closedir(pCur->pDir);
-    pCur->pDir = 0;
-  }
 
   if( idxNum==0 ){
     fsdirSetErrmsg(pCur, "table function fsdir requires an argument");
     return SQLITE_ERROR;
   }
 
-  assert( argc==1 );
+  assert( argc==idxNum && (argc==1 || argc==2) );
   zDir = (const char*)sqlite3_value_text(argv[0]);
   if( zDir==0 ){
     fsdirSetErrmsg(pCur, "table function fsdir requires a non-NULL argument");
     return SQLITE_ERROR;
   }
+  if( argc==2 ){
+    pCur->zBase = (const char*)sqlite3_value_text(argv[1]);
+  }
+  if( pCur->zBase ){
+    pCur->nBase = strlen(pCur->zBase)+1;
+    pCur->zPath = sqlite3_mprintf("%s/%s", pCur->zBase, zDir);
+  }else{
+    pCur->zPath = sqlite3_mprintf("%s", zDir);
+  }
 
-  pCur->zDir = sqlite3_mprintf("%s", zDir);
+  if( pCur->zPath==0 ){
-  if( pCur->zDir==0 ){
     return SQLITE_NOMEM;
   }
-
+  if( lstat(pCur->zPath, &pCur->sStat) ){
-  if( pCur->eType==FSDIR_ENTRY ){
+    fsdirSetErrmsg(pCur, "cannot stat file: %s", pCur->zPath);
-    int rc = lstat(pCur->zDir, &pCur->sStat);
+    return SQLITE_ERROR;
-    if( rc ){
-      fsdirSetErrmsg(pCur, "cannot stat file: %s", pCur->zDir);
-    }else{
-      pCur->zPath = sqlite3_mprintf("%s", pCur->zDir);
-      if( pCur->zPath==0 ) return SQLITE_NOMEM;
-    }
-    return SQLITE_OK;
-  }else{
-    pCur->nDir = strlen(pCur->zDir);
-    pCur->pDir = opendir(zDir);
-    if( pCur->pDir==0 ){
-      fsdirSetErrmsg(pCur, "error in opendir(\"%s\")", zDir);
-      return SQLITE_ERROR;
-    }
-
-    return fsdirNext(cur);
   }
+
+  return SQLITE_OK;
 }
 
 /*
@@ -450,24 +481,33 @@ static int fsdirBestIndex(
   sqlite3_index_info *pIdxInfo
 ){
   int i;                 /* Loop over constraints */
+  int idx4 = -1;
+  int idx5 = -1;
 
   const struct sqlite3_index_constraint *pConstraint;
   pConstraint = pIdxInfo->aConstraint;
   for(i=0; i<pIdxInfo->nConstraint; i++, pConstraint++){
     if( pConstraint->usable==0 ) continue;
     if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue;
-    if( pConstraint->iColumn!=4 ) continue;
+    if( pConstraint->iColumn==4 ) idx4 = i;
-    break;
+    if( pConstraint->iColumn==5 ) idx5 = i;
   }
 
-  if( i<pIdxInfo->nConstraint ){
+  if( idx4<0 ){
-    pIdxInfo->aConstraintUsage[i].omit = 1;
-    pIdxInfo->aConstraintUsage[i].argvIndex = 1;
-    pIdxInfo->idxNum = 1;
-    pIdxInfo->estimatedCost = 10.0;
-  }else{
     pIdxInfo->idxNum = 0;
     pIdxInfo->estimatedCost = (double)(((sqlite3_int64)1) << 50);
+  }else{
+    pIdxInfo->aConstraintUsage[idx4].omit = 1;
+    pIdxInfo->aConstraintUsage[idx4].argvIndex = 1;
+    if( idx5>=0 ){
+      pIdxInfo->aConstraintUsage[idx5].omit = 1;
+      pIdxInfo->aConstraintUsage[idx5].argvIndex = 2;
+      pIdxInfo->idxNum = 2;
+      pIdxInfo->estimatedCost = 10.0;
+    }else{
+      pIdxInfo->idxNum = 1;
+      pIdxInfo->estimatedCost = 100.0;
+    }
   }
 
   return SQLITE_OK;

manifest

@@ -1,5 +1,5 @@
-C Add\ssupport\sfor\s-C\sto\s".ar\sx".
+C Enhance\svirtual\stable\s"fsdir"\sin\sext/misc/fileio.c.\sAdd\ssupport\sfor\s"-C"\sto\nthe\sshell\scommand's\s".ar\sc"\scommand.
-D 2017-12-09T18:28:22.916
+D 2017-12-11T20:22:02.045
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8
@@ -269,7 +269,7 @@ F ext/misc/compress.c 122faa92d25033d6c3f07c39231de074ab3d2e83
 F ext/misc/csv.c 1a009b93650732e22334edc92459c4630b9fa703397cbb3c8ca279921a36ca11
 F ext/misc/dbdump.c 3509fa6b8932d04e932d6b6b827b6a82ca362781b8e8f3c77336f416793e215e
 F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2
-F ext/misc/fileio.c c84ec9c399657bd95914e7c4e9aefcc148cb86fe3ab7f2102e9557c861dd0d51
+F ext/misc/fileio.c 238d0c65a14bda99748285527543273e0248bc428c01c3955ec45acb8738db3b
 F ext/misc/fuzzer.c 7c64b8197bb77b7d64eff7cac7848870235d4c25
 F ext/misc/ieee754.c f190d0cc5182529acb15babd177781be1ac1718c
 F ext/misc/json1.c dbe086615b9546c156bf32b9378fc09383b58bd17513b866cfd24c1e15281984
@@ -474,7 +474,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 17e220191860a64a18c084141e1a8b7309e166a6f2d42c02021af27ea080d157
-F src/shell.c.in 8e57abbd26d7d2344ba752be0f86b2cbf93ad73ca28c651786392fbfd3b512ba
+F src/shell.c.in 4c7a2c1226216dfad7978642c958143f0fdfee4f09aef2add1e73e3601dd7d33
 F src/sqlite.h.in 8fd97993d48b50b9bade38c52f12d175942c9497c960905610c7b03a3e4b5818
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
@@ -1214,7 +1214,7 @@ F test/shell4.test 89ad573879a745974ff2df20ff97c5d6ffffbd5d
 F test/shell5.test 23939a4c51f0421330ea61dbd3c74f9c215f5f8d3d1a94846da6ffc777a35458
 F test/shell6.test 1ceb51b2678c472ba6cf1e5da96679ce8347889fe2c3bf93a0e0fa73f00b00d3
 F test/shell7.test 115132f66d0463417f408562cc2cf534f6bbc6d83a6d50f0072a9eb171bae97f
-F test/shell8.test 0e8e064da50c92df8eb514202dcfd0020f71762250066bf41ed098e0ff5f0e3d
+F test/shell8.test d04773f4e8cdad62d42a1b13bf4c1182109880441689a2f5d5001e190ec04831
 F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3
 F test/show_speedtest1_rtree.tcl 32e6c5f073d7426148a6936a0408f4b5b169aba5
 F test/shrink.test 1b4330b1fd9e818c04726d45cb28db73087535ce
@@ -1682,7 +1682,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 840401cc8ce3a09e0663b46973ecd2856d9607be71d2d1e9b21f7df7a82dcbe5
+P 8cd70960c5ddf0d0b2c40b8b6af4ce6b0277ffdaf04f33fcb33227d2b99ad515
-R e8ce05071b1dec8c5193e0b4d7b226fe
+R ddf59f7efc51e054de53889e9f879e48
 U dan
-Z f3cf78b2d0eddbe511b3f9a244339aaa
+Z c8a2667de38f55deb8cffcf1545f4d0c

manifest.uuid

@@ -1 +1 @@
-8cd70960c5ddf0d0b2c40b8b6af4ce6b0277ffdaf04f33fcb33227d2b99ad515
+0394889afed2479773af594e2d9659cf58b8959004ebcdeaff8e08e5dae684ef

src/shell.c.in

@@ -4285,6 +4285,7 @@ static int arExtractCommand(ShellState *p, ArCommand *pAr){
   return rc;
 }
 
+
 /*
 ** Implementation of .ar "Create" command. 
 **
@@ -4298,11 +4299,7 @@ static int arCreateCommand(
   ArCommand *pAr                  /* Command arguments and options */
 ){
   const char *zSql = 
-    "WITH f(n, m, t, d) AS ("
+    "SELECT name, mode, mtime, data FROM fsdir(?, ?)";
-    "  SELECT name, mode, mtime, data FROM fsentry(:1) UNION ALL "
-    "  SELECT n || '/' || name, mode, mtime, data "
-    "      FROM f, fsdir(n) WHERE (m&?)"
-    ") SELECT * FROM f";
 
   const char *zSqlar = 
       "CREATE TABLE IF NOT EXISTS sqlar("
@@ -4322,17 +4319,17 @@ static int arCreateCommand(
 
   Bytef *aCompress = 0;           /* Compression buffer */
   int nCompress = 0;              /* Size of compression buffer */
-  
+
   rc = sqlite3_exec(p->db, "SAVEPOINT ar;", 0, 0, 0);
   if( rc!=SQLITE_OK ) return rc;
 
   rc = sqlite3_exec(p->db, zSqlar, 0, 0, 0);
   shellPrepare(p, &rc, zInsert, &pInsert);
   shellPrepare(p, &rc, zSql, &pStmt);
+  sqlite3_bind_text(pStmt, 2, pAr->zDir, -1, SQLITE_STATIC);
 
   for(i=0; i<pAr->nArg && rc==SQLITE_OK; i++){
     sqlite3_bind_text(pStmt, 1, pAr->azArg[i], -1, SQLITE_STATIC);
-    sqlite3_bind_int(pStmt, 2, S_IFDIR);
     while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
       int sz;
       const char *zName = (const char*)sqlite3_column_text(pStmt, 0);

test/shell8.test

@@ -87,6 +87,16 @@ do_test 1.2 {
   dir_to_list ar3/ar1
 } $expected
 
+do_test 1.3 {
+  file delete -force ar3
+  file mkdir ar3
+
+  forcedelete test_ar.db
+  catchcmd test_ar.db ".ar cC ar1 file1 file2 dir1"
+  catchcmd test_ar.db ".ar xC ar3"
+  dir_to_list ar3
+} $expected
+
 
 
 finish_test