Extend the importDb() method of both OPFS VFSes to (A) support reading in an async streaming fashion via a callback and (B) automatically disable WAL mode in the imported db.

FossilOrigin-Name: 9b1398c96a4fd0b59e65faa8d5c98de4129f0f0357732f12cb2f5c53a08acdc2

ext/wasm/api/sqlite3-api-prologue.js

@@ -772,8 +772,43 @@ globalThis.sqlite3ApiBootstrap = function sqlite3ApiBootstrap(
     isSharedTypedArray,
     toss: function(...args){throw new Error(args.join(' '))},
     toss3,
-    typedArrayPart
+    typedArrayPart,
-  };
+    /**
+       Given a byte array or ArrayBuffer, this function throws if the
+       lead bytes of that buffer do not hold a SQLite3 database header,
+       else it returns without side effects.
+
+       Added in 3.44.
+    */
+    affirmDbHeader: function(bytes){
+      if(bytes instanceof ArrayBuffer) bytes = new Uint8Array(bytes);
+      const header = "SQLite format 3";
+      if( header.length > bytes.byteLength ){
+        toss3("Input does not contain an SQLite3 database header.");
+      }
+      for(let i = 0; i < header.length; ++i){
+        if( header.charCodeAt(i) !== bytes[i] ){
+          toss3("Input does not contain an SQLite3 database header.");
+        }
+      }
+    },
+    /**
+       Given a byte array or ArrayBuffer, this function throws if the
+       database does not, at a cursory glance, appear to be an SQLite3
+       database. It only examines the size and header, but further
+       checks may be added in the future.
+
+       Added in 3.44.
+    */
+    affirmIsDb: function(bytes){
+      if(bytes instanceof ArrayBuffer) bytes = new Uint8Array(bytes);
+      const n = bytes.byteLength;
+      if(n<512 || n%512!==0) {
+        toss3("Byte array size",n,"is invalid for an SQLite3 db.");
+      }
+      util.affirmDbHeader(bytes);
+    }
+  }/*util*/;
 
   Object.assign(wasm, {
     /**

ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js

@@ -59,6 +59,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
   const toss3 = sqlite3.util.toss3;
   const initPromises = Object.create(null);
   const capi = sqlite3.capi;
+  const util = sqlite3.util;
   const wasm = sqlite3.wasm;
   // Config opts for the VFS...
   const SECTOR_SIZE = 4096;
@@ -869,9 +870,48 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
       return b;
     }
 
+    //! Impl for importDb() when its 2nd arg is a function.
+    async importDbChunked(name, callback){
+      const sah = this.#mapFilenameToSAH.get(name)
+            || this.nextAvailableSAH()
+            || toss("No available handles to import to.");
+      sah.truncate(0);
+      let nWrote = 0, chunk, checkedHeader = false, err = false;
+      try{
+        while( undefined !== (chunk = await callback()) ){
+          if(chunk instanceof ArrayBuffer) chunk = new Uint8Array(chunk);
+          if( 0===nWrote && chunk.byteLength>=15 ){
+            util.affirmDbHeader(chunk);
+            checkedHeader = true;
+          }
+          sah.write(chunk, {at:  HEADER_OFFSET_DATA + nWrote});
+          nWrote += chunk.byteLength;
+        }
+        if( nWrote < 512 || 0!==nWrote % 512 ){
+          toss("Input size",nWrote,"is not correct for an SQLite database.");
+        }
+        if( !checkedHeader ){
+          const header = new Uint8Array(20);
+          sah.read( header, {at: 0} );
+          util.affirmDbHeader( header );
+        }
+        sah.write(new Uint8Array(2), {
+          at: HEADER_OFFSET_DATA + 18
+        }/*force db out of WAL mode*/);
+      }catch(e){
+        this.setAssociatedPath(sah, '', 0);
+      }
+      this.setAssociatedPath(sah, name, capi.SQLITE_OPEN_MAIN_DB);
+      return nWrote;
+    }
+
     //! Documented elsewhere in this file.
     importDb(name, bytes){
-      if(bytes instanceof ArrayBuffer) bytes = new Uint8Array(bytes);
+      if( bytes instanceof ArrayBuffer ) bytes = new Uint8Array(bytes);
+      else if( bytes instanceof Function ) return this.importDbChunked(name, bytes);
+      const sah = this.#mapFilenameToSAH.get(name)
+            || this.nextAvailableSAH()
+            || toss("No available handles to import to.");
       const n = bytes.byteLength;
       if(n<512 || n%512!=0){
         toss("Byte array size is invalid for an SQLite db.");
@@ -882,16 +922,16 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
           toss("Input does not contain an SQLite database header.");
         }
       }
-      const sah = this.#mapFilenameToSAH.get(name)
-            || this.nextAvailableSAH()
-            || toss("No available handles to import to.");
       const nWrote = sah.write(bytes, {at: HEADER_OFFSET_DATA});
       if(nWrote != n){
         this.setAssociatedPath(sah, '', 0);
         toss("Expected to write "+n+" bytes but wrote "+nWrote+".");
       }else{
+        sah.write(new Uint8Array([0,0]), {at: HEADER_OFFSET_DATA+18}
+                   /* force db out of WAL mode */);
         this.setAssociatedPath(sah, name, capi.SQLITE_OPEN_MAIN_DB);
       }
+      return nWrote;
     }
 
   }/*class OpfsSAHPool*/;
@@ -1098,6 +1138,19 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
      automatically clean up any non-database files so importing them
      is pointless.
 
+     If passed a function for its second argument, its behavior
+     changes to asynchronous and it imports its data in chunks fed to
+     it by the given callback function. It calls the callback (which
+     may be async) repeatedly, expecting either a Uint8Array or
+     ArrayBuffer (to denote new input) or undefined (to denote
+     EOF). For so long as the callback continues to return
+     non-undefined, it will append incoming data to the given
+     VFS-hosted database file. The result of the resolved Promise when
+     called this way is the size of the resulting database.
+
+     On succes this routine rewrites the database header bytes in the
+     output file (not the input array) to force disabling of WAL mode.
+
      On a write error, the handle is removed from the pool and made
      available for re-use.
 

ext/wasm/api/sqlite3-vfs-opfs.c-pp.js

@@ -136,6 +136,7 @@ const installOpfsVfs = function callee(options){
     const error =  (...args)=>logImpl(0, ...args);
     const toss = sqlite3.util.toss;
     const capi = sqlite3.capi;
+    const util = sqlite3.util;
     const wasm = sqlite3.wasm;
     const sqlite3_vfs = capi.sqlite3_vfs;
     const sqlite3_file = capi.sqlite3_file;
@@ -1168,40 +1169,98 @@ const installOpfsVfs = function callee(options){
       doDir(opt.directory, 0);
     };
 
+    /**
+       impl of importDb() when it's given a function as its second
+       argument.
+    */
+    const importDbChunked = async function(filename, callback){
+      const [hDir, fnamePart] = await opfsUtil.getDirForFilename(filename, true);
+      const hFile = await hDir.getFileHandle(fnamePart, {create:true});
+      const sah = await hFile.createSyncAccessHandle();
+      sah.truncate(0);
+      let nWrote = 0, chunk, checkedHeader = false, err = false;
+      try{
+        while( undefined !== (chunk = await callback()) ){
+          if(chunk instanceof ArrayBuffer) chunk = new Uint8Array(chunk);
+          if( 0===nWrote && chunk.byteLength>=15 ){
+            util.affirmDbHeader(chunk);
+            checkedHeader = true;
+          }
+          sah.write(chunk, {at: nWrote});
+          nWrote += chunk.byteLength;
+        }
+        if( nWrote < 512 || 0!==nWrote % 512 ){
+          toss("Input size",nWrote,"is not correct for an SQLite database.");
+        }
+        if( !checkedHeader ){
+          const header = new Uint8Array(20);
+          sah.read( header, {at: 0} );
+          util.affirmDbHeader( header );
+        }
+        sah.write(new Uint8Array(2), {at: 18}/*force db out of WAL mode*/);
+        return nWrote;
+      }catch(e){
+        await hDir.removeEntry( fnamePart ).catch(()=>{});
+        throw e;
+      }finally {
+        await sah.close();
+      }
+    };
+
     /**
        Asynchronously imports the given bytes (a byte array or
        ArrayBuffer) into the given database file.
 
+       If passed a function for its second argument, its behaviour
+       changes to async and it imports its data in chunks fed to it by
+       the given callback function. It calls the callback (which may
+       be async) repeatedly, expecting either a Uint8Array or
+       ArrayBuffer (to denote new input) or undefined (to denote
+       EOF). For so long as the callback continues to return
+       non-undefined, it will append incoming data to the given
+       VFS-hosted database file. When called this way, the resolved
+       value of the returned Promise is the number of bytes written to
+       the target file.
+
        It very specifically requires the input to be an SQLite3
        database and throws if that's not the case.  It does so in
        order to prevent this function from taking on a larger scope
        than it is specifically intended to. i.e. we do not want it to
        become a convenience for importing arbitrary files into OPFS.
 
-       Throws on error. Resolves to the number of bytes written.
+       This routine rewrites the database header bytes in the output
+       file (not the input array) to force disabling of WAL mode.
+
+       On error this throws and the state of the input file is
+       undefined (it depends on where the exception was triggered).
+
+       On success, resolves to the number of bytes written.
     */
     opfsUtil.importDb = async function(filename, bytes){
+      if( bytes instanceof Function ){
+        return importDbChunked(filename, bytes);
+      }
       if(bytes instanceof ArrayBuffer) bytes = new Uint8Array(bytes);
+      util.affirmIsDb(bytes);
       const n = bytes.byteLength;
-      if(n<512 || n%512!=0){
-        toss("Byte array size is invalid for an SQLite db.");
-      }
-      const header = "SQLite format 3";
-      for(let i = 0; i < header.length; ++i){
-        if( header.charCodeAt(i) !== bytes[i] ){
-          toss("Input does not contain an SQLite database header.");
-        }
-      }
       const [hDir, fnamePart] = await opfsUtil.getDirForFilename(filename, true);
-      const hFile = await hDir.getFileHandle(fnamePart, {create:true});
+      let sah, err, nWrote = 0;
-      const sah = await hFile.createSyncAccessHandle();
+      try {
-      sah.truncate(0);
+        const hFile = await hDir.getFileHandle(fnamePart, {create:true});
-      const nWrote = sah.write(bytes, {at: 0});
+        sah = await hFile.createSyncAccessHandle();
-      sah.close();
+        sah.truncate(0);
-      if(nWrote != n){
+        nWrote = sah.write(bytes, {at: 0});
-        toss("Expected to write "+n+" bytes but wrote "+nWrote+".");
+        if(nWrote != n){
+          toss("Expected to write "+n+" bytes but wrote "+nWrote+".");
+        }
+        sah.write(new Uint8Array(2), {at: 18}) /* force db out of WAL mode */;
+        return nWrote;
+      }catch(e){
+        await hDir.removeEntry( fnamePart ).catch(()=>{});
+        throw e;
+      }finally{
+        if( sah ) await sah.close();
       }
-      return nWrote;
     };
 
     if(sqlite3.oo1){

ext/wasm/tester1.c-pp.js

@@ -2939,8 +2939,27 @@ globalThis.sqlite3InitModule = sqlite3InitModule;
         let db;
         try {
           const exp = this.opfsDbExport;
+          const filename = this.opfsDbFile;
           delete this.opfsDbExport;
-          this.opfsImportSize = await sqlite3.oo1.OpfsDb.importDb(this.opfsDbFile, exp);
+          this.opfsImportSize = await sqlite3.oo1.OpfsDb.importDb(filename, exp);
+          db = new sqlite3.oo1.OpfsDb(this.opfsDbFile);
+          T.assert(6 === db.selectValue('select count(*) from p')).
+            assert( this.opfsImportSize == exp.byteLength );
+          db.close();
+          this.opfsUnlink(filename);
+          T.assert(!(await sqlite3.opfs.entryExists(filename)));
+          // Try again with a function as an input source:
+          let cursor = 0;
+          const blockSize = 512, end = exp.byteLength;
+          const reader = async function(){
+            if(cursor >= exp.byteLength){
+              return undefined;
+            }
+            const rv = exp.subarray(cursor, cursor+blockSize>end ? end : cursor+blockSize);
+            cursor += blockSize;
+            return rv;
+          };
+          this.opfsImportSize = await sqlite3.oo1.OpfsDb.importDb(filename, reader);
           db = new sqlite3.oo1.OpfsDb(this.opfsDbFile);
           T.assert(6 === db.selectValue('select count(*) from p')).
             assert( this.opfsImportSize == exp.byteLength );
@@ -3059,8 +3078,9 @@ globalThis.sqlite3InitModule = sqlite3InitModule;
           const dbytes = u1.exportFile(dbName);
           T.assert(dbytes.length >= 4096);
           const dbName2 = '/exported.db';
-          u1.importDb(dbName2, dbytes);
+          let nWrote = u1.importDb(dbName2, dbytes);
-          T.assert( 2 == u1.getFileCount() );
+          T.assert( 2 == u1.getFileCount() )
+            .assert( dbytes.byteLength == nWrote );
           let db2 = new u1.OpfsSAHPoolDb(dbName2);
           T.assert(db2 instanceof sqlite3.oo1.DB)
             .assert(3 === db2.selectValue('select count(*) from t'));
@@ -3069,6 +3089,25 @@ globalThis.sqlite3InitModule = sqlite3InitModule;
             .assert(false === u1.unlink(dbName2))
             .assert(1 === u1.getFileCount())
             .assert(1 === u1.getFileNames().length);
+          // Try again with a function as an input source:
+          let cursor = 0;
+          const blockSize = 1024, end = dbytes.byteLength;
+          const reader = async function(){
+            if(cursor >= dbytes.byteLength){
+              return undefined;
+            }
+            const rv = dbytes.subarray(cursor, cursor+blockSize>end ? end : cursor+blockSize);
+            cursor += blockSize;
+            return rv;
+          };
+          nWrote = await u1.importDb(dbName2, reader);
+          T.assert( 2 == u1.getFileCount() );
+          db2 = new u1.OpfsSAHPoolDb(dbName2);
+          T.assert(db2 instanceof sqlite3.oo1.DB)
+            .assert(3 === db2.selectValue('select count(*) from t'));
+          db2.close();
+          T.assert(true === u1.unlink(dbName2))
+            .assert(dbytes.byteLength == nWrote);
         }
 
         T.assert(true === u1.unlink(dbName))

manifest

@@ -1,5 +1,5 @@
-C Add\snote\sabout\sthe\scurrent\sthreading\slimitation\sto\sext/jni/README.md.\sNo\scode\schanges.
+C Extend\sthe\simportDb()\smethod\sof\sboth\sOPFS\sVFSes\sto\s(A)\ssupport\sreading\sin\san\sasync\sstreaming\sfashion\svia\sa\scallback\sand\s(B)\sautomatically\sdisable\sWAL\smode\sin\sthe\simported\sdb.
-D 2023-08-15T13:01:20.690
+D 2023-08-18T14:16:26.669
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -543,13 +543,13 @@ F ext/wasm/api/pre-js.c-pp.js ad906703f7429590f2fbf5e6498513bf727a1a4f0ebfa057af
 F ext/wasm/api/sqlite3-api-cleanup.js d235ad237df6954145404305040991c72ef8b1881715d2a650dda7b3c2576d0e
 F ext/wasm/api/sqlite3-api-glue.js b65e546568f1dfb35205b9792feb5146a6323d71b55cda58e2ed30def6dd52f3
 F ext/wasm/api/sqlite3-api-oo1.js 9678dc4d9a5d39632b6ffe6ea94a023119260815bf32f265bf5f6c36c9516db8
-F ext/wasm/api/sqlite3-api-prologue.js 5f283b096b98bfb1ee2f2201e7ff0489dff00e29e1030c30953bdb4f5b87f4bd
+F ext/wasm/api/sqlite3-api-prologue.js ef6f67c5ea718490806e5e17d2644b8b2f6e6ba5284d23dc1fbfd14d401c1ab5
 F ext/wasm/api/sqlite3-api-worker1.js 9f32af64df1a031071912eea7a201557fe39b1738645c0134562bb84e88e2fec
 F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89
 F ext/wasm/api/sqlite3-opfs-async-proxy.js 8cf8a897726f14071fae6be6648125162b256dfb4f96555b865dbb7a6b65e379
 F ext/wasm/api/sqlite3-v-helper.js 7daa0eab0a513a25b05e9abae7b5beaaa39209b3ed12f86aeae9ef8d2719ed25
-F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js abb69b5e008961026bf5ff433d7116cb046359af92a5daf73208af2e7ac80ae7
+F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 561463ac5380e4ccf1839a1922e6d7a5585660f32e3b9701a270b78cd35566cf
-F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js e04fc2fda6a0200ef80efdbb4ddfa0254453558adb17ec3a230f93d2bf1d711c
+F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 656952a75c36d96e3248b03ae26d6a7f8d6ff31e66432c63e1c0bb021f1234ab
 F ext/wasm/api/sqlite3-wasm.c d4d4c2b349b43b7b861e6d2994299630fb79e07573ea6b61e28e8071b7d16b61
 F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js bc06df0d599e625bde6a10a394e326dc68da9ff07fa5404354580f81566e591f
 F ext/wasm/api/sqlite3-worker1.c-pp.js da509469755035e919c015deea41b4514b5e84c12a1332e6cc8d42cb2cc1fb75
@@ -595,7 +595,7 @@ F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555
 F ext/wasm/test-opfs-vfs.js f09266873e1a34d9bdb6d3981ec8c9e382f31f215c9fd2f9016d2394b8ae9b7b
 F ext/wasm/tester1-worker.html ebc4b820a128963afce328ecf63ab200bd923309eb939f4110510ab449e9814c
 F ext/wasm/tester1.c-pp.html 1c1bc78b858af2019e663b1a31e76657b73dc24bede28ca92fbe917c3a972af2
-F ext/wasm/tester1.c-pp.js 64eb0ee6e695d5638d0f758f31a0ca2231e627ca5d768de3d8b44f9f494de8d4
+F ext/wasm/tester1.c-pp.js 9e0f4da49f02753a73a5f931bfb9b1458175518daa3fec40b5ebdc06c285539c
 F ext/wasm/tests/opfs/concurrency/index.html 0802373d57034d51835ff6041cda438c7a982deea6079efd98098d3e42fbcbc1
 F ext/wasm/tests/opfs/concurrency/test.js a98016113eaf71e81ddbf71655aa29b0fed9a8b79a3cdd3620d1658eb1cc9a5d
 F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2
@@ -2091,8 +2091,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 00ac653562a66aad3112ea322d08be68e05e6bf7413c814dd3f81bf850fcf43b
+P 653ed92dc39185cdedfab3ea518bc7ec2d2826120e5fa4cbdee3343301396184
-R 68ba6514d93093aa325f9f497a3147a9
+R 6b40a7c1cd4b822a33b7a897821b1648
 U stephan
-Z 5cbd8ac0238d9d299017246e54070fc0
+Z b0064c5da7f02d0dcda913e568fa1924
 # Remove this line to create a well-formed Fossil manifest.

manifest.uuid

@@ -1 +1 @@
-653ed92dc39185cdedfab3ea518bc7ec2d2826120e5fa4cbdee3343301396184
+9b1398c96a4fd0b59e65faa8d5c98de4129f0f0357732f12cb2f5c53a08acdc2