Do not use the compress() and uncompress() functions in ext/misc/compress.c -

they are not quite compatible with the spec. Instead use new functions in
ext/misc/sqlar.c.

FossilOrigin-Name: 7652b3c2374084047b6c1da3e525e0cac34fe220597f81e793bc4fd9f33358da

ext/misc/sqlar.c

@@ -0,0 +1,113 @@
+/*
+** 2017-12-17
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** Utility functions sqlar_compress() and sqlar_uncompress(). Useful
+** for working with sqlar archives and used by the shell tool's built-in
+** sqlar support.
+*/
+#include "sqlite3ext.h"
+SQLITE_EXTENSION_INIT1
+#include <zlib.h>
+
+/*
+** Implementation of the "sqlar_compress(X)" SQL function.
+**
+** If the type of X is SQLITE_BLOB, and compressing that blob using
+** zlib utility function compress() yields a smaller blob, return the
+** compressed blob. Otherwise, return a copy of X.
+*/
+static void sqlarCompressFunc(
+  sqlite3_context *context,
+  int argc,
+  sqlite3_value **argv
+){
+  assert( argc==1 );
+  if( sqlite3_value_type(argv[0])==SQLITE_BLOB ){
+    const Bytef *pData = sqlite3_value_blob(argv[0]);
+    uLong nData = sqlite3_value_bytes(argv[0]);
+    uLongf nOut = compressBound(nData);
+    Bytef *pOut;
+
+    pOut = (Bytef*)sqlite3_malloc(nOut);
+    if( pOut==0 ){
+      sqlite3_result_error_nomem(context);
+      return;
+    }else{
+      if( Z_OK!=compress(pOut, &nOut, pData, nData) ){
+        sqlite3_result_error(context, "error in compress()", -1);
+      }else if( nOut<nData ){
+        sqlite3_result_blob(context, pOut, nOut, SQLITE_TRANSIENT);
+      }else{
+        sqlite3_result_value(context, argv[0]);
+      }
+      sqlite3_free(pOut);
+    }
+  }else{
+    sqlite3_result_value(context, argv[0]);
+  }
+}
+
+/*
+** Implementation of the "sqlar_uncompress(X,SZ)" SQL function
+**
+** Parameter SZ is interpreted as an integer. If it is less than or
+** equal to zero, then this function returns a copy of X. Or, if
+** SZ is equal to the size of X when interpreted as a blob, also
+** return a copy of X. Otherwise, decompress blob X using zlib
+** utility function uncompress() and return the results (another
+** blob).
+*/
+static void sqlarUncompressFunc(
+  sqlite3_context *context,
+  int argc,
+  sqlite3_value **argv
+){
+  uLong nData;
+  uLongf sz;
+
+  assert( argc==2 );
+  sz = sqlite3_value_int(argv[1]);
+
+  if( sz<=0 || sz==(nData = sqlite3_value_bytes(argv[0])) ){
+    sqlite3_result_value(context, argv[0]);
+  }else{
+    const Bytef *pData= sqlite3_value_blob(argv[0]);
+    Bytef *pOut = sqlite3_malloc(sz);
+    if( Z_OK!=uncompress(pOut, &sz, pData, nData) ){
+      sqlite3_result_error(context, "error in uncompress()", -1);
+    }else{
+      sqlite3_result_blob(context, pOut, sz, SQLITE_TRANSIENT);
+    }
+    sqlite3_free(pOut);
+  }
+}
+
+
+#ifdef _WIN32
+__declspec(dllexport)
+#endif
+int sqlite3_sqlar_init(
+  sqlite3 *db, 
+  char **pzErrMsg, 
+  const sqlite3_api_routines *pApi
+){
+  int rc = SQLITE_OK;
+  SQLITE_EXTENSION_INIT2(pApi);
+  (void)pzErrMsg;  /* Unused parameter */
+  rc = sqlite3_create_function(db, "sqlar_compress", 1, SQLITE_UTF8, 0,
+                               sqlarCompressFunc, 0, 0);
+  if( rc==SQLITE_OK ){
+    rc = sqlite3_create_function(db, "sqlar_uncompress", 2, SQLITE_UTF8, 0,
+                                 sqlarUncompressFunc, 0, 0);
+  }
+  return rc;
+}

main.mk

@@ -690,7 +690,8 @@ SHELL_SRC = \
 	$(TOP)/src/shell.c.in \
 	$(TOP)/ext/misc/shathree.c \
 	$(TOP)/ext/misc/fileio.c \
-	$(TOP)/ext/misc/completion.c
+	$(TOP)/ext/misc/completion.c \
+	$(TOP)/ext/misc/sqlar.c
 
 shell.c:	$(SHELL_SRC) $(TOP)/tool/mkshellc.tcl
 	tclsh $(TOP)/tool/mkshellc.tcl >shell.c

manifest

@@ -1,5 +1,5 @@
-C Have\sthe\swritefile()\sfunction\soptionally\sset\sthe\smodification-time\sof\sthe\nfiles\sit\swrites\sor\screates.\sAnd\smany\ssmall\sfixes\sto\sthe\snew\scode\son\sthis\nbranch.
+C Do\snot\suse\sthe\scompress()\sand\suncompress()\sfunctions\sin\sext/misc/compress.c\s-\nthey\sare\snot\squite\scompatible\swith\sthe\sspec.\sInstead\suse\snew\sfunctions\sin\next/misc/sqlar.c.
-D 2017-12-14T19:15:07.381
+D 2017-12-16T19:11:26.054
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 6a879cbf01e37f9eac131414955f71774b566502d9a57ded1b8585b507503cb8
@@ -287,6 +287,7 @@ F ext/misc/sha1.c 0b9e9b855354910d3ca467bf39099d570e73db56
 F ext/misc/shathree.c fa185d7aee0ad0aca5e091b4a2db7baff11796170e5793b5de99e511a13af448
 F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52
 F ext/misc/spellfix.c 41cf26c6b89fcaa8798ae10ae64d39c1f1d9d6995152e545bd491c13058b8fac
+F ext/misc/sqlar.c d355cd8b6e7280d2f61d4737672922acb512a2ab1cee52399ffb88980476e31c
 F ext/misc/stmt.c 6f16443abb3551e3f5813bb13ba19a30e7032830015b0f92fe0c0453045c0a11
 F ext/misc/totype.c 4a167594e791abeed95e0a8db028822b5e8fe512
 F ext/misc/unionvtab.c 1e0ebc5078e1a916db191bcd88f87e94ea7ba4aa563ee30ff706261cb4b39461
@@ -397,7 +398,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk 6123b0b2db806ddb482c24786ad6603a289df720382a3bce8f532d76a94c84b1
+F main.mk f5f8e5e6bdd9ab3dabd94c6cf929813a601d5dbf312197fe463190c7d927c27c
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
@@ -475,7 +476,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 17e220191860a64a18c084141e1a8b7309e166a6f2d42c02021af27ea080d157
-F src/shell.c.in 074b2129559a0aa712a367317f7e7daf4740925ec2c123b529800628eb10dc73
+F src/shell.c.in ad8c13b9dee606d1ef11226f90d8e4e084d75986f96e6852e8e5b1c54aa4f0aa
 F src/sqlite.h.in 364515dd186285f3c01f5cab42e7db7edc47c70e87b6a25de389a2e6b8c413fd
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34
@@ -1683,7 +1684,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P b9d2d5d97291bf3d1392232e3705cca89dc7b918db2b08067b2b013ea39320e0
+P 7b51269caebe1492885fe9b965892f49a3f8bdb1d666b0203d594c30f9e83938
-R 975f9981c88e18e3ddfa1fe4bb6e7fae
+R 6fdfedcd54ee170b8eb014864253aebf
 U dan
-Z 1e84576de6ef44efce8abf1d868c1119
+Z 7252a7eb50ba8ff9515237c2fa79b757

manifest.uuid

@@ -1 +1 @@
-7b51269caebe1492885fe9b965892f49a3f8bdb1d666b0203d594c30f9e83938
+7652b3c2374084047b6c1da3e525e0cac34fe220597f81e793bc4fd9f33358da

src/shell.c.in

@@ -797,7 +797,7 @@ INCLUDE ../ext/misc/shathree.c
 INCLUDE ../ext/misc/fileio.c
 INCLUDE ../ext/misc/completion.c
 #ifdef SQLITE_HAVE_ZLIB
-INCLUDE ../ext/misc/compress.c
+INCLUDE ../ext/misc/sqlar.c
 #endif
 
 #if defined(SQLITE_ENABLE_SESSION)
@@ -2901,7 +2901,7 @@ static void open_db(ShellState *p, int keepAlive){
     sqlite3_shathree_init(p->db, 0, 0);
     sqlite3_completion_init(p->db, 0, 0);
 #ifdef SQLITE_HAVE_ZLIB
-    sqlite3_compress_init(p->db, 0, 0);
+    sqlite3_sqlar_init(p->db, 0, 0);
 #endif
     sqlite3_create_function(p->db, "shell_add_schema", 2, SQLITE_UTF8, 0,
                             shellAddSchemaName, 0, 0);
@@ -4482,13 +4482,10 @@ static int arListCommand(ShellState *p, sqlite3 *db, ArCommand *pAr){
 */
 static int arExtractCommand(ShellState *p, sqlite3 *db, ArCommand *pAr){
   const char *zSql1 = 
-    "SELECT :1 || name, writefile(:1 || name, "
+    "SELECT "
-    "CASE WHEN (data AND sz>=0 AND sz!=length(data)) THEN "
+    "  :1 || name, "
-    "    uncompress(data) "
+    "  writefile(:1 || name, sqlar_uncompress(data, sz), mode, mtime) "
-    "ELSE"
+    "FROM sqlar WHERE (%s) AND (data IS NULL OR :2 = 0)";
-    "    data "
-    "END, "
-    "mode, mtime) FROM sqlar WHERE (%s) AND (data IS NULL OR :2 = 0)";
 
   struct timespec times[2];
   sqlite3_stmt *pSql = 0;
@@ -4569,16 +4566,13 @@ static int arCreateUpdate(
       "data BLOB               -- compressed content\n"
   ")";
   const char *zDrop = "DROP TABLE IF EXISTS sqlar";
-  const char *zInsert = "REPLACE INTO sqlar VALUES(?, ?, ?, ?, ?)";
+  const char *zInsert = "REPLACE INTO sqlar VALUES(?,?,?,?,sqlar_compress(?))";
 
   sqlite3_stmt *pStmt = 0;        /* Directory traverser */
   sqlite3_stmt *pInsert = 0;      /* Compilation of zInsert */
   int i;                          /* For iterating through azFile[] */
   int rc;                         /* Return code */
 
-  Bytef *aCompress = 0;           /* Compression buffer */
-  int nCompress = 0;              /* Size of compression buffer */
-
   rc = sqlite3_exec(db, "SAVEPOINT ar;", 0, 0, 0);
   if( rc!=SQLITE_OK ) return rc;
 
@@ -4611,39 +4605,18 @@ static int arCreateUpdate(
       if( S_ISDIR(mode) ){
         sz = 0;
         sqlite3_bind_null(pInsert, 5);
-      }else if( S_ISLNK(mode) ){
-        sz = -1;
-        sqlite3_bind_value(pInsert, 5, sqlite3_column_value(pStmt, 3));
       }else{
-        uLongf nReq;              /* Required size of compression buffer */
+        sqlite3_bind_value(pInsert, 5, sqlite3_column_value(pStmt, 3));
-        const Bytef *pData = (const Bytef*)sqlite3_column_blob(pStmt, 3);
+        if( S_ISLNK(mode) ){
-        sz = sqlite3_column_bytes(pStmt, 3);
+          sz = -1;
-        nReq = compressBound(sz);
-        if( aCompress==0 || nReq>nCompress ){
-          Bytef *aNew = sqlite3_realloc(aCompress, nReq);
-          if( aNew==0 ){
-            rc = SQLITE_NOMEM;
-          }else{
-            aCompress = aNew;
-            nCompress = nReq;
-          }
-        }
-
-        if( Z_OK!=compress(aCompress, &nReq, pData, sz) ){
-          rc = SQLITE_ERROR;
-        }
-        if( nReq<sz ){
-          sqlite3_bind_blob(pInsert, 5, aCompress, nReq, SQLITE_STATIC);
         }else{
-          sqlite3_bind_blob(pInsert, 5, pData, sz, SQLITE_STATIC);
+          sz = sqlite3_column_bytes(pStmt, 3);
         }
       }
 
-      if( rc==SQLITE_OK ){
+      sqlite3_bind_int(pInsert, 4, sz);
-        sqlite3_bind_int(pInsert, 4, sz);
+      sqlite3_step(pInsert);
-        sqlite3_step(pInsert);
+      rc = sqlite3_reset(pInsert);
-        rc = sqlite3_reset(pInsert);
-      }
     }
     shellReset(&rc, pStmt);
   }
@@ -4655,7 +4628,6 @@ static int arCreateUpdate(
   }
   shellFinalize(&rc, pStmt);
   shellFinalize(&rc, pInsert);
-  sqlite3_free(aCompress);
   return rc;
 }
 
@@ -4713,7 +4685,7 @@ static int arDotCommand(
         return rc;
       }
       sqlite3_fileio_init(db, 0, 0);
-      sqlite3_compress_init(db, 0, 0);
+      sqlite3_sqlar_init(db, 0, 0);
     }else{
       db = pState->db;
     }