Enhance the stat VFS to report out the total size of all pages used by

a table, even if the ZIPVFS compression backend is in play.  Update
the sqlite3_analyzer logic to use these new outputs.

FossilOrigin-Name: 19b8eaaf70db82d401d33beb7fd36045d5e8326f

manifest

@@ -1,5 +1,5 @@
-C Changes\sto\ssqlite3_analyzer\sto\stry\sto\savoid\sinteger\soverflow\sproblems\s\nwhen\slinking\sagainst\solder\sversions\sof\sTCL.
+C Enhance\sthe\sstat\sVFS\sto\sreport\sout\sthe\stotal\ssize\sof\sall\spages\sused\sby\na\stable,\seven\sif\sthe\sZIPVFS\scompression\sbackend\sis\sin\splay.\s\sUpdate\nthe\ssqlite3_analyzer\slogic\sto\suse\sthese\snew\soutputs.
-D 2011-09-27T13:40:26.197
+D 2011-09-28T00:50:14.068
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in a162fe39e249b8ed4a65ee947c30152786cfe897
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -223,7 +223,7 @@ F src/test_quota.c a391c866217e92986c6f523f05b08aa6956c8419
 F src/test_rtree.c 6d06306e29946dc36f528a3a2cdc3add794656f1
 F src/test_schema.c 8c06ef9ddb240c7a0fcd31bc221a6a2aade58bf0
 F src/test_server.c 2f99eb2837dfa06a4aacf24af24c6affdf66a84f
-F src/test_stat.c 5bf0dcd0ffa469b2326d90f4a6e52d83e3f2df07
+F src/test_stat.c cf0a0e6d000ee4fbfd0d633d1e774a0267765f05
 F src/test_superlock.c 2b97936ca127d13962c3605dbc9a4ef269c424cd
 F src/test_syscall.c a992d8c80ea91fbf21fb2dd570db40e77dd7e6ae
 F src/test_tclvar.c f4dc67d5f780707210d6bb0eb6016a431c04c7fa
@@ -685,7 +685,7 @@ F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715
 F test/speed4p.explain 6b5f104ebeb34a038b2f714150f51d01143e59aa
 F test/speed4p.test 0e51908951677de5a969b723e03a27a1c45db38b
 F test/sqllimits1.test b1aae27cc98eceb845e7f7adf918561256e31298
-F test/stat.test 0997f6a57a35866b14111ed361ed8851ce7978ae
+F test/stat.test 36bc951bdc2beac4224cc54396fd6a7dc65336f4
 F test/stmt.test 25d64e3dbf9a3ce89558667d7f39d966fe2a71b9
 F test/subquery.test b524f57c9574b2c0347045b4510ef795d4686796
 F test/subquery2.test edcad5c118f0531c2e21bf16a09bbb105252d4cd
@@ -953,7 +953,7 @@ F tool/showjournal.c b62cecaab86a4053d944c276bb5232e4d17ece02
 F tool/showwal.c f09e5a80a293919290ec85a6a37c85a5ddcf37d9
 F tool/soak1.tcl 8d407956e1a45b485a8e072470a3e629a27037fe
 F tool/space_used.tcl f714c41a59e326b8b9042f415b628b561bafa06b
-F tool/spaceanal.tcl 7591cccc6383cd649b4d5e5babe6a3dbba1625d6
+F tool/spaceanal.tcl 58d357384760020443c41d01db612c9809c75de7
 F tool/speedtest.tcl 06c76698485ccf597b9e7dbb1ac70706eb873355
 F tool/speedtest16.c c8a9c793df96db7e4933f0852abb7a03d48f2e81
 F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff
@@ -964,7 +964,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5
 F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06
 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
 F tool/warnings.sh b7fdb2cc525f5ef4fa43c80e771636dd3690f9d2
-P d82cffab6acafcf9d91fea4ac47cad73f6f09775
+P 8c846311a0cd510bbf9361213bb60764e9b797aa
-R 2bc38f424055a926b03e353359683766
+R 3bd6d041dec83962663bcd3d3bb93a06
 U drh
-Z 8c563b03c7524b0b82590cf24a8a41f9
+Z 6010793edcdb8e3508d42d2b73d9c2b0

manifest.uuid

@@ -1 +1 @@
-8c846311a0cd510bbf9361213bb60764e9b797aa
+19b8eaaf70db82d401d33beb7fd36045d5e8326f

src/test_stat.c

@@ -64,20 +64,11 @@
   "  ncell      INTEGER,          /* Cells on page (0 for overflow) */"     \
   "  payload    INTEGER,          /* Bytes of payload on this page */"      \
   "  unused     INTEGER,          /* Bytes of unused space on this page */" \
-  "  mx_payload INTEGER           /* Largest payload size of all cells */"  \
+  "  mx_payload INTEGER,          /* Largest payload size of all cells */"  \
+  "  pgoffset   INTEGER,          /* Offset of page in file */"             \
+  "  pgsize     INTEGER           /* Size of the page */"                   \
   ");"
 
-#if 0
-#define VTAB_SCHEMA2                                                        \
-  "CREATE TABLE yy( "                                                       \
-  "  pageno   INTEGER,            /* B-tree page number */"                 \
-  "  cellno   INTEGER,            /* Cell number within page */"            \
-  "  local    INTEGER,            /* Bytes of content stored locally */"    \
-  "  payload  INTEGER,            /* Total cell payload size */"            \
-  "  novfl    INTEGER             /* Number of overflow pages */"           \
-  ");"
-#endif
-
 
 typedef struct StatTable StatTable;
 typedef struct StatCursor StatCursor;
@@ -126,6 +117,8 @@ struct StatCursor {
   int nPayload;                   /* Value of 'payload' column */
   int nUnused;                    /* Value of 'unused' column */
   int nMxPayload;                 /* Value of 'mx_payload' column */
+  i64 iOffset;                    /* Value of 'pgOffset' column */
+  int szPage;                     /* Value of 'pgSize' column */
 };
 
 struct StatTable {
@@ -283,6 +276,7 @@ static int statDecodePage(Btree *pBt, StatPage *p){
   int iOff;
   int nHdr;
   int isLeaf;
+  int szPage;
 
   u8 *aData = sqlite3PagerGetData(p->pPg);
   u8 *aHdr = &aData[p->iPgno==1 ? 100 : 0];
@@ -303,10 +297,11 @@ static int statDecodePage(Btree *pBt, StatPage *p){
   }
   p->nUnused = nUnused;
   p->iRightChildPg = isLeaf ? 0 : sqlite3Get4byte(&aHdr[8]);
+  szPage = sqlite3BtreeGetPageSize(pBt);
 
   if( p->nCell ){
     int i;                        /* Used to iterate through cells */
-    int nUsable = sqlite3BtreeGetPageSize(pBt) - sqlite3BtreeGetReserve(pBt);
+    int nUsable = szPage - sqlite3BtreeGetReserve(pBt);
 
     p->aCell = sqlite3_malloc((p->nCell+1) * sizeof(StatCell));
     memset(p->aCell, 0, (p->nCell+1) * sizeof(StatCell));
@@ -451,12 +446,28 @@ static int statNext(sqlite3_vtab_cursor *pCursor){
   */
   if( rc==SQLITE_OK ){
     int i;
+    sqlite3_file *fd;
+    sqlite3_int64 x[2];
     StatPage *p = &pCsr->aPage[pCsr->iPage];
     pCsr->zName = (char *)sqlite3_column_text(pCsr->pStmt, 0);
     pCsr->iPageno = p->iPgno;
 
     statDecodePage(pBt, p);
 
+    /* The default page size and offset */
+    pCsr->szPage = sqlite3BtreeGetPageSize(pBt);
+    pCsr->iOffset = pCsr->szPage * (p->iPgno - 1);
+
+    /* If connected to a ZIPVFS backend, override the page size and
+    ** offset with actual values obtained from ZIPVFS.
+    */
+    fd = sqlite3PagerFile(pPager);
+    x[0] = p->iPgno;
+    if( sqlite3OsFileControl(fd, 230440, &x)==SQLITE_OK ){
+      pCsr->iOffset = x[0];
+      pCsr->szPage = x[1];
+    }
+
     switch( p->flags ){
       case 0x05:             /* table internal */
       case 0x02:             /* index internal */
@@ -531,6 +542,12 @@ static int statColumn(
     case 7:            /* mx_payload */
       sqlite3_result_int(ctx, pCsr->nMxPayload);
       break;
+    case 8:            /* pgoffset */
+      sqlite3_result_int64(ctx, pCsr->iOffset);
+      break;
+    case 9:            /* pgsize */
+      sqlite3_result_int(ctx, pCsr->szPage);
+      break;
   }
   return SQLITE_OK;
 }
@@ -607,4 +624,3 @@ int SqlitetestStat_Init(Tcl_Interp *interp){
   return TCL_OK;
 }
 #endif /* if defined(SQLITE_TEST) || TCLSH==2 */
-

test/stat.test

@@ -36,7 +36,8 @@ ifcapable wal {
   do_execsql_test stat-0.1 {
     PRAGMA journal_mode = WAL;
     PRAGMA journal_mode = delete;
-    SELECT * FROM stat;
+    SELECT name, path, pageno, pagetype, ncell, payload, unused, mx_payload
+      FROM stat;
   } {wal delete sqlite_master / 1 leaf 0 0 916 0}
 }
 
@@ -50,17 +51,20 @@ do_test stat-1.0 {
 } {}
 do_test stat-1.1 {
   execsql {
-    SELECT * FROM stat WHERE name = 't1';
+    SELECT name, path, pageno, pagetype, ncell, payload, unused, mx_payload
+      FROM stat WHERE name = 't1';
   }
 } {t1 / 2 leaf 2 10 998 5}
 do_test stat-1.2 {
   execsql {
-    SELECT * FROM stat WHERE name = 'i1';
+    SELECT name, path, pageno, pagetype, ncell, payload, unused, mx_payload
+      FROM stat WHERE name = 'i1';
   }
 } {i1 / 3 leaf 2 10 1000 5}
 do_test stat-1.3 {
   execsql {
-    SELECT * FROM stat WHERE name = 'sqlite_master';
+    SELECT name, path, pageno, pagetype, ncell, payload, unused, mx_payload
+      FROM stat WHERE name = 'sqlite_master';
   }
 } {sqlite_master / 1 leaf 2 77 831 40}
 do_test stat-1.4 {
@@ -77,7 +81,8 @@ do_execsql_test stat-2.1 {
   INSERT INTO t3 SELECT a_string(110+rowid), a_string(221+rowid) FROM t3;
   INSERT INTO t3 SELECT a_string(110+rowid), a_string(221+rowid) FROM t3;
   INSERT INTO t3 SELECT a_string(110+rowid), a_string(221+rowid) FROM t3;
-  SELECT * FROM stat WHERE name != 'sqlite_master';
+  SELECT name, path, pageno, pagetype, ncell, payload, unused, mx_payload
+    FROM stat WHERE name != 'sqlite_master';
 } [list \
   sqlite_autoindex_t3_1 / 3 internal 3 368 623 125       \
   sqlite_autoindex_t3_1 /000/ 8 leaf 8 946 46 123        \
@@ -108,7 +113,8 @@ do_execsql_test stat-3.1 {
   CREATE TABLE t4(x);
   CREATE INDEX i4 ON t4(x);
   INSERT INTO t4(rowid, x) VALUES(2, a_string(7777));
-  SELECT * FROM stat WHERE name != 'sqlite_master';
+  SELECT name, path, pageno, pagetype, ncell, payload, unused, mx_payload
+    FROM stat WHERE name != 'sqlite_master';
 } [list \
   i4 / 3 leaf 1 103 905 7782                 \
   i4 /000+000000 9 overflow 0 1020 0 0       \
@@ -132,7 +138,8 @@ do_execsql_test stat-3.1 {
 do_execsql_test stat-4.1 {
   CREATE TABLE t5(x);
   CREATE INDEX i5 ON t5(x);
-  SELECT * FROM stat WHERE name = 't5' OR name = 'i5';
+  SELECT name, path, pageno, pagetype, ncell, payload, unused, mx_payload
+    FROM stat WHERE name = 't5' OR name = 'i5';
 } [list  \
   i5 / 5 leaf 0 0 1016 0 \
   t5 / 4 leaf 0 0 1016 0 \
@@ -149,7 +156,8 @@ do_execsql_test stat-5.1 {
   CREATE TABLE t1(x);
   INSERT INTO t1 VALUES(zeroblob(1513));
   INSERT INTO t1 VALUES(zeroblob(1514));
-  SELECT * FROM stat WHERE name = 't1';
+  SELECT name, path, pageno, pagetype, ncell, payload, unused, mx_payload
+    FROM stat WHERE name = 't1';
 } [list \
   t1 / 2 leaf 2 993 5 1517                \
   t1 /000+000000 3 overflow 0 1020 0 0    \

tool/spaceanal.tcl

@@ -59,7 +59,8 @@ set tabledef {CREATE TABLE space_used(
    int_unused int,   -- Number of unused bytes on interior pages
    leaf_unused int,  -- Number of unused bytes on primary pages
    ovfl_unused int,  -- Number of unused bytes on overflow pages
-   gap_cnt int       -- Number of gaps in the page layout
+   gap_cnt int,      -- Number of gaps in the page layout
+   compressed_size int  -- Total bytes stored on disk
 );}
 mem eval $tabledef
 
@@ -101,7 +102,8 @@ foreach {name tblname} [concat sqlite_master sqlite_master [db eval $sql]] {
       sum(isoverflow(pagetype, $is_index)) AS ovfl_pages,
       sum(isinternal(pagetype, $is_index) * unused) AS int_unused,
       sum(isleaf(pagetype, $is_index) * unused) AS leaf_unused,
-      sum(isoverflow(pagetype, $is_index) * unused) AS ovfl_unused
+      sum(isoverflow(pagetype, $is_index) * unused) AS ovfl_unused,
+      sum(pgsize) AS compressed_size
     FROM temp.dbstat WHERE name = $name
   } break
 
@@ -138,7 +140,8 @@ foreach {name tblname} [concat sqlite_master sqlite_master [db eval $sql]] {
       $int_unused, 
       $leaf_unused,
       $ovfl_unused,
-      $gap_cnt
+      $gap_cnt,
+      $compressed_size
     );
   }
 }
@@ -224,7 +227,8 @@ proc subreport {title where} {
       int(sum(leaf_unused)) AS leaf_unused,
       int(sum(int_unused)) AS int_unused,
       int(sum(ovfl_unused)) AS ovfl_unused,
-      int(sum(gap_cnt)) AS gap_cnt
+      int(sum(gap_cnt)) AS gap_cnt,
+      int(sum(compressed_size)) AS compressed_size
     FROM space_used WHERE $where" {} {}
 
   # Output the sub-report title, nicely decorated with * characters.
@@ -274,6 +278,11 @@ proc subreport {title where} {
   statline {Percentage of total database} $total_pages_percent
   statline {Number of entries} $nleaf
   statline {Bytes of storage consumed} $storage
+  if {$compressed_size!=$storage} {
+    set pct [expr {$compressed_size*100.0/$storage}]
+    set pct [format {%5.1f%%} $pct]
+    statline {Bytes used after compression} $compressed_size $pct
+  }
   statline {Bytes of payload} $payload $payload_percent
   statline {Average payload per entry} $avg_payload
   statline {Average unused bytes per entry} $avg_unused