Documentation updates, in particular in the new limits.html file is added. (CVS 4053)

FossilOrigin-Name: 4ca6cdae94f6d0a2c95755d4a250f9f3bc7a0d7b

Makefile.in

@@ -584,6 +584,9 @@ formatchng.html:	$(TOP)/www/formatchng.tcl
 index.html:	$(TOP)/www/index.tcl last_change
 	tclsh $(TOP)/www/index.tcl >index.html
 
+limits.html:	$(TOP)/www/limits.tcl last_change
+	tclsh $(TOP)/www/limits.tcl >limits.html
+
 lang.html:	$(TOP)/www/lang.tcl
 	tclsh $(TOP)/www/lang.tcl >lang.html
 
@@ -657,6 +660,7 @@ DOC = \
   formatchng.html \
   index.html \
   lang.html \
+  limits.html \
   lockingv3.html \
   mingw.html \
   nulls.html \

main.mk

@@ -542,6 +542,9 @@ formatchng.html:	$(TOP)/www/formatchng.tcl
 index.html:	$(TOP)/www/index.tcl last_change
 	tclsh $(TOP)/www/index.tcl >index.html
 
+limits.html:	$(TOP)/www/limits.tcl last_change
+	tclsh $(TOP)/www/limits.tcl >limits.html
+
 lang.html:	$(TOP)/www/lang.tcl
 	tclsh $(TOP)/www/lang.tcl doc >lang.html
 
@@ -623,6 +626,7 @@ DOC = \
   fileformat.html \
   formatchng.html \
   index.html \
+  limits.html \
   lang.html \
   lockingv3.html \
   mingw.html \

manifest

@@ -1,6 +1,6 @@
-C Use\sC-sylte\scomments\sexclusively,\snever\sC++\scomments.\s\sTicket\s#2406.\s(CVS\s4052)
+C Documentation\supdates,\sin\sparticular\sin\sthe\snew\slimits.html\sfile\sis\sadded.\s(CVS\s4053)
-D 2007-06-08T18:27:03
+D 2007-06-09T09:53:51
-F Makefile.in a42354804b50c2708ce72cf79e4daa30f50191b5
+F Makefile.in 31d9f7cd42c3d73ae117fcdb4b0ecd029fa8f50b
 F Makefile.linux-gcc 2d8574d1ba75f129aba2019f0b959db380a90935
 F README 9c4e2d6706bdcc3efdd773ce752a8cdab4f90028
 F VERSION 6de5e9812c227f00155cb59af3535017aef3e258
@@ -47,7 +47,7 @@ F ext/fts2/mkfts2amal.tcl 2a9ec76b0760fe7f3669dca5bc0d60728bc1c977
 F ext/icu/icu.c 6b47f5bbaf32bce03112282ecca1f54bec969e42
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895
 F ltmain.sh 56abb507100ed2d4261f6dd1653dec3cf4066387
-F main.mk 4e3bc2b9b069c15c43e4339c0e87eba9388617c1
+F main.mk 5bc9827b6fc59db504210bf68cbe335f3250588a
 F mkdll.sh ed62756baf44babf562a7843588790c02fee2106
 F mkopcodec.awk bd46ad001c98dfbab07b1713cb8e692fa0e5415d
 F mkopcodeh.awk cde995d269aa06c94adbf6455bea0acedb913fa5
@@ -77,7 +77,7 @@ F src/hash.c 67b23e14f0257b69a3e8aa663e4eeadc1a2b6fd5
 F src/hash.h 1b3f7e2609141fd571f62199fc38687d262e9564
 F src/insert.c e595ca26805dfb3a9ebaabc28e7947c479f3b14d
 F src/legacy.c 388c71ad7fbcd898ba1bcbfc98a3ac954bfa5d01
-F src/limits.h 4cb1dbbc528a347fa0e8ad46729dab93297d77ed
+F src/limits.h 71ab25f17e35e0a9f3f6f234b8ed49cc56731d35
 F src/loadext.c afe4f4755dc49c36ef505748bbdddecb9f1d02a2
 F src/main.c 797dc983716c1480f6af78a36be3add8806211a1
 F src/malloc.c 07790014b4b126016ce3c94885a215e45eb07ad5
@@ -472,14 +472,15 @@ F www/direct1b.gif 32b48b764244817b6b591898dc52a04299a7b8a7
 F www/docs.tcl 09eeac4e565789a67abc63f166a9ae7f3050454d
 F www/download.tcl d59a0244f22a975c3f9deafb535fc20549cb8c45
 F www/dynload.tcl 02eb8273aa78cfa9070dd4501dca937fb22b466c
-F www/faq.tcl 705c986e40775cc3d92a1676dcbb55f00f0459c5
+F www/faq.tcl 98179bd65a60b0405b716e554c50bc817a5e39be
 F www/fileformat.tcl 900c95b9633abc3dcfc384d9ddd8eb4876793059
 F www/formatchng.tcl bbb8af1ee494a71031acac4c8d8c51535f23b9df
 F www/fullscanb.gif f7c94cb227f060511f8909e10f570157263e9a25
 F www/index-ex1-x-b.gif f9b1d85c3fa2435cf38b15970c7e3aa1edae23a3
-F www/index.tcl 8f500433525f34ef93427ba5f4f83fb5fde1e0e7
+F www/index.tcl e3b86b8628c9dffd4a60a16c4e4e428a69a0fe25
 F www/indirect1b1.gif adfca361d2df59e34f9c5cac52a670c2bfc303a1
 F www/lang.tcl 8dde95ed76af71579a132c9bb730d3764056d3ff
+F www/limits.tcl fa83cc03631056a504c2e8dd17554fadf1fb5ce1
 F www/lockingv3.tcl e52345bd20323bef6146bfce18ae0829b2b7c87d
 F www/mingw.tcl d96b451568c5d28545fefe0c80bee3431c73f69c
 F www/nulls.tcl ec35193f92485b87b90a994a01d0171b58823fcf
@@ -501,7 +502,7 @@ F www/tclsqlite.tcl bb0d1357328a42b1993d78573e587c6dcbc964b9
 F www/vdbe.tcl 87a31ace769f20d3627a64fa1fade7fed47b90d0
 F www/version3.tcl 890248cf7b70e60c383b0e84d77d5132b3ead42b
 F www/whentouse.tcl fc46eae081251c3c181bd79c5faef8195d7991a5
-P 72612a0373c7abf8aadfdeb46358c0b0ae7b07a0
+P 8f5b789fea23d76128c10b37158de2525a54ce20
-R 4a0de78da8c062d2f4e8738bd1c4d8d0
+R 64d706da97939c009470d72809ed3b81
 U drh
-Z c63d7a71f7409006622c28c29603bd15
+Z dfdc17a078cec5ec9ada07770cf8c9c8

manifest.uuid

@@ -1 +1 @@
-8f5b789fea23d76128c10b37158de2525a54ce20
+4ca6cdae94f6d0a2c95755d4a250f9f3bc7a0d7b

src/limits.h

@@ -12,7 +12,7 @@
 ** 
 ** This file defines various limits of what SQLite can process.
 **
-** @(#) $Id: limits.h,v 1.8 2007/06/07 19:08:34 drh Exp $
+** @(#) $Id: limits.h,v 1.9 2007/06/09 09:53:51 drh Exp $
 */
 
 /*
@@ -63,7 +63,7 @@
 ** any limitation on expression tree depth.
 */
 #ifndef SQLITE_MAX_EXPR_DEPTH
-# define SQLITE_MAX_EXPR_DEPTH 0
+# define SQLITE_MAX_EXPR_DEPTH 1000
 #endif
 
 /*
@@ -75,7 +75,7 @@
 ** any limit on the number of terms in a compount SELECT.
 */
 #ifndef SQLITE_MAX_COMPOUND_SELECT
-# define SQLITE_MAX_COMPOUND_SELECT 100
+# define SQLITE_MAX_COMPOUND_SELECT 500
 #endif
 
 /*

www/faq.tcl

@@ -1,7 +1,7 @@
 #
 # Run this script to generated a faq.html output file
 #
-set rcsid {$Id: faq.tcl,v 1.36 2006/04/05 01:02:08 drh Exp $}
+set rcsid {$Id: faq.tcl,v 1.37 2007/06/09 09:53:51 drh Exp $}
 source common.tcl
 header {SQLite Frequently Asked Questions</title>}
 
@@ -93,14 +93,6 @@ faq {
   </p>
 }
 
-faq {
-  Why does SQLite think that the expression '0'=='00' is TRUE?
-} {
-  <p>As of version 2.7.0, it doesn't.  See the document on
-  <a href="datatype3.html">datatypes in SQLite version 3</a>
-  for details.</p>
-}
-
 faq {
   Why doesn't SQLite allow me to use '0' and '0.0' as the primary
   key on two different rows of the same table?
@@ -114,20 +106,6 @@ faq {
   (See the previous question.)  Hence the values are not unique.</p>
 }
         
-faq {
-  My linux box is not able to read an SQLite database that was created
-  on my SparcStation.
-} {
-  <p>You need to upgrade your SQLite library to version 2.6.3 or later.</p>
-
-  <p>The x86 processor on your linux box is little-endian (meaning that
-  the least significant byte of integers comes first) but the Sparc is
-  big-endian (the most significant bytes comes first).  SQLite databases
-  created on a little-endian architecture cannot be on a big-endian
-  machine by version 2.6.2 or earlier of SQLite.  Beginning with
-  version 2.6.3, SQLite should be able to read and write database files
-  regardless of byte order of the machine on which the file was created.</p>
-}
 
 faq {
   Can multiple applications or multiple instances of the same
@@ -282,37 +260,8 @@ ORDER BY name
 faq {
   Are there any known size limits to SQLite databases?
 } {
-  <p>A database is limited in size to 2 tibibytes (2<sup>41</sup> bytes).
+  <p>See <a href="limits.html">limits.html</a> for a full discussion of
-  That is a theoretical limitation.  In practice, you should try to keep
+  the limits of SQLite.</p>
-  your SQLite databases below 100 gigabytes to avoid performance problems.
-  If you need to store 100 gigabytes or more in a database, consider using
-  an enterprise database engine which is designed for that purpose.</p>
-
-  <p>The theoretical limit on the number of rows in a table is
-  2<sup>64</sup>-1, though obviously you will run into the file size
-  limitation prior to reaching the row limit.  A single row can hold
-  up to 2<sup>30</sup> bytes of data in the current implementation.  The
-  underlying file format supports row sizes up to about 2<sup>62</sup> bytes.
-  </p>
-
-  <p>There are probably limits on the number of tables or indices or
-  the number of columns in a table or index, but nobody is sure what
-  those limits are.  In practice, SQLite must read and parse the original
-  SQL of all table and index declarations everytime a new database file
-  is opened, so for the best performance of
-  <a href="capi3ref.html#sqlite3_open">sqlite3_open()</a> it is best
-  to keep down the number of declared tables.   Likewise, though there
-  is no limit on the number of columns in a table, more than a few hundred
-  seems extreme.  Only the first 31 columns of a table are candidates for
-  certain optimizations.  You can put as many columns in an index as you like
-  but indexes with more than 30 columns will not be used to optimize queries.
-  </p>
-
-  <p>The names of tables, indices, view, triggers, and columns can be
-  as long as desired.  However, the names of SQL functions (as created
-  by the 
-  <a href="capi3ref.html#sqlite3_create_function">sqlite3_create_function()</a>
-  API) may not exceed 255 characters in length.</p>
 }
 
 faq {

www/index.tcl

@@ -27,10 +27,8 @@ Features include:
 <li>A complete database is stored in a single disk file.</li>
 <li>Database files can be freely shared between machines with
     different byte orders.</li>
-<li>Supports databases up to 2 tebibytes
+<li>Supports terabyte-sized databases and gigabyte-sized strings
-    (2<sup><small>41</small></sup> bytes) in size.</li>
+    and blobs.  (See <a href="limits.html">limits.html</a>.)
-<li>Strings and BLOBs up to 2 gibibytes (2<sup><small>31</small></sup> bytes)
-    in size.</li>
 <li>Small code footprint: 
     <a href="http://www.sqlite.org/cvstrac/wiki?p=SizeOfSqlite">
     less than 250KiB</a> fully configured or less
@@ -122,4 +120,4 @@ puts {
 <p align="right"><a href="oldnews.html">Old news...</a></p>
 </td></tr></table>
 }
-footer {$Id: index.tcl,v 1.154 2007/04/25 11:42:33 drh Exp $}
+footer {$Id: index.tcl,v 1.155 2007/06/09 09:53:51 drh Exp $}

www/limits.tcl

@@ -0,0 +1,318 @@
+#
+# Run this script to generate the limits.html output file
+#
+set rcsid {$Id: limits.tcl,v 1.1 2007/06/09 09:53:51 drh Exp $}
+source common.tcl
+header {Implementation Limits For SQLite}
+puts {
+<h2>Limits In SQLite</h2>
+
+<p>
+"Limits" in the context of this article means sizes or
+quantities that can not be exceeded.  We are concerned
+with things like the maximum number of bytes in a
+BLOB or the maximum number of columns in a table.
+</p>
+
+<p>
+SQLite was originally designed with a policy of avoiding
+arbitrary limits.
+Of course, every program that runs on a machine with finite
+memory and disk space has limits of some kind.  But in SQLite, 
+those limits
+were not well defined.  The policy was that if it would fit
+in memory and you could count it with a 32-bit integer, then
+it should work.
+</p>
+
+<p>
+Unfortunately, the no-limits policy has been shown to create
+problems.  Because the upper bounds where not well
+defined, they were not tested, and bugs (including possible
+security exploits) where often found when pushing SQLite to
+extremes.  For this reason, newer versions of SQLite have
+well-defined limits and those limits are tested as part of
+the test suite.
+</p>
+
+<p>
+This article defines what the limits of SQLite are and how they
+can be customized for specific applications.  The default settings
+for limits are normally quite large and adequate for almost every
+application.  Some applications may what to increase a limit here
+or there, but we expect such needs to be rare.  More commonly,
+an application might want to recompile SQLite with much lower
+limits to avoid excess resource utilization in the event of
+bug in higher-level SQL statement generators or to help thwart 
+attackers who inject malicious SQL statements.
+</p>
+}
+proc limititem {title text} {
+  puts "<li><p><b>$title</b></p>\n$text</li>"
+}
+puts {
+<ol>
+}
+
+limititem {Maximum length of a string or BLOB} {
+<p>
+The maximum number of bytes in a string or BLOB in SQLite is defined
+by the preprocessor macro SQLITE_MAX_LENGTH.  The default value
+of this macro is 1 billion (1 thousand million or 1,000,000,000).
+You can raise or lower this value at compile-time using a command-line 
+option like this:
+</p>
+
+<blockquote>-DSQLITE_MAX_LENGTH=123456789</blockquote>
+
+<p>
+The current implementation will only support a string or BLOB
+length up to 2<small><sup>31</sup></small>-1 or 2147483647.  And
+some built-in functions such as hex() might fail well before that
+point.  In security-sensitive applications it is best not to
+try to increase the maximum string and blob length.  In fact,
+you might do well to lower the maximum string and blob length
+to something more in the range of a few million if that is
+possible.
+</p>
+
+<p>
+During part of SQLite's INSERT and SELECT processing, the complete
+content of each row in the database is encoded as a single BLOB.
+So the SQLTIE_MAX_LENGTH parameter also determines the maximum
+number of bytes in a row.
+</p>
+}
+
+limititem {Maximum Number Of Columns} {
+<p>
+The SQLITE_MAX_COLUMN compile-time parameter is used to set an upper
+bound on:
+</p>
+
+<ul>
+<li>The number of columns in a table</li>
+<li>The number of columns in an index</li>
+<li>The number of columns in a view</li>
+<li>The number of terms in the SET clause of an UPDATE statement</li>
+<li>The number of columns in the result set of a SELECT statement</li>
+<li>The number of terms in a GROUP BY or ORDER BY clause</li>
+<li>The number of values in an INSERT statement</li>
+</ul>
+
+<p>
+The default setting for SQLITE_MAX_COLUMN is 2000.  You can change it
+at compile time to values as large as 32676.  You might be able to
+redefine this value to be as large as billions, though nobody has ever
+tried doing that so we do not know if it will work.  On the other hand, there
+are people who will argument that a well-normalized database design
+will never need a value larger than about 100.
+</p>
+
+<p>
+In most applications, the number of columns is small - a few dozen.
+There are places in the SQLite code generator that use algorithms
+that are O(N&sup2;) where N is the number of columns.  
+So if you redefine SQLITE_MAX_COLUMN to be a
+really huge number and you generate SQL that uses a large number of
+columns, you may find that 
+<a href="capi3ref.html#sqlite3_prepare_v2">sqlite3_prepare_v2()</a>
+runs slowly.
+}
+
+limititem {Maximum Length Of An SQL Statement} {
+<p>
+The maximum number of bytes in the text of an SQL statement is 
+limited to SQLITE_MAX_SQL_LENGTH which defaults to 1000000.  You
+can redefine this limit to be as large as the smaller of SQLITE_MAX_LENGTH
+and 1073741824.  
+</p>
+
+<p>
+If an SQL statement is limited to be a million bytes in length, then
+obviously you will not be able to insert multi-million byte strings
+by embedding them as literals inside of INSERT statements.  But
+you should not do that anyway.  Use host parameters 
+for your data.  Prepare short SQL statements like this:
+</p>
+
+<blockquote>
+INSERT INTO tab1 VALUES(?,?,?);
+</blockquote>
+
+<p>
+Then use the
+<a href="capi3ref.html#sqlite3_bind_text">sqlite3_bind_XXXX()</a> functions
+to bind your large string values to the SQL statement.  The use of binding
+obviates the need to escape quote characters in the string, reducing the
+risk of SQL injection attacks.  It is also runs faster since the large
+string does not need to be parsed or copied as much.
+</p>
+}
+
+limititem {Maximum Number Of Tables In A Join} {
+<p>
+SQLite does not support joins containing more than 64 tables.
+This limit arises from the fact that the SQLite code generator
+uses bitmaps with one bit per join-table in the query optimizer.
+</p>
+}
+
+limititem {Maximum Depth Of An Expression Tree} {
+<p>
+SQLite parses expressions into a tree for processing.  During
+code generation, SQLite walks this tree recursively.  The depth
+of expression trees is therefore limited in order to avoid
+using too much stack space.
+</p>
+
+<p>
+The SQLITE_MAX_EXPR_DEPTH parameter determines the maximum expression
+tree depth.  If the value is 0, then no limit is enforced.  The
+current implementation has a default value of 1000.
+</p>
+}
+
+limititem {Maximum Number Of Arguments On A Function} {
+<p>
+The SQLITE_MAX_FUNCTION_ARG parameter determines the maximum number
+of parameters that can be passed to an SQL function.  The default value
+of this limit is 100.  We know of no 
+technical reason why SQLite would not work with functions that have 
+millions of parameters.  However, we suspect that anybody who tries
+to invoke a function with millions of parameters is really
+trying to find security exploits in systems that use SQLite, 
+not do useful work, 
+and so for that reason we have set this parameter relatively low.
+}
+
+limititem {Maximum Number Of Terms In A Compound SELECT Statement} {
+<p>
+A compound SELECT statement is two or more SELECT statements connected
+by operators UNION, UNION ALL, EXCEPT, or INTERSECT.  We call each
+individual SELECT statement within a compound SELECT a "term".
+</p>
+
+<p>
+The code generator in SQLite processes compound SELECT statements using
+a recursive algorithm.  In order to limit the size of the stack, we
+therefore limit the number of terms in a compound SELECT.  The maximum
+number of terms is SQLITE_MAX_COMPOUND_SELECT which defaults to 500.
+We think this is a generous allotment since in practice we almost
+never see the number of terms in a compound select exceed single digits.
+</p>
+}
+
+limititem {Maximum Length Of A LIKE Or GLOB Pattern} {
+<p>
+The pattern matching algorithm used in the default LIKE and GLOB
+implementation of SQLite can exhibit O(N&sup2) performance (where
+N is the number of characters in the pattern) for certain pathological
+cases.  To avoid denial-of-service attacks from miscreants who are able
+to specify their own LIKE or GLOB patterns, the length of the LIKE
+or GLOB pattern is limited to SQLITE_MAX_LIKE_PATTERN_LENGTH bytes.
+The default value of this limit is 50000.  A modern workstation can
+evaluate even a pathological LIKE or GLOB pattern of 50000 bytes
+relatively quickly.  The denial of service problem only comes into
+play when the pattern length gets into millions of bytes.  Nevertheless,
+since most useful LIKE or GLOB patterns are at most a few dozen bytes
+in length, paranoid application developers may want to reduce this
+parameter to something in the range of a few hundred if they know that
+external users are able to generate arbitrary patterns.
+</p>
+}
+
+limititem {Maximum Number Of Host Parameters In A Single SQL Statement} {
+<p>
+A host parameter is a place-holder in an SQL statement that is filled
+in using one of the
+<a href="capi3ref.html#sqlite3_bind_blob">sqlite3_bind_XXXX()</a> interfaces.
+Many SQL programmers are familiar with using a question mark ("?") as a
+host parameter.  SQLite also supports named host parameters prefaced
+by ":", "$", or "@" and numbered host parameters of the form "?123".
+</p>
+
+<p>
+Each host parameter in an SQLite statement is assigned a number.  The
+numbers normally begin with 1 and increase by one with each new
+parameter.  However, when the "?123" form is used, the host parameter
+number is the number that follows the question mark.
+</p>
+
+<p>
+The maximum value of a host parameter number is SQLITE_MAX_VARIABLE_NUMBER.
+This setting defaults to 999.
+</p>
+}
+
+limititem {Maximum Number Of Attached Databases} {
+<p>
+The <a href="lang_attach.html">ATTACH</a> statement is an SQLite extension
+that allows two or more databases to be associated to the same database
+connection and to operate as if they were a single database.  The number
+of simulataneously attached databases is limited to SQLITE_MAX_ATTACHED
+which is set to 10 by default.
+The code generator in SQLite uses bitmaps
+to keep track of attached databases.  That means that the number of
+attached databases cannot be increased above 30 on a 32-bit machine
+or 62 on a 64-bit machine.
+}
+
+limititem {Maximum Database Page Size} {
+<p>
+An SQLite database file is organized as pages.  The size of each
+page is a power of 2 between 512 and SQLITE_MAX_PAGE_SIZE.
+The default value for SQLITE_MAX_PAGE_SIZE is 32768.  The current
+implementation will not support a larger value.
+</p>
+
+<p>
+It used to be the case that SQLite would allocate some stack
+structures whose size was proportional to the maximum page size.
+For this reason, SQLite would sometimes be compiled with a smaller
+maximum page size on embedded devices with limited stack memory.  But
+more recent versions of SQLite put these large structures on the
+heap, not on the stack, so reducing the maximum page size is no
+longer necessary on embedded devices.
+</p>
+}
+
+limititem {Maximum Number Of Pages In A Database File} {
+<p>
+SQLite is able to limit the size of a database file to prevent
+the database file from growing too large and consuming too much
+disk or flash space.
+The SQLITE_MAX_PAGE_COUNT parameter, which is normally set to
+1073741823, is the maximum number of pages allowed in a single
+database file.  An attempt to insert new data that would cause
+the database file to grow larger than this will return
+SQLITE_FULL.
+</p>
+
+<p>
+The <a href="pragma.html#pragma_max_page_count">
+max_page_count PRAGMA</a> can be used to raise or lower this
+limit at run-time.
+</p>
+
+<p>
+Note that the transaction processing in SQLite requires two bits
+of heap memory for every page in the database file.  For databases
+of a few megabytes in size, this amounts to only a few hundred
+bytes of heap memory.  But for gigabyte-sized database the amount
+of heap memory required is getting into the kilobyte range and
+for terabyte-sized databases, megabytes of heap memory must be
+allocated and zeroed at each transaction.  SQLite will
+support very large databases in theory, but the current implementation
+is optimized for the common SQLite use cases of embedded devices
+and persistent stores for desktop applications.  In other words,
+SQLite is designed for use with databases sized in kilobytes or 
+megabytes not gigabytes.  If you are building an application to
+work with databases that are hundreds of gigabytes or more 
+in size, then you should perhaps consider using a different database 
+engine that is explicitly designed for such large data sets.
+</p>
+}
+
+puts {</ol>}
+footer $rcsid