From b0e64f33da6d6e958110c1e40db98ea5e8a11b85 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 9 Jun 2007 09:53:51 +0000 Subject: [PATCH] Documentation updates, in particular in the new limits.html file is added. (CVS 4053) FossilOrigin-Name: 4ca6cdae94f6d0a2c95755d4a250f9f3bc7a0d7b --- Makefile.in | 4 + main.mk | 4 + manifest | 21 ++-- manifest.uuid | 2 +- src/limits.h | 6 +- www/faq.tcl | 57 +-------- www/index.tcl | 8 +- www/limits.tcl | 318 +++++++++++++++++++++++++++++++++++++++++++++++++ 8 files changed, 347 insertions(+), 73 deletions(-) create mode 100644 www/limits.tcl diff --git a/Makefile.in b/Makefile.in index 1f866c668a..55e35042b1 100644 --- a/Makefile.in +++ b/Makefile.in @@ -584,6 +584,9 @@ formatchng.html: $(TOP)/www/formatchng.tcl index.html: $(TOP)/www/index.tcl last_change tclsh $(TOP)/www/index.tcl >index.html +limits.html: $(TOP)/www/limits.tcl last_change + tclsh $(TOP)/www/limits.tcl >limits.html + lang.html: $(TOP)/www/lang.tcl tclsh $(TOP)/www/lang.tcl >lang.html @@ -657,6 +660,7 @@ DOC = \ formatchng.html \ index.html \ lang.html \ + limits.html \ lockingv3.html \ mingw.html \ nulls.html \ diff --git a/main.mk b/main.mk index 34b82c6b68..b5857d2aa3 100644 --- a/main.mk +++ b/main.mk @@ -542,6 +542,9 @@ formatchng.html: $(TOP)/www/formatchng.tcl index.html: $(TOP)/www/index.tcl last_change tclsh $(TOP)/www/index.tcl >index.html +limits.html: $(TOP)/www/limits.tcl last_change + tclsh $(TOP)/www/limits.tcl >limits.html + lang.html: $(TOP)/www/lang.tcl tclsh $(TOP)/www/lang.tcl doc >lang.html @@ -623,6 +626,7 @@ DOC = \ fileformat.html \ formatchng.html \ index.html \ + limits.html \ lang.html \ lockingv3.html \ mingw.html \ diff --git a/manifest b/manifest index 2ed8dd51f6..ef606293e4 100644 --- a/manifest +++ b/manifest @@ -1,6 +1,6 @@ -C Use\sC-sylte\scomments\sexclusively,\snever\sC++\scomments.\s\sTicket\s#2406.\s(CVS\s4052) -D 2007-06-08T18:27:03 -F Makefile.in a42354804b50c2708ce72cf79e4daa30f50191b5 +C Documentation\supdates,\sin\sparticular\sin\sthe\snew\slimits.html\sfile\sis\sadded.\s(CVS\s4053) +D 2007-06-09T09:53:51 +F Makefile.in 31d9f7cd42c3d73ae117fcdb4b0ecd029fa8f50b F Makefile.linux-gcc 2d8574d1ba75f129aba2019f0b959db380a90935 F README 9c4e2d6706bdcc3efdd773ce752a8cdab4f90028 F VERSION 6de5e9812c227f00155cb59af3535017aef3e258 @@ -47,7 +47,7 @@ F ext/fts2/mkfts2amal.tcl 2a9ec76b0760fe7f3669dca5bc0d60728bc1c977 F ext/icu/icu.c 6b47f5bbaf32bce03112282ecca1f54bec969e42 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 F ltmain.sh 56abb507100ed2d4261f6dd1653dec3cf4066387 -F main.mk 4e3bc2b9b069c15c43e4339c0e87eba9388617c1 +F main.mk 5bc9827b6fc59db504210bf68cbe335f3250588a F mkdll.sh ed62756baf44babf562a7843588790c02fee2106 F mkopcodec.awk bd46ad001c98dfbab07b1713cb8e692fa0e5415d F mkopcodeh.awk cde995d269aa06c94adbf6455bea0acedb913fa5 @@ -77,7 +77,7 @@ F src/hash.c 67b23e14f0257b69a3e8aa663e4eeadc1a2b6fd5 F src/hash.h 1b3f7e2609141fd571f62199fc38687d262e9564 F src/insert.c e595ca26805dfb3a9ebaabc28e7947c479f3b14d F src/legacy.c 388c71ad7fbcd898ba1bcbfc98a3ac954bfa5d01 -F src/limits.h 4cb1dbbc528a347fa0e8ad46729dab93297d77ed +F src/limits.h 71ab25f17e35e0a9f3f6f234b8ed49cc56731d35 F src/loadext.c afe4f4755dc49c36ef505748bbdddecb9f1d02a2 F src/main.c 797dc983716c1480f6af78a36be3add8806211a1 F src/malloc.c 07790014b4b126016ce3c94885a215e45eb07ad5 @@ -472,14 +472,15 @@ F www/direct1b.gif 32b48b764244817b6b591898dc52a04299a7b8a7 F www/docs.tcl 09eeac4e565789a67abc63f166a9ae7f3050454d F www/download.tcl d59a0244f22a975c3f9deafb535fc20549cb8c45 F www/dynload.tcl 02eb8273aa78cfa9070dd4501dca937fb22b466c -F www/faq.tcl 705c986e40775cc3d92a1676dcbb55f00f0459c5 +F www/faq.tcl 98179bd65a60b0405b716e554c50bc817a5e39be F www/fileformat.tcl 900c95b9633abc3dcfc384d9ddd8eb4876793059 F www/formatchng.tcl bbb8af1ee494a71031acac4c8d8c51535f23b9df F www/fullscanb.gif f7c94cb227f060511f8909e10f570157263e9a25 F www/index-ex1-x-b.gif f9b1d85c3fa2435cf38b15970c7e3aa1edae23a3 -F www/index.tcl 8f500433525f34ef93427ba5f4f83fb5fde1e0e7 +F www/index.tcl e3b86b8628c9dffd4a60a16c4e4e428a69a0fe25 F www/indirect1b1.gif adfca361d2df59e34f9c5cac52a670c2bfc303a1 F www/lang.tcl 8dde95ed76af71579a132c9bb730d3764056d3ff +F www/limits.tcl fa83cc03631056a504c2e8dd17554fadf1fb5ce1 F www/lockingv3.tcl e52345bd20323bef6146bfce18ae0829b2b7c87d F www/mingw.tcl d96b451568c5d28545fefe0c80bee3431c73f69c F www/nulls.tcl ec35193f92485b87b90a994a01d0171b58823fcf @@ -501,7 +502,7 @@ F www/tclsqlite.tcl bb0d1357328a42b1993d78573e587c6dcbc964b9 F www/vdbe.tcl 87a31ace769f20d3627a64fa1fade7fed47b90d0 F www/version3.tcl 890248cf7b70e60c383b0e84d77d5132b3ead42b F www/whentouse.tcl fc46eae081251c3c181bd79c5faef8195d7991a5 -P 72612a0373c7abf8aadfdeb46358c0b0ae7b07a0 -R 4a0de78da8c062d2f4e8738bd1c4d8d0 +P 8f5b789fea23d76128c10b37158de2525a54ce20 +R 64d706da97939c009470d72809ed3b81 U drh -Z c63d7a71f7409006622c28c29603bd15 +Z dfdc17a078cec5ec9ada07770cf8c9c8 diff --git a/manifest.uuid b/manifest.uuid index 0b6a1eec93..1a7a8f3472 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8f5b789fea23d76128c10b37158de2525a54ce20 \ No newline at end of file +4ca6cdae94f6d0a2c95755d4a250f9f3bc7a0d7b \ No newline at end of file diff --git a/src/limits.h b/src/limits.h index f8f7bfdadf..fc31f34493 100644 --- a/src/limits.h +++ b/src/limits.h @@ -12,7 +12,7 @@ ** ** This file defines various limits of what SQLite can process. ** -** @(#) $Id: limits.h,v 1.8 2007/06/07 19:08:34 drh Exp $ +** @(#) $Id: limits.h,v 1.9 2007/06/09 09:53:51 drh Exp $ */ /* @@ -63,7 +63,7 @@ ** any limitation on expression tree depth. */ #ifndef SQLITE_MAX_EXPR_DEPTH -# define SQLITE_MAX_EXPR_DEPTH 0 +# define SQLITE_MAX_EXPR_DEPTH 1000 #endif /* @@ -75,7 +75,7 @@ ** any limit on the number of terms in a compount SELECT. */ #ifndef SQLITE_MAX_COMPOUND_SELECT -# define SQLITE_MAX_COMPOUND_SELECT 100 +# define SQLITE_MAX_COMPOUND_SELECT 500 #endif /* diff --git a/www/faq.tcl b/www/faq.tcl index 0e93651ba2..c68c6de4b5 100644 --- a/www/faq.tcl +++ b/www/faq.tcl @@ -1,7 +1,7 @@ # # Run this script to generated a faq.html output file # -set rcsid {$Id: faq.tcl,v 1.36 2006/04/05 01:02:08 drh Exp $} +set rcsid {$Id: faq.tcl,v 1.37 2007/06/09 09:53:51 drh Exp $} source common.tcl header {SQLite Frequently Asked Questions} @@ -93,14 +93,6 @@ faq {

} -faq { - Why does SQLite think that the expression '0'=='00' is TRUE? -} { -

As of version 2.7.0, it doesn't. See the document on - datatypes in SQLite version 3 - for details.

-} - faq { Why doesn't SQLite allow me to use '0' and '0.0' as the primary key on two different rows of the same table? @@ -114,20 +106,6 @@ faq { (See the previous question.) Hence the values are not unique.

} -faq { - My linux box is not able to read an SQLite database that was created - on my SparcStation. -} { -

You need to upgrade your SQLite library to version 2.6.3 or later.

- -

The x86 processor on your linux box is little-endian (meaning that - the least significant byte of integers comes first) but the Sparc is - big-endian (the most significant bytes comes first). SQLite databases - created on a little-endian architecture cannot be on a big-endian - machine by version 2.6.2 or earlier of SQLite. Beginning with - version 2.6.3, SQLite should be able to read and write database files - regardless of byte order of the machine on which the file was created.

-} faq { Can multiple applications or multiple instances of the same @@ -282,37 +260,8 @@ ORDER BY name faq { Are there any known size limits to SQLite databases? } { -

A database is limited in size to 2 tibibytes (241 bytes). - That is a theoretical limitation. In practice, you should try to keep - your SQLite databases below 100 gigabytes to avoid performance problems. - If you need to store 100 gigabytes or more in a database, consider using - an enterprise database engine which is designed for that purpose.

- -

The theoretical limit on the number of rows in a table is - 264-1, though obviously you will run into the file size - limitation prior to reaching the row limit. A single row can hold - up to 230 bytes of data in the current implementation. The - underlying file format supports row sizes up to about 262 bytes. -

- -

There are probably limits on the number of tables or indices or - the number of columns in a table or index, but nobody is sure what - those limits are. In practice, SQLite must read and parse the original - SQL of all table and index declarations everytime a new database file - is opened, so for the best performance of - sqlite3_open() it is best - to keep down the number of declared tables. Likewise, though there - is no limit on the number of columns in a table, more than a few hundred - seems extreme. Only the first 31 columns of a table are candidates for - certain optimizations. You can put as many columns in an index as you like - but indexes with more than 30 columns will not be used to optimize queries. -

- -

The names of tables, indices, view, triggers, and columns can be - as long as desired. However, the names of SQL functions (as created - by the - sqlite3_create_function() - API) may not exceed 255 characters in length.

+

See limits.html for a full discussion of + the limits of SQLite.

} faq { diff --git a/www/index.tcl b/www/index.tcl index 724881b7b4..b1aa5453b4 100644 --- a/www/index.tcl +++ b/www/index.tcl @@ -27,10 +27,8 @@ Features include:
  • A complete database is stored in a single disk file.
  • Database files can be freely shared between machines with different byte orders.
    • -
  • Supports databases up to 2 tebibytes - (241 bytes) in size.
    • -
  • Strings and BLOBs up to 2 gibibytes (231 bytes) - in size.
    • +
  • Supports terabyte-sized databases and gigabyte-sized strings + and blobs. (See limits.html.)
  • Small code footprint: less than 250KiB fully configured or less @@ -122,4 +120,4 @@ puts {

    Old news...

    } -footer {$Id: index.tcl,v 1.154 2007/04/25 11:42:33 drh Exp $} +footer {$Id: index.tcl,v 1.155 2007/06/09 09:53:51 drh Exp $} diff --git a/www/limits.tcl b/www/limits.tcl new file mode 100644 index 0000000000..47ab270a34 --- /dev/null +++ b/www/limits.tcl @@ -0,0 +1,318 @@ +# +# Run this script to generate the limits.html output file +# +set rcsid {$Id: limits.tcl,v 1.1 2007/06/09 09:53:51 drh Exp $} +source common.tcl +header {Implementation Limits For SQLite} +puts { +

    Limits In SQLite

    + +

    +"Limits" in the context of this article means sizes or +quantities that can not be exceeded. We are concerned +with things like the maximum number of bytes in a +BLOB or the maximum number of columns in a table. +

    + +

    +SQLite was originally designed with a policy of avoiding +arbitrary limits. +Of course, every program that runs on a machine with finite +memory and disk space has limits of some kind. But in SQLite, +those limits +were not well defined. The policy was that if it would fit +in memory and you could count it with a 32-bit integer, then +it should work. +

    + +

    +Unfortunately, the no-limits policy has been shown to create +problems. Because the upper bounds where not well +defined, they were not tested, and bugs (including possible +security exploits) where often found when pushing SQLite to +extremes. For this reason, newer versions of SQLite have +well-defined limits and those limits are tested as part of +the test suite. +

    + +

    +This article defines what the limits of SQLite are and how they +can be customized for specific applications. The default settings +for limits are normally quite large and adequate for almost every +application. Some applications may what to increase a limit here +or there, but we expect such needs to be rare. More commonly, +an application might want to recompile SQLite with much lower +limits to avoid excess resource utilization in the event of +bug in higher-level SQL statement generators or to help thwart +attackers who inject malicious SQL statements. +

    +} +proc limititem {title text} { + puts "

  • $title

    \n$text
    • " +} +puts { +
      +} + +limititem {Maximum length of a string or BLOB} { +

      +The maximum number of bytes in a string or BLOB in SQLite is defined +by the preprocessor macro SQLITE_MAX_LENGTH. The default value +of this macro is 1 billion (1 thousand million or 1,000,000,000). +You can raise or lower this value at compile-time using a command-line +option like this: +

      + +
      -DSQLITE_MAX_LENGTH=123456789
      + +

      +The current implementation will only support a string or BLOB +length up to 231-1 or 2147483647. And +some built-in functions such as hex() might fail well before that +point. In security-sensitive applications it is best not to +try to increase the maximum string and blob length. In fact, +you might do well to lower the maximum string and blob length +to something more in the range of a few million if that is +possible. +

      + +

      +During part of SQLite's INSERT and SELECT processing, the complete +content of each row in the database is encoded as a single BLOB. +So the SQLTIE_MAX_LENGTH parameter also determines the maximum +number of bytes in a row. +

      +} + +limititem {Maximum Number Of Columns} { +

      +The SQLITE_MAX_COLUMN compile-time parameter is used to set an upper +bound on: +

      + + + +

      +The default setting for SQLITE_MAX_COLUMN is 2000. You can change it +at compile time to values as large as 32676. You might be able to +redefine this value to be as large as billions, though nobody has ever +tried doing that so we do not know if it will work. On the other hand, there +are people who will argument that a well-normalized database design +will never need a value larger than about 100. +

      + +

      +In most applications, the number of columns is small - a few dozen. +There are places in the SQLite code generator that use algorithms +that are O(N²) where N is the number of columns. +So if you redefine SQLITE_MAX_COLUMN to be a +really huge number and you generate SQL that uses a large number of +columns, you may find that +sqlite3_prepare_v2() +runs slowly. +} + +limititem {Maximum Length Of An SQL Statement} { +

      +The maximum number of bytes in the text of an SQL statement is +limited to SQLITE_MAX_SQL_LENGTH which defaults to 1000000. You +can redefine this limit to be as large as the smaller of SQLITE_MAX_LENGTH +and 1073741824. +

      + +

      +If an SQL statement is limited to be a million bytes in length, then +obviously you will not be able to insert multi-million byte strings +by embedding them as literals inside of INSERT statements. But +you should not do that anyway. Use host parameters +for your data. Prepare short SQL statements like this: +

      + +
      +INSERT INTO tab1 VALUES(?,?,?); +
      + +

      +Then use the +sqlite3_bind_XXXX() functions +to bind your large string values to the SQL statement. The use of binding +obviates the need to escape quote characters in the string, reducing the +risk of SQL injection attacks. It is also runs faster since the large +string does not need to be parsed or copied as much. +

      +} + +limititem {Maximum Number Of Tables In A Join} { +

      +SQLite does not support joins containing more than 64 tables. +This limit arises from the fact that the SQLite code generator +uses bitmaps with one bit per join-table in the query optimizer. +

      +} + +limititem {Maximum Depth Of An Expression Tree} { +

      +SQLite parses expressions into a tree for processing. During +code generation, SQLite walks this tree recursively. The depth +of expression trees is therefore limited in order to avoid +using too much stack space. +

      + +

      +The SQLITE_MAX_EXPR_DEPTH parameter determines the maximum expression +tree depth. If the value is 0, then no limit is enforced. The +current implementation has a default value of 1000. +

      +} + +limititem {Maximum Number Of Arguments On A Function} { +

      +The SQLITE_MAX_FUNCTION_ARG parameter determines the maximum number +of parameters that can be passed to an SQL function. The default value +of this limit is 100. We know of no +technical reason why SQLite would not work with functions that have +millions of parameters. However, we suspect that anybody who tries +to invoke a function with millions of parameters is really +trying to find security exploits in systems that use SQLite, +not do useful work, +and so for that reason we have set this parameter relatively low. +} + +limititem {Maximum Number Of Terms In A Compound SELECT Statement} { +

      +A compound SELECT statement is two or more SELECT statements connected +by operators UNION, UNION ALL, EXCEPT, or INTERSECT. We call each +individual SELECT statement within a compound SELECT a "term". +

      + +

      +The code generator in SQLite processes compound SELECT statements using +a recursive algorithm. In order to limit the size of the stack, we +therefore limit the number of terms in a compound SELECT. The maximum +number of terms is SQLITE_MAX_COMPOUND_SELECT which defaults to 500. +We think this is a generous allotment since in practice we almost +never see the number of terms in a compound select exceed single digits. +

      +} + +limititem {Maximum Length Of A LIKE Or GLOB Pattern} { +

      +The pattern matching algorithm used in the default LIKE and GLOB +implementation of SQLite can exhibit O(N²) performance (where +N is the number of characters in the pattern) for certain pathological +cases. To avoid denial-of-service attacks from miscreants who are able +to specify their own LIKE or GLOB patterns, the length of the LIKE +or GLOB pattern is limited to SQLITE_MAX_LIKE_PATTERN_LENGTH bytes. +The default value of this limit is 50000. A modern workstation can +evaluate even a pathological LIKE or GLOB pattern of 50000 bytes +relatively quickly. The denial of service problem only comes into +play when the pattern length gets into millions of bytes. Nevertheless, +since most useful LIKE or GLOB patterns are at most a few dozen bytes +in length, paranoid application developers may want to reduce this +parameter to something in the range of a few hundred if they know that +external users are able to generate arbitrary patterns. +

      +} + +limititem {Maximum Number Of Host Parameters In A Single SQL Statement} { +

      +A host parameter is a place-holder in an SQL statement that is filled +in using one of the +sqlite3_bind_XXXX() interfaces. +Many SQL programmers are familiar with using a question mark ("?") as a +host parameter. SQLite also supports named host parameters prefaced +by ":", "$", or "@" and numbered host parameters of the form "?123". +

      + +

      +Each host parameter in an SQLite statement is assigned a number. The +numbers normally begin with 1 and increase by one with each new +parameter. However, when the "?123" form is used, the host parameter +number is the number that follows the question mark. +

      + +

      +The maximum value of a host parameter number is SQLITE_MAX_VARIABLE_NUMBER. +This setting defaults to 999. +

      +} + +limititem {Maximum Number Of Attached Databases} { +

      +The ATTACH statement is an SQLite extension +that allows two or more databases to be associated to the same database +connection and to operate as if they were a single database. The number +of simulataneously attached databases is limited to SQLITE_MAX_ATTACHED +which is set to 10 by default. +The code generator in SQLite uses bitmaps +to keep track of attached databases. That means that the number of +attached databases cannot be increased above 30 on a 32-bit machine +or 62 on a 64-bit machine. +} + +limititem {Maximum Database Page Size} { +

      +An SQLite database file is organized as pages. The size of each +page is a power of 2 between 512 and SQLITE_MAX_PAGE_SIZE. +The default value for SQLITE_MAX_PAGE_SIZE is 32768. The current +implementation will not support a larger value. +

      + +

      +It used to be the case that SQLite would allocate some stack +structures whose size was proportional to the maximum page size. +For this reason, SQLite would sometimes be compiled with a smaller +maximum page size on embedded devices with limited stack memory. But +more recent versions of SQLite put these large structures on the +heap, not on the stack, so reducing the maximum page size is no +longer necessary on embedded devices. +

      +} + +limititem {Maximum Number Of Pages In A Database File} { +

      +SQLite is able to limit the size of a database file to prevent +the database file from growing too large and consuming too much +disk or flash space. +The SQLITE_MAX_PAGE_COUNT parameter, which is normally set to +1073741823, is the maximum number of pages allowed in a single +database file. An attempt to insert new data that would cause +the database file to grow larger than this will return +SQLITE_FULL. +

      + +

      +The +max_page_count PRAGMA can be used to raise or lower this +limit at run-time. +

      + +

      +Note that the transaction processing in SQLite requires two bits +of heap memory for every page in the database file. For databases +of a few megabytes in size, this amounts to only a few hundred +bytes of heap memory. But for gigabyte-sized database the amount +of heap memory required is getting into the kilobyte range and +for terabyte-sized databases, megabytes of heap memory must be +allocated and zeroed at each transaction. SQLite will +support very large databases in theory, but the current implementation +is optimized for the common SQLite use cases of embedded devices +and persistent stores for desktop applications. In other words, +SQLite is designed for use with databases sized in kilobytes or +megabytes not gigabytes. If you are building an application to +work with databases that are hundreds of gigabytes or more +in size, then you should perhaps consider using a different database +engine that is explicitly designed for such large data sets. +

      +} + +puts {
    } +footer $rcsid