database/sqlite
1
0
Fork 0
mirror of https://github.com/sqlite/sqlite.git synced 2025-08-08 14:02:16 +03:00
Code Activity

Merge version 3.14 plus some subsequent patches (including the page-cache

Browse Source 
performance patch) from trunk.

FossilOrigin-Name: d9f8918c5b7b6c8540b3f433142e1b4aa4433885
This commit is contained in:
drh
2016-08-11 19:12:25 +00:00
parent 21ac1de75b 1bf4ca7c42
commit 3d4ad18cbe
206 changed files with 10183 additions and 2194 deletions
Download Patch File Download Diff File Expand all files Collapse all files

506
tool/dbhash.c Normal file
View File

@@ -0,0 +1,506 @@
/*
** 2016-06-07
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
**
** This is a utility program that computes an SHA1 hash on the content
** of an SQLite database.
**
** The hash is computed over just the content of the database. Free
** space inside of the database file, and alternative on-disk representations
** of the same content (ex: UTF8 vs UTF16) do not affect the hash. So,
** for example, the database file page size, encoding, and auto_vacuum setting
** can all be changed without changing the hash.
*/
#include <stdio.h>
#include <stdlib.h>
#include <stdarg.h>
#include <ctype.h>
#include <string.h>
#include <assert.h>
#include "sqlite3.h"
/* Context for the SHA1 hash */
typedef struct SHA1Context SHA1Context;
struct SHA1Context {
unsigned int state[5];
unsigned int count[2];
unsigned char buffer[64];
};
/*
** All global variables are gathered into the "g" singleton.
*/
struct GlobalVars {
const char *zArgv0; /* Name of program */
unsigned fDebug; /* Debug flags */
sqlite3 *db; /* The database connection */
SHA1Context cx; /* SHA1 hash context */
} g;
/*
** Debugging flags
*/
#define DEBUG_FULLTRACE 0x00000001 /* Trace hash to stderr */
/******************************************************************************
** The Hash Engine
**
** Modify these routines (and appropriate state fields in global variable 'g')
** in order to compute a different (better?) hash of the database.
*/
/*
* blk0() and blk() perform the initial expand.
* I got the idea of expanding during the round function from SSLeay
*
* blk0le() for little-endian and blk0be() for big-endian.
*/
#if __GNUC__ && (defined(__i386__) || defined(__x86_64__))
/*
* GCC by itself only generates left rotates. Use right rotates if
* possible to be kinder to dinky implementations with iterative rotate
* instructions.
*/
#define SHA_ROT(op, x, k) \
({ unsigned int y; asm(op " %1,%0" : "=r" (y) : "I" (k), "0" (x)); y; })
#define rol(x,k) SHA_ROT("roll", x, k)
#define ror(x,k) SHA_ROT("rorl", x, k)
#else
/* Generic C equivalent */
#define SHA_ROT(x,l,r) ((x) << (l) | (x) >> (r))
#define rol(x,k) SHA_ROT(x,k,32-(k))
#define ror(x,k) SHA_ROT(x,32-(k),k)
#endif
#define blk0le(i) (block[i] = (ror(block[i],8)&0xFF00FF00) \
|(rol(block[i],8)&0x00FF00FF))
#define blk0be(i) block[i]
#define blk(i) (block[i&15] = rol(block[(i+13)&15]^block[(i+8)&15] \
^block[(i+2)&15]^block[i&15],1))
/*
* (R0+R1), R2, R3, R4 are the different operations (rounds) used in SHA1
*
* Rl0() for little-endian and Rb0() for big-endian. Endianness is
* determined at run-time.
*/
#define Rl0(v,w,x,y,z,i) \
z+=((w&(x^y))^y)+blk0le(i)+0x5A827999+rol(v,5);w=ror(w,2);
#define Rb0(v,w,x,y,z,i) \
z+=((w&(x^y))^y)+blk0be(i)+0x5A827999+rol(v,5);w=ror(w,2);
#define R1(v,w,x,y,z,i) \
z+=((w&(x^y))^y)+blk(i)+0x5A827999+rol(v,5);w=ror(w,2);
#define R2(v,w,x,y,z,i) \
z+=(w^x^y)+blk(i)+0x6ED9EBA1+rol(v,5);w=ror(w,2);
#define R3(v,w,x,y,z,i) \
z+=(((w|x)&y)|(w&x))+blk(i)+0x8F1BBCDC+rol(v,5);w=ror(w,2);
#define R4(v,w,x,y,z,i) \
z+=(w^x^y)+blk(i)+0xCA62C1D6+rol(v,5);w=ror(w,2);
/*
* Hash a single 512-bit block. This is the core of the algorithm.
*/
#define a qq[0]
#define b qq[1]
#define c qq[2]
#define d qq[3]
#define e qq[4]
void SHA1Transform(unsigned int state[5], const unsigned char buffer[64]){
unsigned int qq[5]; /* a, b, c, d, e; */
static int one = 1;
unsigned int block[16];
memcpy(block, buffer, 64);
memcpy(qq,state,5*sizeof(unsigned int));
/* Copy g.cx.state[] to working vars */
/*
a = state[0];
b = state[1];
c = state[2];
d = state[3];
e = state[4];
*/
/* 4 rounds of 20 operations each. Loop unrolled. */
if( 1 == *(unsigned char*)&one ){
Rl0(a,b,c,d,e, 0); Rl0(e,a,b,c,d, 1); Rl0(d,e,a,b,c, 2); Rl0(c,d,e,a,b, 3);
Rl0(b,c,d,e,a, 4); Rl0(a,b,c,d,e, 5); Rl0(e,a,b,c,d, 6); Rl0(d,e,a,b,c, 7);
Rl0(c,d,e,a,b, 8); Rl0(b,c,d,e,a, 9); Rl0(a,b,c,d,e,10); Rl0(e,a,b,c,d,11);
Rl0(d,e,a,b,c,12); Rl0(c,d,e,a,b,13); Rl0(b,c,d,e,a,14); Rl0(a,b,c,d,e,15);
}else{
Rb0(a,b,c,d,e, 0); Rb0(e,a,b,c,d, 1); Rb0(d,e,a,b,c, 2); Rb0(c,d,e,a,b, 3);
Rb0(b,c,d,e,a, 4); Rb0(a,b,c,d,e, 5); Rb0(e,a,b,c,d, 6); Rb0(d,e,a,b,c, 7);
Rb0(c,d,e,a,b, 8); Rb0(b,c,d,e,a, 9); Rb0(a,b,c,d,e,10); Rb0(e,a,b,c,d,11);
Rb0(d,e,a,b,c,12); Rb0(c,d,e,a,b,13); Rb0(b,c,d,e,a,14); Rb0(a,b,c,d,e,15);
}
R1(e,a,b,c,d,16); R1(d,e,a,b,c,17); R1(c,d,e,a,b,18); R1(b,c,d,e,a,19);
R2(a,b,c,d,e,20); R2(e,a,b,c,d,21); R2(d,e,a,b,c,22); R2(c,d,e,a,b,23);
R2(b,c,d,e,a,24); R2(a,b,c,d,e,25); R2(e,a,b,c,d,26); R2(d,e,a,b,c,27);
R2(c,d,e,a,b,28); R2(b,c,d,e,a,29); R2(a,b,c,d,e,30); R2(e,a,b,c,d,31);
R2(d,e,a,b,c,32); R2(c,d,e,a,b,33); R2(b,c,d,e,a,34); R2(a,b,c,d,e,35);
R2(e,a,b,c,d,36); R2(d,e,a,b,c,37); R2(c,d,e,a,b,38); R2(b,c,d,e,a,39);
R3(a,b,c,d,e,40); R3(e,a,b,c,d,41); R3(d,e,a,b,c,42); R3(c,d,e,a,b,43);
R3(b,c,d,e,a,44); R3(a,b,c,d,e,45); R3(e,a,b,c,d,46); R3(d,e,a,b,c,47);
R3(c,d,e,a,b,48); R3(b,c,d,e,a,49); R3(a,b,c,d,e,50); R3(e,a,b,c,d,51);
R3(d,e,a,b,c,52); R3(c,d,e,a,b,53); R3(b,c,d,e,a,54); R3(a,b,c,d,e,55);
R3(e,a,b,c,d,56); R3(d,e,a,b,c,57); R3(c,d,e,a,b,58); R3(b,c,d,e,a,59);
R4(a,b,c,d,e,60); R4(e,a,b,c,d,61); R4(d,e,a,b,c,62); R4(c,d,e,a,b,63);
R4(b,c,d,e,a,64); R4(a,b,c,d,e,65); R4(e,a,b,c,d,66); R4(d,e,a,b,c,67);
R4(c,d,e,a,b,68); R4(b,c,d,e,a,69); R4(a,b,c,d,e,70); R4(e,a,b,c,d,71);
R4(d,e,a,b,c,72); R4(c,d,e,a,b,73); R4(b,c,d,e,a,74); R4(a,b,c,d,e,75);
R4(e,a,b,c,d,76); R4(d,e,a,b,c,77); R4(c,d,e,a,b,78); R4(b,c,d,e,a,79);
/* Add the working vars back into context.state[] */
state[0] += a;
state[1] += b;
state[2] += c;
state[3] += d;
state[4] += e;
}
/* Initialize the SHA1 hash */
static void hash_init(void){
/* SHA1 initialization constants */
g.cx.state[0] = 0x67452301;
g.cx.state[1] = 0xEFCDAB89;
g.cx.state[2] = 0x98BADCFE;
g.cx.state[3] = 0x10325476;
g.cx.state[4] = 0xC3D2E1F0;
g.cx.count[0] = g.cx.count[1] = 0;
}
/* Add new content to the SHA1 hash */
static void hash_step(const unsigned char *data, unsigned int len){
unsigned int i, j;
j = g.cx.count[0];
if( (g.cx.count[0] += len << 3) < j ){
g.cx.count[1] += (len>>29)+1;
}
j = (j >> 3) & 63;
if( (j + len) > 63 ){
(void)memcpy(&g.cx.buffer[j], data, (i = 64-j));
SHA1Transform(g.cx.state, g.cx.buffer);
for(; i + 63 < len; i += 64){
SHA1Transform(g.cx.state, &data[i]);
}
j = 0;
}else{
i = 0;
}
(void)memcpy(&g.cx.buffer[j], &data[i], len - i);
}
/* Add padding and compute and output the message digest. */
static void hash_finish(const char *zName){
unsigned int i;
unsigned char finalcount[8];
unsigned char digest[20];
static const char zEncode[] = "0123456789abcdef";
char zOut[41];
for (i = 0; i < 8; i++){
finalcount[i] = (unsigned char)((g.cx.count[(i >= 4 ? 0 : 1)]
>> ((3-(i & 3)) * 8) ) & 255); /* Endian independent */
}
hash_step((const unsigned char *)"\200", 1);
while ((g.cx.count[0] & 504) != 448){
hash_step((const unsigned char *)"\0", 1);
}
hash_step(finalcount, 8); /* Should cause a SHA1Transform() */
for (i = 0; i < 20; i++){
digest[i] = (unsigned char)((g.cx.state[i>>2] >> ((3-(i & 3)) * 8) ) & 255);
}
for(i=0; i<20; i++){
zOut[i*2] = zEncode[(digest[i]>>4)&0xf];
zOut[i*2+1] = zEncode[digest[i] & 0xf];
}
zOut[i*2]= 0;
printf("%s %s\n", zOut, zName);
}
/* End of the hashing logic
*******************************************************************************/
/*
** Print an error resulting from faulting command-line arguments and
** abort the program.
*/
static void cmdlineError(const char *zFormat, ...){
va_list ap;
fprintf(stderr, "%s: ", g.zArgv0);
va_start(ap, zFormat);
vfprintf(stderr, zFormat, ap);
va_end(ap);
fprintf(stderr, "\n\"%s --help\" for more help\n", g.zArgv0);
exit(1);
}
/*
** Print an error message for an error that occurs at runtime, then
** abort the program.
*/
static void runtimeError(const char *zFormat, ...){
va_list ap;
fprintf(stderr, "%s: ", g.zArgv0);
va_start(ap, zFormat);
vfprintf(stderr, zFormat, ap);
va_end(ap);
fprintf(stderr, "\n");
exit(1);
}
/*
** Prepare a new SQL statement. Print an error and abort if anything
** goes wrong.
*/
static sqlite3_stmt *db_vprepare(const char *zFormat, va_list ap){
char *zSql;
int rc;
sqlite3_stmt *pStmt;
zSql = sqlite3_vmprintf(zFormat, ap);
if( zSql==0 ) runtimeError("out of memory");
rc = sqlite3_prepare_v2(g.db, zSql, -1, &pStmt, 0);
if( rc ){
runtimeError("SQL statement error: %s\n\"%s\"", sqlite3_errmsg(g.db),
zSql);
}
sqlite3_free(zSql);
return pStmt;
}
static sqlite3_stmt *db_prepare(const char *zFormat, ...){
va_list ap;
sqlite3_stmt *pStmt;
va_start(ap, zFormat);
pStmt = db_vprepare(zFormat, ap);
va_end(ap);
return pStmt;
}
/*
** Compute the hash for all rows of the query formed from the printf-style
** zFormat and its argument.
*/
static void hash_one_query(const char *zFormat, ...){
va_list ap;
sqlite3_stmt *pStmt; /* The query defined by zFormat and "..." */
int nCol; /* Number of columns in the result set */
int i; /* Loop counter */
/* Prepare the query defined by zFormat and "..." */
va_start(ap, zFormat);
pStmt = db_vprepare(zFormat, ap);
va_end(ap);
nCol = sqlite3_column_count(pStmt);
/* Compute a hash over the result of the query */
while( SQLITE_ROW==sqlite3_step(pStmt) ){
for(i=0; i<nCol; i++){
switch( sqlite3_column_type(pStmt,i) ){
case SQLITE_NULL: {
hash_step((const unsigned char*)"0",1);
if( g.fDebug & DEBUG_FULLTRACE ) fprintf(stderr, "NULL\n");
break;
}
case SQLITE_INTEGER: {
sqlite3_uint64 u;
int j;
unsigned char x[8];
sqlite3_int64 v = sqlite3_column_int64(pStmt,i);
memcpy(&u, &v, 8);
for(j=7; j>=0; j--){
x[j] = u & 0xff;
u >>= 8;
}
hash_step((const unsigned char*)"1",1);
hash_step(x,8);
if( g.fDebug & DEBUG_FULLTRACE ){
fprintf(stderr, "INT %s\n", sqlite3_column_text(pStmt,i));
}
break;
}
case SQLITE_FLOAT: {
sqlite3_uint64 u;
int j;
unsigned char x[8];
double r = sqlite3_column_double(pStmt,i);
memcpy(&u, &r, 8);
for(j=7; j>=0; j--){
x[j] = u & 0xff;
u >>= 8;
}
hash_step((const unsigned char*)"2",1);
hash_step(x,8);
if( g.fDebug & DEBUG_FULLTRACE ){
fprintf(stderr, "FLOAT %s\n", sqlite3_column_text(pStmt,i));
}
break;
}
case SQLITE_TEXT: {
int n = sqlite3_column_bytes(pStmt, i);
const unsigned char *z = sqlite3_column_text(pStmt, i);
hash_step((const unsigned char*)"3", 1);
hash_step(z, n);
if( g.fDebug & DEBUG_FULLTRACE ){
fprintf(stderr, "TEXT '%s'\n", sqlite3_column_text(pStmt,i));
}
break;
}
case SQLITE_BLOB: {
int n = sqlite3_column_bytes(pStmt, i);
const unsigned char *z = sqlite3_column_blob(pStmt, i);
hash_step((const unsigned char*)"4", 1);
hash_step(z, n);
if( g.fDebug & DEBUG_FULLTRACE ){
fprintf(stderr, "BLOB (%d bytes)\n", n);
}
break;
}
}
}
}
sqlite3_finalize(pStmt);
}
/*
** Print sketchy documentation for this utility program
*/
static void showHelp(void){
printf("Usage: %s [options] FILE ...\n", g.zArgv0);
printf(
"Compute a SHA1 hash on the content of database FILE. System tables such as\n"
"sqlite_stat1, sqlite_stat4, and sqlite_sequence are omitted from the hash.\n"
"Options:\n"
" --debug N Set debugging flags to N (experts only)\n"
" --like PATTERN Only hash tables whose name is LIKE the pattern\n"
" --schema-only Only hash the schema - omit table content\n"
" --without-schema Only hash table content - omit the schema\n"
);
}
int main(int argc, char **argv){
const char *zDb = 0; /* Name of the database currently being hashed */
int i; /* Loop counter */
int rc; /* Subroutine return code */
char *zErrMsg; /* Error message when opening database */
sqlite3_stmt *pStmt; /* An SQLite query */
const char *zLike = 0; /* LIKE pattern of tables to hash */
int omitSchema = 0; /* True to compute hash on content only */
int omitContent = 0; /* True to compute hash on schema only */
int nFile = 0; /* Number of input filenames seen */
g.zArgv0 = argv[0];
sqlite3_config(SQLITE_CONFIG_SINGLETHREAD);
for(i=1; i<argc; i++){
const char *z = argv[i];
if( z[0]=='-' ){
z++;
if( z[0]=='-' ) z++;
if( strcmp(z,"debug")==0 ){
if( i==argc-1 ) cmdlineError("missing argument to %s", argv[i]);
g.fDebug = strtol(argv[++i], 0, 0);
}else
if( strcmp(z,"help")==0 ){
showHelp();
return 0;
}else
if( strcmp(z,"like")==0 ){
if( i==argc-1 ) cmdlineError("missing argument to %s", argv[i]);
if( zLike!=0 ) cmdlineError("only one --like allowed");
zLike = argv[++i];
}else
if( strcmp(z,"schema-only")==0 ){
omitContent = 1;
}else
if( strcmp(z,"without-schema")==0 ){
omitSchema = 1;
}else
{
cmdlineError("unknown option: %s", argv[i]);
}
}else{
nFile++;
if( nFile<i ) argv[nFile] = argv[i];
}
}
if( nFile==0 ){
cmdlineError("no input files specified - nothing to do");
}
if( omitSchema && omitContent ){
cmdlineError("only one of --without-schema and --omit-schema allowed");
}
if( zLike==0 ) zLike = "%";
for(i=1; i<=nFile; i++){
static const int openFlags =
SQLITE_OPEN_READWRITE | /* Read/write so hot journals can recover */
SQLITE_OPEN_URI
;
zDb = argv[i];
rc = sqlite3_open_v2(zDb, &g.db, openFlags, 0);
if( rc ){
fprintf(stderr, "cannot open database file '%s'\n", zDb);
continue;
}
rc = sqlite3_exec(g.db, "SELECT * FROM sqlite_master", 0, 0, &zErrMsg);
if( rc || zErrMsg ){
sqlite3_close(g.db);
g.db = 0;
fprintf(stderr, "'%s' is not a valid SQLite database\n", zDb);
continue;
}
/* Start the hash */
hash_init();
/* Hash table content */
if( !omitContent ){
pStmt = db_prepare(
"SELECT name FROM sqlite_master\n"
" WHERE type='table' AND sql NOT LIKE 'CREATE VIRTUAL%%'\n"
" AND name NOT LIKE 'sqlite_%%'\n"
" AND name LIKE '%q'\n"
" ORDER BY name COLLATE nocase;\n",
zLike
);
while( SQLITE_ROW==sqlite3_step(pStmt) ){
/* We want rows of the table to be hashed in PRIMARY KEY order.
** Technically, an ORDER BY clause is required to guarantee that
** order. However, though not guaranteed by the documentation, every
** historical version of SQLite has always output rows in PRIMARY KEY
** order when there is no WHERE or GROUP BY clause, so the ORDER BY
** can be safely omitted. */
hash_one_query("SELECT * FROM \"%w\"", sqlite3_column_text(pStmt,0));
}
sqlite3_finalize(pStmt);
}
/* Hash the database schema */
if( !omitSchema ){
hash_one_query(
"SELECT type, name, tbl_name, sql FROM sqlite_master\n"
" WHERE tbl_name LIKE '%q'\n"
" ORDER BY name COLLATE nocase;\n",
zLike
);
}
/* Finish and output the hash and close the database connection. */
hash_finish(zDb);
sqlite3_close(g.db);
}
return 0;
}

123
tool/lemon.c
View File

@@ -288,10 +288,13 @@ struct rule {
const char *code; /* The code executed when this rule is reduced */
const char *codePrefix; /* Setup code before code[] above */
const char *codeSuffix; /* Breakdown code after code[] above */
int noCode; /* True if this rule has no associated C code */
int codeEmitted; /* True if the code has been emitted already */
struct symbol *precsym; /* Precedence symbol for this rule */
int index; /* An index number for this rule */
int iRule; /* Rule number as used in the generated tables */
Boolean canReduce; /* True if this rule is ever reduced */
Boolean doesReduce; /* Reduce actions occur after optimization */
struct rule *nextlhs; /* Next rule with the same LHS */
struct rule *next; /* Next rule in the global list */
};
@@ -339,6 +342,7 @@ struct action {
struct state *stp; /* The new state, if a shift */
struct rule *rp; /* The rule, if a reduce */
} x;
struct symbol *spOpt; /* SHIFTREDUCE optimization to this symbol */
struct action *next; /* Next action for this state */
struct action *collide; /* Next action with the same hash */
};
@@ -349,7 +353,7 @@ struct state {
struct config *bp; /* The basis configurations for this state */
struct config *cfp; /* All configurations in this set */
int statenum; /* Sequential number for this state */
struct action *ap; /* Array of actions for this state */
struct action *ap; /* List of actions for this state */
int nTknAct, nNtAct; /* Number of actions on terminals and nonterminals */
int iTknOfst, iNtOfst; /* yy_action[] offset for terminals and nonterms */
int iDfltReduce; /* Default action is to REDUCE by this rule */
@@ -530,6 +534,7 @@ void Action_add(
*app = newaction;
newaction->type = type;
newaction->sp = sp;
newaction->spOpt = 0;
if( type==SHIFT ){
newaction->x.stp = (struct state *)arg;
}else{
@@ -1499,7 +1504,7 @@ static void handle_T_option(char *z){
lemon_strcpy(user_templatename, z);
}
/* Merge together to lists of rules order by rule.iRule */
/* Merge together to lists of rules ordered by rule.iRule */
static struct rule *Rule_merge(struct rule *pA, struct rule *pB){
struct rule *pFirst = 0;
struct rule **ppPrev = &pFirst;
@@ -1642,7 +1647,10 @@ int main(int argc, char **argv)
for(i=1; ISUPPER(lem.symbols[i]->name[0]); i++);
lem.nterminal = i;
/* Assign sequential rule numbers */
/* Assign sequential rule numbers. Start with 0. Put rules that have no
** reduce action C-code associated with them last, so that the switch()
** statement that selects reduction actions will have a smaller jump table.
*/
for(i=0, rp=lem.rule; rp; rp=rp->next){
rp->iRule = rp->code ? i++ : -1;
}
@@ -2211,6 +2219,7 @@ to follow the previous rule.");
}else{
psp->prevrule->line = psp->tokenlineno;
psp->prevrule->code = &x[1];
psp->prevrule->noCode = 0;
}
}else if( x[0]=='[' ){
psp->state = PRECEDENCE_MARK_1;
@@ -2317,6 +2326,7 @@ to follow the previous rule.");
rp->lhsalias = psp->lhsalias;
rp->nrhs = psp->nrhs;
rp->code = 0;
rp->noCode = 1;
rp->precsym = 0;
rp->index = psp->gp->nrule++;
rp->nextlhs = rp->lhs->rule;
@@ -3160,6 +3170,9 @@ int PrintAction(
result = 0;
break;
}
if( result && ap->spOpt ){
fprintf(fp," /* because %s==%s */", ap->sp->name, ap->spOpt->name);
}
return result;
}
@@ -3527,9 +3540,8 @@ PRIVATE char *append_str(const char *zText, int n, int p1, int p2){
}
/*
** zCode is a string that is the action associated with a rule. Expand
** the symbols in this string so that the refer to elements of the parser
** stack.
** Write and transform the rp->code string so that symbols are expanded.
** Populate the rp->codePrefix and rp->codeSuffix strings, as appropriate.
**
** Return 1 if the expanded code requires that "yylhsminor" local variable
** to be defined.
@@ -3553,17 +3565,17 @@ PRIVATE int translate_code(struct lemon *lemp, struct rule *rp){
static char newlinestr[2] = { '\n', '\0' };
rp->code = newlinestr;
rp->line = rp->ruleline;
rp->noCode = 1;
}else{
rp->noCode = 0;
}
if( rp->lhsalias==0 ){
/* There is no LHS value symbol. */
lhsdirect = 1;
}else if( rp->nrhs==0 ){
if( rp->nrhs==0 ){
/* If there are no RHS symbols, then writing directly to the LHS is ok */
lhsdirect = 1;
}else if( rp->rhsalias[0]==0 ){
/* The left-most RHS symbol has not value. LHS direct is ok. But
/* The left-most RHS symbol has no value. LHS direct is ok. But
** we have to call the distructor on the RHS symbol first. */
lhsdirect = 1;
if( has_destructor(rp->rhs[0],lemp) ){
@@ -3571,7 +3583,11 @@ PRIVATE int translate_code(struct lemon *lemp, struct rule *rp){
append_str(" yy_destructor(yypParser,%d,&yymsp[%d].minor);\n", 0,
rp->rhs[0]->index,1-rp->nrhs);
rp->codePrefix = Strsafe(append_str(0,0,0,0));
rp->noCode = 0;
}
}else if( rp->lhsalias==0 ){
/* There is no LHS value symbol. */
lhsdirect = 1;
}else if( strcmp(rp->lhsalias,rp->rhsalias[0])==0 ){
/* The LHS symbol and the left-most RHS symbol are the same, so
** direct writing is allowed */
@@ -3715,7 +3731,10 @@ PRIVATE int translate_code(struct lemon *lemp, struct rule *rp){
/* Suffix code generation complete */
cp = append_str(0,0,0,0);
if( cp ) rp->codeSuffix = Strsafe(cp);
if( cp && cp[0] ){
rp->codeSuffix = Strsafe(cp);
rp->noCode = 0;
}
return rc;
}
@@ -4134,6 +4153,19 @@ void ReportTable(
}
free(ax);
/* Mark rules that are actually used for reduce actions after all
** optimizations have been applied
*/
for(rp=lemp->rule; rp; rp=rp->next) rp->doesReduce = LEMON_FALSE;
for(i=0; i<lemp->nxstate; i++){
struct action *ap;
for(ap=lemp->sorted[i]->ap; ap; ap=ap->next){
if( ap->type==REDUCE || ap->type==SHIFTREDUCE ){
ap->x.rp->doesReduce = i;
}
}
}
/* Finish rendering the constants now that the action table has
** been computed */
fprintf(out,"#define YYNSTATE %d\n",lemp->nxstate); lineno++;
@@ -4199,20 +4231,21 @@ void ReportTable(
fprintf(out, "};\n"); lineno++;
/* Output the yy_shift_ofst[] table */
fprintf(out, "#define YY_SHIFT_USE_DFLT (%d)\n", mnTknOfst-1); lineno++;
n = lemp->nxstate;
while( n>0 && lemp->sorted[n-1]->iTknOfst==NO_OFFSET ) n--;
fprintf(out, "#define YY_SHIFT_COUNT (%d)\n", n-1); lineno++;
fprintf(out, "#define YY_SHIFT_MIN (%d)\n", mnTknOfst); lineno++;
fprintf(out, "#define YY_SHIFT_MAX (%d)\n", mxTknOfst); lineno++;
fprintf(out, "#define YY_SHIFT_USE_DFLT (%d)\n", lemp->nactiontab); lineno++;
fprintf(out, "#define YY_SHIFT_COUNT (%d)\n", n-1); lineno++;
fprintf(out, "#define YY_SHIFT_MIN (%d)\n", mnTknOfst); lineno++;
fprintf(out, "#define YY_SHIFT_MAX (%d)\n", mxTknOfst); lineno++;
fprintf(out, "static const %s yy_shift_ofst[] = {\n",
minimum_size_type(mnTknOfst-1, mxTknOfst, &sz)); lineno++;
minimum_size_type(mnTknOfst, lemp->nterminal+lemp->nactiontab, &sz));
lineno++;
lemp->tablesize += n*sz;
for(i=j=0; i<n; i++){
int ofst;
stp = lemp->sorted[i];
ofst = stp->iTknOfst;
if( ofst==NO_OFFSET ) ofst = mnTknOfst - 1;
if( ofst==NO_OFFSET ) ofst = lemp->nactiontab;
if( j==0 ) fprintf(out," /* %5d */ ", i);
fprintf(out, " %4d,", ofst);
if( j==9 || i==n-1 ){
@@ -4396,8 +4429,11 @@ void ReportTable(
/* First output rules other than the default: rule */
for(rp=lemp->rule; rp; rp=rp->next){
struct rule *rp2; /* Other rules with the same action */
if( rp->code==0 ) continue;
if( rp->code[0]=='\n' && rp->code[1]==0 ) continue; /* Will be default: */
if( rp->codeEmitted ) continue;
if( rp->noCode ){
/* No C code actions, so this will be part of the "default:" rule */
continue;
}
fprintf(out," case %d: /* ", rp->iRule);
writeRuleText(out, rp);
fprintf(out, " */\n"); lineno++;
@@ -4407,22 +4443,27 @@ void ReportTable(
fprintf(out," case %d: /* ", rp2->iRule);
writeRuleText(out, rp2);
fprintf(out," */ yytestcase(yyruleno==%d);\n", rp2->iRule); lineno++;
rp2->code = 0;
rp2->codeEmitted = 1;
}
}
emit_code(out,rp,lemp,&lineno);
fprintf(out," break;\n"); lineno++;
rp->code = 0;
rp->codeEmitted = 1;
}
/* Finally, output the default: rule. We choose as the default: all
** empty actions. */
fprintf(out," default:\n"); lineno++;
for(rp=lemp->rule; rp; rp=rp->next){
if( rp->code==0 ) continue;
assert( rp->code[0]=='\n' && rp->code[1]==0 );
if( rp->codeEmitted ) continue;
assert( rp->noCode );
fprintf(out," /* (%d) ", rp->iRule);
writeRuleText(out, rp);
fprintf(out, " */ yytestcase(yyruleno==%d);\n", rp->iRule); lineno++;
if( rp->doesReduce ){
fprintf(out, " */ yytestcase(yyruleno==%d);\n", rp->iRule); lineno++;
}else{
fprintf(out, " (OPTIMIZED OUT) */ assert(yyruleno!=%d);\n",
rp->iRule); lineno++;
}
}
fprintf(out," break;\n"); lineno++;
tplt_xfer(lemp->name,in,out,&lineno);
@@ -4493,7 +4534,7 @@ void ReportHeader(struct lemon *lemp)
void CompressTables(struct lemon *lemp)
{
struct state *stp;
struct action *ap, *ap2;
struct action *ap, *ap2, *nextap;
struct rule *rp, *rp2, *rbest;
int nbest, n;
int i;
@@ -4570,6 +4611,36 @@ void CompressTables(struct lemon *lemp)
}
}
}
/* If a SHIFTREDUCE action specifies a rule that has a single RHS term
** (meaning that the SHIFTREDUCE will land back in the state where it
** started) and if there is no C-code associated with the reduce action,
** then we can go ahead and convert the action to be the same as the
** action for the RHS of the rule.
*/
for(i=0; i<lemp->nstate; i++){
stp = lemp->sorted[i];
for(ap=stp->ap; ap; ap=nextap){
nextap = ap->next;
if( ap->type!=SHIFTREDUCE ) continue;
rp = ap->x.rp;
if( rp->noCode==0 ) continue;
if( rp->nrhs!=1 ) continue;
#if 1
/* Only apply this optimization to non-terminals. It would be OK to
** apply it to terminal symbols too, but that makes the parser tables
** larger. */
if( ap->sp->index<lemp->nterminal ) continue;
#endif
/* If we reach this point, it means the optimization can be applied */
nextap = ap;
for(ap2=stp->ap; ap2 && (ap2==ap || ap2->sp!=rp->lhs); ap2=ap2->next){}
assert( ap2!=0 );
ap->spOpt = ap2->sp;
ap->type = ap2->type;
ap->x = ap2->x;
}
}
}

239
tool/lempar.c
View File

@@ -116,7 +116,7 @@
**
** N between YY_MIN_REDUCE Reduce by rule N-YY_MIN_REDUCE
** and YY_MAX_REDUCE
**
** N == YY_ERROR_ACTION A syntax error has occurred.
**
** N == YY_ACCEPT_ACTION The parser accepts its input.
@@ -125,16 +125,20 @@
** slots in the yy_action[] table.
**
** The action table is constructed as a single large table named yy_action[].
** Given state S and lookahead X, the action is computed as
** Given state S and lookahead X, the action is computed as either:
**
** yy_action[ yy_shift_ofst[S] + X ]
** (A) N = yy_action[ yy_shift_ofst[S] + X ]
** (B) N = yy_default[S]
**
** If the index value yy_shift_ofst[S]+X is out of range or if the value
** yy_lookahead[yy_shift_ofst[S]+X] is not equal to X or if yy_shift_ofst[S]
** is equal to YY_SHIFT_USE_DFLT, it means that the action is not in the table
** and that yy_default[S] should be used instead.
** The (A) formula is preferred. The B formula is used instead if:
** (1) The yy_shift_ofst[S]+X value is out of range, or
** (2) yy_lookahead[yy_shift_ofst[S]+X] is not equal to X, or
** (3) yy_shift_ofst[S] equal YY_SHIFT_USE_DFLT.
** (Implementation note: YY_SHIFT_USE_DFLT is chosen so that
** YY_SHIFT_USE_DFLT+X will be out of range for all possible lookaheads X.
** Hence only tests (1) and (2) need to be evaluated.)
**
** The formula above is for computing the action when the lookahead is
** The formulas above are for computing the action when the lookahead is
** a terminal symbol. If the lookahead is a non-terminal (as occurs after
** a reduce action) then the yy_reduce_ofst[] array is used in place of
** the yy_shift_ofst[] array and YY_REDUCE_USE_DFLT is used in place of
@@ -203,9 +207,9 @@ typedef struct yyStackEntry yyStackEntry;
/* The state of the parser is completely contained in an instance of
** the following structure */
struct yyParser {
int yyidx; /* Index of top element in stack */
yyStackEntry *yytos; /* Pointer to top element of the stack */
#ifdef YYTRACKMAXSTACKDEPTH
int yyidxMax; /* Maximum value of yyidx */
int yyhwm; /* High-water mark of the stack */
#endif
#ifndef YYNOERRORRECOVERY
int yyerrcnt; /* Shifts left before out of the error */
@@ -214,6 +218,7 @@ struct yyParser {
#if YYSTACKDEPTH<=0
int yystksz; /* Current side of the stack */
yyStackEntry *yystack; /* The parser's stack */
yyStackEntry yystk0; /* First stack entry */
#else
yyStackEntry yystack[YYSTACKDEPTH]; /* The parser's stack */
#endif
@@ -271,24 +276,34 @@ static const char *const yyRuleName[] = {
#if YYSTACKDEPTH<=0
/*
** Try to increase the size of the parser stack.
** Try to increase the size of the parser stack. Return the number
** of errors. Return 0 on success.
*/
static void yyGrowStack(yyParser *p){
static int yyGrowStack(yyParser *p){
int newSize;
int idx;
yyStackEntry *pNew;
newSize = p->yystksz*2 + 100;
pNew = realloc(p->yystack, newSize*sizeof(pNew[0]));
idx = p->yytos ? (int)(p->yytos - p->yystack) : 0;
if( p->yystack==&p->yystk0 ){
pNew = malloc(newSize*sizeof(pNew[0]));
if( pNew ) pNew[0] = p->yystk0;
}else{
pNew = realloc(p->yystack, newSize*sizeof(pNew[0]));
}
if( pNew ){
p->yystack = pNew;
p->yystksz = newSize;
p->yytos = &p->yystack[idx];
#ifndef NDEBUG
if( yyTraceFILE ){
fprintf(yyTraceFILE,"%sStack grows to %d entries!\n",
yyTracePrompt, p->yystksz);
fprintf(yyTraceFILE,"%sStack grows from %d to %d entries.\n",
yyTracePrompt, p->yystksz, newSize);
}
#endif
p->yystksz = newSize;
}
return pNew==0;
}
#endif
@@ -317,15 +332,24 @@ void *ParseAlloc(void *(*mallocProc)(YYMALLOCARGTYPE)){
yyParser *pParser;
pParser = (yyParser*)(*mallocProc)( (YYMALLOCARGTYPE)sizeof(yyParser) );
if( pParser ){
pParser->yyidx = -1;
#ifdef YYTRACKMAXSTACKDEPTH
pParser->yyidxMax = 0;
pParser->yyhwm = 0;
#endif
#if YYSTACKDEPTH<=0
pParser->yytos = NULL;
pParser->yystack = NULL;
pParser->yystksz = 0;
yyGrowStack(pParser);
if( yyGrowStack(pParser) ){
pParser->yystack = &pParser->yystk0;
pParser->yystksz = 1;
}
#endif
#ifndef YYNOERRORRECOVERY
pParser->yyerrcnt = -1;
#endif
pParser->yytos = pParser->yystack;
pParser->yystack[0].stateno = 0;
pParser->yystack[0].major = 0;
}
return pParser;
}
@@ -369,8 +393,9 @@ static void yy_destructor(
*/
static void yy_pop_parser_stack(yyParser *pParser){
yyStackEntry *yytos;
assert( pParser->yyidx>=0 );
yytos = &pParser->yystack[pParser->yyidx--];
assert( pParser->yytos!=0 );
assert( pParser->yytos > pParser->yystack );
yytos = pParser->yytos--;
#ifndef NDEBUG
if( yyTraceFILE ){
fprintf(yyTraceFILE,"%sPopping %s\n",
@@ -397,9 +422,9 @@ void ParseFree(
#ifndef YYPARSEFREENEVERNULL
if( pParser==0 ) return;
#endif
while( pParser->yyidx>=0 ) yy_pop_parser_stack(pParser);
while( pParser->yytos>pParser->yystack ) yy_pop_parser_stack(pParser);
#if YYSTACKDEPTH<=0
free(pParser->yystack);
if( pParser->yystack!=&pParser->yystk0 ) free(pParser->yystack);
#endif
(*freeProc)((void*)pParser);
}
@@ -410,7 +435,7 @@ void ParseFree(
#ifdef YYTRACKMAXSTACKDEPTH
int ParseStackPeak(void *p){
yyParser *pParser = (yyParser*)p;
return pParser->yyidxMax;
return pParser->yyhwm;
}
#endif
@@ -423,56 +448,53 @@ static unsigned int yy_find_shift_action(
YYCODETYPE iLookAhead /* The look-ahead token */
){
int i;
int stateno = pParser->yystack[pParser->yyidx].stateno;
int stateno = pParser->yytos->stateno;
if( stateno>=YY_MIN_REDUCE ) return stateno;
assert( stateno <= YY_SHIFT_COUNT );
do{
i = yy_shift_ofst[stateno];
if( i==YY_SHIFT_USE_DFLT ) return yy_default[stateno];
assert( iLookAhead!=YYNOCODE );
i += iLookAhead;
if( i<0 || i>=YY_ACTTAB_COUNT || yy_lookahead[i]!=iLookAhead ){
if( iLookAhead>0 ){
#ifdef YYFALLBACK
YYCODETYPE iFallback; /* Fallback token */
if( iLookAhead<sizeof(yyFallback)/sizeof(yyFallback[0])
&& (iFallback = yyFallback[iLookAhead])!=0 ){
YYCODETYPE iFallback; /* Fallback token */
if( iLookAhead<sizeof(yyFallback)/sizeof(yyFallback[0])
&& (iFallback = yyFallback[iLookAhead])!=0 ){
#ifndef NDEBUG
if( yyTraceFILE ){
fprintf(yyTraceFILE, "%sFALLBACK %s => %s\n",
yyTracePrompt, yyTokenName[iLookAhead], yyTokenName[iFallback]);
}
#endif
assert( yyFallback[iFallback]==0 ); /* Fallback loop must terminate */
iLookAhead = iFallback;
continue;
if( yyTraceFILE ){
fprintf(yyTraceFILE, "%sFALLBACK %s => %s\n",
yyTracePrompt, yyTokenName[iLookAhead], yyTokenName[iFallback]);
}
#endif
assert( yyFallback[iFallback]==0 ); /* Fallback loop must terminate */
iLookAhead = iFallback;
continue;
}
#endif
#ifdef YYWILDCARD
{
int j = i - iLookAhead + YYWILDCARD;
if(
{
int j = i - iLookAhead + YYWILDCARD;
if(
#if YY_SHIFT_MIN+YYWILDCARD<0
j>=0 &&
j>=0 &&
#endif
#if YY_SHIFT_MAX+YYWILDCARD>=YY_ACTTAB_COUNT
j<YY_ACTTAB_COUNT &&
j<YY_ACTTAB_COUNT &&
#endif
yy_lookahead[j]==YYWILDCARD
){
yy_lookahead[j]==YYWILDCARD && iLookAhead>0
){
#ifndef NDEBUG
if( yyTraceFILE ){
fprintf(yyTraceFILE, "%sWILDCARD %s => %s\n",
yyTracePrompt, yyTokenName[iLookAhead],
yyTokenName[YYWILDCARD]);
}
#endif /* NDEBUG */
return yy_action[j];
if( yyTraceFILE ){
fprintf(yyTraceFILE, "%sWILDCARD %s => %s\n",
yyTracePrompt, yyTokenName[iLookAhead],
yyTokenName[YYWILDCARD]);
}
#endif /* NDEBUG */
return yy_action[j];
}
#endif /* YYWILDCARD */
}
#endif /* YYWILDCARD */
return yy_default[stateno];
}else{
return yy_action[i];
@@ -516,13 +538,13 @@ static int yy_find_reduce_action(
*/
static void yyStackOverflow(yyParser *yypParser){
ParseARG_FETCH;
yypParser->yyidx--;
yypParser->yytos--;
#ifndef NDEBUG
if( yyTraceFILE ){
fprintf(yyTraceFILE,"%sStack Overflow!\n",yyTracePrompt);
}
#endif
while( yypParser->yyidx>=0 ) yy_pop_parser_stack(yypParser);
while( yypParser->yytos>yypParser->yystack ) yy_pop_parser_stack(yypParser);
/* Here code is inserted which will execute if the parser
** stack every overflows */
/******** Begin %stack_overflow code ******************************************/
@@ -539,11 +561,11 @@ static void yyTraceShift(yyParser *yypParser, int yyNewState){
if( yyTraceFILE ){
if( yyNewState<YYNSTATE ){
fprintf(yyTraceFILE,"%sShift '%s', go to state %d\n",
yyTracePrompt,yyTokenName[yypParser->yystack[yypParser->yyidx].major],
yyTracePrompt,yyTokenName[yypParser->yytos->major],
yyNewState);
}else{
fprintf(yyTraceFILE,"%sShift '%s'\n",
yyTracePrompt,yyTokenName[yypParser->yystack[yypParser->yyidx].major]);
yyTracePrompt,yyTokenName[yypParser->yytos->major]);
}
}
}
@@ -561,27 +583,30 @@ static void yy_shift(
ParseTOKENTYPE yyMinor /* The minor token to shift in */
){
yyStackEntry *yytos;
yypParser->yyidx++;
yypParser->yytos++;
#ifdef YYTRACKMAXSTACKDEPTH
if( yypParser->yyidx>yypParser->yyidxMax ){
yypParser->yyidxMax = yypParser->yyidx;
if( (int)(yypParser->yytos - yypParser->yystack)>yypParser->yyhwm ){
yypParser->yyhwm++;
assert( yypParser->yyhwm == (int)(yypParser->yytos - yypParser->yystack) );
}
#endif
#if YYSTACKDEPTH>0
if( yypParser->yyidx>=YYSTACKDEPTH ){
if( yypParser->yytos>=&yypParser->yystack[YYSTACKDEPTH] ){
yyStackOverflow(yypParser);
return;
}
#else
if( yypParser->yyidx>=yypParser->yystksz ){
yyGrowStack(yypParser);
if( yypParser->yyidx>=yypParser->yystksz ){
if( yypParser->yytos>=&yypParser->yystack[yypParser->yystksz] ){
if( yyGrowStack(yypParser) ){
yyStackOverflow(yypParser);
return;
}
}
#endif
yytos = &yypParser->yystack[yypParser->yyidx];
if( yyNewState > YY_MAX_SHIFT ){
yyNewState += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE;
}
yytos = yypParser->yytos;
yytos->stateno = (YYACTIONTYPE)yyNewState;
yytos->major = (YYCODETYPE)yyMajor;
yytos->minor.yy0 = yyMinor;
@@ -613,7 +638,7 @@ static void yy_reduce(
yyStackEntry *yymsp; /* The top of the parser's stack */
int yysize; /* Amount to pop the stack */
ParseARG_FETCH;
yymsp = &yypParser->yystack[yypParser->yyidx];
yymsp = yypParser->yytos;
#ifndef NDEBUG
if( yyTraceFILE && yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ){
yysize = yyRuleInfo[yyruleno].nrhs;
@@ -627,22 +652,23 @@ static void yy_reduce(
** enough on the stack to push the LHS value */
if( yyRuleInfo[yyruleno].nrhs==0 ){
#ifdef YYTRACKMAXSTACKDEPTH
if( yypParser->yyidx>yypParser->yyidxMax ){
yypParser->yyidxMax = yypParser->yyidx;
if( (int)(yypParser->yytos - yypParser->yystack)>yypParser->yyhwm ){
yypParser->yyhwm++;
assert( yypParser->yyhwm == (int)(yypParser->yytos - yypParser->yystack));
}
#endif
#if YYSTACKDEPTH>0
if( yypParser->yyidx>=YYSTACKDEPTH-1 ){
if( yypParser->yytos>=&yypParser->yystack[YYSTACKDEPTH-1] ){
yyStackOverflow(yypParser);
return;
}
#else
if( yypParser->yyidx>=yypParser->yystksz-1 ){
yyGrowStack(yypParser);
if( yypParser->yyidx>=yypParser->yystksz-1 ){
if( yypParser->yytos>=&yypParser->yystack[yypParser->yystksz-1] ){
if( yyGrowStack(yypParser) ){
yyStackOverflow(yypParser);
return;
}
yymsp = yypParser->yytos;
}
#endif
}
@@ -665,15 +691,17 @@ static void yy_reduce(
yysize = yyRuleInfo[yyruleno].nrhs;
yyact = yy_find_reduce_action(yymsp[-yysize].stateno,(YYCODETYPE)yygoto);
if( yyact <= YY_MAX_SHIFTREDUCE ){
if( yyact>YY_MAX_SHIFT ) yyact += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE;
yypParser->yyidx -= yysize - 1;
if( yyact>YY_MAX_SHIFT ){
yyact += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE;
}
yymsp -= yysize-1;
yypParser->yytos = yymsp;
yymsp->stateno = (YYACTIONTYPE)yyact;
yymsp->major = (YYCODETYPE)yygoto;
yyTraceShift(yypParser, yyact);
}else{
assert( yyact == YY_ACCEPT_ACTION );
yypParser->yyidx -= yysize;
yypParser->yytos -= yysize;
yy_accept(yypParser);
}
}
@@ -691,7 +719,7 @@ static void yy_parse_failed(
fprintf(yyTraceFILE,"%sFail!\n",yyTracePrompt);
}
#endif
while( yypParser->yyidx>=0 ) yy_pop_parser_stack(yypParser);
while( yypParser->yytos>yypParser->yystack ) yy_pop_parser_stack(yypParser);
/* Here code is inserted which will be executed whenever the
** parser fails */
/************ Begin %parse_failure code ***************************************/
@@ -729,7 +757,10 @@ static void yy_accept(
fprintf(yyTraceFILE,"%sAccept!\n",yyTracePrompt);
}
#endif
while( yypParser->yyidx>=0 ) yy_pop_parser_stack(yypParser);
#ifndef YYNOERRORRECOVERY
yypParser->yyerrcnt = -1;
#endif
assert( yypParser->yytos==yypParser->yystack );
/* Here code is inserted which will be executed whenever the
** parser accepts */
/*********** Begin %parse_accept code *****************************************/
@@ -773,28 +804,8 @@ void Parse(
#endif
yyParser *yypParser; /* The parser */
/* (re)initialize the parser, if necessary */
yypParser = (yyParser*)yyp;
if( yypParser->yyidx<0 ){
#if YYSTACKDEPTH<=0
if( yypParser->yystksz <=0 ){
yyStackOverflow(yypParser);
return;
}
#endif
yypParser->yyidx = 0;
#ifndef YYNOERRORRECOVERY
yypParser->yyerrcnt = -1;
#endif
yypParser->yystack[0].stateno = 0;
yypParser->yystack[0].major = 0;
#ifndef NDEBUG
if( yyTraceFILE ){
fprintf(yyTraceFILE,"%sInitialize. Empty stack. State 0\n",
yyTracePrompt);
}
#endif
}
assert( yypParser->yytos!=0 );
#if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY)
yyendofinput = (yymajor==0);
#endif
@@ -809,7 +820,6 @@ void Parse(
do{
yyact = yy_find_shift_action(yypParser,(YYCODETYPE)yymajor);
if( yyact <= YY_MAX_SHIFTREDUCE ){
if( yyact > YY_MAX_SHIFT ) yyact += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE;
yy_shift(yypParser,yyact,yymajor,yyminor);
#ifndef YYNOERRORRECOVERY
yypParser->yyerrcnt--;
@@ -851,7 +861,7 @@ void Parse(
if( yypParser->yyerrcnt<0 ){
yy_syntax_error(yypParser,yymajor,yyminor);
}
yymx = yypParser->yystack[yypParser->yyidx].major;
yymx = yypParser->yytos->major;
if( yymx==YYERRORSYMBOL || yyerrorhit ){
#ifndef NDEBUG
if( yyTraceFILE ){
@@ -862,18 +872,20 @@ void Parse(
yy_destructor(yypParser, (YYCODETYPE)yymajor, &yyminorunion);
yymajor = YYNOCODE;
}else{
while(
yypParser->yyidx >= 0 &&
yymx != YYERRORSYMBOL &&
(yyact = yy_find_reduce_action(
yypParser->yystack[yypParser->yyidx].stateno,
while( yypParser->yytos >= &yypParser->yystack
&& yymx != YYERRORSYMBOL
&& (yyact = yy_find_reduce_action(
yypParser->yytos->stateno,
YYERRORSYMBOL)) >= YY_MIN_REDUCE
){
yy_pop_parser_stack(yypParser);
}
if( yypParser->yyidx < 0 || yymajor==0 ){
if( yypParser->yytos < yypParser->yystack || yymajor==0 ){
yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion);
yy_parse_failed(yypParser);
#ifndef YYNOERRORRECOVERY
yypParser->yyerrcnt = -1;
#endif
yymajor = YYNOCODE;
}else if( yymx!=YYERRORSYMBOL ){
yy_shift(yypParser,yyact,YYERRORSYMBOL,yyminor);
@@ -910,18 +922,23 @@ void Parse(
yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion);
if( yyendofinput ){
yy_parse_failed(yypParser);
#ifndef YYNOERRORRECOVERY
yypParser->yyerrcnt = -1;
#endif
}
yymajor = YYNOCODE;
#endif
}
}while( yymajor!=YYNOCODE && yypParser->yyidx>=0 );
}while( yymajor!=YYNOCODE && yypParser->yytos>yypParser->yystack );
#ifndef NDEBUG
if( yyTraceFILE ){
int i;
yyStackEntry *i;
char cDiv = '[';
fprintf(yyTraceFILE,"%sReturn. Stack=",yyTracePrompt);
for(i=1; i<=yypParser->yyidx; i++)
fprintf(yyTraceFILE,"%c%s", i==1 ? '[' : ' ',
yyTokenName[yypParser->yystack[i].major]);
for(i=&yypParser->yystack[1]; i<=yypParser->yytos; i++){
fprintf(yyTraceFILE,"%c%s", cDiv, yyTokenName[i->major]);
cDiv = ' ';
}
fprintf(yyTraceFILE,"]\n");
}
#endif

15
tool/libvers.c Normal file
View File

@@ -0,0 +1,15 @@
/*
** Compile this program against an SQLite library of unknown version
** and then run this program, and it will print out the SQLite version
** information.
*/
#include <stdio.h>
extern const char *sqlite3_libversion(void);
extern const char *sqlite3_sourceid(void);
int main(int argc, char **argv){
printf("SQLite version %s\n", sqlite3_libversion());
printf("SQLite source %s\n", sqlite3_sourceid());
return 0;
}

2
tool/mksqlite3c-noext.tcl
View File

@@ -220,7 +220,7 @@ proc copy_file {filename} {
if {[lsearch -exact $cdecllist $funcname] >= 0} {
append line SQLITE_CDECL
} else {
append line SQLITE_STDCALL
append line SQLITE_APICALL
}
append line " " $funcname $rest
puts $out $line

34
tool/opcodesum.tcl Normal file
View File

@@ -0,0 +1,34 @@
#!/usr/bin/tclsh
#
# Run this script, redirecting input from cachegrind output, to compute the
# number of CPU cycles used by each VDBE opcode.
#
# The cachegrind output should be configured so that it reports a single
# column of Ir at the left margin. Ex:
#
# cg_annotation --show=Ir --auto=yes cachegrind.out.* | tclsh opcodesum.tcl
#
set currentop x
set ncycle(x) 0
while {![eof stdin]} {
set line [string map {\173 x \175 x \042 x} [gets stdin]]
if {[regexp { \. case OP_.*:} $line]} {
regexp {OP_(.+):} $line all currentop
set ncycle($currentop) 0
} elseif {[lindex $line 1]=="default:"
&& [regexp {really OP_Noop and OP_Explain} $line]} {
break
} elseif {[lindex $line 0]!="."} {
regsub -all {[^0-9]} [lindex $line 0] {} n
if {$n!=""} {incr ncycle($currentop) $n}
}
}
unset ncycle(x)
set results {}
foreach op [lsort [array names ncycle]] {
if {$ncycle($op)==0} continue
lappend results [list $ncycle($op) $op]
}
foreach entry [lsort -index 0 -int -decr $results] {
puts [format {%-16s %10d} [lindex $entry 1] [lindex $entry 0]]
}

5
tool/replace.tcl
View File

@@ -4,16 +4,19 @@
# only lines successfully modified with a regular
# expression.
#
fconfigure stdout -translation binary -encoding binary
fconfigure stderr -translation binary -encoding binary
set mode [string tolower [lindex $argv 0]]
set from [lindex $argv 1]
set to [lindex $argv 2]
if {$mode ni [list exact include]} {exit 1}
if {$mode ni [list exact regsub include]} {exit 1}
if {[string length $from]==0} {exit 2}
while {![eof stdin]} {
set line [gets stdin]
if {[eof stdin]} break
switch -exact $mode {
exact {set line [string map [list $from $to] $line]}
regsub {regsub -all -- $from $line $to line}
include {if {[regsub -all -- $from $line $to line]==0} continue}
}
puts stdout $line

21
tool/spaceanal.tcl
View File

@@ -152,6 +152,7 @@ set tabledef {CREATE TABLE space_used(
name clob, -- Name of a table or index in the database file
tblname clob, -- Name of associated table
is_index boolean, -- TRUE if it is an index, false for a table
is_without_rowid boolean, -- TRUE if WITHOUT ROWID table
nentry int, -- Number of entries in the BTree
leaf_entries int, -- Number of leaf entries
depth int, -- Depth of the b-tree
@@ -184,7 +185,7 @@ set sql { SELECT name, tbl_name FROM sqlite_master WHERE rootpage>0 }
foreach {name tblname} [concat sqlite_master sqlite_master [db eval $sql]] {
set is_index [expr {$name!=$tblname}]
set idx_btree [expr {$is_index || [is_without_rowid $name]}]
set is_without_rowid [is_without_rowid $name]
db eval {
SELECT
sum(ncell) AS nentry,
@@ -235,6 +236,7 @@ foreach {name tblname} [concat sqlite_master sqlite_master [db eval $sql]] {
$name,
$tblname,
$is_index,
$is_without_rowid,
$nentry,
$leaf_entries,
$depth,
@@ -330,12 +332,15 @@ proc subreport {title where showFrag} {
# following query returns exactly one row (because it is an aggregate).
#
# The results of the query are stored directly by SQLite into local
# variables (i.e. $nentry, $nleaf etc.).
# variables (i.e. $nentry, $payload etc.).
#
mem eval "
SELECT
int(sum(nentry)) AS nentry,
int(sum(leaf_entries)) AS nleaf,
int(sum(
CASE WHEN (is_without_rowid OR is_index) THEN nentry
ELSE leaf_entries
END
)) AS nentry,
int(sum(payload)) AS payload,
int(sum(ovfl_payload)) AS ovfl_payload,
max(mx_payload) AS mx_payload,
@@ -375,8 +380,8 @@ proc subreport {title where showFrag} {
set storage [expr {$total_pages*$pageSize}]
set payload_percent [percent $payload $storage {of storage consumed}]
set total_unused [expr {$ovfl_unused+$int_unused+$leaf_unused}]
set avg_payload [divide $payload $nleaf]
set avg_unused [divide $total_unused $nleaf]
set avg_payload [divide $payload $nentry]
set avg_unused [divide $total_unused $nentry]
if {$int_pages>0} {
# TODO: Is this formula correct?
set nTab [mem eval "
@@ -390,12 +395,12 @@ proc subreport {title where showFrag} {
"]
set avg_fanout [format %.2f $avg_fanout]
}
set ovfl_cnt_percent [percent $ovfl_cnt $nleaf {of all entries}]
set ovfl_cnt_percent [percent $ovfl_cnt $nentry {of all entries}]
# Print out the sub-report statistics.
#
statline {Percentage of total database} $total_pages_percent
statline {Number of entries} $nleaf
statline {Number of entries} $nentry
statline {Bytes of storage consumed} $storage
if {$compressed_size!=$storage} {
set compressed_size [expr {$compressed_size+$compressOverhead*$total_pages}]

105
tool/speed-check.sh Normal file
View File

@@ -0,0 +1,105 @@
#!/bin/bash
#
# This is a template for a script used for day-to-day size and
# performance monitoring of SQLite. Typical usage:
#
# sh run-speed-test.sh trunk # Baseline measurement of trunk
# sh run-speed-test.sh x1 # Measure some experimental change
# fossil test-diff --tk cout-trunk.txt cout-x1.txt # View chanages
#
# There are multiple output files, all with a base name given by
# the first argument:
#
# summary-$BASE.txt # Copy of standard output
# cout-$BASE.txt # cachegrind output
# explain-$BASE.txt # EXPLAIN listings (only with --explain)
#
if test "$1" = ""
then
echo "Usage: $0 OUTPUTFILE [OPTIONS]"
exit
fi
NAME=$1
shift
CC_OPTS="-DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_MEMSYS5"
SPEEDTEST_OPTS="--shrink-memory --reprepare --heap 10000000 64"
SIZE=5
doExplain=0
doCachegrind=1
while test "$1" != ""; do
case $1 in
--reprepare)
SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1"
;;
--autovacuum)
SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1"
;;
--utf16be)
SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1"
;;
--stats)
SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1"
;;
--without-rowid)
SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1"
;;
--nomemstat)
SPEEDTEST_OPTS="$SPEEDTEST_OPTS $1"
;;
--temp)
SPEEDTEST_OPTS="$SPEEDTEST_OPTS --temp 6"
;;
--wal)
SPEEDTEST_OPTS="$SPEEDTEST_OPTS --journal wal"
;;
--size)
shift; SIZE=$1
;;
--explain)
doExplain=1
;;
--vdbeprofile)
rm -f vdbe_profile.out
CC_OPTS="$CC_OPTS -DVDBE_PROFILE"
doCachegrind=0
;;
--heap)
CC_OPTS="$CC_OPTS -DSQLITE_ENABLE_MEMSYS5"
shift;
SPEEDTEST_OPTS="$SPEEDTEST_OPTS --heap $1 64"
;;
*)
CC_OPTS="$CC_OPTS $1"
;;
esac
shift
done
SPEEDTEST_OPTS="$SPEEDTEST_OPTS --size $SIZE"
echo "NAME = $NAME" | tee summary-$NAME.txt
echo "SPEEDTEST_OPTS = $SPEEDTEST_OPTS" | tee -a summary-$NAME.txt
echo "CC_OPTS = $CC_OPTS" | tee -a summary-$NAME.txt
rm -f cachegrind.out.* speedtest1 speedtest1.db sqlite3.o
gcc -g -Os -Wall -I. $CC_OPTS -c sqlite3.c
size sqlite3.o | tee -a summary-$NAME.txt
if test $doExplain -eq 1; then
gcc -g -Os -Wall -I. $CC_OPTS \
-DSQLITE_ENABLE_EXPLAIN_COMMENTS \
./shell.c ./sqlite3.c -o sqlite3 -ldl -lpthread
fi
SRC=./speedtest1.c
gcc -g -Os -Wall -I. $CC_OPTS $SRC ./sqlite3.o -o speedtest1 -ldl -lpthread
ls -l speedtest1 | tee -a summary-$NAME.txt
if test $doCachegrind -eq 1; then
valgrind --tool=cachegrind ./speedtest1 speedtest1.db \
$SPEEDTEST_OPTS 2>&1 | tee -a summary-$NAME.txt
else
./speedtest1 speedtest1.db $SPEEDTEST_OPTS 2>&1 | tee -a summary-$NAME.txt
fi
size sqlite3.o | tee -a summary-$NAME.txt
wc sqlite3.c
if test $doCachegrind -eq 1; then
cg_anno.tcl cachegrind.out.* >cout-$NAME.txt
fi
if test $doExplain -eq 1; then
./speedtest1 --explain $SPEEDTEST_OPTS | ./sqlite3 >explain-$NAME.txt
fi

158
tool/sqldiff.c
View File

@@ -33,6 +33,7 @@ struct GlobalVars {
const char *zArgv0; /* Name of program */
int bSchemaOnly; /* Only show schema differences */
int bSchemaPK; /* Use the schema-defined PK, not the true PK */
int bHandleVtab; /* Handle fts3, fts4, fts5 and rtree vtabs */
unsigned fDebug; /* Debug flags */
sqlite3 *db; /* The database connection */
} g;
@@ -683,7 +684,7 @@ static void diff_one_table(const char *zTab, FILE *out){
/* Run the query and output differences */
if( !g.bSchemaOnly ){
pStmt = db_prepare(sql.z);
pStmt = db_prepare("%s", sql.z);
while( SQLITE_ROW==sqlite3_step(pStmt) ){
int iType = sqlite3_column_int(pStmt, nPk);
if( iType==1 || iType==2 ){
@@ -1301,7 +1302,7 @@ static void rbudiff_one_table(const char *zTab, FILE *out){
char *zOtaControl;
int nOtaControl = sqlite3_column_bytes(pStmt, nCol);
zOtaControl = (char*)sqlite3_malloc(nOtaControl);
zOtaControl = (char*)sqlite3_malloc(nOtaControl+1);
memcpy(zOtaControl, sqlite3_column_text(pStmt, nCol), nOtaControl+1);
for(i=0; i<nCol; i++){
@@ -1460,7 +1461,7 @@ static void summarize_one_table(const char *zTab, FILE *out){
}
/* Run the query and output difference summary */
pStmt = db_prepare(sql.z);
pStmt = db_prepare("%s", sql.z);
nUpdate = 0;
nInsert = 0;
nDelete = 0;
@@ -1735,6 +1736,144 @@ end_changeset_one_table:
sqlite3_free(zId);
}
/*
** Extract the next SQL keyword or quoted string from buffer zIn and copy it
** (or a prefix of it if it will not fit) into buffer zBuf, size nBuf bytes.
** Return a pointer to the character within zIn immediately following
** the token or quoted string just extracted.
*/
const char *gobble_token(const char *zIn, char *zBuf, int nBuf){
const char *p = zIn;
char *pOut = zBuf;
char *pEnd = &pOut[nBuf-1];
char q = 0; /* quote character, if any */
if( p==0 ) return 0;
while( *p==' ' ) p++;
switch( *p ){
case '"': q = '"'; break;
case '\'': q = '\''; break;
case '`': q = '`'; break;
case '[': q = ']'; break;
}
if( q ){
p++;
while( *p && pOut<pEnd ){
if( *p==q ){
p++;
if( *p!=q ) break;
}
if( pOut<pEnd ) *pOut++ = *p;
p++;
}
}else{
while( *p && *p!=' ' && *p!='(' ){
if( pOut<pEnd ) *pOut++ = *p;
p++;
}
}
*pOut = '\0';
return p;
}
/*
** This function is the implementation of SQL scalar function "module_name":
**
** module_name(SQL)
**
** The only argument should be an SQL statement of the type that may appear
** in the sqlite_master table. If the statement is a "CREATE VIRTUAL TABLE"
** statement, then the value returned is the name of the module that it
** uses. Otherwise, if the statement is not a CVT, NULL is returned.
*/
static void module_name_func(
sqlite3_context *pCtx,
int nVal, sqlite3_value **apVal
){
const char *zSql;
char zToken[32];
assert( nVal==1 );
zSql = (const char*)sqlite3_value_text(apVal[0]);
zSql = gobble_token(zSql, zToken, sizeof(zToken));
if( zSql==0 || sqlite3_stricmp(zToken, "create") ) return;
zSql = gobble_token(zSql, zToken, sizeof(zToken));
if( zSql==0 || sqlite3_stricmp(zToken, "virtual") ) return;
zSql = gobble_token(zSql, zToken, sizeof(zToken));
if( zSql==0 || sqlite3_stricmp(zToken, "table") ) return;
zSql = gobble_token(zSql, zToken, sizeof(zToken));
if( zSql==0 ) return;
zSql = gobble_token(zSql, zToken, sizeof(zToken));
if( zSql==0 || sqlite3_stricmp(zToken, "using") ) return;
zSql = gobble_token(zSql, zToken, sizeof(zToken));
sqlite3_result_text(pCtx, zToken, -1, SQLITE_TRANSIENT);
}
/*
** Return the text of an SQL statement that itself returns the list of
** tables to process within the database.
*/
const char *all_tables_sql(){
if( g.bHandleVtab ){
int rc;
rc = sqlite3_exec(g.db,
"CREATE TEMP TABLE tblmap(module COLLATE nocase, postfix);"
"INSERT INTO temp.tblmap VALUES"
"('fts3', '_content'), ('fts3', '_segments'), ('fts3', '_segdir'),"
"('fts4', '_content'), ('fts4', '_segments'), ('fts4', '_segdir'),"
"('fts4', '_docsize'), ('fts4', '_stat'),"
"('fts5', '_data'), ('fts5', '_idx'), ('fts5', '_content'),"
"('fts5', '_docsize'), ('fts5', '_config'),"
"('rtree', '_node'), ('rtree', '_rowid'), ('rtree', '_parent');"
, 0, 0, 0
);
assert( rc==SQLITE_OK );
rc = sqlite3_create_function(
g.db, "module_name", 1, SQLITE_UTF8, 0, module_name_func, 0, 0
);
assert( rc==SQLITE_OK );
return
"SELECT name FROM main.sqlite_master\n"
" WHERE type='table' AND (\n"
" module_name(sql) IS NULL OR \n"
" module_name(sql) IN (SELECT module FROM temp.tblmap)\n"
" ) AND name NOT IN (\n"
" SELECT a.name || b.postfix \n"
"FROM main.sqlite_master AS a, temp.tblmap AS b \n"
"WHERE module_name(a.sql) = b.module\n"
" )\n"
"UNION \n"
"SELECT name FROM aux.sqlite_master\n"
" WHERE type='table' AND (\n"
" module_name(sql) IS NULL OR \n"
" module_name(sql) IN (SELECT module FROM temp.tblmap)\n"
" ) AND name NOT IN (\n"
" SELECT a.name || b.postfix \n"
"FROM aux.sqlite_master AS a, temp.tblmap AS b \n"
"WHERE module_name(a.sql) = b.module\n"
" )\n"
" ORDER BY name";
}else{
return
"SELECT name FROM main.sqlite_master\n"
" WHERE type='table' AND sql NOT LIKE 'CREATE VIRTUAL%%'\n"
" UNION\n"
"SELECT name FROM aux.sqlite_master\n"
" WHERE type='table' AND sql NOT LIKE 'CREATE VIRTUAL%%'\n"
" ORDER BY name";
}
}
/*
** Print sketchy documentation for this utility program
*/
@@ -1751,6 +1890,7 @@ static void showHelp(void){
" --summary Show only a summary of the differences\n"
" --table TAB Show only differences in table TAB\n"
" --transaction Show SQL output inside a transaction\n"
" --vtab Handle fts3, fts4, fts5 and rtree tables\n"
);
}
@@ -1821,6 +1961,9 @@ int main(int argc, char **argv){
if( strcmp(z,"transaction")==0 ){
useTransaction = 1;
}else
if( strcmp(z,"vtab")==0 ){
g.bHandleVtab = 1;
}else
{
cmdlineError("unknown option: %s", argv[i]);
}
@@ -1875,14 +2018,7 @@ int main(int argc, char **argv){
xDiff(zTab, out);
}else{
/* Handle tables one by one */
pStmt = db_prepare(
"SELECT name FROM main.sqlite_master\n"
" WHERE type='table' AND sql NOT LIKE 'CREATE VIRTUAL%%'\n"
" UNION\n"
"SELECT name FROM aux.sqlite_master\n"
" WHERE type='table' AND sql NOT LIKE 'CREATE VIRTUAL%%'\n"
" ORDER BY name"
);
pStmt = db_prepare("%s", all_tables_sql() );
while( SQLITE_ROW==sqlite3_step(pStmt) ){
xDiff((const char*)sqlite3_column_text(pStmt,0), out);
}

10
tool/srcck1.c
View File

@@ -58,7 +58,7 @@ static char *readFile(const char *zFilename){
return z;
}
/* Change the C code in the argument to see if it might have
/* Check the C code in the argument to see if it might have
** side effects. The only accurate way to know this is to do a full
** parse of the C code, which this routine does not do. This routine
** uses a simple heuristic of looking for:
@@ -111,11 +111,11 @@ static unsigned int findCloseParen(const char *z){
** Print error messages whenever a side effect is found. Return the number
** of problems seen.
*/
static unsigned int findAllSideEffects(const unsigned char *z){
static unsigned int findAllSideEffects(const char *z){
unsigned int lineno = 1; /* Line number */
unsigned int i;
unsigned int nErr = 0;
unsigned char c, prevC = 0;
char c, prevC = 0;
for(i=0; (c = z[i])!=0; prevC=c, i++){
if( c=='\n' ){ lineno++; continue; }
if( isalpha(c) && !isalpha(prevC) ){
@@ -125,7 +125,7 @@ static unsigned int findAllSideEffects(const unsigned char *z){
|| strncmp(&z[i],"testcase(",9)==0
){
unsigned int n;
unsigned const char *z2 = &z[i+5];
const char *z2 = &z[i+5];
while( z2[0]!='(' ){ z2++; }
z2++;
n = findCloseParen(z2);
@@ -141,7 +141,7 @@ static unsigned int findAllSideEffects(const unsigned char *z){
}
int main(int argc, char **argv){
unsigned char *z;
char *z;
unsigned int nErr = 0;
if( argc!=2 ){
fprintf(stderr, "Usage: %s FILENAME\n", argv[0]);