database/sqlite
1
0
Fork 0
mirror of https://github.com/sqlite/sqlite.git synced 2025-07-29 08:01:23 +03:00
Code Activity

Merge current trunk into this branch.

Browse Source 
FossilOrigin-Name: ed47d7f9a44b2af8ab7dd956495e71ea9a159cec438d1909f7022254a779e068
This commit is contained in:
stephan
2024-09-23 17:06:06 +00:00
parent 8ade1d2f97 5e419c1048
commit d69b4249ff
84 changed files with 5084 additions and 1364 deletions
Download Patch File Download Diff File Expand all files Collapse all files

91
ext/consio/console_io.c
View File

@ -595,6 +595,93 @@ oPutbUtf8(const char *cBuf, int nAccept){
# endif
}
/*
** Flush the given output stream. Return non-zero for success, else 0.
*/
#if !defined(SQLITE_CIO_NO_FLUSH) && !defined(SQLITE_CIO_NO_SETMODE)
SQLITE_INTERNAL_LINKAGE int
fFlushBuffer(FILE *pfOut){
# if CIO_WIN_WC_XLATE && !defined(SHELL_OMIT_FIO_DUPE)
return FlushFileBuffers(handleOfFile(pfOut))? 1 : 0;
# else
return fflush(pfOut);
# endif
}
#endif
#if CIO_WIN_WC_XLATE \
&& !defined(SHELL_OMIT_FIO_DUPE) \
&& defined(SQLITE_USE_ONLY_WIN32)
static struct FileAltIds {
int fd;
HANDLE fh;
} altIdsOfFile(FILE *pf){
struct FileAltIds rv = { _fileno(pf) };
union { intptr_t osfh; HANDLE fh; } fid = {
(rv.fd>=0)? _get_osfhandle(rv.fd) : (intptr_t)INVALID_HANDLE_VALUE
};
rv.fh = fid.fh;
return rv;
}
SQLITE_INTERNAL_LINKAGE size_t
cfWrite(const void *buf, size_t osz, size_t ocnt, FILE *pf){
size_t rv = 0;
struct FileAltIds fai = altIdsOfFile(pf);
int fmode = _setmode(fai.fd, _O_BINARY);
_setmode(fai.fd, fmode);
while( rv < ocnt ){
size_t nbo = osz;
while( nbo > 0 ){
DWORD dwno = (nbo>(1L<<24))? 1L<<24 : (DWORD)nbo;
BOOL wrc = TRUE;
BOOL genCR = (fmode & _O_TEXT)!=0;
if( genCR ){
const char *pnl = (const char*)memchr(buf, '\n', nbo);
if( pnl ) nbo = pnl - (const char*)buf;
else genCR = 0;
}
if( dwno>0 ) wrc = WriteFile(fai.fh, buf, dwno, 0,0);
if( genCR && wrc ){
wrc = WriteFile(fai.fh, "\r\n", 2, 0,0);
++dwno; /* Skip over the LF */
}
if( !wrc ) return rv;
buf = (const char*)buf + dwno;
nbo += dwno;
}
++rv;
}
return rv;
}
/* An fgets() equivalent, using Win32 file API for actual input.
** Input ends when given buffer is filled or a newline is read.
** If the FILE object is in text mode, swallows 0x0D. (ASCII CR)
*/
SQLITE_INTERNAL_LINKAGE char *
cfGets(char *cBuf, int n, FILE *pf){
int nci = 0;
struct FileAltIds fai = altIdsOfFile(pf);
int fmode = _setmode(fai.fd, _O_BINARY);
BOOL eatCR = (fmode & _O_TEXT)!=0;
_setmode(fai.fd, fmode);
while( nci < n-1 ){
DWORD nr;
if( !ReadFile(fai.fh, cBuf+nci, 1, &nr, 0) || nr==0 ) break;
if( nr>0 && (!eatCR || cBuf[nci]!='\r') ){
nci += nr;
if( cBuf[nci-nr]=='\n' ) break;
}
}
if( nci < n ) cBuf[nci] = 0;
return (nci>0)? cBuf : 0;
}
# else
# define cfWrite(b,os,no,f) fwrite(b,os,no,f)
# define cfGets(b,n,f) fgets(b,n,f)
# endif
# ifdef CONSIO_EPUTB
SQLITE_INTERNAL_LINKAGE int
ePutbUtf8(const char *cBuf, int nAccept){
@ -606,7 +693,7 @@ ePutbUtf8(const char *cBuf, int nAccept){
return conZstrEmit(ppst, cBuf, nAccept);
}else {
# endif
return (int)fwrite(cBuf, 1, nAccept, pfErr);
return (int)cfWrite(cBuf, 1, nAccept, pfErr);
# if CIO_WIN_WC_XLATE
}
# endif
@ -672,7 +759,7 @@ SQLITE_INTERNAL_LINKAGE char* fGetsUtf8(char *cBuf, int ncMax, FILE *pfIn){
# endif
}else{
# endif
return fgets(cBuf, ncMax, pfIn);
return cfGets(cBuf, ncMax, pfIn);
# if CIO_WIN_WC_XLATE
}
# endif

15
ext/consio/console_io.h
View File

@ -176,12 +176,19 @@ SQLITE_INTERNAL_LINKAGE int
ePutbUtf8(const char *cBuf, int nAccept);
#endif
/*
** Flush the given output stream. Return non-zero for success, else 0.
*/
#if !defined(SQLITE_CIO_NO_FLUSH) && !defined(SQLITE_CIO_NO_SETMODE)
SQLITE_INTERNAL_LINKAGE int
fFlushBuffer(FILE *pfOut);
#endif
/*
** Collect input like fgets(...) with special provisions for input
** from the console on platforms that require same. Defers to the
** C library fgets() when input is not from the console. Newline
** translation may be done as set by set{Binary,Text}Mode(). As a
** convenience, pfIn==NULL is treated as stdin.
** from the console on such platforms as require same. Newline
** translation may be done as set by set{Binary,Text}Mode().
** As a convenience, pfIn==NULL is treated as stdin.
*/
SQLITE_INTERNAL_LINKAGE char* fGetsUtf8(char *cBuf, int ncMax, FILE *pfIn);
/* Like fGetsUtf8 except stream is always the designated input. */

13
ext/fts5/fts5.h
View File

@ -493,18 +493,19 @@ struct Fts5ExtensionApi {
**
** FTS5_TOKENIZER
**
** There is also an fts5_tokenizer object. This is an older version of
** fts5_tokenizer_v2. It is similar except that:
** There is also an fts5_tokenizer object. This is an older, deprecated,
** version of fts5_tokenizer_v2. It is similar except that:
**
** <ul>
** <li> There is no "iVersion" field, and
** <li> The xTokenize() method does not take a locale argument.
** </ul>
**
** fts5_tokenizer tokenizers should be registered with the xCreateTokenizer()
** function, instead of xCreateTokenizer_v2(). Tokenizers implementations
** registered using either API may be retrieved using both xFindTokenizer()
** and xFindTokenizer_v2().
** Legacy fts5_tokenizer tokenizers must be registered using the
** legacy xCreateTokenizer() function, instead of xCreateTokenizer_v2().
**
** Tokenizer implementations registered using either API may be retrieved
** using both xFindTokenizer() and xFindTokenizer_v2().
**
** SYNONYM SUPPORT
**

15
ext/fts5/fts5Int.h
View File

@ -636,16 +636,13 @@ Fts5Table *sqlite3Fts5TableFromCsrid(Fts5Global*, i64);
int sqlite3Fts5FlushToDisk(Fts5Table*);
int sqlite3Fts5ExtractText(
Fts5Config *pConfig,
sqlite3_value *pVal, /* Value to extract text from */
int bContent, /* Loaded from content table */
int *pbResetTokenizer, /* OUT: True if ClearLocale() required */
const char **ppText, /* OUT: Pointer to text buffer */
int *pnText /* OUT: Size of (*ppText) in bytes */
);
void sqlite3Fts5ClearLocale(Fts5Config *pConfig);
void sqlite3Fts5SetLocale(Fts5Config *pConfig, const char *pLoc, int nLoc);
int sqlite3Fts5IsLocaleValue(Fts5Config *pConfig, sqlite3_value *pVal);
int sqlite3Fts5DecodeLocaleValue(sqlite3_value *pVal,
const char **ppText, int *pnText, const char **ppLoc, int *pnLoc
);
/*
** End of interface to code in fts5.c.

9
ext/fts5/fts5_config.c
View File

@ -516,6 +516,15 @@ static int fts5ConfigMakeExprlist(Fts5Config *p){
}
}
}
if( p->eContent==FTS5_CONTENT_NORMAL && p->bLocale ){
for(i=0; i<p->nCol; i++){
if( p->abUnindexed[i]==0 ){
sqlite3Fts5BufferAppendPrintf(&rc, &buf, ", T.l%d", i);
}else{
sqlite3Fts5BufferAppendPrintf(&rc, &buf, ", NULL");
}
}
}
assert( p->zContentExprlist==0 );
p->zContentExprlist = (char*)buf.p;

2
ext/fts5/fts5_index.c
View File

@ -2185,7 +2185,7 @@ static void fts5SegIterNext_None(
if( iOff<pIter->iEndofDoclist ){
/* Next entry is on the current page */
i64 iDelta;
u64 iDelta;
iOff += sqlite3Fts5GetVarint(&pIter->pLeaf->p[iOff], (u64*)&iDelta);
pIter->iLeafOffset = iOff;
pIter->iRowid += iDelta;

463
ext/fts5/fts5_main.c
View File

@ -83,8 +83,17 @@ struct Fts5Global {
Fts5TokenizerModule *pTok; /* First in list of all tokenizer modules */
Fts5TokenizerModule *pDfltTok; /* Default tokenizer module */
Fts5Cursor *pCsr; /* First in list of all open cursors */
u32 aLocaleHdr[4];
};
/*
** Size of header on fts5_locale() values. And macro to access a buffer
** containing a copy of the header from an Fts5Config pointer.
*/
#define FTS5_LOCALE_HDR_SIZE ((int)sizeof( ((Fts5Global*)0)->aLocaleHdr ))
#define FTS5_LOCALE_HDR(pConfig) ((const u8*)(pConfig->pGlobal->aLocaleHdr))
/*
** Each auxiliary function registered with the FTS5 module is represented
** by an object of the following type. All such objects are stored as part
@ -247,12 +256,6 @@ struct Fts5Cursor {
#define BitFlagAllTest(x,y) (((x) & (y))==(y))
#define BitFlagTest(x,y) (((x) & (y))!=0)
/*
** The subtype value and header bytes used by fts5_locale().
*/
#define FTS5_LOCALE_SUBTYPE ((unsigned int)'L')
#define FTS5_LOCALE_HEADER "\x00\xE0\xB2\xEB"
/*
** Macros to Set(), Clear() and Test() cursor flags.
@ -427,8 +430,7 @@ static int fts5InitVtab(
/* Load the initial configuration */
if( rc==SQLITE_OK ){
rc = sqlite3Fts5IndexLoadConfig(pTab->p.pIndex);
sqlite3Fts5IndexRollback(pTab->p.pIndex);
rc = sqlite3Fts5ConfigLoad(pTab->p.pConfig, pTab->p.pConfig->iCookie-1);
}
if( rc==SQLITE_OK && pConfig->eContent==FTS5_CONTENT_NORMAL ){
@ -1257,7 +1259,7 @@ static void fts5SetVtabError(Fts5FullTable *p, const char *zFormat, ...){
** valid until after the final call to sqlite3Fts5Tokenize() that will use
** the locale.
*/
static void fts5SetLocale(
static void sqlite3Fts5SetLocale(
Fts5Config *pConfig,
const char *zLocale,
int nLocale
@ -1268,127 +1270,74 @@ static void fts5SetLocale(
}
/*
** Clear any locale configured by an earlier call to fts5SetLocale() or
** sqlite3Fts5ExtractText().
** Clear any locale configured by an earlier call to sqlite3Fts5SetLocale().
*/
void sqlite3Fts5ClearLocale(Fts5Config *pConfig){
fts5SetLocale(pConfig, 0, 0);
sqlite3Fts5SetLocale(pConfig, 0, 0);
}
/*
** This function is used to extract utf-8 text from an sqlite3_value. This
** is usually done in order to tokenize it. For example, when:
**
** * a value is written to an fts5 table,
** * a value is deleted from an FTS5_CONTENT_NORMAL table,
** * a value containing a query expression is passed to xFilter()
**
** and so on.
**
** This function handles 2 cases:
**
** 1) Ordinary values. The text can be extracted from these using
** sqlite3_value_text().
**
** 2) Combination text/locale blobs created by fts5_locale(). There
** are several cases for these:
**
** * Blobs tagged with FTS5_LOCALE_SUBTYPE.
** * Blobs read from the content table of a locale=1 external-content
** table, and
** * Blobs read from the content table of a locale=1 regular
** content table.
**
** The first two cases above should have the 4 byte FTS5_LOCALE_HEADER
** header. It is an error if a blob with the subtype or a blob read
** from the content table of an external content table does not have
** the required header. A blob read from the content table of a regular
** locale=1 table does not have the header. This is to save space.
**
** If successful, SQLITE_OK is returned and output parameters (*ppText)
** and (*pnText) are set to point to a buffer containing the extracted utf-8
** text and its length in bytes, respectively. The buffer is not
** nul-terminated. It has the same lifetime as the sqlite3_value object
** from which it is extracted.
**
** Parameter bContent must be true if the value was read from an indexed
** column (i.e. not UNINDEXED) of the on disk content.
**
** If pbResetTokenizer is not NULL and if case (2) is used, then
** fts5SetLocale() is called to ensure subsequent sqlite3Fts5Tokenize() calls
** use the locale. In this case (*pbResetTokenizer) is set to true before
** returning, to indicate that the caller must call sqlite3Fts5ClearLocale()
** to clear the locale after tokenizing the text.
** Return true if the value passed as the only argument is an
** fts5_locale() value.
*/
int sqlite3Fts5ExtractText(
Fts5Config *pConfig,
sqlite3_value *pVal, /* Value to extract text from */
int bContent, /* True if indexed table content */
int *pbResetTokenizer, /* OUT: True if xSetLocale(NULL) required */
const char **ppText, /* OUT: Pointer to text buffer */
int *pnText /* OUT: Size of (*ppText) in bytes */
){
const char *pText = 0;
int nText = 0;
int rc = SQLITE_OK;
int bDecodeBlob = 0;
assert( pbResetTokenizer==0 || *pbResetTokenizer==0 );
assert( bContent==0 || pConfig->eContent!=FTS5_CONTENT_NONE );
assert( bContent==0 || sqlite3_value_subtype(pVal)==0 );
int sqlite3Fts5IsLocaleValue(Fts5Config *pConfig, sqlite3_value *pVal){
int ret = 0;
if( sqlite3_value_type(pVal)==SQLITE_BLOB ){
if( sqlite3_value_subtype(pVal)==FTS5_LOCALE_SUBTYPE
|| (bContent && pConfig->bLocale)
){
bDecodeBlob = 1;
}
}
if( bDecodeBlob ){
const int SZHDR = sizeof(FTS5_LOCALE_HEADER)-1;
/* Call sqlite3_value_bytes() after sqlite3_value_blob() in this case.
** If the blob was created using zeroblob(), then sqlite3_value_blob()
** may call malloc(). If this malloc() fails, then the values returned
** by both value_blob() and value_bytes() will be 0. If value_bytes() were
** called first, then the NULL pointer returned by value_blob() might
** be dereferenced. */
const u8 *pBlob = sqlite3_value_blob(pVal);
int nBlob = sqlite3_value_bytes(pVal);
/* Unless this blob was read from the %_content table of an
** FTS5_CONTENT_NORMAL table, it should have the 4 byte fts5_locale()
** header. Check for this. If it is not found, return an error. */
if( (!bContent || pConfig->eContent!=FTS5_CONTENT_NORMAL) ){
if( nBlob<SZHDR || memcmp(FTS5_LOCALE_HEADER, pBlob, SZHDR) ){
rc = SQLITE_ERROR;
}else{
pBlob += 4;
nBlob -= 4;
}
int nBlob = sqlite3_value_bytes(pVal);
if( nBlob>FTS5_LOCALE_HDR_SIZE
&& 0==memcmp(pBlob, FTS5_LOCALE_HDR(pConfig), FTS5_LOCALE_HDR_SIZE)
){
ret = 1;
}
if( rc==SQLITE_OK ){
int nLocale = 0;
for(nLocale=0; nLocale<nBlob; nLocale++){
if( pBlob[nLocale]==0x00 ) break;
}
if( nLocale==nBlob || nLocale==0 ){
rc = SQLITE_ERROR;
}else{
pText = (const char*)&pBlob[nLocale+1];
nText = nBlob-nLocale-1;
if( pbResetTokenizer ){
fts5SetLocale(pConfig, (const char*)pBlob, nLocale);
*pbResetTokenizer = 1;
}
}
}
}else{
pText = (const char*)sqlite3_value_text(pVal);
nText = sqlite3_value_bytes(pVal);
}
return ret;
}
*ppText = pText;
*pnText = nText;
return rc;
/*
** Value pVal is guaranteed to be an fts5_locale() value, according to
** sqlite3Fts5IsLocaleValue(). This function extracts the text and locale
** from the value and returns them separately.
**
** If successful, SQLITE_OK is returned and (*ppText) and (*ppLoc) set
** to point to buffers containing the text and locale, as utf-8,
** respectively. In this case output parameters (*pnText) and (*pnLoc) are
** set to the sizes in bytes of these two buffers.
**
** Or, if an error occurs, then an SQLite error code is returned. The final
** value of the four output parameters is undefined in this case.
*/
int sqlite3Fts5DecodeLocaleValue(
sqlite3_value *pVal,
const char **ppText,
int *pnText,
const char **ppLoc,
int *pnLoc
){
const char *p = sqlite3_value_blob(pVal);
int n = sqlite3_value_bytes(pVal);
int nLoc = 0;
assert( sqlite3_value_type(pVal)==SQLITE_BLOB );
assert( n>FTS5_LOCALE_HDR_SIZE );
for(nLoc=FTS5_LOCALE_HDR_SIZE; p[nLoc]; nLoc++){
if( nLoc==(n-1) ){
return SQLITE_MISMATCH;
}
}
*ppLoc = &p[FTS5_LOCALE_HDR_SIZE];
*pnLoc = nLoc - FTS5_LOCALE_HDR_SIZE;
*ppText = &p[nLoc+1];
*pnText = n - nLoc - 1;
return SQLITE_OK;
}
/*
@ -1397,8 +1346,8 @@ int sqlite3Fts5ExtractText(
** the text of the expression, and sets output variable (*pzText) to
** point to a nul-terminated buffer containing the expression.
**
** If pVal was an fts5_locale() value, then fts5SetLocale() is called to
** set the tokenizer to use the specified locale.
** If pVal was an fts5_locale() value, then sqlite3Fts5SetLocale() is called
** to set the tokenizer to use the specified locale.
**
** If output variable (*pbFreeAndReset) is set to true, then the caller
** is required to (a) call sqlite3Fts5ClearLocale() to reset the tokenizer
@ -1410,24 +1359,22 @@ static int fts5ExtractExprText(
char **pzText, /* OUT: nul-terminated buffer of text */
int *pbFreeAndReset /* OUT: Free (*pzText) and clear locale */
){
const char *zText = 0;
int nText = 0;
int rc = SQLITE_OK;
int bReset = 0;
*pbFreeAndReset = 0;
rc = sqlite3Fts5ExtractText(pConfig, pVal, 0, &bReset, &zText, &nText);
if( rc==SQLITE_OK ){
if( bReset ){
*pzText = sqlite3Fts5Mprintf(&rc, "%.*s", nText, zText);
if( rc!=SQLITE_OK ){
sqlite3Fts5ClearLocale(pConfig);
}else{
*pbFreeAndReset = 1;
}
}else{
*pzText = (char*)zText;
if( sqlite3Fts5IsLocaleValue(pConfig, pVal) ){
const char *pText = 0;
int nText = 0;
const char *pLoc = 0;
int nLoc = 0;
rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc);
*pzText = sqlite3Fts5Mprintf(&rc, "%.*s", nText, pText);
if( rc==SQLITE_OK ){
sqlite3Fts5SetLocale(pConfig, pLoc, nLoc);
}
*pbFreeAndReset = 1;
}else{
*pzText = (char*)sqlite3_value_text(pVal);
*pbFreeAndReset = 0;
}
return rc;
@ -1967,19 +1914,14 @@ static int fts5UpdateMethod(
else{
int eType1 = sqlite3_value_numeric_type(apVal[1]);
/* Ensure that no fts5_locale() values are written to locale=0 tables.
** And that no blobs except fts5_locale() blobs are written to indexed
** (i.e. not UNINDEXED) columns of locale=1 tables. */
int ii;
for(ii=0; ii<pConfig->nCol; ii++){
if( sqlite3_value_type(apVal[ii+2])==SQLITE_BLOB ){
int bSub = (sqlite3_value_subtype(apVal[ii+2])==FTS5_LOCALE_SUBTYPE);
if( (pConfig->bLocale && !bSub && pConfig->abUnindexed[ii]==0)
|| (pConfig->bLocale==0 && bSub)
){
if( pConfig->bLocale==0 ){
fts5SetVtabError(pTab, "fts5_locale() requires locale=1");
}
/* It is an error to write an fts5_locale() value to a table without
** the locale=1 option. */
if( pConfig->bLocale==0 ){
int ii;
for(ii=0; ii<pConfig->nCol; ii++){
sqlite3_value *pVal = apVal[ii+2];
if( sqlite3Fts5IsLocaleValue(pConfig, pVal) ){
fts5SetVtabError(pTab, "fts5_locale() requires locale=1");
rc = SQLITE_MISMATCH;
goto update_out;
}
@ -2068,9 +2010,11 @@ static int fts5SyncMethod(sqlite3_vtab *pVtab){
** Implementation of xBegin() method.
*/
static int fts5BeginMethod(sqlite3_vtab *pVtab){
fts5CheckTransactionState((Fts5FullTable*)pVtab, FTS5_BEGIN, 0);
fts5NewTransaction((Fts5FullTable*)pVtab);
return SQLITE_OK;
int rc = fts5NewTransaction((Fts5FullTable*)pVtab);
if( rc==SQLITE_OK ){
fts5CheckTransactionState((Fts5FullTable*)pVtab, FTS5_BEGIN, 0);
}
return rc;
}
/*
@ -2138,11 +2082,11 @@ static int fts5ApiTokenize_v2(
Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab);
int rc = SQLITE_OK;
fts5SetLocale(pTab->pConfig, pLoc, nLoc);
sqlite3Fts5SetLocale(pTab->pConfig, pLoc, nLoc);
rc = sqlite3Fts5Tokenize(pTab->pConfig,
FTS5_TOKENIZE_AUX, pText, nText, pUserData, xToken
);
fts5SetLocale(pTab->pConfig, 0, 0);
sqlite3Fts5SetLocale(pTab->pConfig, 0, 0);
return rc;
}
@ -2170,6 +2114,49 @@ static int fts5ApiPhraseSize(Fts5Context *pCtx, int iPhrase){
return sqlite3Fts5ExprPhraseSize(pCsr->pExpr, iPhrase);
}
/*
** Argument pStmt is an SQL statement of the type used by Fts5Cursor. This
** function extracts the text value of column iCol of the current row.
** Additionally, if there is an associated locale, it invokes
** sqlite3Fts5SetLocale() to configure the tokenizer. In all cases the caller
** should invoke sqlite3Fts5ClearLocale() to clear the locale at some point
** after this function returns.
**
** If successful, (*ppText) is set to point to a buffer containing the text
** value as utf-8 and SQLITE_OK returned. (*pnText) is set to the size of that
** buffer in bytes. It is not guaranteed to be nul-terminated. If an error
** occurs, an SQLite error code is returned. The final values of the two
** output parameters are undefined in this case.
*/
static int fts5TextFromStmt(
Fts5Config *pConfig,
sqlite3_stmt *pStmt,
int iCol,
const char **ppText,
int *pnText
){
sqlite3_value *pVal = sqlite3_column_value(pStmt, iCol+1);
const char *pLoc = 0;
int nLoc = 0;
int rc = SQLITE_OK;
if( pConfig->bLocale
&& pConfig->eContent==FTS5_CONTENT_EXTERNAL
&& sqlite3Fts5IsLocaleValue(pConfig, pVal)
){
rc = sqlite3Fts5DecodeLocaleValue(pVal, ppText, pnText, &pLoc, &nLoc);
}else{
*ppText = (const char*)sqlite3_value_text(pVal);
*pnText = sqlite3_value_bytes(pVal);
if( pConfig->bLocale && pConfig->eContent==FTS5_CONTENT_NORMAL ){
pLoc = (const char*)sqlite3_column_text(pStmt, iCol+1+pConfig->nCol);
nLoc = sqlite3_column_bytes(pStmt, iCol+1+pConfig->nCol);
}
}
sqlite3Fts5SetLocale(pConfig, pLoc, nLoc);
return rc;
}
static int fts5ApiColumnText(
Fts5Context *pCtx,
int iCol,
@ -2189,10 +2176,8 @@ static int fts5ApiColumnText(
}else{
rc = fts5SeekCursor(pCsr, 0);
if( rc==SQLITE_OK ){
Fts5Config *pConfig = pTab->pConfig;
int bContent = (pConfig->abUnindexed[iCol]==0);
sqlite3_value *pVal = sqlite3_column_value(pCsr->pStmt, iCol+1);
sqlite3Fts5ExtractText(pConfig, pVal, bContent, 0, pz, pn);
rc = fts5TextFromStmt(pTab->pConfig, pCsr->pStmt, iCol, pz, pn);
sqlite3Fts5ClearLocale(pTab->pConfig);
}
}
return rc;
@ -2234,17 +2219,15 @@ static int fts5CsrPoslist(
rc = fts5SeekCursor(pCsr, 0);
}
for(i=0; i<pConfig->nCol && rc==SQLITE_OK; i++){
sqlite3_value *pVal = sqlite3_column_value(pCsr->pStmt, i+1);
const char *z = 0;
int n = 0;
int bReset = 0;
rc = sqlite3Fts5ExtractText(pConfig, pVal, 1, &bReset, &z, &n);
rc = fts5TextFromStmt(pConfig, pCsr->pStmt, i, &z, &n);
if( rc==SQLITE_OK ){
rc = sqlite3Fts5ExprPopulatePoslists(
pConfig, pCsr->pExpr, aPopulator, i, z, n
);
}
if( bReset ) sqlite3Fts5ClearLocale(pConfig);
sqlite3Fts5ClearLocale(pConfig);
}
sqlite3_free(aPopulator);
@ -2430,17 +2413,14 @@ static int fts5ApiColumnSize(Fts5Context *pCtx, int iCol, int *pnToken){
if( pConfig->abUnindexed[i]==0 ){
const char *z = 0;
int n = 0;
int bReset = 0;
sqlite3_value *pVal = sqlite3_column_value(pCsr->pStmt, i+1);
pCsr->aColumnSize[i] = 0;
rc = sqlite3Fts5ExtractText(pConfig, pVal, 1, &bReset, &z, &n);
rc = fts5TextFromStmt(pConfig, pCsr->pStmt, i, &z, &n);
if( rc==SQLITE_OK ){
rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_AUX,
z, n, (void*)&pCsr->aColumnSize[i], fts5ColumnSizeCb
);
if( bReset ) sqlite3Fts5ClearLocale(pConfig);
}
sqlite3Fts5ClearLocale(pConfig);
}
}
}
@ -2712,37 +2692,14 @@ static int fts5ApiColumnLocale(
){
rc = fts5SeekCursor(pCsr, 0);
if( rc==SQLITE_OK ){
/* Load the value into pVal. pVal is a locale/text pair iff:
**
** 1) It is an SQLITE_BLOB, and
** 2) Either the subtype is FTS5_LOCALE_SUBTYPE, or else the
** value was loaded from an FTS5_CONTENT_NORMAL table, and
** 3) It does not begin with an 0x00 byte.
*/
sqlite3_value *pVal = sqlite3_column_value(pCsr->pStmt, iCol+1);
if( sqlite3_value_type(pVal)==SQLITE_BLOB ){
const u8 *pBlob = (const u8*)sqlite3_value_blob(pVal);
int nBlob = sqlite3_value_bytes(pVal);
if( pConfig->eContent==FTS5_CONTENT_EXTERNAL ){
const int SZHDR = sizeof(FTS5_LOCALE_HEADER)-1;
if( nBlob<SZHDR || memcmp(FTS5_LOCALE_HEADER, pBlob, SZHDR) ){
rc = SQLITE_ERROR;
}
pBlob += 4;
nBlob -= 4;
}
if( rc==SQLITE_OK ){
int nLocale = 0;
for(nLocale=0; nLocale<nBlob && pBlob[nLocale]!=0x00; nLocale++);
if( nLocale==nBlob || nLocale==0 ){
rc = SQLITE_ERROR;
}else{
/* A locale/text pair */
*pzLocale = (const char*)pBlob;
*pnLocale = nLocale;
}
}
const char *zDummy = 0;
int nDummy = 0;
rc = fts5TextFromStmt(pConfig, pCsr->pStmt, iCol, &zDummy, &nDummy);
if( rc==SQLITE_OK ){
*pzLocale = pConfig->t.pLocale;
*pnLocale = pConfig->t.nLocale;
}
sqlite3Fts5ClearLocale(pConfig);
}
}
@ -2963,57 +2920,6 @@ static int fts5PoslistBlob(sqlite3_context *pCtx, Fts5Cursor *pCsr){
return rc;
}
/*
** Value pVal was read from column iCol of the FTS5 table. This function
** returns it to the owner of pCtx via a call to an sqlite3_result_xxx()
** function. This function deals with the same cases as
** sqlite3Fts5ExtractText():
**
** 1) Ordinary values. These can be returned using sqlite3_result_value().
**
** 2) Blobs from fts5_locale(). The text is extracted from these and
** returned via sqlite3_result_text(). The locale is discarded.
*/
static void fts5ExtractValueFromColumn(
sqlite3_context *pCtx,
Fts5Config *pConfig,
int iCol,
sqlite3_value *pVal
){
assert( pConfig->eContent!=FTS5_CONTENT_NONE );
if( pConfig->bLocale
&& sqlite3_value_type(pVal)==SQLITE_BLOB
&& pConfig->abUnindexed[iCol]==0
){
const int SZHDR = sizeof(FTS5_LOCALE_HEADER)-1;
const u8 *pBlob = sqlite3_value_blob(pVal);
int nBlob = sqlite3_value_bytes(pVal);
int ii;
if( pConfig->eContent==FTS5_CONTENT_EXTERNAL ){
if( nBlob<SZHDR || memcmp(pBlob, FTS5_LOCALE_HEADER, SZHDR) ){
sqlite3_result_error_code(pCtx, SQLITE_ERROR);
return;
}else{
pBlob += 4;
nBlob -= 4;
}
}
for(ii=0; ii<nBlob && pBlob[ii]; ii++);
if( ii==0 || ii==nBlob ){
sqlite3_result_error_code(pCtx, SQLITE_ERROR);
}else{
const char *pText = (const char*)&pBlob[ii+1];
sqlite3_result_text(pCtx, pText, nBlob-ii-1, SQLITE_TRANSIENT);
}
return;
}
sqlite3_result_value(pCtx, pVal);
}
/*
** This is the xColumn method, called by SQLite to request a value from
** the row that the supplied cursor currently points to.
@ -3070,8 +2976,22 @@ static int fts5ColumnMethod(
rc = fts5SeekCursor(pCsr, 1);
if( rc==SQLITE_OK ){
sqlite3_value *pVal = sqlite3_column_value(pCsr->pStmt, iCol+1);
fts5ExtractValueFromColumn(pCtx, pConfig, iCol, pVal);
if( pConfig->bLocale
&& pConfig->eContent==FTS5_CONTENT_EXTERNAL
&& sqlite3Fts5IsLocaleValue(pConfig, pVal)
){
const char *z = 0;
int n = 0;
rc = fts5TextFromStmt(pConfig, pCsr->pStmt, iCol, &z, &n);
if( rc==SQLITE_OK ){
sqlite3_result_text(pCtx, z, n, SQLITE_TRANSIENT);
}
sqlite3Fts5ClearLocale(pConfig);
}else{
sqlite3_result_value(pCtx, pVal);
}
}
pConfig->pzErrmsg = 0;
}
}
@ -3267,7 +3187,9 @@ static int fts5NewTokenizerModule(
*/
typedef struct Fts5VtoVTokenizer Fts5VtoVTokenizer;
struct Fts5VtoVTokenizer {
Fts5TokenizerModule *pMod;
int bV2Native; /* True if v2 native tokenizer */
fts5_tokenizer x1; /* Tokenizer functions */
fts5_tokenizer_v2 x2; /* V2 tokenizer functions */
Fts5Tokenizer *pReal;
};
@ -3287,7 +3209,9 @@ static int fts5VtoVCreate(
pNew = (Fts5VtoVTokenizer*)sqlite3Fts5MallocZero(&rc, sizeof(*pNew));
if( rc==SQLITE_OK ){
pNew->pMod = pMod;
pNew->x1 = pMod->x1;
pNew->x2 = pMod->x2;
pNew->bV2Native = pMod->bV2Native;
if( pMod->bV2Native ){
rc = pMod->x2.xCreate(pMod->pUserData, azArg, nArg, &pNew->pReal);
}else{
@ -3309,11 +3233,10 @@ static int fts5VtoVCreate(
static void fts5VtoVDelete(Fts5Tokenizer *pTok){
Fts5VtoVTokenizer *p = (Fts5VtoVTokenizer*)pTok;
if( p ){
Fts5TokenizerModule *pMod = p->pMod;
if( pMod->bV2Native ){
pMod->x2.xDelete(p->pReal);
if( p->bV2Native ){
p->x2.xDelete(p->pReal);
}else{
pMod->x1.xDelete(p->pReal);
p->x1.xDelete(p->pReal);
}
sqlite3_free(p);
}
@ -3331,9 +3254,8 @@ static int fts5V1toV2Tokenize(
int (*xToken)(void*, int, const char*, int, int, int)
){
Fts5VtoVTokenizer *p = (Fts5VtoVTokenizer*)pTok;
Fts5TokenizerModule *pMod = p->pMod;
assert( pMod->bV2Native );
return pMod->x2.xTokenize(p->pReal, pCtx, flags, pText, nText, 0, 0, xToken);
assert( p->bV2Native );
return p->x2.xTokenize(p->pReal, pCtx, flags, pText, nText, 0, 0, xToken);
}
/*
@ -3348,10 +3270,9 @@ static int fts5V2toV1Tokenize(
int (*xToken)(void*, int, const char*, int, int, int)
){
Fts5VtoVTokenizer *p = (Fts5VtoVTokenizer*)pTok;
Fts5TokenizerModule *pMod = p->pMod;
assert( pMod->bV2Native==0 );
assert( p->bV2Native==0 );
UNUSED_PARAM2(pLocale,nLocale);
return pMod->x1.xTokenize(p->pReal, pCtx, flags, pText, nText, xToken);
return p->x1.xTokenize(p->pReal, pCtx, flags, pText, nText, xToken);
}
/*
@ -3632,13 +3553,12 @@ static void fts5LocaleFunc(
if( zLocale==0 || zLocale[0]=='\0' ){
sqlite3_result_text(pCtx, zText, nText, SQLITE_TRANSIENT);
}else{
Fts5Global *p = (Fts5Global*)sqlite3_user_data(pCtx);
u8 *pBlob = 0;
u8 *pCsr = 0;
int nBlob = 0;
const int nHdr = 4;
assert( sizeof(FTS5_LOCALE_HEADER)==nHdr+1 );
nBlob = nHdr + nLocale + 1 + nText;
nBlob = FTS5_LOCALE_HDR_SIZE + nLocale + 1 + nText;
pBlob = (u8*)sqlite3_malloc(nBlob);
if( pBlob==0 ){
sqlite3_result_error_nomem(pCtx);
@ -3646,8 +3566,8 @@ static void fts5LocaleFunc(
}
pCsr = pBlob;
memcpy(pCsr, FTS5_LOCALE_HEADER, nHdr);
pCsr += nHdr;
memcpy(pCsr, (const u8*)p->aLocaleHdr, FTS5_LOCALE_HDR_SIZE);
pCsr += FTS5_LOCALE_HDR_SIZE;
memcpy(pCsr, zLocale, nLocale);
pCsr += nLocale;
(*pCsr++) = 0x00;
@ -3655,7 +3575,6 @@ static void fts5LocaleFunc(
assert( &pCsr[nText]==&pBlob[nBlob] );
sqlite3_result_blob(pCtx, pBlob, nBlob, sqlite3_free);
sqlite3_result_subtype(pCtx, FTS5_LOCALE_SUBTYPE);
}
}
@ -3757,6 +3676,16 @@ static int fts5Init(sqlite3 *db){
pGlobal->api.xFindTokenizer = fts5FindTokenizer;
pGlobal->api.xCreateTokenizer_v2 = fts5CreateTokenizer_v2;
pGlobal->api.xFindTokenizer_v2 = fts5FindTokenizer_v2;
/* Initialize pGlobal->aLocaleHdr[] to a 128-bit pseudo-random vector.
** The constants below were generated randomly. */
sqlite3_randomness(sizeof(pGlobal->aLocaleHdr), pGlobal->aLocaleHdr);
pGlobal->aLocaleHdr[0] ^= 0xF924976D;
pGlobal->aLocaleHdr[1] ^= 0x16596E13;
pGlobal->aLocaleHdr[2] ^= 0x7C80BEAA;
pGlobal->aLocaleHdr[3] ^= 0x9B03A67F;
assert( sizeof(pGlobal->aLocaleHdr)==16 );
rc = sqlite3_create_module_v2(db, "fts5", &fts5Mod, p, fts5ModuleDestroy);
if( rc==SQLITE_OK ) rc = sqlite3Fts5IndexInit(db);
if( rc==SQLITE_OK ) rc = sqlite3Fts5ExprInit(pGlobal, db);

238
ext/fts5/fts5_storage.c
View File

@ -73,6 +73,30 @@ struct Fts5Storage {
#define FTS5_STMT_REPLACE_CONFIG 10
#define FTS5_STMT_SCAN 11
/*
** Return a pointer to a buffer obtained from sqlite3_malloc() that contains
** nBind comma-separated question marks. e.g. if nBind is passed 5, this
** function returns "?,?,?,?,?".
**
** If *pRc is not SQLITE_OK when this function is called, it is a no-op and
** NULL is returned immediately. Or, if the attempt to malloc a buffer
** fails, then *pRc is set to SQLITE_NOMEM and NULL is returned. Otherwise,
** if it is SQLITE_OK when this function is called and the malloc() succeeds,
** *pRc is left unchanged.
*/
static char *fts5BindingsList(int *pRc, int nBind){
char *zBind = sqlite3Fts5MallocZero(pRc, 1 + nBind*2);
if( zBind ){
int ii;
for(ii=0; ii<nBind; ii++){
zBind[ii*2] = '?';
zBind[ii*2 + 1] = ',';
}
zBind[ii*2-1] = '\0';
}
return zBind;
}
/*
** Prepare the two insert statements - Fts5Storage.pInsertContent and
** Fts5Storage.pInsertDocsize - if they have not already been prepared.
@ -141,19 +165,20 @@ static int fts5StorageGetStmt(
);
break;
case FTS5_STMT_INSERT_CONTENT:
case FTS5_STMT_REPLACE_CONTENT: {
int nCol = pC->nCol + 1;
case FTS5_STMT_INSERT_CONTENT: {
int nCol = 0;
char *zBind;
int i;
zBind = sqlite3_malloc64(1 + nCol*2);
if( zBind ){
for(i=0; i<nCol; i++){
zBind[i*2] = '?';
zBind[i*2 + 1] = ',';
nCol = 1 + pC->nCol;
if( pC->bLocale ){
for(i=0; i<pC->nCol; i++){
if( pC->abUnindexed[i]==0 ) nCol++;
}
zBind[i*2-1] = '\0';
}
zBind = fts5BindingsList(&rc, nCol);
if( zBind ){
zSql = sqlite3_mprintf(azStmt[eStmt], pC->zDb, pC->zName, zBind);
sqlite3_free(zBind);
}
@ -344,7 +369,7 @@ int sqlite3Fts5StorageOpen(
if( bCreate ){
if( pConfig->eContent==FTS5_CONTENT_NORMAL ){
int nDefn = 32 + pConfig->nCol*10;
char *zDefn = sqlite3_malloc64(32 + (sqlite3_int64)pConfig->nCol * 10);
char *zDefn = sqlite3_malloc64(32 + (sqlite3_int64)pConfig->nCol * 20);
if( zDefn==0 ){
rc = SQLITE_NOMEM;
}else{
@ -356,6 +381,14 @@ int sqlite3Fts5StorageOpen(
sqlite3_snprintf(nDefn-iOff, &zDefn[iOff], ", c%d", i);
iOff += (int)strlen(&zDefn[iOff]);
}
if( pConfig->bLocale ){
for(i=0; i<pConfig->nCol; i++){
if( pConfig->abUnindexed[i]==0 ){
sqlite3_snprintf(nDefn-iOff, &zDefn[iOff], ", l%d", i);
iOff += (int)strlen(&zDefn[iOff]);
}
}
}
rc = sqlite3Fts5CreateTable(pConfig, "content", zDefn, 0, pzErr);
}
sqlite3_free(zDefn);
@ -507,7 +540,8 @@ static int fts5StorageDeleteFromIndex(
sqlite3_value *pVal = 0;
const char *pText = 0;
int nText = 0;
int bReset = 0;
const char *pLoc = 0;
int nLoc = 0;
assert( pSeek==0 || apVal==0 );
assert( pSeek!=0 || apVal!=0 );
@ -517,10 +551,19 @@ static int fts5StorageDeleteFromIndex(
pVal = apVal[iCol-1];
}
rc = sqlite3Fts5ExtractText(
pConfig, pVal, pSeek!=0, &bReset, &pText, &nText
);
if( pConfig->bLocale && sqlite3Fts5IsLocaleValue(pConfig, pVal) ){
rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc);
}else{
pText = (const char*)sqlite3_value_text(pVal);
nText = sqlite3_value_bytes(pVal);
if( pConfig->bLocale && pSeek ){
pLoc = (const char*)sqlite3_column_text(pSeek, iCol + pConfig->nCol);
nLoc = sqlite3_column_bytes(pSeek, iCol + pConfig->nCol);
}
}
if( rc==SQLITE_OK ){
sqlite3Fts5SetLocale(pConfig, pLoc, nLoc);
ctx.szCol = 0;
rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT,
pText, nText, (void*)&ctx, fts5StorageInsertCallback
@ -529,7 +572,7 @@ static int fts5StorageDeleteFromIndex(
if( rc==SQLITE_OK && p->aTotalSize[iCol-1]<0 ){
rc = FTS5_CORRUPT;
}
if( bReset ) sqlite3Fts5ClearLocale(pConfig);
sqlite3Fts5ClearLocale(pConfig);
}
}
}
@ -788,20 +831,35 @@ int sqlite3Fts5StorageRebuild(Fts5Storage *p){
for(ctx.iCol=0; rc==SQLITE_OK && ctx.iCol<pConfig->nCol; ctx.iCol++){
ctx.szCol = 0;
if( pConfig->abUnindexed[ctx.iCol]==0 ){
int bReset = 0; /* True if tokenizer locale must be reset */
int nText = 0; /* Size of pText in bytes */
const char *pText = 0; /* Pointer to buffer containing text value */
sqlite3_value *pVal = sqlite3_column_value(pScan, ctx.iCol+1);
int nLoc = 0; /* Size of pLoc in bytes */
const char *pLoc = 0; /* Pointer to buffer containing text value */
sqlite3_value *pVal = sqlite3_column_value(pScan, ctx.iCol+1);
if( pConfig->eContent==FTS5_CONTENT_EXTERNAL
&& sqlite3Fts5IsLocaleValue(pConfig, pVal)
){
rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc);
}else{
pText = (const char*)sqlite3_value_text(pVal);
nText = sqlite3_value_bytes(pVal);
if( pConfig->bLocale ){
int iCol = ctx.iCol + 1 + pConfig->nCol;
pLoc = (const char*)sqlite3_column_text(pScan, iCol);
nLoc = sqlite3_column_bytes(pScan, iCol);
}
}
rc = sqlite3Fts5ExtractText(pConfig, pVal, 1, &bReset, &pText, &nText);
if( rc==SQLITE_OK ){
sqlite3Fts5SetLocale(pConfig, pLoc, nLoc);
rc = sqlite3Fts5Tokenize(pConfig,
FTS5_TOKENIZE_DOCUMENT,
pText, nText,
(void*)&ctx,
fts5StorageInsertCallback
);
if( bReset ) sqlite3Fts5ClearLocale(pConfig);
sqlite3Fts5ClearLocale(pConfig);
}
}
sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol);
@ -884,29 +942,45 @@ int sqlite3Fts5StorageContentInsert(
}else{
sqlite3_stmt *pInsert = 0; /* Statement to write %_content table */
int i; /* Counter variable */
int nIndexed = 0; /* Number indexed columns seen */
rc = fts5StorageGetStmt(p, FTS5_STMT_INSERT_CONTENT, &pInsert, 0);
for(i=1; rc==SQLITE_OK && i<=pConfig->nCol+1; i++){
if( pInsert ) sqlite3_clear_bindings(pInsert);
/* Bind the rowid value */
sqlite3_bind_value(pInsert, 1, apVal[1]);
/* Loop through values for user-defined columns. i=2 is the leftmost
** user-defined column. As is column 1 of pSavedRow. */
for(i=2; rc==SQLITE_OK && i<=pConfig->nCol+1; i++){
int bUnindexed = pConfig->abUnindexed[i-2];
sqlite3_value *pVal = apVal[i];
nIndexed += !bUnindexed;
if( sqlite3_value_nochange(pVal) && p->pSavedRow ){
/* This is an UPDATE statement, and column (i-2) was not modified.
** Retrieve the value from Fts5Storage.pSavedRow instead. */
pVal = sqlite3_column_value(p->pSavedRow, i-1);
}else if( sqlite3_value_subtype(pVal)==FTS5_LOCALE_SUBTYPE ){
assert( pConfig->bLocale );
assert( i>1 );
if( pConfig->abUnindexed[i-2] ){
/* At attempt to insert an fts5_locale() value into an UNINDEXED
** column. Strip the locale away and just bind the text. */
const char *pText = 0;
int nText = 0;
rc = sqlite3Fts5ExtractText(pConfig, pVal, 0, 0, &pText, &nText);
sqlite3_bind_text(pInsert, i, pText, nText, SQLITE_TRANSIENT);
}else{
const u8 *pBlob = (const u8*)sqlite3_value_blob(pVal);
int nBlob = sqlite3_value_bytes(pVal);
assert( nBlob>4 );
sqlite3_bind_blob(pInsert, i, pBlob+4, nBlob-4, SQLITE_TRANSIENT);
if( pConfig->bLocale && bUnindexed==0 ){
sqlite3_bind_value(pInsert, pConfig->nCol + 1 + nIndexed,
sqlite3_column_value(p->pSavedRow, pConfig->nCol + i - 1)
);
}
}else if( sqlite3Fts5IsLocaleValue(pConfig, pVal) ){
const char *pText = 0;
const char *pLoc = 0;
int nText = 0;
int nLoc = 0;
assert( pConfig->bLocale );
rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc);
if( rc==SQLITE_OK ){
sqlite3_bind_text(pInsert, i, pText, nText, SQLITE_TRANSIENT);
if( bUnindexed==0 ){
int iLoc = pConfig->nCol + 1 + nIndexed;
sqlite3_bind_text(pInsert, iLoc, pLoc, nLoc, SQLITE_TRANSIENT);
}
}
continue;
}
@ -945,23 +1019,37 @@ int sqlite3Fts5StorageIndexInsert(
for(ctx.iCol=0; rc==SQLITE_OK && ctx.iCol<pConfig->nCol; ctx.iCol++){
ctx.szCol = 0;
if( pConfig->abUnindexed[ctx.iCol]==0 ){
int bReset = 0; /* True if tokenizer locale must be reset */
int nText = 0; /* Size of pText in bytes */
const char *pText = 0; /* Pointer to buffer containing text value */
int nLoc = 0; /* Size of pText in bytes */
const char *pLoc = 0; /* Pointer to buffer containing text value */
sqlite3_value *pVal = apVal[ctx.iCol+2];
int bDisk = 0;
if( p->pSavedRow && sqlite3_value_nochange(pVal) ){
pVal = sqlite3_column_value(p->pSavedRow, ctx.iCol+1);
bDisk = 1;
if( pConfig->eContent==FTS5_CONTENT_NORMAL && pConfig->bLocale ){
int iCol = ctx.iCol + 1 + pConfig->nCol;
pLoc = (const char*)sqlite3_column_text(p->pSavedRow, iCol);
nLoc = sqlite3_column_bytes(p->pSavedRow, iCol);
}
}else{
pVal = apVal[ctx.iCol+2];
}
rc = sqlite3Fts5ExtractText(pConfig, pVal, bDisk, &bReset, &pText,&nText);
if( pConfig->bLocale && sqlite3Fts5IsLocaleValue(pConfig, pVal) ){
rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc);
}else{
pText = (const char*)sqlite3_value_text(pVal);
nText = sqlite3_value_bytes(pVal);
}
if( rc==SQLITE_OK ){
assert( bReset==0 || pConfig->bLocale );
sqlite3Fts5SetLocale(pConfig, pLoc, nLoc);
rc = sqlite3Fts5Tokenize(pConfig,
FTS5_TOKENIZE_DOCUMENT, pText, nText, (void*)&ctx,
fts5StorageInsertCallback
);
if( bReset ) sqlite3Fts5ClearLocale(pConfig);
sqlite3Fts5ClearLocale(pConfig);
}
}
sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol);
@ -1126,37 +1214,61 @@ int sqlite3Fts5StorageIntegrity(Fts5Storage *p, int iArg){
rc = sqlite3Fts5TermsetNew(&ctx.pTermset);
}
for(i=0; rc==SQLITE_OK && i<pConfig->nCol; i++){
if( pConfig->abUnindexed[i] ) continue;
ctx.iCol = i;
ctx.szCol = 0;
if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){
rc = sqlite3Fts5TermsetNew(&ctx.pTermset);
}
if( rc==SQLITE_OK ){
int bReset = 0; /* True if tokenizer locale must be reset */
int nText = 0; /* Size of pText in bytes */
const char *pText = 0; /* Pointer to buffer containing text value */
if( pConfig->abUnindexed[i]==0 ){
const char *pText = 0;
int nText = 0;
const char *pLoc = 0;
int nLoc = 0;
sqlite3_value *pVal = sqlite3_column_value(pScan, i+1);
if( pConfig->eContent==FTS5_CONTENT_EXTERNAL
&& sqlite3Fts5IsLocaleValue(pConfig, pVal)
){
rc = sqlite3Fts5DecodeLocaleValue(
pVal, &pText, &nText, &pLoc, &nLoc
);
}else{
if( pConfig->eContent==FTS5_CONTENT_NORMAL && pConfig->bLocale ){
int iCol = i + 1 + pConfig->nCol;
pLoc = (const char*)sqlite3_column_text(pScan, iCol);
nLoc = sqlite3_column_bytes(pScan, iCol);
}
pText = (const char*)sqlite3_value_text(pVal);
nText = sqlite3_value_bytes(pVal);
}
ctx.iCol = i;
ctx.szCol = 0;
if( rc==SQLITE_OK && pConfig->eDetail==FTS5_DETAIL_COLUMNS ){
rc = sqlite3Fts5TermsetNew(&ctx.pTermset);
}
rc = sqlite3Fts5ExtractText(pConfig,
sqlite3_column_value(pScan, i+1), 1, &bReset, &pText, &nText
);
if( rc==SQLITE_OK ){
sqlite3Fts5SetLocale(pConfig, pLoc, nLoc);
rc = sqlite3Fts5Tokenize(pConfig,
FTS5_TOKENIZE_DOCUMENT,
pText, nText,
(void*)&ctx,
fts5StorageIntegrityCallback
);
if( bReset ) sqlite3Fts5ClearLocale(pConfig);
sqlite3Fts5ClearLocale(pConfig);
}
/* If this is not a columnsize=0 database, check that the number
** of tokens in the value matches the aColSize[] value read from
** the %_docsize table. */
if( rc==SQLITE_OK
&& pConfig->bColumnsize
&& ctx.szCol!=aColSize[i]
){
rc = FTS5_CORRUPT;
}
aTotalSize[i] += ctx.szCol;
if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){
sqlite3Fts5TermsetFree(ctx.pTermset);
ctx.pTermset = 0;
}
}
if( rc==SQLITE_OK && pConfig->bColumnsize && ctx.szCol!=aColSize[i] ){
rc = FTS5_CORRUPT;
}
aTotalSize[i] += ctx.szCol;
if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){
sqlite3Fts5TermsetFree(ctx.pTermset);
ctx.pTermset = 0;
}
}
sqlite3Fts5TermsetFree(ctx.pTermset);

111
ext/fts5/fts5_tcl.c
View File

@ -96,14 +96,14 @@ static int SQLITE_TCLAPI f5tDbAndApi(
rc = sqlite3_prepare_v2(db, "SELECT fts5(?1)", -1, &pStmt, 0);
if( rc!=SQLITE_OK ){
Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), 0);
Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), (char*)0);
return TCL_ERROR;
}
sqlite3_bind_pointer(pStmt, 1, (void*)&pApi, "fts5_api_ptr", 0);
sqlite3_step(pStmt);
if( sqlite3_finalize(pStmt)!=SQLITE_OK ){
Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), 0);
Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), (char*)0);
return TCL_ERROR;
}
@ -392,7 +392,7 @@ static int SQLITE_TCLAPI xF5tApi(
CASE(12, "xSetAuxdata") {
F5tAuxData *pData = (F5tAuxData*)sqlite3_malloc(sizeof(F5tAuxData));
if( pData==0 ){
Tcl_AppendResult(interp, "out of memory", 0);
Tcl_AppendResult(interp, "out of memory", (char*)0);
return TCL_ERROR;
}
pData->pObj = objv[2];
@ -452,7 +452,7 @@ static int SQLITE_TCLAPI xF5tApi(
rc = p->pApi->xPhraseFirst(p->pFts, iPhrase, &iter, &iCol, &iOff);
if( rc!=SQLITE_OK ){
Tcl_AppendResult(interp, sqlite3ErrName(rc), 0);
Tcl_AppendResult(interp, sqlite3ErrName(rc), (char*)0);
return TCL_ERROR;
}
for( ;iCol>=0; p->pApi->xPhraseNext(p->pFts, &iter, &iCol, &iOff) ){
@ -683,7 +683,7 @@ static int SQLITE_TCLAPI f5tCreateFunction(
pApi, zName, (void*)pCtx, xF5tFunction, xF5tDestroy
);
if( rc!=SQLITE_OK ){
Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), 0);
Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), (char*)0);
return TCL_ERROR;
}
@ -750,7 +750,7 @@ static int SQLITE_TCLAPI f5tTokenize(
if( objc==5 ){
char *zOpt = Tcl_GetString(objv[1]);
if( strcmp("-subst", zOpt) ){
Tcl_AppendResult(interp, "unrecognized option: ", zOpt, 0);
Tcl_AppendResult(interp, "unrecognized option: ", zOpt, (char*)0);
return TCL_ERROR;
}
}
@ -759,7 +759,7 @@ static int SQLITE_TCLAPI f5tTokenize(
return TCL_ERROR;
}
if( nArg==0 ){
Tcl_AppendResult(interp, "no such tokenizer: ", 0);
Tcl_AppendResult(interp, "no such tokenizer: ", (char*)0);
Tcl_Free((void*)azArg);
return TCL_ERROR;
}
@ -767,13 +767,13 @@ static int SQLITE_TCLAPI f5tTokenize(
rc = pApi->xFindTokenizer(pApi, azArg[0], &pUserdata, &tokenizer);
if( rc!=SQLITE_OK ){
Tcl_AppendResult(interp, "no such tokenizer: ", azArg[0], 0);
Tcl_AppendResult(interp, "no such tokenizer: ", azArg[0], (char*)0);
return TCL_ERROR;
}
rc = tokenizer.xCreate(pUserdata, &azArg[1], (int)(nArg-1), &pTok);
if( rc!=SQLITE_OK ){
Tcl_AppendResult(interp, "error in tokenizer.xCreate()", 0);
Tcl_AppendResult(interp, "error in tokenizer.xCreate()", (char*)0);
return TCL_ERROR;
}
@ -787,7 +787,7 @@ static int SQLITE_TCLAPI f5tTokenize(
);
tokenizer.xDelete(pTok);
if( rc!=SQLITE_OK ){
Tcl_AppendResult(interp, "error in tokenizer.xTokenize()", 0);
Tcl_AppendResult(interp, "error in tokenizer.xTokenize()", (char*)0);
Tcl_DecrRefCount(pRet);
return TCL_ERROR;
}
@ -1049,7 +1049,7 @@ static int SQLITE_TCLAPI f5tTokenizerLocale(
if( p->xToken==0 ){
Tcl_AppendResult(interp,
"sqlite3_fts5_locale may only be used by tokenizer callback", 0
"sqlite3_fts5_locale may only be used by tokenizer callback", (char*)0
);
return TCL_ERROR;
}
@ -1098,7 +1098,7 @@ static int SQLITE_TCLAPI f5tTokenizerReturn(
if( p->xToken==0 ){
Tcl_AppendResult(interp,
"sqlite3_fts5_token may only be used by tokenizer callback", 0
"sqlite3_fts5_token may only be used by tokenizer callback", (char*)0
);
return TCL_ERROR;
}
@ -1250,7 +1250,7 @@ static int SQLITE_TCLAPI f5tCreateTokenizer(
Tcl_AppendResult(interp, (
bV2 ? "error in fts5_api.xCreateTokenizer_v2()"
: "error in fts5_api.xCreateTokenizer()"
), 0);
), (char*)0);
return TCL_ERROR;
}
@ -1540,12 +1540,92 @@ static int SQLITE_TCLAPI f5tRegisterOriginText(
Tcl_ResetResult(interp);
if( rc!=SQLITE_OK ){
Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), 0);
Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), (void*)0);
return TCL_ERROR;
}
return TCL_OK;
}
/*
** This function is used to DROP an fts5 table. It works even if the data
** structures fts5 stores within the database are corrupt, which sometimes
** prevents a straight "DROP TABLE" command from succeeding.
**
** The first parameter is the database handle to use for the DROP TABLE
** operation. The second is the name of the database to drop the fts5 table
** from (i.e. "main", "temp" or the name of an attached database). The
** third parameter is the name of the fts5 table to drop.
**
** SQLITE_OK is returned if the table is successfully dropped. Or, if an
** error occurs, an SQLite error code.
*/
static int sqlite3_fts5_drop_corrupt_table(
sqlite3 *db, /* Database handle */
const char *zDb, /* Database name ("main", "temp" etc.) */
const char *zTab /* Name of fts5 table to drop */
){
int rc = SQLITE_OK;
int bDef = 0;
rc = sqlite3_db_config(db, SQLITE_DBCONFIG_DEFENSIVE, -1, &bDef);
if( rc==SQLITE_OK ){
char *zScript = sqlite3_mprintf(
"DELETE FROM %Q.'%q_data';"
"DELETE FROM %Q.'%q_config';"
"INSERT INTO %Q.'%q_data' VALUES(10, X'0000000000');"
"INSERT INTO %Q.'%q_config' VALUES('version', 4);"
"DROP TABLE %Q.'%q';",
zDb, zTab, zDb, zTab, zDb, zTab, zDb, zTab, zDb, zTab
);
if( zScript==0 ){
rc = SQLITE_NOMEM;
}else{
if( bDef ) sqlite3_db_config(db, SQLITE_DBCONFIG_DEFENSIVE, 0, 0);
rc = sqlite3_exec(db, zScript, 0, 0, 0);
if( bDef ) sqlite3_db_config(db, SQLITE_DBCONFIG_DEFENSIVE, 1, 0);
sqlite3_free(zScript);
}
}
return rc;
}
/*
** sqlite3_fts5_drop_corrupt_table DB DATABASE TABLE
**
** Description...
*/
static int SQLITE_TCLAPI f5tDropCorruptTable(
void * clientData,
Tcl_Interp *interp,
int objc,
Tcl_Obj *CONST objv[]
){
sqlite3 *db = 0;
const char *zDb = 0;
const char *zTab = 0;
int rc = SQLITE_OK;
if( objc!=4 ){
Tcl_WrongNumArgs(interp, 1, objv, "DB DATABASE TABLE");
return TCL_ERROR;
}
if( f5tDbPointer(interp, objv[1], &db) ){
return TCL_ERROR;
}
zDb = Tcl_GetString(objv[2]);
zTab = Tcl_GetString(objv[3]);
rc = sqlite3_fts5_drop_corrupt_table(db, zDb, zTab);
if( rc!=SQLITE_OK ){
Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), (void*)0);
return TCL_ERROR;
}
return TCL_OK;
}
/*
** Entry point.
*/
@ -1564,7 +1644,8 @@ int Fts5tcl_Init(Tcl_Interp *interp){
{ "sqlite3_fts5_token_hash", f5tTokenHash, 0 },
{ "sqlite3_fts5_register_matchinfo", f5tRegisterMatchinfo, 0 },
{ "sqlite3_fts5_register_fts5tokenize", f5tRegisterTok, 0 },
{ "sqlite3_fts5_register_origintext",f5tRegisterOriginText, 0 }
{ "sqlite3_fts5_register_origintext",f5tRegisterOriginText, 0 },
{ "sqlite3_fts5_drop_corrupt_table", f5tDropCorruptTable, 0 }
};
int i;
F5tTokenizerContext *pContext;

6
ext/fts5/test/fts5blob.test
View File

@ -103,13 +103,13 @@ do_execsql_test 3.0 {
do_catchsql_test 3.1 {
INSERT INTO x1(rowid, a, b) VALUES(113, 'hello world', X'123456');
} {1 {datatype mismatch}}
} {0 {}}
do_catchsql_test 3.2 {
INSERT INTO x2(rowid, a, b) VALUES(113, 'hello world', X'123456');
} {1 {datatype mismatch}}
} {0 {}}
do_catchsql_test 3.3 {
INSERT INTO x3(rowid, a, b) VALUES(113, 'hello world', X'123456');
} {1 {datatype mismatch}}
} {0 {}}
#--------------------------------------------------------------------------

14
ext/fts5/test/fts5corrupt3.test
View File

@ -680,11 +680,11 @@ do_test 12.0 {
do_catchsql_test 11.1 {
SELECT * FROM t1 WHERE t1 MATCH 'abandon';
} {1 {vtable constructor failed: t1}}
} {1 {database disk image is malformed}}
do_catchsql_test 11.2 {
INSERT INTO t1(t1, rank) VALUES('merge', 500);
} {1 {vtable constructor failed: t1}}
} {1 {database disk image is malformed}}
#-------------------------------------------------------------------------
#
@ -1040,7 +1040,7 @@ do_test 16.0 {
do_catchsql_test 16.1 {
INSERT INTO t1(t1) VALUES('integrity-check');
} {1 {vtable constructor failed: t1}}
} {1 {database disk image is malformed}}
#--------------------------------------------------------------------------
reset_db
@ -1126,7 +1126,7 @@ do_test 17.0 {
do_catchsql_test 17.1 {
SELECT * FROM t1 WHERE t1 MATCH 'abandon';
} {1 {vtable constructor failed: t1}}
} {1 {database disk image is malformed}}
#--------------------------------------------------------------------------
reset_db
@ -1630,7 +1630,7 @@ do_test 20.0 {
do_catchsql_test 20.1 {
SELECT * FROM t1 WHERE t1 MATCH 'abandon';
} {1 {vtable constructor failed: t1}}
} {1 {database disk image is malformed}}
#-------------------------------------------------------------------------
reset_db
@ -2100,7 +2100,7 @@ do_test 22.0 {
do_catchsql_test 22.1 {
INSERT INTO t1(t1) VALUES('optimize');
} {1 {vtable constructor failed: t1}}
} {1 {database disk image is malformed}}
#--------------------------------------------------------------------------
reset_db
@ -3700,7 +3700,7 @@ do_catchsql_test 32.1 {
highlight(t1, 2, '[', ']')
FROM t1('g + h')
WHERE rank MATCH 'bm25(1.0, 1.0)' ORDER BY rank;
} {1 {vtable constructor failed: t1}}
} {1 {database disk image is malformed}}
do_catchsql_test 32.2 {
SELECT * FROM t3;

94
ext/fts5/test/fts5corrupt8.test Normal file
View File

@ -0,0 +1,94 @@
# 2024 Aug 28
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5corrupt8
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x);
}
do_execsql_test 1.1 {
UPDATE t1_data SET block='hello world' WHERE id=10
}
db close
sqlite3 db test.db
do_catchsql_test 1.2 {
SELECT * FROM t1
} {1 {database disk image is malformed}}
do_catchsql_test 1.3 {
DROP TABLE t1
} {0 {}}
do_execsql_test 1.4 {
SELECT * FROM sqlite_schema
}
do_execsql_test 2.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x);
}
do_execsql_test 2.1 {
UPDATE t1_config SET v=555 WHERE k='version'
}
db close
sqlite3 db test.db
do_catchsql_test 2.2 {
SELECT * FROM t1
} {1 {invalid fts5 file format (found 555, expected 4 or 5) - run 'rebuild'}}
do_catchsql_test 2.3 {
DROP TABLE t1
} {1 {invalid fts5 file format (found 555, expected 4 or 5) - run 'rebuild'}}
do_test 2.4 {
sqlite3_fts5_drop_corrupt_table db main t1
} {}
do_execsql_test 2.5 {
SELECT * FROM sqlite_schema
}
do_execsql_test 3.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x);
}
do_execsql_test 3.1 {
DELETE FROM t1_config;
}
db close
sqlite3 db test.db
do_catchsql_test 3.2 {
SELECT * FROM t1
} {1 {invalid fts5 file format (found 0, expected 4 or 5) - run 'rebuild'}}
do_catchsql_test 3.3 {
DROP TABLE t1
} {1 {invalid fts5 file format (found 0, expected 4 or 5) - run 'rebuild'}}
do_test 3.4 {
sqlite3_db_config db DEFENSIVE 1
} {1}
do_test 3.5 {
sqlite3_fts5_drop_corrupt_table db main t1
} {}
do_test 3.6 {
sqlite3_db_config db DEFENSIVE -1
} {1}
do_execsql_test 3.7 {
SELECT * FROM sqlite_schema
}
finish_test

2
ext/fts5/test/fts5fault4.test
View File

@ -90,7 +90,7 @@ set ::res [db eval {SELECT rowid, x1 FROM x1 WHERE x1 MATCH '*reads'}]
do_faultsim_test 4 -faults oom-* -body {
db eval {SELECT rowid, x, x1 FROM x1 WHERE x1 MATCH '*reads'}
} -test {
faultsim_test_result {0 {0 {} 3}}
faultsim_test_result {0 {0 {} 2}}
}
#-------------------------------------------------------------------------

35
ext/fts5/test/fts5faultI.test
View File

@ -246,7 +246,7 @@ do_execsql_test 10.1 {
} {hello}
faultsim_save_and_close
do_faultsim_test 10 -faults oom* -prep {
do_faultsim_test 10.1 -faults oom* -prep {
faultsim_restore_and_reopen
} -body {
execsql {
@ -256,6 +256,39 @@ do_faultsim_test 10 -faults oom* -prep {
faultsim_test_result {0 hello}
}
breakpoint
faultsim_save_and_close
do_faultsim_test 10.2 -faults oom-t* -prep {
faultsim_restore_and_reopen
} -body {
execsql {
INSERT INTO ft VALUES(zeroblob(10000));
}
} -test {
faultsim_test_result {0 {}}
}
#-------------------------------------------------------------------------
reset_db
do_execsql_test 11.0 {
CREATE VIRTUAL TABLE f1 USING fts5(content);
CREATE TABLE g1(id, content);
INSERT INTO g1 VALUES(30000, 'a b c');
INSERT INTO g1 VALUES(40000, 'd e f');
}
faultsim_save_and_close
do_faultsim_test 11 -faults oom* -prep {
faultsim_restore_and_reopen
} -body {
execsql {
INSERT INTO f1(rowid, content) SELECT id, content FROM g1;
}
} -test {
faultsim_test_result {0 {}}
}
finish_test

56
ext/fts5/test/fts5integrity2.test Normal file
View File

@ -0,0 +1,56 @@
# 2024 September 3
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
# This file contains tests focused on the integrity-check procedure.
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5integrity2
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
do_execsql_test 2.0 {
CREATE VIRTUAL TABLE t2 USING fts5(a, detail='none');
BEGIN;
INSERT INTO t2(rowid, a) VALUES(-1, 'hello world');
INSERT INTO t2(rowid, a) VALUES(9223372036854775807, 'hello world');
COMMIT;
}
do_execsql_test 2.1 {
SELECT rowid FROM t2('hello AND world');
} {-1 9223372036854775807}
#-------------------------------------------------------------------------
do_execsql_test 2.0 {
CREATE VIRTUAL TABLE t1 USING fts5(a, detail='none');
CREATE TABLE r1(r);
WITH c(x) AS (VALUES(1) UNION SELECT x<<1 FROM c)
INSERT INTO r1(r) SELECT -1-x FROM c;
INSERT INTO t1(rowid, a) SELECT r, 'abc' FROM r1;
}
do_execsql_test 2.1 {
PRAGMA integrity_check;
} {ok}
do_execsql_test 2.2 {
SELECT rowid FROM t1('abc') ORDER BY +rowid;
} [db eval {SELECT r FROM r1 ORDER BY r}]
finish_test

131
ext/fts5/test/fts5locale.test
View File

@ -73,6 +73,7 @@ do_execsql_test 1.2 {
SELECT rowid, a FROM t1( fts5_locale('reverse', 'abc') );
} {2 cba}
#-------------------------------------------------------------------------
# Test that the locale= option exists and seems to accept values. And
# that fts5_locale() values may only be inserted into an internal-content
@ -99,8 +100,11 @@ do_catchsql_test 2.3 {
INSERT INTO b1(b1, rank) VALUES('locale', 0);
} {1 {SQL logic error}}
do_execsql_test 2.4 {
do_execsql_test 2.4.1 {
INSERT INTO b1 VALUES('abc', 'one two three');
}
do_execsql_test 2.4.2 {
INSERT INTO b1 VALUES('def', fts5_locale('reverse', 'four five six'));
}
@ -131,6 +135,7 @@ do_execsql_test 2.12 { SELECT quote(y) FROM b1('ruof') } {
do_execsql_test 2.13 {
INSERT INTO b1(b1) VALUES('integrity-check');
}
do_execsql_test 2.14 {
INSERT INTO b1(b1) VALUES('rebuild');
}
@ -149,7 +154,6 @@ do_execsql_test 2.18 {
INSERT INTO b1(rowid, x, y) VALUES(
test_setsubtype(45, 76), 'abc def', 'def abc'
);
INSERT INTO b1(b1) VALUES('integrity-check');
}
#-------------------------------------------------------------------------
@ -278,9 +282,9 @@ do_execsql_test 5.2 {
}
do_execsql_test 5.3 {
SELECT typeof(c0), typeof(c1) FROM t1_content
SELECT typeof(c0), typeof(c1), typeof(l0) FROM t1_content
} {
blob text
text text text
}
#-------------------------------------------------------------------------
@ -305,37 +309,37 @@ foreach {tn opt} {
fts5_aux_test_functions db
do_execsql_test 5.$tn.3 {
do_execsql_test 6.$tn.3 {
SELECT fts5_test_columnsize(y1) FROM y1
} {
2 3 2 4
}
do_execsql_test 5.$tn.4 {
do_execsql_test 6.$tn.4 {
SELECT rowid, fts5_test_columnsize(y1) FROM y1('shall');
} {
2 3
}
do_execsql_test 5.$tn.5 {
do_execsql_test 6.$tn.5 {
SELECT rowid, fts5_test_columnsize(y1) FROM y1('shall');
} {
2 3
}
do_execsql_test 5.$tn.6 {
do_execsql_test 6.$tn.6 {
SELECT rowid, fts5_test_columnsize(y1) FROM y1('have');
} {
4 4
}
do_execsql_test 5.$tn.7 {
do_execsql_test 6.$tn.7 {
SELECT rowid, highlight(y1, 0, '[', ']') FROM y1('have');
} {
4 {which it hath been used to [have]}
}
do_execsql_test 5.$tn.8 {
do_execsql_test 6.$tn.8 {
SELECT rowid,
highlight(y1, 0, '[', ']'),
snippet(y1, 0, '[', ']', '...', 10)
@ -473,7 +477,7 @@ foreach_detail_mode $::testprefix {
}
foreach {tn v} {
1 X'001122'
1 X'001152'
2 X'0011223344'
3 X'00E0B2EB68656c6c6f'
4 X'00E0B2EB0068656c6c6f'
@ -484,33 +488,33 @@ foreach_detail_mode $::testprefix {
do_catchsql_test 10.2.$tn.3 {
INSERT INTO ft(ft) VALUES('rebuild');
} {1 {SQL logic error}}
} {0 {}}
do_catchsql_test 10.2.$tn.4 "
SELECT * FROM ft( test_setsubtype($v, 76) );
" {1 {SQL logic error}}
" {1 {fts5: syntax error near ""}}
do_execsql_test 10.2.$tn.5 {
INSERT INTO ft(rowid, x) VALUES(1, 'hello world');
}
if {"%DETAIL%"!="full"} {
do_catchsql_test 10.2.$tn.6 {
if {"%DETAIL%"=="full"} {
do_execsql_test 10.2.$tn.6 {
SELECT fts5_test_poslist(ft) FROM ft('world');
} {1 SQLITE_ERROR}
} {0.0.1}
do_catchsql_test 10.2.$tn.7 {
do_execsql_test 10.2.$tn.7.1 {
SELECT fts5_test_columnsize(ft) FROM ft('world');
} {1 SQLITE_ERROR}
} {1}
do_catchsql_test 10.2.$tn.7 {
do_execsql_test 10.2.$tn.7.2 {
SELECT fts5_test_columnlocale(ft) FROM ft('world');
} {1 SQLITE_ERROR}
} {{{}}}
}
do_catchsql_test 10.2.$tn.8 {
SELECT * FROM ft('hello')
} {1 {SQL logic error}}
SELECT count(*) FROM ft('hello')
} {0 1}
do_catchsql_test 10.2.$tn.9 {
PRAGMA integrity_check;
@ -523,11 +527,11 @@ foreach_detail_mode $::testprefix {
do_catchsql_test 10.2.$tn.11 "
INSERT INTO ft(ft, rowid, x) VALUES('delete', 1, test_setsubtype($v,76) )
" {1 {SQL logic error}}
" {0 {}}
do_catchsql_test 10.2.$tn.12 "
INSERT INTO ft(rowid, x) VALUES(2, test_setsubtype($v,76) )
" {1 {SQL logic error}}
" {0 {}}
do_execsql_test 10.2.$tn.13 {
INSERT INTO ft2(rowid, x) VALUES(1, 'hello world');
@ -536,7 +540,7 @@ foreach_detail_mode $::testprefix {
do_catchsql_test 10.2.$tn.15 {
PRAGMA integrity_check;
} {1 {SQL logic error}}
} {0 {{malformed inverted index for FTS5 table main.ft2}}}
do_execsql_test 10.2.$tn.16 {
DELETE FROM ft2_content;
@ -663,5 +667,82 @@ do_catchsql_test 13.2.7 {
FROM ft('one AND three') ORDER BY rowid
} {1 {non-integer argument passed to function fts5_get_locale()}}
#-------------------------------------------------------------------------
# Check that UPDATE statements that may affect more than one row work.
#
reset_db
do_execsql_test 14.1 {
CREATE VIRTUAL TABLE ft USING fts5(a, b, locale=1);
}
do_execsql_test 14.2 {
INSERT INTO ft VALUES('hello', 'world');
}
do_execsql_test 14.3 {
UPDATE ft SET b = fts5_locale('en_AU', 'world');
}
do_execsql_test 14.4 {
INSERT INTO ft VALUES(X'abcd', X'1234');
} {}
do_execsql_test 14.5 {
SELECT quote(a), quote(b) FROM ft
} {'hello' 'world' X'ABCD' X'1234'}
do_execsql_test 14.6 {
DELETE FROM ft;
INSERT INTO ft VALUES(NULL, 'null');
INSERT INTO ft VALUES(123, 'int');
INSERT INTO ft VALUES(345.0, 'real');
INSERT INTO ft VALUES('abc', 'text');
INSERT INTO ft VALUES(fts5_locale('abc', 'def'), 'text');
SELECT a, typeof(a), b FROM ft
} {
{} null null
123 integer int
345.0 real real
abc text text
def text text
}
do_execsql_test 14.7 {
SELECT quote(c0), typeof(c0) FROM ft_content
} {
NULL null
123 integer
345.0 real
'abc' text
'def' text
}
#-------------------------------------------------------------------------
# Check that inserting UNINDEXED columns between indexed columns of a
# locale=1 table does not cause a problem.
#
reset_db
sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create
fts5_aux_test_functions db
do_execsql_test 15.1 {
CREATE VIRTUAL TABLE ft USING fts5(a, b UNINDEXED, c, locale=1, tokenize=tcl);
}
do_execsql_test 15.2 {
INSERT INTO ft VALUES('one', 'two', 'three');
INSERT INTO ft VALUES('one', 'two', fts5_locale('loc', 'three'));
}
do_execsql_test 15.3 {
SELECT c2, l2 FROM ft_content
} {three {} three loc}
do_execsql_test 15.4 {
SELECT c, fts5_columnlocale(ft, 2) FROM ft
} {three {} three loc}
finish_test

8
ext/fts5/test/fts5misc.test
View File

@ -44,12 +44,12 @@ do_catchsql_test 1.2.2 {
do_catchsql_test 1.3.1 {
SELECT highlight(t1, 4, '<b>', '</b>') FROM t1('*reads');
} {1 {no such cursor: 1}}
} {1 {no such cursor: 0}}
do_catchsql_test 1.3.2 {
SELECT a FROM t1
WHERE rank = (SELECT highlight(t1, 4, '<b>', '</b>') FROM t1('*reads'));
} {1 {no such cursor: 1}}
} {1 {no such cursor: 0}}
db close
sqlite3 db test.db
@ -57,12 +57,12 @@ sqlite3 db test.db
do_catchsql_test 1.3.3 {
SELECT a FROM t1
WHERE rank = (SELECT highlight(t1, 4, '<b>', '</b>') FROM t1('*reads'));
} {1 {no such cursor: 1}}
} {1 {no such cursor: 0}}
fts5_aux_test_functions db
do_catchsql_test 1.3.4 {
SELECT fts5_columntext(t1) FROM t1('*reads');
} {1 {no such cursor: 1}}
} {1 {no such cursor: 0}}
#-------------------------------------------------------------------------
reset_db

16
ext/fts5/test/fts5phrase.test
View File

@ -93,15 +93,21 @@ foreach {tn cols tokens} {
10 {b} "i e"
11 {a} "i e"
} {
set fts "{$cols}:[join $tokens +]"
set where [list]
foreach c $cols { lappend where "pmatch($c, '$tokens')" }
set where [join $where " OR "]
set res [db eval "SELECT rowid FROM t3 WHERE $where"]
do_execsql_test "1.$tn.$fts->([llength $res] rows)" {
SELECT rowid FROM t3($fts)
} $res
foreach fts [list \
"{$cols}:[join $tokens +]" \
"{$cols}:NEAR([join $tokens +])" \
"{$cols}:NEAR([join $tokens +],1)" \
"{$cols}:NEAR([join $tokens +],111)" \
] {
set res [db eval "SELECT rowid FROM t3 WHERE $where"]
do_execsql_test "1.$tn.$fts->([llength $res] rows)" {
SELECT rowid FROM t3($fts)
} $res
}
}
do_execsql_test 2.0 {

2
ext/fts5/test/fts5simple.test
View File

@ -350,7 +350,7 @@ do_execsql_test 14.3 {
do_execsql_test 14.4 {
SELECT rowid, x, x1 FROM x1 WHERE x1 MATCH '*reads'
} {0 {} 3}
} {0 {} 2}
#-------------------------------------------------------------------------
reset_db

2
ext/misc/fileio.c
View File

@ -40,7 +40,7 @@
** modification-time of the target file is set to this value before
** returning.
**
** If three or more arguments are passed to this function and an
** If five or more arguments are passed to this function and an
** error is encountered, an exception is raised.
**
** READFILE(FILE):

392
ext/misc/percentile.c
View File

@ -11,7 +11,7 @@
******************************************************************************
**
** This file contains code to implement the percentile(Y,P) SQL function
** as described below:
** and similar as described below:
**
** (1) The percentile(Y,P) function is an aggregate function taking
** exactly two arguments.
@ -60,31 +60,105 @@
**
** (13) A separate median(Y) function is the equivalent percentile(Y,50).
**
** (14) A separate percentile_cond(Y,X) function is the equivalent of
** percentile(Y,X*100.0).
** (14) A separate percentile_cont(Y,P) function is equivalent to
** percentile(Y,P/100.0). In other words, the fraction value in
** the second argument is in the range of 0 to 1 instead of 0 to 100.
**
** (15) A separate percentile_disc(Y,P) function is like
** percentile_cont(Y,P) except that instead of returning the weighted
** average of the nearest two input values, it returns the next lower
** value. So the percentile_disc(Y,P) will always return a value
** that was one of the inputs.
**
** (16) All of median(), percentile(Y,P), percentile_cont(Y,P) and
** percentile_disc(Y,P) can be used as window functions.
**
** Differences from standard SQL:
**
** * The percentile_cont(X,P) function is equivalent to the following in
** standard SQL:
**
** (percentile_cont(P) WITHIN GROUP (ORDER BY X))
**
** The SQLite syntax is much more compact. The standard SQL syntax
** is also supported if SQLite is compiled with the
** -DSQLITE_ENABLE_ORDERED_SET_AGGREGATES option.
**
** * No median(X) function exists in the SQL standard. App developers
** are expected to write "percentile_cont(0.5)WITHIN GROUP(ORDER BY X)".
**
** * No percentile(Y,P) function exists in the SQL standard. Instead of
** percential(Y,P), developers must write this:
** "percentile_cont(P/100.0) WITHIN GROUP (ORDER BY Y)". Note that
** the fraction parameter to percentile() goes from 0 to 100 whereas
** the fraction parameter in SQL standard percentile_cont() goes from
** 0 to 1.
**
** Implementation notes as of 2024-08-31:
**
** * The regular aggregate-function versions of these routines work
** by accumulating all values in an array of doubles, then sorting
** that array using quicksort before computing the answer. Thus
** the runtime is O(NlogN) where N is the number of rows of input.
**
** * For the window-function versions of these routines, the array of
** inputs is sorted as soon as the first value is computed. Thereafter,
** the array is kept in sorted order using an insert-sort. This
** results in O(N*K) performance where K is the size of the window.
** One can imagine alternative implementations that give O(N*logN*logK)
** performance, but they require more complex logic and data structures.
** The developers have elected to keep the asymptotically slower
** algorithm for now, for simplicity, under the theory that window
** functions are seldom used and when they are, the window size K is
** often small. The developers might revisit that decision later,
** should the need arise.
*/
#include "sqlite3ext.h"
SQLITE_EXTENSION_INIT1
#if defined(SQLITE3_H)
/* no-op */
#elif defined(SQLITE_STATIC_PERCENTILE)
# include "sqlite3.h"
#else
# include "sqlite3ext.h"
SQLITE_EXTENSION_INIT1
#endif
#include <assert.h>
#include <string.h>
#include <stdlib.h>
/* The following object is the session context for a single percentile()
** function. We have to remember all input Y values until the very end.
/* The following object is the group context for a single percentile()
** aggregate. Remember all input Y values until the very end.
** Those values are accumulated in the Percentile.a[] array.
*/
typedef struct Percentile Percentile;
struct Percentile {
unsigned nAlloc; /* Number of slots allocated for a[] */
unsigned nUsed; /* Number of slots actually used in a[] */
double rPct; /* 1.0 more than the value for P */
char bSorted; /* True if a[] is already in sorted order */
char bKeepSorted; /* True if advantageous to keep a[] sorted */
char bPctValid; /* True if rPct is valid */
double rPct; /* Fraction. 0.0 to 1.0 */
double *a; /* Array of Y values */
};
/* Details of each function in the percentile family */
typedef struct PercentileFunc PercentileFunc;
struct PercentileFunc {
const char *zName; /* Function name */
char nArg; /* Number of arguments */
char mxFrac; /* Maximum value of the "fraction" input */
char bDiscrete; /* True for percentile_disc() */
};
static const PercentileFunc aPercentFunc[] = {
{ "median", 1, 1, 0 },
{ "percentile", 2, 100, 0 },
{ "percentile_cont", 2, 1, 0 },
{ "percentile_disc", 2, 1, 1 },
};
/*
** Return TRUE if the input floating-point number is an infinity.
*/
static int isInfinity(double r){
static int percentIsInfinity(double r){
sqlite3_uint64 u;
assert( sizeof(u)==sizeof(r) );
memcpy(&u, &r, sizeof(u));
@ -92,13 +166,64 @@ static int isInfinity(double r){
}
/*
** Return TRUE if two doubles differ by 0.001 or less
** Return TRUE if two doubles differ by 0.001 or less.
*/
static int sameValue(double a, double b){
static int percentSameValue(double a, double b){
a -= b;
return a>=-0.001 && a<=0.001;
}
/*
** Search p (which must have p->bSorted) looking for an entry with
** value y. Return the index of that entry.
**
** If bExact is true, return -1 if the entry is not found.
**
** If bExact is false, return the index at which a new entry with
** value y should be insert in order to keep the values in sorted
** order. The smallest return value in this case will be 0, and
** the largest return value will be p->nUsed.
*/
static int percentBinarySearch(Percentile *p, double y, int bExact){
int iFirst = 0; /* First element of search range */
int iLast = p->nUsed - 1; /* Last element of search range */
while( iLast>=iFirst ){
int iMid = (iFirst+iLast)/2;
double x = p->a[iMid];
if( x<y ){
iFirst = iMid + 1;
}else if( x>y ){
iLast = iMid - 1;
}else{
return iMid;
}
}
if( bExact ) return -1;
return iFirst;
}
/*
** Generate an error for a percentile function.
**
** The error format string must have exactly one occurrance of "%%s()"
** (with two '%' characters). That substring will be replaced by the name
** of the function.
*/
static void percentError(sqlite3_context *pCtx, const char *zFormat, ...){
PercentileFunc *pFunc = (PercentileFunc*)sqlite3_user_data(pCtx);
char *zMsg1;
char *zMsg2;
va_list ap;
va_start(ap, zFormat);
zMsg1 = sqlite3_vmprintf(zFormat, ap);
va_end(ap);
zMsg2 = zMsg1 ? sqlite3_mprintf(zMsg1, pFunc->zName) : 0;
sqlite3_result_error(pCtx, zMsg2, -1);
sqlite3_free(zMsg1);
sqlite3_free(zMsg2);
}
/*
** The "step" function for percentile(Y,P) is called once for each
** input row.
@ -112,28 +237,20 @@ static void percentStep(sqlite3_context *pCtx, int argc, sqlite3_value **argv){
if( argc==1 ){
/* Requirement 13: median(Y) is the same as percentile(Y,50). */
rPct = 50.0;
}else if( sqlite3_user_data(pCtx)==0 ){
/* Requirement 3: P must be a number between 0 and 100 */
eType = sqlite3_value_numeric_type(argv[1]);
rPct = sqlite3_value_double(argv[1]);
if( (eType!=SQLITE_INTEGER && eType!=SQLITE_FLOAT)
|| rPct<0.0 || rPct>100.0 ){
sqlite3_result_error(pCtx, "2nd argument to percentile() is not "
"a number between 0.0 and 100.0", -1);
return;
}
rPct = 0.5;
}else{
/* Requirement 3: P must be a number between 0 and 1 */
/* Requirement 3: P must be a number between 0 and 100 */
PercentileFunc *pFunc = (PercentileFunc*)sqlite3_user_data(pCtx);
eType = sqlite3_value_numeric_type(argv[1]);
rPct = sqlite3_value_double(argv[1]);
rPct = sqlite3_value_double(argv[1])/(double)pFunc->mxFrac;
if( (eType!=SQLITE_INTEGER && eType!=SQLITE_FLOAT)
|| rPct<0.0 || rPct>1.0 ){
sqlite3_result_error(pCtx, "2nd argument to percentile_cont() is not "
"a number between 0.0 and 1.0", -1);
|| rPct<0.0 || rPct>1.0
){
percentError(pCtx, "the fraction argument to %%s()"
" is not between 0.0 and %.1f",
(double)pFunc->mxFrac);
return;
}
rPct *= 100.0;
}
/* Allocate the session context. */
@ -142,11 +259,12 @@ static void percentStep(sqlite3_context *pCtx, int argc, sqlite3_value **argv){
/* Remember the P value. Throw an error if the P value is different
** from any prior row, per Requirement (2). */
if( p->rPct==0.0 ){
p->rPct = rPct+1.0;
}else if( !sameValue(p->rPct,rPct+1.0) ){
sqlite3_result_error(pCtx, "2nd argument to percentile() is not the "
"same for all input rows", -1);
if( !p->bPctValid ){
p->rPct = rPct;
p->bPctValid = 1;
}else if( !percentSameValue(p->rPct,rPct) ){
percentError(pCtx, "the fraction argument to %%s()"
" is not the same for all input rows");
return;
}
@ -157,15 +275,14 @@ static void percentStep(sqlite3_context *pCtx, int argc, sqlite3_value **argv){
/* If not NULL, then Y must be numeric. Otherwise throw an error.
** Requirement 4 */
if( eType!=SQLITE_INTEGER && eType!=SQLITE_FLOAT ){
sqlite3_result_error(pCtx, "1st argument to percentile() is not "
"numeric", -1);
percentError(pCtx, "input to %%s() is not numeric");
return;
}
/* Throw an error if the Y value is infinity or NaN */
y = sqlite3_value_double(argv[0]);
if( isInfinity(y) ){
sqlite3_result_error(pCtx, "Inf input to percentile()", -1);
if( percentIsInfinity(y) ){
percentError(pCtx, "Inf input to %%s()");
return;
}
@ -182,50 +299,80 @@ static void percentStep(sqlite3_context *pCtx, int argc, sqlite3_value **argv){
p->nAlloc = n;
p->a = a;
}
p->a[p->nUsed++] = y;
if( p->nUsed==0 ){
p->a[p->nUsed++] = y;
p->bSorted = 1;
}else if( !p->bSorted || y>=p->a[p->nUsed-1] ){
p->a[p->nUsed++] = y;
}else if( p->bKeepSorted ){
int i;
i = percentBinarySearch(p, y, 0);
if( i<(int)p->nUsed ){
memmove(&p->a[i+1], &p->a[i], (p->nUsed-i)*sizeof(p->a[0]));
}
p->a[i] = y;
p->nUsed++;
}else{
p->a[p->nUsed++] = y;
p->bSorted = 0;
}
}
/*
** Sort an array of doubles.
** Interchange two doubles.
*/
static void sortDoubles(double *a, int n){
int iLt; /* Entries with index less than iLt are less than rPivot */
int iGt; /* Entries with index iGt or more are greater than rPivot */
#define SWAP_DOUBLE(X,Y) {double ttt=(X);(X)=(Y);(Y)=ttt;}
/*
** Sort an array of doubles.
**
** Algorithm: quicksort
**
** This is implemented separately rather than using the qsort() routine
** from the standard library because:
**
** (1) To avoid a dependency on qsort()
** (2) To avoid the function call to the comparison routine for each
** comparison.
*/
static void percentSort(double *a, unsigned int n){
int iLt; /* Entries before a[iLt] are less than rPivot */
int iGt; /* Entries at or after a[iGt] are greater than rPivot */
int i; /* Loop counter */
double rPivot; /* The pivot value */
double rTmp; /* Temporary used to swap two values */
if( n<2 ) return;
if( n>5 ){
rPivot = (a[0] + a[n/2] + a[n-1])/3.0;
}else{
rPivot = a[n/2];
assert( n>=2 );
if( a[0]>a[n-1] ){
SWAP_DOUBLE(a[0],a[n-1])
}
iLt = i = 0;
iGt = n;
while( i<iGt ){
if( n==2 ) return;
iGt = n-1;
i = n/2;
if( a[0]>a[i] ){
SWAP_DOUBLE(a[0],a[i])
}else if( a[i]>a[iGt] ){
SWAP_DOUBLE(a[i],a[iGt])
}
if( n==3 ) return;
rPivot = a[i];
iLt = i = 1;
do{
if( a[i]<rPivot ){
if( i>iLt ){
rTmp = a[i];
a[i] = a[iLt];
a[iLt] = rTmp;
}
if( i>iLt ) SWAP_DOUBLE(a[i],a[iLt])
iLt++;
i++;
}else if( a[i]>rPivot ){
do{
iGt--;
}while( iGt>i && a[iGt]>rPivot );
rTmp = a[i];
a[i] = a[iGt];
a[iGt] = rTmp;
SWAP_DOUBLE(a[i],a[iGt])
}else{
i++;
}
}
if( iLt>=2 ) sortDoubles(a, iLt);
if( n-iGt>=2 ) sortDoubles(a+iGt, n-iGt);
}while( i<iGt );
if( iLt>=2 ) percentSort(a, iLt);
if( n-iGt>=2 ) percentSort(a+iGt, n-iGt);
/* Uncomment for testing */
#if 0
for(i=0; i<n-1; i++){
@ -234,12 +381,61 @@ static void sortDoubles(double *a, int n){
#endif
}
/*
** Called to compute the final output of percentile() and to clean
** up all allocated memory.
** The "inverse" function for percentile(Y,P) is called to remove a
** row that was previously inserted by "step".
*/
static void percentFinal(sqlite3_context *pCtx){
static void percentInverse(sqlite3_context *pCtx,int argc,sqlite3_value **argv){
Percentile *p;
int eType;
double y;
int i;
assert( argc==2 || argc==1 );
/* Allocate the session context. */
p = (Percentile*)sqlite3_aggregate_context(pCtx, sizeof(*p));
assert( p!=0 );
/* Ignore rows for which Y is NULL */
eType = sqlite3_value_type(argv[0]);
if( eType==SQLITE_NULL ) return;
/* If not NULL, then Y must be numeric. Otherwise throw an error.
** Requirement 4 */
if( eType!=SQLITE_INTEGER && eType!=SQLITE_FLOAT ){
return;
}
/* Ignore the Y value if it is infinity or NaN */
y = sqlite3_value_double(argv[0]);
if( percentIsInfinity(y) ){
return;
}
if( p->bSorted==0 ){
assert( p->nUsed>1 );
percentSort(p->a, p->nUsed);
p->bSorted = 1;
}
p->bKeepSorted = 1;
/* Find and remove the row */
i = percentBinarySearch(p, y, 1);
if( i>=0 ){
p->nUsed--;
if( i<(int)p->nUsed ){
memmove(&p->a[i], &p->a[i+1], (p->nUsed - i)*sizeof(p->a[0]));
}
}
}
/*
** Compute the final output of percentile(). Clean up all allocated
** memory if and only if bIsFinal is true.
*/
static void percentCompute(sqlite3_context *pCtx, int bIsFinal){
Percentile *p;
PercentileFunc *pFunc = (PercentileFunc*)sqlite3_user_data(pCtx);
unsigned i1, i2;
double v1, v2;
double ix, vx;
@ -247,21 +443,38 @@ static void percentFinal(sqlite3_context *pCtx){
if( p==0 ) return;
if( p->a==0 ) return;
if( p->nUsed ){
sortDoubles(p->a, p->nUsed);
ix = (p->rPct-1.0)*(p->nUsed-1)*0.01;
if( p->bSorted==0 ){
assert( p->nUsed>1 );
percentSort(p->a, p->nUsed);
p->bSorted = 1;
}
ix = p->rPct*(p->nUsed-1);
i1 = (unsigned)ix;
i2 = ix==(double)i1 || i1==p->nUsed-1 ? i1 : i1+1;
v1 = p->a[i1];
v2 = p->a[i2];
vx = v1 + (v2-v1)*(ix-i1);
if( pFunc->bDiscrete ){
vx = p->a[i1];
}else{
i2 = ix==(double)i1 || i1==p->nUsed-1 ? i1 : i1+1;
v1 = p->a[i1];
v2 = p->a[i2];
vx = v1 + (v2-v1)*(ix-i1);
}
sqlite3_result_double(pCtx, vx);
}
sqlite3_free(p->a);
memset(p, 0, sizeof(*p));
if( bIsFinal ){
sqlite3_free(p->a);
memset(p, 0, sizeof(*p));
}else{
p->bKeepSorted = 1;
}
}
static void percentFinal(sqlite3_context *pCtx){
percentCompute(pCtx, 1);
}
static void percentValue(sqlite3_context *pCtx){
percentCompute(pCtx, 0);
}
#ifdef _WIN32
#if defined(_WIN32) && !defined(SQLITE3_H) && !defined(SQLITE_STATIC_PERCENTILE)
__declspec(dllexport)
#endif
int sqlite3_percentile_init(
@ -270,20 +483,21 @@ int sqlite3_percentile_init(
const sqlite3_api_routines *pApi
){
int rc = SQLITE_OK;
unsigned int i;
#if defined(SQLITE3_H) || defined(SQLITE_STATIC_PERCENTILE)
(void)pApi; /* Unused parameter */
#else
SQLITE_EXTENSION_INIT2(pApi);
#endif
(void)pzErrMsg; /* Unused parameter */
rc = sqlite3_create_function(db, "percentile", 2,
SQLITE_UTF8|SQLITE_INNOCUOUS, 0,
0, percentStep, percentFinal);
if( rc==SQLITE_OK ){
rc = sqlite3_create_function(db, "median", 1,
SQLITE_UTF8|SQLITE_INNOCUOUS, 0,
0, percentStep, percentFinal);
}
if( rc==SQLITE_OK ){
rc = sqlite3_create_function(db, "percentile_cont", 2,
SQLITE_UTF8|SQLITE_INNOCUOUS, &percentStep,
0, percentStep, percentFinal);
for(i=0; i<sizeof(aPercentFunc)/sizeof(aPercentFunc[0]); i++){
rc = sqlite3_create_window_function(db,
aPercentFunc[i].zName,
aPercentFunc[i].nArg,
SQLITE_UTF8|SQLITE_INNOCUOUS|SQLITE_SELFORDER1,
(void*)&aPercentFunc[i],
percentStep, percentFinal, percentValue, percentInverse, 0);
if( rc ) break;
}
return rc;
}

34
ext/misc/sha1.c
View File

@ -196,7 +196,8 @@ static void hash_step_vformat(
** zOut[]. zOut[] must be at least 41 bytes long. */
static void hash_finish(
SHA1Context *p, /* The SHA1 context to finish and render */
char *zOut /* Store hexadecimal hash here */
char *zOut, /* Store hex or binary hash here */
int bAsBinary /* 1 for binary hash, 0 for hex hash */
){
unsigned int i;
unsigned char finalcount[8];
@ -215,11 +216,15 @@ static void hash_finish(
for (i = 0; i < 20; i++){
digest[i] = (unsigned char)((p->state[i>>2] >> ((3-(i & 3)) * 8) ) & 255);
}
for(i=0; i<20; i++){
zOut[i*2] = zEncode[(digest[i]>>4)&0xf];
zOut[i*2+1] = zEncode[digest[i] & 0xf];
if( bAsBinary ){
memcpy(zOut, digest, 20);
}else{
for(i=0; i<20; i++){
zOut[i*2] = zEncode[(digest[i]>>4)&0xf];
zOut[i*2+1] = zEncode[digest[i] & 0xf];
}
zOut[i*2]= 0;
}
zOut[i*2]= 0;
}
/* End of the hashing logic
*****************************************************************************/
@ -251,8 +256,13 @@ static void sha1Func(
}else{
hash_step(&cx, sqlite3_value_text(argv[0]), nByte);
}
hash_finish(&cx, zOut);
sqlite3_result_text(context, zOut, 40, SQLITE_TRANSIENT);
if( sqlite3_user_data(context)!=0 ){
hash_finish(&cx, zOut, 1);
sqlite3_result_blob(context, zOut, 20, SQLITE_TRANSIENT);
}else{
hash_finish(&cx, zOut, 0);
sqlite3_result_blob(context, zOut, 40, SQLITE_TRANSIENT);
}
}
/*
@ -365,7 +375,7 @@ static void sha1QueryFunc(
}
sqlite3_finalize(pStmt);
}
hash_finish(&cx, zOut);
hash_finish(&cx, zOut, 0);
sqlite3_result_text(context, zOut, 40, SQLITE_TRANSIENT);
}
@ -379,11 +389,17 @@ int sqlite3_sha_init(
const sqlite3_api_routines *pApi
){
int rc = SQLITE_OK;
static int one = 1;
SQLITE_EXTENSION_INIT2(pApi);
(void)pzErrMsg; /* Unused parameter */
rc = sqlite3_create_function(db, "sha1", 1,
SQLITE_UTF8 | SQLITE_INNOCUOUS | SQLITE_DETERMINISTIC,
0, sha1Func, 0, 0);
0, sha1Func, 0, 0);
if( rc==SQLITE_OK ){
rc = sqlite3_create_function(db, "sha1b", 1,
SQLITE_UTF8 | SQLITE_INNOCUOUS | SQLITE_DETERMINISTIC,
(void*)&one, sha1Func, 0, 0);
}
if( rc==SQLITE_OK ){
rc = sqlite3_create_function(db, "sha1_query", 1,
SQLITE_UTF8|SQLITE_DIRECTONLY, 0,

987
ext/misc/vfstrace.c Normal file
View File

@ -0,0 +1,987 @@
/*
** 2011 March 16
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
******************************************************************************
**
** This file contains code implements a VFS shim that writes diagnostic
** output for each VFS call, similar to "strace".
**
** USAGE:
**
** This source file exports a single symbol which is the name of a
** function:
**
** int vfstrace_register(
** const char *zTraceName, // Name of the newly constructed VFS
** const char *zOldVfsName, // Name of the underlying VFS
** int (*xOut)(const char*,void*), // Output routine. ex: fputs
** void *pOutArg, // 2nd argument to xOut. ex: stderr
** int makeDefault // Make the new VFS the default
** );
**
** Applications that want to trace their VFS usage must provide a callback
** function with this prototype:
**
** int traceOutput(const char *zMessage, void *pAppData);
**
** This function will "output" the trace messages, where "output" can
** mean different things to different applications. The traceOutput function
** for the command-line shell (see shell.c) is "fputs" from the standard
** library, which means that all trace output is written on the stream
** specified by the second argument. In the case of the command-line shell
** the second argument is stderr. Other applications might choose to output
** trace information to a file, over a socket, or write it into a buffer.
**
** The vfstrace_register() function creates a new "shim" VFS named by
** the zTraceName parameter. A "shim" VFS is an SQLite backend that does
** not really perform the duties of a true backend, but simply filters or
** interprets VFS calls before passing them off to another VFS which does
** the actual work. In this case the other VFS - the one that does the
** real work - is identified by the second parameter, zOldVfsName. If
** the 2nd parameter is NULL then the default VFS is used. The common
** case is for the 2nd parameter to be NULL.
**
** The third and fourth parameters are the pointer to the output function
** and the second argument to the output function. For the SQLite
** command-line shell, when the -vfstrace option is used, these parameters
** are fputs and stderr, respectively.
**
** The fifth argument is true (non-zero) to cause the newly created VFS
** to become the default VFS. The common case is for the fifth parameter
** to be true.
**
** The call to vfstrace_register() simply creates the shim VFS that does
** tracing. The application must also arrange to use the new VFS for
** all database connections that are created and for which tracing is
** desired. This can be done by specifying the trace VFS using URI filename
** notation, or by specifying the trace VFS as the 4th parameter to
** sqlite3_open_v2() or by making the trace VFS be the default (by setting
** the 5th parameter of vfstrace_register() to 1).
**
**
** ENABLING VFSTRACE IN A COMMAND-LINE SHELL
**
** The SQLite command line shell implemented by the shell.c source file
** can be used with this module. To compile in -vfstrace support, first
** gather this file (test_vfstrace.c), the shell source file (shell.c),
** and the SQLite amalgamation source files (sqlite3.c, sqlite3.h) into
** the working directory. Then compile using a command like the following:
**
** gcc -o sqlite3 -Os -I. -DSQLITE_ENABLE_VFSTRACE \
** -DSQLITE_THREADSAFE=0 -DSQLITE_ENABLE_FTS3 -DSQLITE_ENABLE_RTREE \
** -DHAVE_READLINE -DHAVE_USLEEP=1 \
** shell.c test_vfstrace.c sqlite3.c -ldl -lreadline -lncurses
**
** The gcc command above works on Linux and provides (in addition to the
** -vfstrace option) support for FTS3 and FTS4, RTREE, and command-line
** editing using the readline library. The command-line shell does not
** use threads so we added -DSQLITE_THREADSAFE=0 just to make the code
** run a little faster. For compiling on a Mac, you'll probably need
** to omit the -DHAVE_READLINE, the -lreadline, and the -lncurses options.
** The compilation could be simplified to just this:
**
** gcc -DSQLITE_ENABLE_VFSTRACE \
** shell.c test_vfstrace.c sqlite3.c -ldl -lpthread
**
** In this second example, all unnecessary options have been removed
** Note that since the code is now threadsafe, we had to add the -lpthread
** option to pull in the pthreads library.
**
** To cross-compile for windows using MinGW, a command like this might
** work:
**
** /opt/mingw/bin/i386-mingw32msvc-gcc -o sqlite3.exe -Os -I \
** -DSQLITE_THREADSAFE=0 -DSQLITE_ENABLE_VFSTRACE \
** shell.c test_vfstrace.c sqlite3.c
**
** Similar compiler commands will work on different systems. The key
** invariants are (1) you must have -DSQLITE_ENABLE_VFSTRACE so that
** the shell.c source file will know to include the -vfstrace command-line
** option and (2) you must compile and link the three source files
** shell,c, test_vfstrace.c, and sqlite3.c.
*/
#include <stdlib.h>
#include <string.h>
#include "sqlite3.h"
/*
** An instance of this structure is attached to the each trace VFS to
** provide auxiliary information.
*/
typedef struct vfstrace_info vfstrace_info;
struct vfstrace_info {
sqlite3_vfs *pRootVfs; /* The underlying real VFS */
int (*xOut)(const char*, void*); /* Send output here */
void *pOutArg; /* First argument to xOut */
const char *zVfsName; /* Name of this trace-VFS */
sqlite3_vfs *pTraceVfs; /* Pointer back to the trace VFS */
};
/*
** The sqlite3_file object for the trace VFS
*/
typedef struct vfstrace_file vfstrace_file;
struct vfstrace_file {
sqlite3_file base; /* Base class. Must be first */
vfstrace_info *pInfo; /* The trace-VFS to which this file belongs */
const char *zFName; /* Base name of the file */
sqlite3_file *pReal; /* The real underlying file */
};
/*
** Method declarations for vfstrace_file.
*/
static int vfstraceClose(sqlite3_file*);
static int vfstraceRead(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst);
static int vfstraceWrite(sqlite3_file*,const void*,int iAmt, sqlite3_int64);
static int vfstraceTruncate(sqlite3_file*, sqlite3_int64 size);
static int vfstraceSync(sqlite3_file*, int flags);
static int vfstraceFileSize(sqlite3_file*, sqlite3_int64 *pSize);
static int vfstraceLock(sqlite3_file*, int);
static int vfstraceUnlock(sqlite3_file*, int);
static int vfstraceCheckReservedLock(sqlite3_file*, int *);
static int vfstraceFileControl(sqlite3_file*, int op, void *pArg);
static int vfstraceSectorSize(sqlite3_file*);
static int vfstraceDeviceCharacteristics(sqlite3_file*);
static int vfstraceShmLock(sqlite3_file*,int,int,int);
static int vfstraceShmMap(sqlite3_file*,int,int,int, void volatile **);
static void vfstraceShmBarrier(sqlite3_file*);
static int vfstraceShmUnmap(sqlite3_file*,int);
/*
** Method declarations for vfstrace_vfs.
*/
static int vfstraceOpen(sqlite3_vfs*, const char *, sqlite3_file*, int , int *);
static int vfstraceDelete(sqlite3_vfs*, const char *zName, int syncDir);
static int vfstraceAccess(sqlite3_vfs*, const char *zName, int flags, int *);
static int vfstraceFullPathname(sqlite3_vfs*, const char *zName, int, char *);
static void *vfstraceDlOpen(sqlite3_vfs*, const char *zFilename);
static void vfstraceDlError(sqlite3_vfs*, int nByte, char *zErrMsg);
static void (*vfstraceDlSym(sqlite3_vfs*,void*, const char *zSymbol))(void);
static void vfstraceDlClose(sqlite3_vfs*, void*);
static int vfstraceRandomness(sqlite3_vfs*, int nByte, char *zOut);
static int vfstraceSleep(sqlite3_vfs*, int microseconds);
static int vfstraceCurrentTime(sqlite3_vfs*, double*);
static int vfstraceGetLastError(sqlite3_vfs*, int, char*);
static int vfstraceCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*);
static int vfstraceSetSystemCall(sqlite3_vfs*,const char*, sqlite3_syscall_ptr);
static sqlite3_syscall_ptr vfstraceGetSystemCall(sqlite3_vfs*, const char *);
static const char *vfstraceNextSystemCall(sqlite3_vfs*, const char *zName);
/*
** Return a pointer to the tail of the pathname. Examples:
**
** /home/drh/xyzzy.txt -> xyzzy.txt
** xyzzy.txt -> xyzzy.txt
*/
static const char *fileTail(const char *z){
size_t i;
if( z==0 ) return 0;
i = strlen(z)-1;
while( i>0 && z[i-1]!='/' ){ i--; }
return &z[i];
}
/*
** Send trace output defined by zFormat and subsequent arguments.
*/
static void vfstrace_printf(
vfstrace_info *pInfo,
const char *zFormat,
...
){
va_list ap;
char *zMsg;
va_start(ap, zFormat);
zMsg = sqlite3_vmprintf(zFormat, ap);
va_end(ap);
pInfo->xOut(zMsg, pInfo->pOutArg);
sqlite3_free(zMsg);
}
/*
** Try to convert an error code into a symbolic name for that error code.
*/
static const char *vfstrace_errcode_name(int rc ){
const char *zVal = 0;
switch( rc ){
case SQLITE_OK: zVal = "SQLITE_OK"; break;
case SQLITE_INTERNAL: zVal = "SQLITE_INTERNAL"; break;
case SQLITE_ERROR: zVal = "SQLITE_ERROR"; break;
case SQLITE_PERM: zVal = "SQLITE_PERM"; break;
case SQLITE_ABORT: zVal = "SQLITE_ABORT"; break;
case SQLITE_BUSY: zVal = "SQLITE_BUSY"; break;
case SQLITE_LOCKED: zVal = "SQLITE_LOCKED"; break;
case SQLITE_NOMEM: zVal = "SQLITE_NOMEM"; break;
case SQLITE_READONLY: zVal = "SQLITE_READONLY"; break;
case SQLITE_INTERRUPT: zVal = "SQLITE_INTERRUPT"; break;
case SQLITE_IOERR: zVal = "SQLITE_IOERR"; break;
case SQLITE_CORRUPT: zVal = "SQLITE_CORRUPT"; break;
case SQLITE_NOTFOUND: zVal = "SQLITE_NOTFOUND"; break;
case SQLITE_FULL: zVal = "SQLITE_FULL"; break;
case SQLITE_CANTOPEN: zVal = "SQLITE_CANTOPEN"; break;
case SQLITE_PROTOCOL: zVal = "SQLITE_PROTOCOL"; break;
case SQLITE_EMPTY: zVal = "SQLITE_EMPTY"; break;
case SQLITE_SCHEMA: zVal = "SQLITE_SCHEMA"; break;
case SQLITE_TOOBIG: zVal = "SQLITE_TOOBIG"; break;
case SQLITE_CONSTRAINT: zVal = "SQLITE_CONSTRAINT"; break;
case SQLITE_MISMATCH: zVal = "SQLITE_MISMATCH"; break;
case SQLITE_MISUSE: zVal = "SQLITE_MISUSE"; break;
case SQLITE_NOLFS: zVal = "SQLITE_NOLFS"; break;
case SQLITE_IOERR_READ: zVal = "SQLITE_IOERR_READ"; break;
case SQLITE_IOERR_SHORT_READ: zVal = "SQLITE_IOERR_SHORT_READ"; break;
case SQLITE_IOERR_WRITE: zVal = "SQLITE_IOERR_WRITE"; break;
case SQLITE_IOERR_FSYNC: zVal = "SQLITE_IOERR_FSYNC"; break;
case SQLITE_IOERR_DIR_FSYNC: zVal = "SQLITE_IOERR_DIR_FSYNC"; break;
case SQLITE_IOERR_TRUNCATE: zVal = "SQLITE_IOERR_TRUNCATE"; break;
case SQLITE_IOERR_FSTAT: zVal = "SQLITE_IOERR_FSTAT"; break;
case SQLITE_IOERR_UNLOCK: zVal = "SQLITE_IOERR_UNLOCK"; break;
case SQLITE_IOERR_RDLOCK: zVal = "SQLITE_IOERR_RDLOCK"; break;
case SQLITE_IOERR_DELETE: zVal = "SQLITE_IOERR_DELETE"; break;
case SQLITE_IOERR_BLOCKED: zVal = "SQLITE_IOERR_BLOCKED"; break;
case SQLITE_IOERR_NOMEM: zVal = "SQLITE_IOERR_NOMEM"; break;
case SQLITE_IOERR_ACCESS: zVal = "SQLITE_IOERR_ACCESS"; break;
case SQLITE_IOERR_CHECKRESERVEDLOCK:
zVal = "SQLITE_IOERR_CHECKRESERVEDLOCK"; break;
case SQLITE_IOERR_LOCK: zVal = "SQLITE_IOERR_LOCK"; break;
case SQLITE_IOERR_CLOSE: zVal = "SQLITE_IOERR_CLOSE"; break;
case SQLITE_IOERR_DIR_CLOSE: zVal = "SQLITE_IOERR_DIR_CLOSE"; break;
case SQLITE_IOERR_SHMOPEN: zVal = "SQLITE_IOERR_SHMOPEN"; break;
case SQLITE_IOERR_SHMSIZE: zVal = "SQLITE_IOERR_SHMSIZE"; break;
case SQLITE_IOERR_SHMLOCK: zVal = "SQLITE_IOERR_SHMLOCK"; break;
case SQLITE_IOERR_SHMMAP: zVal = "SQLITE_IOERR_SHMMAP"; break;
case SQLITE_IOERR_SEEK: zVal = "SQLITE_IOERR_SEEK"; break;
case SQLITE_IOERR_GETTEMPPATH: zVal = "SQLITE_IOERR_GETTEMPPATH"; break;
case SQLITE_IOERR_CONVPATH: zVal = "SQLITE_IOERR_CONVPATH"; break;
case SQLITE_READONLY_DBMOVED: zVal = "SQLITE_READONLY_DBMOVED"; break;
case SQLITE_LOCKED_SHAREDCACHE: zVal = "SQLITE_LOCKED_SHAREDCACHE"; break;
case SQLITE_BUSY_RECOVERY: zVal = "SQLITE_BUSY_RECOVERY"; break;
case SQLITE_CANTOPEN_NOTEMPDIR: zVal = "SQLITE_CANTOPEN_NOTEMPDIR"; break;
}
return zVal;
}
/*
** Convert value rc into a string and print it using zFormat. zFormat
** should have exactly one %s
*/
static void vfstrace_print_errcode(
vfstrace_info *pInfo,
const char *zFormat,
int rc
){
const char *zVal;
char zBuf[50];
zVal = vfstrace_errcode_name(rc);
if( zVal==0 ){
zVal = vfstrace_errcode_name(rc&0xff);
if( zVal ){
sqlite3_snprintf(sizeof(zBuf), zBuf, "%s | 0x%x", zVal, rc&0xffff00);
}else{
sqlite3_snprintf(sizeof(zBuf), zBuf, "%d (0x%x)", rc, rc);
}
zVal = zBuf;
}
vfstrace_printf(pInfo, zFormat, zVal);
}
/*
** Append to a buffer.
*/
static void strappend(char *z, int *pI, const char *zAppend){
int i = *pI;
while( zAppend[0] ){ z[i++] = *(zAppend++); }
z[i] = 0;
*pI = i;
}
/*
** Close an vfstrace-file.
*/
static int vfstraceClose(sqlite3_file *pFile){
vfstrace_file *p = (vfstrace_file *)pFile;
vfstrace_info *pInfo = p->pInfo;
int rc;
vfstrace_printf(pInfo, "%s.xClose(%s)", pInfo->zVfsName, p->zFName);
rc = p->pReal->pMethods->xClose(p->pReal);
vfstrace_print_errcode(pInfo, " -> %s\n", rc);
if( rc==SQLITE_OK ){
sqlite3_free((void*)p->base.pMethods);
p->base.pMethods = 0;
}
return rc;
}
/*
** Read data from an vfstrace-file.
*/
static int vfstraceRead(
sqlite3_file *pFile,
void *zBuf,
int iAmt,
sqlite_int64 iOfst
){
vfstrace_file *p = (vfstrace_file *)pFile;
vfstrace_info *pInfo = p->pInfo;
int rc;
vfstrace_printf(pInfo, "%s.xRead(%s,n=%d,ofst=%lld)",
pInfo->zVfsName, p->zFName, iAmt, iOfst);
rc = p->pReal->pMethods->xRead(p->pReal, zBuf, iAmt, iOfst);
vfstrace_print_errcode(pInfo, " -> %s\n", rc);
return rc;
}
/*
** Write data to an vfstrace-file.
*/
static int vfstraceWrite(
sqlite3_file *pFile,
const void *zBuf,
int iAmt,
sqlite_int64 iOfst
){
vfstrace_file *p = (vfstrace_file *)pFile;
vfstrace_info *pInfo = p->pInfo;
int rc;
vfstrace_printf(pInfo, "%s.xWrite(%s,n=%d,ofst=%lld)",
pInfo->zVfsName, p->zFName, iAmt, iOfst);
rc = p->pReal->pMethods->xWrite(p->pReal, zBuf, iAmt, iOfst);
vfstrace_print_errcode(pInfo, " -> %s\n", rc);
return rc;
}
/*
** Truncate an vfstrace-file.
*/
static int vfstraceTruncate(sqlite3_file *pFile, sqlite_int64 size){
vfstrace_file *p = (vfstrace_file *)pFile;
vfstrace_info *pInfo = p->pInfo;
int rc;
vfstrace_printf(pInfo, "%s.xTruncate(%s,%lld)", pInfo->zVfsName, p->zFName,
size);
rc = p->pReal->pMethods->xTruncate(p->pReal, size);
vfstrace_printf(pInfo, " -> %d\n", rc);
return rc;
}
/*
** Sync an vfstrace-file.
*/
static int vfstraceSync(sqlite3_file *pFile, int flags){
vfstrace_file *p = (vfstrace_file *)pFile;
vfstrace_info *pInfo = p->pInfo;
int rc;
int i;
char zBuf[100];
memcpy(zBuf, "|0", 3);
i = 0;
if( flags & SQLITE_SYNC_FULL ) strappend(zBuf, &i, "|FULL");
else if( flags & SQLITE_SYNC_NORMAL ) strappend(zBuf, &i, "|NORMAL");
if( flags & SQLITE_SYNC_DATAONLY ) strappend(zBuf, &i, "|DATAONLY");
if( flags & ~(SQLITE_SYNC_FULL|SQLITE_SYNC_DATAONLY) ){
sqlite3_snprintf(sizeof(zBuf)-i, &zBuf[i], "|0x%x", flags);
}
vfstrace_printf(pInfo, "%s.xSync(%s,%s)", pInfo->zVfsName, p->zFName,
&zBuf[1]);
rc = p->pReal->pMethods->xSync(p->pReal, flags);
vfstrace_printf(pInfo, " -> %d\n", rc);
return rc;
}
/*
** Return the current file-size of an vfstrace-file.
*/
static int vfstraceFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){
vfstrace_file *p = (vfstrace_file *)pFile;
vfstrace_info *pInfo = p->pInfo;
int rc;
vfstrace_printf(pInfo, "%s.xFileSize(%s)", pInfo->zVfsName, p->zFName);
rc = p->pReal->pMethods->xFileSize(p->pReal, pSize);
vfstrace_print_errcode(pInfo, " -> %s,", rc);
vfstrace_printf(pInfo, " size=%lld\n", *pSize);
return rc;
}
/*
** Return the name of a lock.
*/
static const char *lockName(int eLock){
const char *azLockNames[] = {
"NONE", "SHARED", "RESERVED", "PENDING", "EXCLUSIVE"
};
if( eLock<0 || eLock>=(int)(sizeof(azLockNames)/sizeof(azLockNames[0])) ){
return "???";
}else{
return azLockNames[eLock];
}
}
/*
** Lock an vfstrace-file.
*/
static int vfstraceLock(sqlite3_file *pFile, int eLock){
vfstrace_file *p = (vfstrace_file *)pFile;
vfstrace_info *pInfo = p->pInfo;
int rc;
vfstrace_printf(pInfo, "%s.xLock(%s,%s)", pInfo->zVfsName, p->zFName,
lockName(eLock));
rc = p->pReal->pMethods->xLock(p->pReal, eLock);
vfstrace_print_errcode(pInfo, " -> %s\n", rc);
return rc;
}
/*
** Unlock an vfstrace-file.
*/
static int vfstraceUnlock(sqlite3_file *pFile, int eLock){
vfstrace_file *p = (vfstrace_file *)pFile;
vfstrace_info *pInfo = p->pInfo;
int rc;
vfstrace_printf(pInfo, "%s.xUnlock(%s,%s)", pInfo->zVfsName, p->zFName,
lockName(eLock));
rc = p->pReal->pMethods->xUnlock(p->pReal, eLock);
vfstrace_print_errcode(pInfo, " -> %s\n", rc);
return rc;
}
/*
** Check if another file-handle holds a RESERVED lock on an vfstrace-file.
*/
static int vfstraceCheckReservedLock(sqlite3_file *pFile, int *pResOut){
vfstrace_file *p = (vfstrace_file *)pFile;
vfstrace_info *pInfo = p->pInfo;
int rc;
vfstrace_printf(pInfo, "%s.xCheckReservedLock(%s,%d)",
pInfo->zVfsName, p->zFName);
rc = p->pReal->pMethods->xCheckReservedLock(p->pReal, pResOut);
vfstrace_print_errcode(pInfo, " -> %s", rc);
vfstrace_printf(pInfo, ", out=%d\n", *pResOut);
return rc;
}
/*
** File control method. For custom operations on an vfstrace-file.
*/
static int vfstraceFileControl(sqlite3_file *pFile, int op, void *pArg){
vfstrace_file *p = (vfstrace_file *)pFile;
vfstrace_info *pInfo = p->pInfo;
int rc;
char zBuf[100];
char zBuf2[100];
char *zOp;
char *zRVal = 0;
switch( op ){
case SQLITE_FCNTL_LOCKSTATE: zOp = "LOCKSTATE"; break;
case SQLITE_GET_LOCKPROXYFILE: zOp = "GET_LOCKPROXYFILE"; break;
case SQLITE_SET_LOCKPROXYFILE: zOp = "SET_LOCKPROXYFILE"; break;
case SQLITE_LAST_ERRNO: zOp = "LAST_ERRNO"; break;
case SQLITE_FCNTL_SIZE_HINT: {
sqlite3_snprintf(sizeof(zBuf), zBuf, "SIZE_HINT,%lld",
*(sqlite3_int64*)pArg);
zOp = zBuf;
break;
}
case SQLITE_FCNTL_CHUNK_SIZE: {
sqlite3_snprintf(sizeof(zBuf), zBuf, "CHUNK_SIZE,%d", *(int*)pArg);
zOp = zBuf;
break;
}
case SQLITE_FCNTL_FILE_POINTER: zOp = "FILE_POINTER"; break;
case SQLITE_FCNTL_WIN32_AV_RETRY: zOp = "WIN32_AV_RETRY"; break;
case SQLITE_FCNTL_PERSIST_WAL: {
sqlite3_snprintf(sizeof(zBuf), zBuf, "PERSIST_WAL,%d", *(int*)pArg);
zOp = zBuf;
break;
}
case SQLITE_FCNTL_OVERWRITE: zOp = "OVERWRITE"; break;
case SQLITE_FCNTL_VFSNAME: zOp = "VFSNAME"; break;
case SQLITE_FCNTL_POWERSAFE_OVERWRITE: zOp = "POWERSAFE_OVERWRITE"; break;
case SQLITE_FCNTL_PRAGMA: {
const char *const* a = (const char*const*)pArg;
sqlite3_snprintf(sizeof(zBuf), zBuf, "PRAGMA,[%s,%s]",a[1],a[2]);
zOp = zBuf;
break;
}
case SQLITE_FCNTL_BUSYHANDLER: zOp = "BUSYHANDLER"; break;
case SQLITE_FCNTL_TEMPFILENAME: zOp = "TEMPFILENAME"; break;
case SQLITE_FCNTL_MMAP_SIZE: {
sqlite3_int64 iMMap = *(sqlite3_int64*)pArg;
sqlite3_snprintf(sizeof(zBuf), zBuf, "MMAP_SIZE,%lld",iMMap);
zOp = zBuf;
break;
}
case SQLITE_FCNTL_TRACE: zOp = "TRACE"; break;
case SQLITE_FCNTL_HAS_MOVED: zOp = "HAS_MOVED"; break;
case SQLITE_FCNTL_SYNC: zOp = "SYNC"; break;
case SQLITE_FCNTL_COMMIT_PHASETWO: zOp = "COMMIT_PHASETWO"; break;
case SQLITE_FCNTL_WIN32_SET_HANDLE: zOp = "WIN32_SET_HANDLE"; break;
case SQLITE_FCNTL_WAL_BLOCK: zOp = "WAL_BLOCK"; break;
case SQLITE_FCNTL_ZIPVFS: zOp = "ZIPVFS"; break;
case SQLITE_FCNTL_RBU: zOp = "RBU"; break;
case SQLITE_FCNTL_VFS_POINTER: zOp = "VFS_POINTER"; break;
case SQLITE_FCNTL_JOURNAL_POINTER: zOp = "JOURNAL_POINTER"; break;
case SQLITE_FCNTL_WIN32_GET_HANDLE: zOp = "WIN32_GET_HANDLE"; break;
case SQLITE_FCNTL_PDB: zOp = "PDB"; break;
case SQLITE_FCNTL_BEGIN_ATOMIC_WRITE: zOp = "BEGIN_ATOMIC_WRITE"; break;
case SQLITE_FCNTL_COMMIT_ATOMIC_WRITE: zOp = "COMMIT_ATOMIC_WRITE"; break;
case SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE: {
zOp = "ROLLBACK_ATOMIC_WRITE";
break;
}
case SQLITE_FCNTL_LOCK_TIMEOUT: {
sqlite3_snprintf(sizeof(zBuf), zBuf, "LOCK_TIMEOUT,%d", *(int*)pArg);
zOp = zBuf;
break;
}
case SQLITE_FCNTL_DATA_VERSION: zOp = "DATA_VERSION"; break;
case SQLITE_FCNTL_SIZE_LIMIT: zOp = "SIZE_LIMIT"; break;
case SQLITE_FCNTL_CKPT_DONE: zOp = "CKPT_DONE"; break;
case SQLITE_FCNTL_RESERVE_BYTES: zOp = "RESERVED_BYTES"; break;
case SQLITE_FCNTL_CKPT_START: zOp = "CKPT_START"; break;
case SQLITE_FCNTL_EXTERNAL_READER: zOp = "EXTERNAL_READER"; break;
case SQLITE_FCNTL_CKSM_FILE: zOp = "CKSM_FILE"; break;
case SQLITE_FCNTL_RESET_CACHE: zOp = "RESET_CACHE"; break;
case 0xca093fa0: zOp = "DB_UNCHANGED"; break;
default: {
sqlite3_snprintf(sizeof zBuf, zBuf, "%d", op);
zOp = zBuf;
break;
}
}
vfstrace_printf(pInfo, "%s.xFileControl(%s,%s)",
pInfo->zVfsName, p->zFName, zOp);
rc = p->pReal->pMethods->xFileControl(p->pReal, op, pArg);
if( rc==SQLITE_OK ){
switch( op ){
case SQLITE_FCNTL_VFSNAME: {
*(char**)pArg = sqlite3_mprintf("vfstrace.%s/%z",
pInfo->zVfsName, *(char**)pArg);
zRVal = *(char**)pArg;
break;
}
case SQLITE_FCNTL_MMAP_SIZE: {
sqlite3_snprintf(sizeof(zBuf2), zBuf2, "%lld", *(sqlite3_int64*)pArg);
zRVal = zBuf2;
break;
}
case SQLITE_FCNTL_HAS_MOVED:
case SQLITE_FCNTL_PERSIST_WAL: {
sqlite3_snprintf(sizeof(zBuf2), zBuf2, "%d", *(int*)pArg);
zRVal = zBuf2;
break;
}
case SQLITE_FCNTL_PRAGMA:
case SQLITE_FCNTL_TEMPFILENAME: {
zRVal = *(char**)pArg;
break;
}
}
}
if( zRVal ){
vfstrace_print_errcode(pInfo, " -> %s", rc);
vfstrace_printf(pInfo, ", %s\n", zRVal);
}else{
vfstrace_print_errcode(pInfo, " -> %s\n", rc);
}
return rc;
}
/*
** Return the sector-size in bytes for an vfstrace-file.
*/
static int vfstraceSectorSize(sqlite3_file *pFile){
vfstrace_file *p = (vfstrace_file *)pFile;
vfstrace_info *pInfo = p->pInfo;
int rc;
vfstrace_printf(pInfo, "%s.xSectorSize(%s)", pInfo->zVfsName, p->zFName);
rc = p->pReal->pMethods->xSectorSize(p->pReal);
vfstrace_printf(pInfo, " -> %d\n", rc);
return rc;
}
/*
** Return the device characteristic flags supported by an vfstrace-file.
*/
static int vfstraceDeviceCharacteristics(sqlite3_file *pFile){
vfstrace_file *p = (vfstrace_file *)pFile;
vfstrace_info *pInfo = p->pInfo;
int rc;
vfstrace_printf(pInfo, "%s.xDeviceCharacteristics(%s)",
pInfo->zVfsName, p->zFName);
rc = p->pReal->pMethods->xDeviceCharacteristics(p->pReal);
vfstrace_printf(pInfo, " -> 0x%08x\n", rc);
return rc;
}
/*
** Shared-memory operations.
*/
static int vfstraceShmLock(sqlite3_file *pFile, int ofst, int n, int flags){
vfstrace_file *p = (vfstrace_file *)pFile;
vfstrace_info *pInfo = p->pInfo;
int rc;
char zLck[100];
int i = 0;
memcpy(zLck, "|0", 3);
if( flags & SQLITE_SHM_UNLOCK ) strappend(zLck, &i, "|UNLOCK");
if( flags & SQLITE_SHM_LOCK ) strappend(zLck, &i, "|LOCK");
if( flags & SQLITE_SHM_SHARED ) strappend(zLck, &i, "|SHARED");
if( flags & SQLITE_SHM_EXCLUSIVE ) strappend(zLck, &i, "|EXCLUSIVE");
if( flags & ~(0xf) ){
sqlite3_snprintf(sizeof(zLck)-i, &zLck[i], "|0x%x", flags);
}
vfstrace_printf(pInfo, "%s.xShmLock(%s,ofst=%d,n=%d,%s)",
pInfo->zVfsName, p->zFName, ofst, n, &zLck[1]);
rc = p->pReal->pMethods->xShmLock(p->pReal, ofst, n, flags);
vfstrace_print_errcode(pInfo, " -> %s\n", rc);
return rc;
}
static int vfstraceShmMap(
sqlite3_file *pFile,
int iRegion,
int szRegion,
int isWrite,
void volatile **pp
){
vfstrace_file *p = (vfstrace_file *)pFile;
vfstrace_info *pInfo = p->pInfo;
int rc;
vfstrace_printf(pInfo, "%s.xShmMap(%s,iRegion=%d,szRegion=%d,isWrite=%d,*)",
pInfo->zVfsName, p->zFName, iRegion, szRegion, isWrite);
rc = p->pReal->pMethods->xShmMap(p->pReal, iRegion, szRegion, isWrite, pp);
vfstrace_print_errcode(pInfo, " -> %s\n", rc);
return rc;
}
static void vfstraceShmBarrier(sqlite3_file *pFile){
vfstrace_file *p = (vfstrace_file *)pFile;
vfstrace_info *pInfo = p->pInfo;
vfstrace_printf(pInfo, "%s.xShmBarrier(%s)\n", pInfo->zVfsName, p->zFName);
p->pReal->pMethods->xShmBarrier(p->pReal);
}
static int vfstraceShmUnmap(sqlite3_file *pFile, int delFlag){
vfstrace_file *p = (vfstrace_file *)pFile;
vfstrace_info *pInfo = p->pInfo;
int rc;
vfstrace_printf(pInfo, "%s.xShmUnmap(%s,delFlag=%d)",
pInfo->zVfsName, p->zFName, delFlag);
rc = p->pReal->pMethods->xShmUnmap(p->pReal, delFlag);
vfstrace_print_errcode(pInfo, " -> %s\n", rc);
return rc;
}
/*
** Open an vfstrace file handle.
*/
static int vfstraceOpen(
sqlite3_vfs *pVfs,
const char *zName,
sqlite3_file *pFile,
int flags,
int *pOutFlags
){
int rc;
vfstrace_file *p = (vfstrace_file *)pFile;
vfstrace_info *pInfo = (vfstrace_info*)pVfs->pAppData;
sqlite3_vfs *pRoot = pInfo->pRootVfs;
p->pInfo = pInfo;
p->zFName = zName ? fileTail(zName) : "<temp>";
p->pReal = (sqlite3_file *)&p[1];
rc = pRoot->xOpen(pRoot, zName, p->pReal, flags, pOutFlags);
vfstrace_printf(pInfo, "%s.xOpen(%s,flags=0x%x)",
pInfo->zVfsName, p->zFName, flags);
if( p->pReal->pMethods ){
sqlite3_io_methods *pNew = sqlite3_malloc( sizeof(*pNew) );
const sqlite3_io_methods *pSub = p->pReal->pMethods;
memset(pNew, 0, sizeof(*pNew));
pNew->iVersion = pSub->iVersion;
pNew->xClose = vfstraceClose;
pNew->xRead = vfstraceRead;
pNew->xWrite = vfstraceWrite;
pNew->xTruncate = vfstraceTruncate;
pNew->xSync = vfstraceSync;
pNew->xFileSize = vfstraceFileSize;
pNew->xLock = vfstraceLock;
pNew->xUnlock = vfstraceUnlock;
pNew->xCheckReservedLock = vfstraceCheckReservedLock;
pNew->xFileControl = vfstraceFileControl;
pNew->xSectorSize = vfstraceSectorSize;
pNew->xDeviceCharacteristics = vfstraceDeviceCharacteristics;
if( pNew->iVersion>=2 ){
pNew->xShmMap = pSub->xShmMap ? vfstraceShmMap : 0;
pNew->xShmLock = pSub->xShmLock ? vfstraceShmLock : 0;
pNew->xShmBarrier = pSub->xShmBarrier ? vfstraceShmBarrier : 0;
pNew->xShmUnmap = pSub->xShmUnmap ? vfstraceShmUnmap : 0;
}
pFile->pMethods = pNew;
}
vfstrace_print_errcode(pInfo, " -> %s", rc);
if( pOutFlags ){
vfstrace_printf(pInfo, ", outFlags=0x%x\n", *pOutFlags);
}else{
vfstrace_printf(pInfo, "\n");
}
return rc;
}
/*
** Delete the file located at zPath. If the dirSync argument is true,
** ensure the file-system modifications are synced to disk before
** returning.
*/
static int vfstraceDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){
vfstrace_info *pInfo = (vfstrace_info*)pVfs->pAppData;
sqlite3_vfs *pRoot = pInfo->pRootVfs;
int rc;
vfstrace_printf(pInfo, "%s.xDelete(\"%s\",%d)",
pInfo->zVfsName, zPath, dirSync);
rc = pRoot->xDelete(pRoot, zPath, dirSync);
vfstrace_print_errcode(pInfo, " -> %s\n", rc);
return rc;
}
/*
** Test for access permissions. Return true if the requested permission
** is available, or false otherwise.
*/
static int vfstraceAccess(
sqlite3_vfs *pVfs,
const char *zPath,
int flags,
int *pResOut
){
vfstrace_info *pInfo = (vfstrace_info*)pVfs->pAppData;
sqlite3_vfs *pRoot = pInfo->pRootVfs;
int rc;
vfstrace_printf(pInfo, "%s.xAccess(\"%s\",%d)",
pInfo->zVfsName, zPath, flags);
rc = pRoot->xAccess(pRoot, zPath, flags, pResOut);
vfstrace_print_errcode(pInfo, " -> %s", rc);
vfstrace_printf(pInfo, ", out=%d\n", *pResOut);
return rc;
}
/*
** Populate buffer zOut with the full canonical pathname corresponding
** to the pathname in zPath. zOut is guaranteed to point to a buffer
** of at least (DEVSYM_MAX_PATHNAME+1) bytes.
*/
static int vfstraceFullPathname(
sqlite3_vfs *pVfs,
const char *zPath,
int nOut,
char *zOut
){
vfstrace_info *pInfo = (vfstrace_info*)pVfs->pAppData;
sqlite3_vfs *pRoot = pInfo->pRootVfs;
int rc;
vfstrace_printf(pInfo, "%s.xFullPathname(\"%s\")",
pInfo->zVfsName, zPath);
rc = pRoot->xFullPathname(pRoot, zPath, nOut, zOut);
vfstrace_print_errcode(pInfo, " -> %s", rc);
vfstrace_printf(pInfo, ", out=\"%.*s\"\n", nOut, zOut);
return rc;
}
/*
** Open the dynamic library located at zPath and return a handle.
*/
static void *vfstraceDlOpen(sqlite3_vfs *pVfs, const char *zPath){
vfstrace_info *pInfo = (vfstrace_info*)pVfs->pAppData;
sqlite3_vfs *pRoot = pInfo->pRootVfs;
vfstrace_printf(pInfo, "%s.xDlOpen(\"%s\")\n", pInfo->zVfsName, zPath);
return pRoot->xDlOpen(pRoot, zPath);
}
/*
** Populate the buffer zErrMsg (size nByte bytes) with a human readable
** utf-8 string describing the most recent error encountered associated
** with dynamic libraries.
*/
static void vfstraceDlError(sqlite3_vfs *pVfs, int nByte, char *zErrMsg){
vfstrace_info *pInfo = (vfstrace_info*)pVfs->pAppData;
sqlite3_vfs *pRoot = pInfo->pRootVfs;
vfstrace_printf(pInfo, "%s.xDlError(%d)", pInfo->zVfsName, nByte);
pRoot->xDlError(pRoot, nByte, zErrMsg);
vfstrace_printf(pInfo, " -> \"%s\"", zErrMsg);
}
/*
** Return a pointer to the symbol zSymbol in the dynamic library pHandle.
*/
static void (*vfstraceDlSym(sqlite3_vfs *pVfs,void *p,const char *zSym))(void){
vfstrace_info *pInfo = (vfstrace_info*)pVfs->pAppData;
sqlite3_vfs *pRoot = pInfo->pRootVfs;
vfstrace_printf(pInfo, "%s.xDlSym(\"%s\")\n", pInfo->zVfsName, zSym);
return pRoot->xDlSym(pRoot, p, zSym);
}
/*
** Close the dynamic library handle pHandle.
*/
static void vfstraceDlClose(sqlite3_vfs *pVfs, void *pHandle){
vfstrace_info *pInfo = (vfstrace_info*)pVfs->pAppData;
sqlite3_vfs *pRoot = pInfo->pRootVfs;
vfstrace_printf(pInfo, "%s.xDlOpen()\n", pInfo->zVfsName);
pRoot->xDlClose(pRoot, pHandle);
}
/*
** Populate the buffer pointed to by zBufOut with nByte bytes of
** random data.
*/
static int vfstraceRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){
vfstrace_info *pInfo = (vfstrace_info*)pVfs->pAppData;
sqlite3_vfs *pRoot = pInfo->pRootVfs;
vfstrace_printf(pInfo, "%s.xRandomness(%d)\n", pInfo->zVfsName, nByte);
return pRoot->xRandomness(pRoot, nByte, zBufOut);
}
/*
** Sleep for nMicro microseconds. Return the number of microseconds
** actually slept.
*/
static int vfstraceSleep(sqlite3_vfs *pVfs, int nMicro){
vfstrace_info *pInfo = (vfstrace_info*)pVfs->pAppData;
sqlite3_vfs *pRoot = pInfo->pRootVfs;
return pRoot->xSleep(pRoot, nMicro);
}
/*
** Return the current time as a Julian Day number in *pTimeOut.
*/
static int vfstraceCurrentTime(sqlite3_vfs *pVfs, double *pTimeOut){
vfstrace_info *pInfo = (vfstrace_info*)pVfs->pAppData;
sqlite3_vfs *pRoot = pInfo->pRootVfs;
return pRoot->xCurrentTime(pRoot, pTimeOut);
}
static int vfstraceCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *pTimeOut){
vfstrace_info *pInfo = (vfstrace_info*)pVfs->pAppData;
sqlite3_vfs *pRoot = pInfo->pRootVfs;
return pRoot->xCurrentTimeInt64(pRoot, pTimeOut);
}
/*
** Return th3 most recent error code and message
*/
static int vfstraceGetLastError(sqlite3_vfs *pVfs, int iErr, char *zErr){
vfstrace_info *pInfo = (vfstrace_info*)pVfs->pAppData;
sqlite3_vfs *pRoot = pInfo->pRootVfs;
return pRoot->xGetLastError(pRoot, iErr, zErr);
}
/*
** Override system calls.
*/
static int vfstraceSetSystemCall(
sqlite3_vfs *pVfs,
const char *zName,
sqlite3_syscall_ptr pFunc
){
vfstrace_info *pInfo = (vfstrace_info*)pVfs->pAppData;
sqlite3_vfs *pRoot = pInfo->pRootVfs;
return pRoot->xSetSystemCall(pRoot, zName, pFunc);
}
static sqlite3_syscall_ptr vfstraceGetSystemCall(
sqlite3_vfs *pVfs,
const char *zName
){
vfstrace_info *pInfo = (vfstrace_info*)pVfs->pAppData;
sqlite3_vfs *pRoot = pInfo->pRootVfs;
return pRoot->xGetSystemCall(pRoot, zName);
}
static const char *vfstraceNextSystemCall(sqlite3_vfs *pVfs, const char *zName){
vfstrace_info *pInfo = (vfstrace_info*)pVfs->pAppData;
sqlite3_vfs *pRoot = pInfo->pRootVfs;
return pRoot->xNextSystemCall(pRoot, zName);
}
/*
** Clients invoke this routine to construct a new trace-vfs shim.
**
** Return SQLITE_OK on success.
**
** SQLITE_NOMEM is returned in the case of a memory allocation error.
** SQLITE_NOTFOUND is returned if zOldVfsName does not exist.
*/
int vfstrace_register(
const char *zTraceName, /* Name of the newly constructed VFS */
const char *zOldVfsName, /* Name of the underlying VFS */
int (*xOut)(const char*,void*), /* Output routine. ex: fputs */
void *pOutArg, /* 2nd argument to xOut. ex: stderr */
int makeDefault /* True to make the new VFS the default */
){
sqlite3_vfs *pNew;
sqlite3_vfs *pRoot;
vfstrace_info *pInfo;
size_t nName;
size_t nByte;
pRoot = sqlite3_vfs_find(zOldVfsName);
if( pRoot==0 ) return SQLITE_NOTFOUND;
nName = strlen(zTraceName);
nByte = sizeof(*pNew) + sizeof(*pInfo) + nName + 1;
pNew = sqlite3_malloc64( nByte );
if( pNew==0 ) return SQLITE_NOMEM;
memset(pNew, 0, nByte);
pInfo = (vfstrace_info*)&pNew[1];
pNew->iVersion = pRoot->iVersion;
pNew->szOsFile = pRoot->szOsFile + sizeof(vfstrace_file);
pNew->mxPathname = pRoot->mxPathname;
pNew->zName = (char*)&pInfo[1];
memcpy((char*)&pInfo[1], zTraceName, nName+1);
pNew->pAppData = pInfo;
pNew->xOpen = vfstraceOpen;
pNew->xDelete = vfstraceDelete;
pNew->xAccess = vfstraceAccess;
pNew->xFullPathname = vfstraceFullPathname;
pNew->xDlOpen = pRoot->xDlOpen==0 ? 0 : vfstraceDlOpen;
pNew->xDlError = pRoot->xDlError==0 ? 0 : vfstraceDlError;
pNew->xDlSym = pRoot->xDlSym==0 ? 0 : vfstraceDlSym;
pNew->xDlClose = pRoot->xDlClose==0 ? 0 : vfstraceDlClose;
pNew->xRandomness = vfstraceRandomness;
pNew->xSleep = vfstraceSleep;
pNew->xCurrentTime = vfstraceCurrentTime;
pNew->xGetLastError = pRoot->xGetLastError==0 ? 0 : vfstraceGetLastError;
if( pNew->iVersion>=2 ){
pNew->xCurrentTimeInt64 = pRoot->xCurrentTimeInt64==0 ? 0 :
vfstraceCurrentTimeInt64;
if( pNew->iVersion>=3 ){
pNew->xSetSystemCall = pRoot->xSetSystemCall==0 ? 0 :
vfstraceSetSystemCall;
pNew->xGetSystemCall = pRoot->xGetSystemCall==0 ? 0 :
vfstraceGetSystemCall;
pNew->xNextSystemCall = pRoot->xNextSystemCall==0 ? 0 :
vfstraceNextSystemCall;
}
}
pInfo->pRootVfs = pRoot;
pInfo->xOut = xOut;
pInfo->pOutArg = pOutArg;
pInfo->zVfsName = pNew->zName;
pInfo->pTraceVfs = pNew;
vfstrace_printf(pInfo, "%s.enabled_for(\"%s\")\n",
pInfo->zVfsName, pRoot->zName);
return sqlite3_vfs_register(pNew, makeDefault);
}
/*
** Look for the named VFS. If it is a TRACEVFS, then unregister it
** and delete it.
*/
void vfstrace_unregister(const char *zTraceName){
sqlite3_vfs *pVfs = sqlite3_vfs_find(zTraceName);
if( pVfs==0 ) return;
if( pVfs->xOpen!=vfstraceOpen ) return;
sqlite3_vfs_unregister(pVfs);
sqlite3_free(pVfs);
}

41
ext/rbu/sqlite3rbu.c
View File

@ -336,6 +336,27 @@ struct RbuFrame {
u32 iWalFrame;
};
#ifndef UNUSED_PARAMETER
/*
** The following macros are used to suppress compiler warnings and to
** make it clear to human readers when a function parameter is deliberately
** left unused within the body of a function. This usually happens when
** a function is called via a function pointer. For example the
** implementation of an SQL aggregate step callback may not use the
** parameter indicating the number of arguments passed to the aggregate,
** if it knows that this is enforced elsewhere.
**
** When a function parameter is not used at all within the body of a function,
** it is generally named "NotUsed" or "NotUsed2" to make things even clearer.
** However, these macros may also be used to suppress warnings related to
** parameters that may or may not be used depending on compilation options.
** For example those parameters only used in assert() statements. In these
** cases the parameters are named as per the usual conventions.
*/
#define UNUSED_PARAMETER(x) (void)(x)
#define UNUSED_PARAMETER2(x,y) UNUSED_PARAMETER(x),UNUSED_PARAMETER(y)
#endif
/*
** RBU handle.
**
@ -387,7 +408,7 @@ struct sqlite3rbu {
int rc; /* Value returned by last rbu_step() call */
char *zErrmsg; /* Error message if rc!=SQLITE_OK */
int nStep; /* Rows processed for current object */
int nProgress; /* Rows processed for all objects */
sqlite3_int64 nProgress; /* Rows processed for all objects */
RbuObjIter objiter; /* Iterator for skipping through tbl/idx */
const char *zVfsName; /* Name of automatically created rbu vfs */
rbu_file *pTargetFd; /* File handle open on target db */
@ -504,7 +525,7 @@ static unsigned int rbuDeltaGetInt(const char **pz, int *pLen){
v = (v<<6) + c;
}
z--;
*pLen -= z - zStart;
*pLen -= (int)(z - zStart);
*pz = (char*)z;
return v;
}
@ -689,6 +710,7 @@ static void rbuFossilDeltaFunc(
char *aOut;
assert( argc==2 );
UNUSED_PARAMETER(argc);
nOrig = sqlite3_value_bytes(argv[0]);
aOrig = (const char*)sqlite3_value_blob(argv[0]);
@ -2268,13 +2290,13 @@ static char *rbuObjIterGetIndexWhere(sqlite3rbu *p, RbuObjIter *pIter){
else if( c==')' ){
nParen--;
if( nParen==0 ){
int nSpan = &zSql[i] - pIter->aIdxCol[iIdxCol].zSpan;
int nSpan = (int)(&zSql[i] - pIter->aIdxCol[iIdxCol].zSpan);
pIter->aIdxCol[iIdxCol++].nSpan = nSpan;
i++;
break;
}
}else if( c==',' && nParen==1 ){
int nSpan = &zSql[i] - pIter->aIdxCol[iIdxCol].zSpan;
int nSpan = (int)(&zSql[i] - pIter->aIdxCol[iIdxCol].zSpan);
pIter->aIdxCol[iIdxCol++].nSpan = nSpan;
pIter->aIdxCol[iIdxCol].zSpan = &zSql[i+1];
}else if( c=='"' || c=='\'' || c=='`' ){
@ -2964,6 +2986,8 @@ static void rbuFileSuffix3(const char *zBase, char *z){
for(i=sz-1; i>0 && z[i]!='/' && z[i]!='.'; i--){}
if( z[i]=='.' && sz>i+4 ) memmove(&z[i+1], &z[sz-3], 4);
}
#else
UNUSED_PARAMETER2(zBase,z);
#endif
}
@ -3548,7 +3572,7 @@ static void rbuSaveState(sqlite3rbu *p, int eStage){
"(%d, %Q), "
"(%d, %Q), "
"(%d, %d), "
"(%d, %d), "
"(%d, %lld), "
"(%d, %lld), "
"(%d, %lld), "
"(%d, %lld), "
@ -3906,6 +3930,7 @@ static void rbuIndexCntFunc(
sqlite3 *db = (rbuIsVacuum(p) ? p->dbRbu : p->dbMain);
assert( nVal==1 );
UNUSED_PARAMETER(nVal);
rc = prepareFreeAndCollectError(db, &pStmt, &zErrmsg,
sqlite3_mprintf("SELECT count(*) FROM sqlite_schema "
@ -4181,7 +4206,7 @@ sqlite3rbu *sqlite3rbu_vacuum(
){
if( zTarget==0 ){ return rbuMisuseError(); }
if( zState ){
int n = strlen(zState);
size_t n = strlen(zState);
if( n>=7 && 0==memcmp("-vactmp", &zState[n-7], 7) ){
return rbuMisuseError();
}
@ -4398,6 +4423,7 @@ int sqlite3rbu_savestate(sqlite3rbu *p){
*/
static int xDefaultRename(void *pArg, const char *zOld, const char *zNew){
int rc = SQLITE_OK;
UNUSED_PARAMETER(pArg);
#if defined(_WIN32_WCE)
{
LPWSTR zWideOld;
@ -5302,6 +5328,9 @@ static int rbuVfsCurrentTime(sqlite3_vfs *pVfs, double *pTimeOut){
** No-op.
*/
static int rbuVfsGetLastError(sqlite3_vfs *pVfs, int a, char *b){
UNUSED_PARAMETER(pVfs);
UNUSED_PARAMETER(a);
UNUSED_PARAMETER(b);
return 0;
}

1
ext/session/sessionalter.test
View File

@ -202,6 +202,7 @@ foreach {tn sql1 at sql2} {
sqlite3changegroup grp
grp schema db main
breakpoint
grp add $C1
grp add $C2
set T1 [grp output]

24
ext/session/sessionfault3.test
View File

@ -56,4 +56,28 @@ do_faultsim_test 1 -faults oom* -prep {
faultsim_test_result {0 {}} {1 SQLITE_NOMEM}
}
#-------------------------------------------------------------------------
reset_db
do_execsql_test 2.0 {
CREATE TABLE t1(a INTEGER PRIMARY KEY, b);
INSERT INTO t1 VALUES(1, 'one');
INSERT INTO t1 VALUES(2, 'two');
ALTER TABLE t1 ADD COLUMN c DEFAULT 'abcdefghijklmnopqrstuvwxyz';
}
faultsim_save_and_close
do_faultsim_test 2 -faults oom-t* -prep {
faultsim_restore_and_reopen
db eval {SELECT * FROM sqlite_schema}
} -body {
sqlite3session S db main
S attach *
execsql {
DELETE FROM t1 WHERE a = 1;
}
} -test {
faultsim_test_result {0 {}} {1 SQLITE_NOMEM}
catch { S delete }
}
finish_test

17
ext/session/sqlite3session.c
View File

@ -1757,16 +1757,19 @@ static void sessionPreupdateOneChange(
for(i=0; i<(pTab->nCol-pTab->bRowid); i++){
sqlite3_value *p = 0;
if( op!=SQLITE_INSERT ){
TESTONLY(int trc = ) pSession->hook.xOld(pSession->hook.pCtx, i, &p);
assert( trc==SQLITE_OK );
/* This may fail if the column has a non-NULL default and was added
** using ALTER TABLE ADD COLUMN after this record was created. */
rc = pSession->hook.xOld(pSession->hook.pCtx, i, &p);
}else if( pTab->abPK[i] ){
TESTONLY(int trc = ) pSession->hook.xNew(pSession->hook.pCtx, i, &p);
assert( trc==SQLITE_OK );
}
/* This may fail if SQLite value p contains a utf-16 string that must
** be converted to utf-8 and an OOM error occurs while doing so. */
rc = sessionSerializeValue(0, p, &nByte);
if( rc==SQLITE_OK ){
/* This may fail if SQLite value p contains a utf-16 string that must
** be converted to utf-8 and an OOM error occurs while doing so. */
rc = sessionSerializeValue(0, p, &nByte);
}
if( rc!=SQLITE_OK ) goto error_out;
}
if( pTab->bRowid ){
@ -5660,6 +5663,9 @@ static int sessionChangesetExtendRecord(
sessionAppendBlob(pOut, aRec, nRec, &rc);
if( rc==SQLITE_OK && pTab->pDfltStmt==0 ){
rc = sessionPrepareDfltStmt(pGrp->db, pTab, &pTab->pDfltStmt);
if( rc==SQLITE_OK && SQLITE_ROW!=sqlite3_step(pTab->pDfltStmt) ){
rc = sqlite3_errcode(pGrp->db);
}
}
for(ii=nCol; rc==SQLITE_OK && ii<pTab->nCol; ii++){
int eType = sqlite3_column_type(pTab->pDfltStmt, ii);
@ -5676,6 +5682,7 @@ static int sessionChangesetExtendRecord(
}
if( SQLITE_OK==sessionBufferGrow(pOut, 8, &rc) ){
sessionPutI64(&pOut->aBuf[pOut->nBuf], iVal);
pOut->nBuf += 8;
}
break;
}

8
ext/session/test_session.c
View File

@ -383,10 +383,10 @@ static int SQLITE_TCLAPI test_session_cmd(
{ "rowid", SQLITE_SESSION_OBJCONFIG_ROWID },
{ 0, 0 }
};
size_t sz = sizeof(aOpt[0]);
int sz = (int)sizeof(aOpt[0]);
int iArg;
int iOpt;
Tcl_Size iOpt;
if( Tcl_GetIndexFromObjStruct(interp,objv[2],aOpt,sz,"option",0,&iOpt) ){
return TCL_ERROR;
}
@ -803,7 +803,7 @@ static int SQLITE_TCLAPI testSqlite3changesetApply(
if( bV2 ){
while( objc>1 ){
const char *z1 = Tcl_GetString(objv[1]);
int n = strlen(z1);
int n = (int)strlen(z1);
if( n>3 && n<=12 && 0==sqlite3_strnicmp("-nosavepoint", z1, n) ){
flags |= SQLITE_CHANGESETAPPLY_NOSAVEPOINT;
}
@ -1119,7 +1119,7 @@ static int SQLITE_TCLAPI test_sqlite3session_foreach(
while( objc>1 ){
char *zOpt = Tcl_GetString(objv[1]);
int nOpt = strlen(zOpt);
int nOpt = (int)strlen(zOpt);
if( zOpt[0]!='-' ) break;
if( nOpt<=7 && 0==sqlite3_strnicmp(zOpt, "-invert", nOpt) ){
isInvert = 1;