diff --git a/ext/misc/base64.c b/ext/misc/base64.c index 1ed1a4f174..d6f7834a85 100644 --- a/ext/misc/base64.c +++ b/ext/misc/base64.c @@ -12,7 +12,7 @@ ** ** This is a SQLite extension for converting in either direction ** between a (binary) blob and base64 text. Base64 can transit a -** sane ASCII channel unmolested. It also plays nicely in CSV or +** sane USASCII channel unmolested. It also plays nicely in CSV or ** written as TCL brace-enclosed literals or SQL string literals, ** and can be used unmodified in XML-like documents. ** @@ -37,10 +37,10 @@ ** error will be thrown for other input argument types. ** ** This code relies on UTF-8 encoding only with respect to the -** meaning of the first 128 (7-bit) codes being the same as ASCII. +** meaning of the first 128 (7-bit) codes matching that of USASCII. ** It will fail miserably if somehow made to try to convert EBCDIC. -** Because it is table-driven, it could be enhanced to handle that. -** But the world and SQLite have moved on from that anachronism. +** Because it is table-driven, it could be enhanced to handle that, +** but the world and SQLite have moved on from that anachronism. ** ** To build the extension: ** Set shell variable SQDIR= @@ -52,12 +52,16 @@ #include +#ifndef SQLITE_SHELL_EXTFUNCS /* Guard for #include as built-in extension. */ #include "sqlite3ext.h" +#endif + SQLITE_EXTENSION_INIT1; #define PC 0x80 /* pad character */ #define WS 0x81 /* whitespace */ #define ND 0x82 /* Not above or digit-value */ +#define PAD_CHAR '=' typedef unsigned char ubyte; @@ -89,10 +93,10 @@ static const char b64Numerals[64] #define IS_BX_PAD(bdp) ((bdp)==PC) #define BX_NUMERAL(dv) (b64Numerals[dv]) /* Width of base64 lines. Should be an integer multiple of 4. */ -#define DARK_MAX 72 +#define B64_DARK_MAX 72 /* Encode a byte buffer into base64 text. If pSep!=0, it's a C string -** to be appended to encoded groups to limit their length to DARK_MAX +** to be appended to encoded groups to limit their length to B64_DARK_MAX ** or to terminate the last group (to aid concatenation.) */ static char* toBase64( ubyte *pIn, int nbIn, char *pOut, char *pSep ){ @@ -111,12 +115,12 @@ static char* toBase64( ubyte *pIn, int nbIn, char *pOut, char *pSep ){ nc = ncio[nbi]; nbIn -= nbi; for( nbe=3; nbe>=0; --nbe ){ - char ce = (nbe0 && *pIn!='=' ){ + while( ncIn>0 && *pIn!=PAD_CHAR ){ static signed char nboi[] = { 0, 0, 1, 2, 3 }; char *pUse = skipNonB64(pIn); unsigned long qv = 0L; - int nti, nbo; + int nti, nbo, nac; ncIn -= (pUse - pIn); - if( ncIn<=0 ) break; pIn = pUse; nti = (ncIn>4)? 4 : ncIn; nbo = nboi[nti]; - while( nti>0 ){ - char c = *pIn++; + if( nbo==0 ) break; + for( nac=0; nac<4; ++nac ){ + char c = (nac<=nti)? *pIn++ : PAD_CHAR; ubyte bdp = BX_DV_PROTO(c); --ncIn; switch( bdp ){ @@ -157,15 +161,16 @@ static ubyte* fromBase64( char *pIn, int ncIn, ubyte *pOut ){ break; case PC: bdp = 0; + --nbo; /* fall thru */ - default: /* It's the digit value. */ + default: /* bdp is the digit value. */ qv = qv<<6 | bdp; - --nti; break; } } + nti = 2; while( nbo-- > 0 ){ - *pOut++ = (qv >> (8*nbo))&0xff; + *pOut++ = (qv >> (8*nti--))&0xff; } } return pOut; @@ -183,7 +188,7 @@ static void base64(sqlite3_context *context, int na, sqlite3_value *av[]){ case SQLITE_BLOB: nb = nv; nc = 4*(nv+2/3); /* quads needed */ - nc += (nc+(DARK_MAX-1))/DARK_MAX + 1; /* LFs and a 0-terminator */ + nc += (nc+(B64_DARK_MAX-1))/B64_DARK_MAX + 1; /* LFs and a 0-terminator */ if( nvMax < nc ){ sqlite3_result_error(context, "blob expanded to base64 too big.", -1); } @@ -219,11 +224,15 @@ static void base64(sqlite3_context *context, int na, sqlite3_value *av[]){ /* ** Establish linkage to running SQLite library. */ +#ifndef SQLITE_SHELL_EXTFUNCS #ifdef _WIN32 __declspec(dllexport) #endif -int sqlite3_base_init(sqlite3 *db, char **pzErr, - const sqlite3_api_routines *pApi){ +int sqlite3_base_init +#else +static int sqlite3_base64_init +#endif +(sqlite3 *db, char **pzErr, const sqlite3_api_routines *pApi){ SQLITE_EXTENSION_INIT2(pApi); (void)pzErr; return sqlite3_create_function @@ -231,3 +240,11 @@ int sqlite3_base_init(sqlite3 *db, char **pzErr, SQLITE_DETERMINISTIC|SQLITE_INNOCUOUS|SQLITE_DIRECTONLY|SQLITE_UTF8, 0, base64, 0, 0); } + +/* +** Define some macros to allow this extension to be built into the shell +** conveniently, in conjunction with use of SQLITE_SHELL_EXTFUNCS. This +** allows shell.c, as distributed, to have this extension built in. +*/ +#define BASE64_INIT(db) sqlite3_base64_init(db, 0, 0) +#define BASE64_EXPOSE(db, pzErr) /* Not needed, ..._init() does this. */ diff --git a/ext/misc/base85.c b/ext/misc/base85.c index 365db80f2b..dfa9bd0e31 100644 --- a/ext/misc/base85.c +++ b/ext/misc/base85.c @@ -14,7 +14,7 @@ ** It can be built as a standalone program or an SQLite3 extension. ** ** Much like base64 representations, base85 can be sent through a -** sane ASCII channel unmolested. It also plays nicely in CSV or +** sane USASCII channel unmolested. It also plays nicely in CSV or ** written as TCL brace-enclosed literals or SQL string literals. ** It is not suited for unmodified use in XML-like documents. ** @@ -24,7 +24,7 @@ ** Further, this is an independent implementation of a base85 system. ** Hence, the author has rightfully put this into the public domain. ** -** Base85 numerals are taken from the set of 7-bit ASCII codes, +** Base85 numerals are taken from the set of 7-bit USASCII codes, ** excluding control characters and Space ! " ' ( ) { | } ~ Del ** in code order representing digit values 0 to 84 (base 10.) ** @@ -32,8 +32,11 @@ ** are represented as 5-digit base85 numbers with MS to LS digit ** order. Groups of 1-3 bytes are represented with 2-4 digits, ** still big-endian but 8-24 bit values. (Using big-endian yields -** the simplest transition to byte groups smaller than 4 bytes.) +** the simplest transition to byte groups smaller than 4 bytes. +** These byte groups can also be considered base-256 numbers.) ** Groups of 0 bytes are represented with 0 digits and vice-versa. +** No pad characters are used; Encoded base85 numeral sequence +** (aka "group") length maps 1-to-1 to the decoded binary length. ** ** Any character not in the base85 numeral set delimits groups. ** When base85 is streamed or stored in containers of indefinite @@ -82,7 +85,10 @@ #ifndef BASE85_STANDALONE +#ifndef SQLITE_SHELL_EXTFUNCS /* Guard for #include as built-in extension. */ # include "sqlite3ext.h" +#endif + SQLITE_EXTENSION_INIT1; #else @@ -107,7 +113,7 @@ static void sayHelp(){ } #endif -/* Classify c according to interval within ASCII set w.r.t. base85 +/* Classify c according to interval within USASCII set w.r.t. base85 * Values of 1 and 3 are base85 numerals. Values of 0, 2, or 4 are not. */ #define B85_CLASS( c ) (((c)>='#')+((c)>'&')+((c)>='*')+((c)>'z')) @@ -127,16 +133,25 @@ static unsigned char base85DigitValue( char c ){ } #endif +/* Width of base64 lines. Should be an integer multiple of 5. */ +#define B85_DARK_MAX 80 + + static char * skipNonB85( char *s ){ char c; while( (c = *s) && !IS_B85(c) ) ++s; return s; } +/* Convert small integer, known to be in 0..84 inclusive, to base85 numeral.*/ static char base85Numeral( unsigned char b ){ return (b < 4)? (char)(b + '#') : (char)(b - 4 + '*'); } +/* Encode a byte buffer into base85 text. If pSep!=0, it's a C string +** to be appended to encoded groups to limit their length to B85_DARK_MAX +** or to terminate the last group (to aid concatenation.) +*/ static char* toBase85( unsigned char *pIn, int nbIn, char *pOut, char *pSep ){ int nCol = 0; *pOut = 0; @@ -157,7 +172,7 @@ static char* toBase85( unsigned char *pIn, int nbIn, char *pOut, char *pSep ){ pOut[--nco] = base85Numeral(dv); } pOut += ncio[nbi]; - if( pSep && ((nCol += ncio[nbi])>=80 || nbIn<=0) ){ + if( pSep && ((nCol += ncio[nbi])>=B85_DARK_MAX || nbIn<=0) ){ char *p = pSep; while( *p ) *pOut++ = *p++; nCol = 0; @@ -167,6 +182,7 @@ static char* toBase85( unsigned char *pIn, int nbIn, char *pOut, char *pSep ){ return pOut; } +/* Decode base85 text into a byte buffer. */ static unsigned char* fromBase85( char *pIn, int ncIn, unsigned char *pOut ){ if( ncIn>0 && pIn[ncIn-1]=='\n' ) --ncIn; while( ncIn>0 ){ @@ -196,6 +212,7 @@ static unsigned char* fromBase85( char *pIn, int ncIn, unsigned char *pOut ){ } #ifndef OMIT_BASE85_CHECKER +/* Say whether input char sequence is all (base85 and/or whitespace).*/ static int allBase85( char *p, int len ){ char c; while( len-- > 0 && (c = *p++) != 0 ){ @@ -208,6 +225,7 @@ static int allBase85( char *p, int len ){ #ifndef BASE85_STANDALONE # ifndef OMIT_BASE85_CHECKER +/* This function does the work for the SQLite is_base85(t) UDF. */ static void is_base85(sqlite3_context *context, int na, sqlite3_value *av[]){ assert(na==1); switch( sqlite3_value_type(av[0]) ){ @@ -228,6 +246,7 @@ static void is_base85(sqlite3_context *context, int na, sqlite3_value *av[]){ } # endif +/* This function does the work for the SQLite base85(x) UDF. */ static void base85(sqlite3_context *context, int na, sqlite3_value *av[]){ int nb, nc, nv = sqlite3_value_bytes(av[0]); int nvMax = sqlite3_limit(sqlite3_context_db_handle(context), @@ -272,11 +291,18 @@ static void base85(sqlite3_context *context, int na, sqlite3_value *av[]){ sqlite3_result_error(context, "base85 OOM", -1); } +/* +** Establish linkage to running SQLite library. +*/ +#ifndef SQLITE_SHELL_EXTFUNCS #ifdef _WIN32 __declspec(dllexport) #endif -int sqlite3_base_init(sqlite3 *db, char **pzErr, - const sqlite3_api_routines *pApi){ +int sqlite3_base_init +#else +static int sqlite3_base85_init +#endif +(sqlite3 *db, char **pzErr, const sqlite3_api_routines *pApi){ SQLITE_EXTENSION_INIT2(pApi); (void)pzErr; # ifndef OMIT_BASE85_CHECKER @@ -294,12 +320,20 @@ int sqlite3_base_init(sqlite3 *db, char **pzErr, 0, base85, 0, 0); } +/* +** Define some macros to allow this extension to be built into the shell +** conveniently, in conjunction with use of SQLITE_SHELL_EXTFUNCS. This +** allows shell.c, as distributed, to have this extension built in. +*/ +# define BASE85_INIT(db) sqlite3_base85_init(db, 0, 0) +# define BASE85_EXPOSE(db, pzErr) /* Not needed, ..._init() does this. */ + #else /* standalone program */ int main(int na, char *av[]){ int cin; int rc = 0; - unsigned char bBuf[64]; + unsigned char bBuf[4*(B85_DARK_MAX/5)]; char cBuf[5*(sizeof(bBuf)/4)+2]; size_t nio; # ifndef OMIT_BASE85_CHECKER diff --git a/ext/misc/basexx.c b/ext/misc/basexx.c new file mode 100644 index 0000000000..700a99409c --- /dev/null +++ b/ext/misc/basexx.c @@ -0,0 +1,64 @@ +/* +** 2022-11-20 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This source allows multiple SQLite extensions to be either: combined +** into a single runtime-loadable library; or built into the SQLite shell +** using a preprocessing convention set by src/shell.c.in (and shell.c). +** +** Presently, it combines the base64.c and base85.c extensions. However, +** it can be used as a template for other combinations. +*/ + +#ifndef SQLITE_SHELL_EXTFUNCS /* Guard for #include as built-in extension. */ +# include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1; +#endif + +static void init_api_ptr(const sqlite3_api_routines *pApi){ + SQLITE_EXTENSION_INIT2(pApi); +} + +#undef SQLITE_EXTENSION_INIT1 +#define SQLITE_EXTENSION_INIT1 /* */ +#undef SQLITE_EXTENSION_INIT2 +#define SQLITE_EXTENSION_INIT2(v) (void)v + +/* These next 2 undef's are only needed because the entry point names + * collide when formulated per the rules stated for loadable extension + * entry point names that will be deduced from the file basenames. + */ +#undef sqlite3_base_init +#define sqlite3_base_init sqlite3_base64_init +#include "base64.c" + +#undef sqlite3_base_init +#define sqlite3_base_init sqlite3_base85_init +#include "base85.c" + +static int sqlite3_basexx_init(sqlite3 *db, char **pzErr, + const sqlite3_api_routines *pApi){ + init_api_ptr(pApi); + int rc1 = BASE64_INIT(db); + int rc2 = BASE85_INIT(db); + int rc = SQLITE_OK; + + if( rc1==SQLITE_OK && rc2==SQLITE_OK ){ + BASE64_EXPOSE(db, pzErr); + BASE64_EXPOSE(db, pzErr); + return SQLITE_OK; + }else{ + return SQLITE_ERROR; + } +} + +# define BASEXX_INIT(db) sqlite3_basexx_init(db, 0, 0) +# define BASEXX_EXPOSE(db, pzErr) /* Not needed, ..._init() does this. */ diff --git a/manifest b/manifest index faf2f3f49e..f02b4834b7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C New\sextensions\sfor\sbase85\sand\sbase64\sconversion\sUDFs -D 2022-11-19T02:39:16.296 +C Fix\sa\sbase64\sdecode\sbug.\sProvide\sfor\sconvenient\sinclusion\sof\sextension(s)\sbuilt\sinto\sthe\sCLI,\sto\ssimplify\stesting\sand\sfor\sits\sown\ssake.\sImprove\scomments.\sCure\scollision\sbetween\sbase64.c\sand\sbase85.c\swhen\sboth\sare\sin\sthe\ssame\stranslation\sunit. +D 2022-11-21T00:11:09.323 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -289,8 +289,9 @@ F ext/misc/README.md d6dd0fe1d8af77040216798a6a2b0c46c73054d2f0ea544fbbcdccf6f23 F ext/misc/amatch.c e3ad5532799cee9a97647f483f67f43b38796b84b5a8c60594fe782a4338f358 F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb F ext/misc/appendvfs.c 9642c7a194a2a25dca7ad3e36af24a0a46d7702168c4ad7e59c9f9b0e16a3824 -F ext/misc/base64.c e4d3f13bb59aa903734b557e1400f3c5961f0a5abc40400e0e2aa3ad362c0fd7 -F ext/misc/base85.c 3e07aea038129ee646b129ac2ed736344ea03859ff447019e67250b80722c528 +F ext/misc/base64.c 05da915d991f24e59515d0566f9a206c338032410d42472a3d9348da8bc166f9 +F ext/misc/base85.c 2c680ca7733f9a86f5d292fec71d10777290e68e7ae59d90597ae75fc44a88b6 +F ext/misc/basexx.c d32037f1414d9da11e0be334395cd4d82438b756933c99fc4075b6346cd11fd2 F ext/misc/blobio.c a867c4c4617f6ec223a307ebfe0eabb45e0992f74dd47722b96f3e631c0edb2a F ext/misc/btreeinfo.c d28ce349b40054eaa9473e835837bad7a71deec33ba13e39f963d50933bfa0f9 F ext/misc/carray.c b752f46411e4e47e34dce6f0c88bc8e51bb821ba9e49bfcd882506451c928f69 @@ -640,7 +641,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 9886d6669f5787471aab6ae52af76fad90b53edb1c218fc9ed9d953363bc5184 -F src/shell.c.in 16740a86346ba9823f92528ec588f2b74f68166dac965dabd19883ace230f11d +F src/shell.c.in 49b6aeaf950b9b7a74aca7b80e6d004ed32f0a16d2c9471e34587cc4755d5eaa F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f @@ -2057,8 +2058,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0cbf55407a3a94b1c9c0ada52fa2995088bac3739876fa8d465dfb4dfcc4a6ea -R 4ecad003b3a6f60dd6b11d354f8edbe2 +P 5cc1fe1ddc2a33c59d3c006057e474c7c7975c483395ddea530df6968fe15341 +R 0260a27c5340f9b412a0f6e0981a0d6d U larrybr -Z d182470ca73df70df09238ac5a5e4382 +Z 18487c3a1bcdc27a89418b38ad3e3f97 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b47c8a0962..41284b2a7c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5cc1fe1ddc2a33c59d3c006057e474c7c7975c483395ddea530df6968fe15341 \ No newline at end of file +07543d23a98c2a851393a2674e59d3cf1df37c244fb451cb7436f49c95c1423f \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index db8d987660..478c055357 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -1070,6 +1070,9 @@ INCLUDE ../ext/recover/dbdata.c INCLUDE ../ext/recover/sqlite3recover.h INCLUDE ../ext/recover/sqlite3recover.c #endif +#ifdef SQLITE_SHELL_EXTSRC +# include SHELL_STRINGIFY(SQLITE_SHELL_EXTSRC) +#endif #if defined(SQLITE_ENABLE_SESSION) /* @@ -5115,6 +5118,7 @@ static void open_db(ShellState *p, int openFlags){ } exit(1); } + #ifndef SQLITE_OMIT_LOAD_EXTENSION sqlite3_enable_load_extension(p->db, 1); #endif @@ -5137,6 +5141,34 @@ static void open_db(ShellState *p, int openFlags){ sqlite3_sqlar_init(p->db, 0, 0); } #endif +#ifdef SQLITE_SHELL_EXTFUNCS + /* Create a preprocessing mechanism for extensions to make + * their own provisions for being built into the shell. + * This is a short-span macro. See further below for usage. + */ +#define SHELL_SUB_MACRO(base, variant) base ## _ ## variant +#define SHELL_SUBMACRO(base, variant) SHELL_SUB_MACRO(base, variant) + /* Let custom-included extensions get their ..._init() called. + * The WHATEVER_INIT( db, pzErrorMsg, pApi ) macro should cause + * the extension's sqlite3_*_init( db, pzErrorMsg, pApi ) + * inititialization routine to be called. + */ + { + int irc = SHELL_SUBMACRO(SQLITE_SHELL_EXTFUNCS, INIT)(p->db); + /* Let custom-included extensions expose their functionality. + * The WHATEVER_EXPOSE( db, pzErrorMsg ) macro should cause + * the SQL functions, virtual tables, collating sequences or + * VFS's implemented by the extension to be registered. + */ + if( irc==SQLITE_OK + || irc==SQLITE_OK_LOAD_PERMANENTLY ){ + SHELL_SUBMACRO(SQLITE_SHELL_EXTFUNCS, EXPOSE)(p->db, 0); + } +#undef SHELL_SUB_MACRO +#undef SHELL_SUBMACRO + } +#endif + sqlite3_create_function(p->db, "shell_add_schema", 3, SQLITE_UTF8, 0, shellAddSchemaName, 0, 0); sqlite3_create_function(p->db, "shell_module_schema", 1, SQLITE_UTF8, 0, @@ -5157,6 +5189,7 @@ static void open_db(ShellState *p, int openFlags){ sqlite3_create_function(p->db, "edit", 2, SQLITE_UTF8, 0, editFunc, 0, 0); #endif + if( p->openMode==SHELL_OPEN_ZIPFILE ){ char *zSql = sqlite3_mprintf( "CREATE VIRTUAL TABLE zip USING zipfile(%Q);", zDbFilename);