From 38b26d82e6009bf58b150311a2038e1bdc3c7065 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 10 Sep 2024 12:09:03 +0000 Subject: [PATCH 01/24] Add the ability for sqlite_dbpage to truncate the database file by writing a NULL page. Experimental. FossilOrigin-Name: eb3c89ee2e4c5425be75deaf46a06a9cd8b210c695b918dd63a78f930c6e6b63 --- manifest | 15 ++++++++----- manifest.uuid | 2 +- src/dbpage.c | 62 ++++++++++++++++++++++++++++++++++++++++++--------- 3 files changed, 62 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index c5f3a5e79e..fc47d2846a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Generalize\sthe\ssqlite3_dbpage\svirtual\stable\sso\sthat\sit\sis\sable\sto\swrite\nnew\spages\sonto\sthe\send\sof\sthe\sdatabase\sfile\susing\sINSERT. -D 2024-09-09T18:45:58.205 +C Add\sthe\sability\sfor\ssqlite_dbpage\sto\struncate\sthe\sdatabase\sfile\sby\swriting\na\sNULL\spage.\s\sExperimental. +D 2024-09-10T12:09:03.839 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -714,7 +714,7 @@ F src/callback.c db3a45e376deff6a16c0058163fe0ae2b73a2945f3f408ca32cf74960b28d49 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c b224d3db0f28c4a5f1407c50107a0a8133bd244ff3c7f6f8cedeb896a8cf1b64 F src/date.c 89ce1ff20512a7fa5070ba6e7dd5c171148ca7d580955795bf97c79c2456144a -F src/dbpage.c f8c93e845d1093554247c1e757cb443fc48ffbcb112cecfdebeca4b6aa6e5c6e +F src/dbpage.c 3c437630c2933b9eefca915d191f8dea9da135195593bb17f553be58ffcd3634 F src/dbstat.c 73362c0df0f40ad5523a6f5501224959d0976757b511299bf892313e79d14f5c F src/delete.c 444c4d1eaac40103461e3b6f0881846dd3aafc1cec1dd169d3482fa331667da7 F src/expr.c 6d5f2c38fe3ec06a7eac599dac822788b36064124e20112a844e9cd5156cb239 @@ -2212,8 +2212,11 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 123cb1f579daec3ed092fe9dd1bc0d3250f2b56d4cda1efa92af139029e112e2 -R 3c119894ad399726f0c98d7eab61a4dc +P fe0d67e72d4228661c021f227bfc0d5ddb1b726db0f36c7221ead8dd8bd1dc73 +R fb6f94f4d85e2b2f81d969e85d52cc7e +T *branch * dbpage +T *sym-dbpage * +T -sym-trunk * U drh -Z 54e5dc5728a062a0fc19b7e3f1b82dff +Z 44b34a478a24f6c5d7a75b5688f34391 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 49f91d3e2e..653a750d52 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fe0d67e72d4228661c021f227bfc0d5ddb1b726db0f36c7221ead8dd8bd1dc73 +eb3c89ee2e4c5425be75deaf46a06a9cd8b210c695b918dd63a78f930c6e6b63 diff --git a/src/dbpage.c b/src/dbpage.c index 9740b418a3..4da81f8ca1 100644 --- a/src/dbpage.c +++ b/src/dbpage.c @@ -28,7 +28,13 @@ ** ** The data field of sqlite_dbpage table can be updated. The new ** value must be a BLOB which is the correct page size, otherwise the -** update fails. Rows may not be deleted or inserted. +** update fails. INSERT operations also work, and operate as if they +** where REPLACE. The size of the database can be extended by INSERT-ing +** new pages on the end. +** +** Rows may not be deleted. However, doing an INSERT to page number N +** with NULL page data causes the N-th page and all subsequent pages to be +** deleted and the database to be truncated. */ #include "sqliteInt.h" /* Requires access to internal data structures */ @@ -51,6 +57,8 @@ struct DbpageCursor { struct DbpageTable { sqlite3_vtab base; /* Base class. Must be first */ sqlite3 *db; /* The database */ + int nTrunc; /* Entries in aTrunc[] */ + Pgno *aTrunc; /* Truncation size for each database */ }; /* Columns */ @@ -59,7 +67,6 @@ struct DbpageTable { #define DBPAGE_COLUMN_SCHEMA 2 - /* ** Connect to or create a dbpagevfs virtual table. */ @@ -100,6 +107,8 @@ static int dbpageConnect( ** Disconnect from or destroy a dbpagevfs virtual table. */ static int dbpageDisconnect(sqlite3_vtab *pVtab){ + DbpageTable *pTab = (DbpageTable *)pVtab; + sqlite3_free(pTab->aTrunc); sqlite3_free(pVtab); return SQLITE_OK; } @@ -325,6 +334,7 @@ static int dbpageUpdate( Btree *pBt; Pager *pPager; int szPage; + int isInsert; (void)pRowid; if( pTab->db->flags & SQLITE_Defensive ){ @@ -337,12 +347,14 @@ static int dbpageUpdate( } if( sqlite3_value_type(argv[0])==SQLITE_NULL ){ pgno = (Pgno)sqlite3_value_int(argv[2]); + isInsert = 1; }else{ pgno = sqlite3_value_int(argv[0]); if( (Pgno)sqlite3_value_int(argv[1])!=pgno ){ zErr = "cannot insert"; goto update_fail; } + isInsert = 0; } if( sqlite3_value_type(argv[4])==SQLITE_NULL ){ iDb = 0; @@ -363,18 +375,28 @@ static int dbpageUpdate( if( sqlite3_value_type(argv[3])!=SQLITE_BLOB || sqlite3_value_bytes(argv[3])!=szPage ){ - zErr = "bad page value"; - goto update_fail; + if( sqlite3_value_type(argv[3])==SQLITE_NULL && isInsert ){ + if( iDb>=pTab->nTrunc ){ + pTab->aTrunc = sqlite3_realloc(pTab->aTrunc, (iDb+1)*sizeof(Pgno)); + if( pTab->aTrunc ){ + pTab->nTrunc = iDb+1; + }else{ + return SQLITE_NOMEM; + } + } + pTab->aTrunc[iDb] = pgno; + }else{ + zErr = "bad page value"; + goto update_fail; + } } pPager = sqlite3BtreePager(pBt); rc = sqlite3PagerGet(pPager, pgno, (DbPage**)&pDbPage, 0); if( rc==SQLITE_OK ){ const void *pData = sqlite3_value_blob(argv[3]); - assert( pData!=0 || pTab->db->mallocFailed ); - if( pData - && (rc = sqlite3PagerWrite(pDbPage))==SQLITE_OK - ){ - memcpy(sqlite3PagerGetData(pDbPage), pData, szPage); + if( (rc = sqlite3PagerWrite(pDbPage))==SQLITE_OK && pData ){ + unsigned char *aPage = sqlite3PagerGetData(pDbPage); + memcpy(aPage, pData, szPage); } } sqlite3PagerUnref(pDbPage); @@ -398,6 +420,26 @@ static int dbpageBegin(sqlite3_vtab *pVtab){ Btree *pBt = db->aDb[i].pBt; if( pBt ) (void)sqlite3BtreeBeginTrans(pBt, 1, 0); } + if( pTab->nTrunc>0 ){ + memset(pTab->aTrunc, 0, sizeof(pTab->aTrunc[0])*pTab->nTrunc); + } + return SQLITE_OK; +} + +/* Invoke sqlite3PagerTruncate() as necessary, just prior to COMMIT +*/ +static int dbpageSync(sqlite3_vtab *pVtab){ + int iDb; + DbpageTable *pTab = (DbpageTable *)pVtab; + + for(iDb=0; iDbnTrunc; iDb++){ + if( pTab->aTrunc[iDb]>0 ){ + Btree *pBt = pTab->db->aDb[iDb].pBt; + Pager *pPager = sqlite3BtreePager(pBt); + sqlite3PagerTruncateImage(pPager, pTab->aTrunc[iDb]); + pTab->aTrunc[iDb] = 0; + } + } return SQLITE_OK; } @@ -422,7 +464,7 @@ int sqlite3DbpageRegister(sqlite3 *db){ dbpageRowid, /* xRowid - read data */ dbpageUpdate, /* xUpdate */ dbpageBegin, /* xBegin */ - 0, /* xSync */ + dbpageSync, /* xSync */ 0, /* xCommit */ 0, /* xRollback */ 0, /* xFindMethod */ From a9c8f7cf34545f410e947b59b8298633324bde97 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 10 Sep 2024 17:05:12 +0000 Subject: [PATCH 02/24] Initial infrastructure for the sqlite3-rsync utility. Prototype only. Does not work. FossilOrigin-Name: 397b2d37b7a6619b0c1eee201065585d03496f94786b21540f613e4716d56612 --- Makefile.in | 3 + Makefile.msc | 3 + main.mk | 4 + manifest | 23 +- manifest.uuid | 2 +- tool/sqlite3-rsync.c | 684 +++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 707 insertions(+), 12 deletions(-) create mode 100644 tool/sqlite3-rsync.c diff --git a/Makefile.in b/Makefile.in index e359975d87..f57fb9fa10 100644 --- a/Makefile.in +++ b/Makefile.in @@ -700,6 +700,9 @@ sqldiff$(TEXE): $(TOP)/tool/sqldiff.c sqlite3.lo sqlite3.h dbhash$(TEXE): $(TOP)/tool/dbhash.c sqlite3.lo sqlite3.h $(LTLINK) -o $@ $(TOP)/tool/dbhash.c sqlite3.lo $(TLIBS) +sqlite3-rsync(TEXE): $(TOP)/tool/sqlite3-rsync.c sqlite3.lo sqlite3.h + $(LTLINK) -o $@ $(TOP)/tool/sqlite3-rsync.c sqlite3.lo $(TLIBS) + scrub$(TEXE): $(TOP)/ext/misc/scrub.c sqlite3.lo $(LTLINK) -o $@ -I. -DSCRUB_STANDALONE \ $(TOP)/ext/misc/scrub.c sqlite3.lo $(TLIBS) diff --git a/Makefile.msc b/Makefile.msc index 41e5c1081a..e60df047e6 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1867,6 +1867,9 @@ sqldiff.exe: $(TOP)\tool\sqldiff.c $(TOP)\ext\consio\console_io.h $(TOP)\ext\con dbhash.exe: $(TOP)\tool\dbhash.c $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) $(TOP)\tool\dbhash.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) +sqlite3-rsync.exe: $(TOP)\tool\sqlite3-rsync.c $(SQLITE3C) $(SQLITE3H) $(LIBRESOBJS) + $(LTLINK) $(NO_WARN) $(TOP)\tool\sqlite3-rsync.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) $(LIBRESOBJS) + scrub.exe: $(TOP)\ext\misc\scrub.c $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) -DSCRUB_STANDALONE=1 $(TOP)\ext\misc\scrub.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) diff --git a/main.mk b/main.mk index 17284dd9f3..4c395e6e61 100644 --- a/main.mk +++ b/main.mk @@ -568,6 +568,10 @@ dbhash$(EXE): $(TOP)/tool/dbhash.c sqlite3.c sqlite3.h $(TCCX) -o dbhash$(EXE) -DSQLITE_THREADSAFE=0 \ $(TOP)/tool/dbhash.c sqlite3.c $(TLIBS) $(THREADLIB) +sqlite3-rsync$(EXE): $(TOP)/tool/sqlite3-rsync.c sqlite3.o + $(TCCX) -o sqlite3-rsync$(EXE) -DSQLITE_THREADSAFE=0 \ + $(TOP)/tool/sqlite3-rsync.c sqlite3.o $(TLIBS) $(THREADLIB) + scrub$(EXE): $(TOP)/ext/misc/scrub.c sqlite3.o $(TCC) -I. -DSCRUB_STANDALONE -o scrub$(EXE) $(TOP)/ext/misc/scrub.c sqlite3.o $(THREADLIB) diff --git a/manifest b/manifest index fc47d2846a..be390eef5f 100644 --- a/manifest +++ b/manifest @@ -1,11 +1,11 @@ -C Add\sthe\sability\sfor\ssqlite_dbpage\sto\struncate\sthe\sdatabase\sfile\sby\swriting\na\sNULL\spage.\s\sExperimental. -D 2024-09-10T12:09:03.839 +C Initial\sinfrastructure\sfor\sthe\ssqlite3-rsync\sutility.\s\sPrototype\sonly.\nDoes\snot\swork. +D 2024-09-10T17:05:12.819 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 -F Makefile.in 7753650b4204e3ccd55a4e6a0d73a5a01f737dcefb099d901ce1de5df9d0b82c +F Makefile.in 496dab6c49377c3acdb69cc8869227eddb0614b2bee01300e4c8ed9fabcb2a63 F Makefile.linux-gcc f3842a0b1efbfbb74ac0ef60e56b301836d05b4d867d014f714fa750048f1ab6 -F Makefile.msc 6e8925dca6dc8c3e9cce042bbf347d20164653e63aeafcf6f6a28e27cf976d8b +F Makefile.msc 4ecdd8ec6bb3264cc2f6c4b154cf9ddd2647e4c6fcb2a294c9725a1483cb2862 F README.md c3c0f19532ce28f6297a71870f3c7b424729f0e6d9ab889616d3587dd2332159 F VERSION 0db40f92c04378404eb45bff93e9e42c148c7e54fd3da99469ed21e22411f5a6 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -687,7 +687,7 @@ F ext/wasm/wasmfs.make 8a4955882aaa0783b3f60a9484a1f0f3d8b6f775c0fcd17c082f31966 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 5ade0bc977aa135e79e3faaea894d5671b26107cc91e70783aa7dc83f22f3ba0 -F main.mk 391342c3c0907f57bbb9ab60ce4b3cfe1ea61161996b449033984673d18980fd +F main.mk 936f535d99e70cf7c1f51c485a3fad7c7c858abad34d41ce100729befc2b2afe F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 F mptest/crash01.test 61e61469e257df0850df4293d7d4d6c2af301421 @@ -2174,6 +2174,7 @@ F tool/speedtest8inst1.c 7ce07da76b5e745783e703a834417d725b7d45fd F tool/spellsift.tcl 52b4b04dc4333c7ab024f09d9d66ed6b6f7c6eb00b38497a09f338fa55d40618 x F tool/split-sqlite3c.tcl 5aa60643afca558bc732b1444ae81a522326f91e1dc5665b369c54f09e20de60 F tool/sqldiff.c 847fc8fcfddf5ce4797b7394cad6372f2f5dc17d8186e2ef8fb44d50fae4f44a +F tool/sqlite3-rsync.c 951956172494a5091868126f36772d9c732f748e301429609638a682ffd0619f F tool/sqlite3_analyzer.c.in 8da2b08f56eeac331a715036cf707cc20f879f231362be0c22efd682e2b89b4f F tool/sqltclsh.c.in 1bcc2e9da58fadf17b0bf6a50e68c1159e602ce057210b655d50bad5aaaef898 F tool/sqltclsh.tcl 862f4cf1418df5e1315b5db3b5ebe88969e2a784525af5fbf9596592f14ed848 @@ -2212,11 +2213,11 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P fe0d67e72d4228661c021f227bfc0d5ddb1b726db0f36c7221ead8dd8bd1dc73 -R fb6f94f4d85e2b2f81d969e85d52cc7e -T *branch * dbpage -T *sym-dbpage * -T -sym-trunk * +P eb3c89ee2e4c5425be75deaf46a06a9cd8b210c695b918dd63a78f930c6e6b63 +R 603b2a7d0991fde4fb9573d941ee4937 +T *branch * sqlite3-rsync +T *sym-sqlite3-rsync * +T -sym-dbpage * U drh -Z 44b34a478a24f6c5d7a75b5688f34391 +Z e29851e41b6e21f917894502487616d4 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 653a750d52..0685ed49d1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -eb3c89ee2e4c5425be75deaf46a06a9cd8b210c695b918dd63a78f930c6e6b63 +397b2d37b7a6619b0c1eee201065585d03496f94786b21540f613e4716d56612 diff --git a/tool/sqlite3-rsync.c b/tool/sqlite3-rsync.c new file mode 100644 index 0000000000..2f3002cafc --- /dev/null +++ b/tool/sqlite3-rsync.c @@ -0,0 +1,684 @@ +/* +** 2024-09-10 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This is a utility program that makes a copy of a live SQLite database +** using a bandwidth-efficient protocol, similar to "rsync". +*/ +#include +#include +#include +#include +#include "sqlite3.h" + +static const char zUsage[] = + "sqlite3-rsync ORIGIN REPLICA\n" + "\n" + "One of ORIGIN or REPLICA is a pathname to a database on the local\n" + "machine and the other is of the form \"USER@HOST:PATH\" describing\n" + "a database on a remote machine. This utility makes REPLICA into a\n" + "copy of ORIGIN\n" +; + +/* Context for the run */ +typedef struct SQLiteRsync SQLiteRsync; +struct SQLiteRsync { + const char *zOrigin; /* Name of the origin */ + const char *zReplica; /* Name of the replica */ + FILE *pOut; /* Transmit to the other side */ + FILE *pIn; /* Receive from the other side */ + sqlite3_uint64 nOut; /* Bytes transmitted */ + sqlite3_uint64 nIn; /* Bytes received */ + int eVerbose; /* Bigger for more output. 0 means none. */ + int bCommCheck; /* True to debug the communication protocol */ +}; + +/**************************************************************************** +** Beginning of the popen2() implementation copied from Fossil ************* +****************************************************************************/ +#ifdef _WIN32 +#include +#include +/* +** Print a fatal error and quit. +*/ +static void win32_fatal_error(const char *zMsg){ + fprintf(stderr, "%s", zMsg); + exit(1); +} +#else +#include +#include +#include +#endif + +/* +** The following macros are used to cast pointers to integers and +** integers to pointers. The way you do this varies from one compiler +** to the next, so we have developed the following set of #if statements +** to generate appropriate macros for a wide range of compilers. +** +** The correct "ANSI" way to do this is to use the intptr_t type. +** Unfortunately, that typedef is not available on all compilers, or +** if it is available, it requires an #include of specific headers +** that vary from one machine to the next. +** +** This code is copied out of SQLite. +*/ +#if defined(__PTRDIFF_TYPE__) /* This case should work for GCC */ +# define INT_TO_PTR(X) ((void*)(__PTRDIFF_TYPE__)(X)) +# define PTR_TO_INT(X) ((int)(__PTRDIFF_TYPE__)(X)) +#elif !defined(__GNUC__) /* Works for compilers other than LLVM */ +# define INT_TO_PTR(X) ((void*)&((char*)0)[X]) +# define PTR_TO_INT(X) ((int)(((char*)X)-(char*)0)) +#elif defined(HAVE_STDINT_H) /* Use this case if we have ANSI headers */ +# define INT_TO_PTR(X) ((void*)(intptr_t)(X)) +# define PTR_TO_INT(X) ((int)(intptr_t)(X)) +#else /* Generates a warning - but it always works */ +# define INT_TO_PTR(X) ((void*)(X)) +# define PTR_TO_INT(X) ((int)(X)) +#endif + + +#ifdef _WIN32 +/* +** On windows, create a child process and specify the stdin, stdout, +** and stderr channels for that process to use. +** +** Return the number of errors. +*/ +static int win32_create_child_process( + wchar_t *zCmd, /* The command that the child process will run */ + HANDLE hIn, /* Standard input */ + HANDLE hOut, /* Standard output */ + HANDLE hErr, /* Standard error */ + DWORD *pChildPid /* OUT: Child process handle */ +){ + STARTUPINFOW si; + PROCESS_INFORMATION pi; + BOOL rc; + + memset(&si, 0, sizeof(si)); + si.cb = sizeof(si); + si.dwFlags = STARTF_USESTDHANDLES; + SetHandleInformation(hIn, HANDLE_FLAG_INHERIT, TRUE); + si.hStdInput = hIn; + SetHandleInformation(hOut, HANDLE_FLAG_INHERIT, TRUE); + si.hStdOutput = hOut; + SetHandleInformation(hErr, HANDLE_FLAG_INHERIT, TRUE); + si.hStdError = hErr; + rc = CreateProcessW( + NULL, /* Application Name */ + zCmd, /* Command-line */ + NULL, /* Process attributes */ + NULL, /* Thread attributes */ + TRUE, /* Inherit Handles */ + 0, /* Create flags */ + NULL, /* Environment */ + NULL, /* Current directory */ + &si, /* Startup Info */ + &pi /* Process Info */ + ); + if( rc ){ + CloseHandle( pi.hProcess ); + CloseHandle( pi.hThread ); + *pChildPid = pi.dwProcessId; + }else{ + win32_fatal_error("cannot create child process"); + } + return rc!=0; +} +void *win32_utf8_to_unicode(const char *zUtf8){ + int nByte = MultiByteToWideChar(CP_UTF8, 0, zUtf8, -1, 0, 0); + wchar_t *zUnicode = malloc( nByte*2 ); + MultiByteToWideChar(CP_UTF8, 0, zUtf8, -1, zUnicode, nByte); + return zUnicode; +} +#endif + +/* +** Create a child process running shell command "zCmd". *ppOut is +** a FILE that becomes the standard input of the child process. +** (The caller writes to *ppOut in order to send text to the child.) +** *ppIn is stdout from the child process. (The caller +** reads from *ppIn in order to receive input from the child.) +** Note that *ppIn is an unbuffered file descriptor, not a FILE. +** The process ID of the child is written into *pChildPid. +** +** Return the number of errors. +*/ +static int popen2( + const char *zCmd, /* Command to run in the child process */ + FILE **ppIn, /* Read from child using this file descriptor */ + FILE **ppOut, /* Write to child using this file descriptor */ + int *pChildPid, /* PID of the child process */ + int bDirect /* 0: run zCmd as a shell cmd. 1: run directly */ +){ +#ifdef _WIN32 + HANDLE hStdinRd, hStdinWr, hStdoutRd, hStdoutWr, hStderr; + SECURITY_ATTRIBUTES saAttr; + DWORD childPid = 0; + int fd; + + saAttr.nLength = sizeof(saAttr); + saAttr.bInheritHandle = TRUE; + saAttr.lpSecurityDescriptor = NULL; + hStderr = GetStdHandle(STD_ERROR_HANDLE); + if( !CreatePipe(&hStdoutRd, &hStdoutWr, &saAttr, 4096) ){ + win32_fatal_error("cannot create pipe for stdout"); + } + SetHandleInformation( hStdoutRd, HANDLE_FLAG_INHERIT, FALSE); + + if( !CreatePipe(&hStdinRd, &hStdinWr, &saAttr, 4096) ){ + win32_fatal_error("cannot create pipe for stdin"); + } + SetHandleInformation( hStdinWr, HANDLE_FLAG_INHERIT, FALSE); + + win32_create_child_process(win32_utf8_to_unicode(zCmd), + hStdinRd, hStdoutWr, hStderr,&childPid); + *pChildPid = childPid; + fd = _open_osfhandle(PTR_TO_INT(hStdoutRd), 0); + *ppIn = fdopen(fd, "r"); + fd = _open_osfhandle(PTR_TO_INT(hStdinWr), 0); + *ppOut = _fdopen(fd, "w"); + CloseHandle(hStdinRd); + CloseHandle(hStdoutWr); + return 0; +#else + int pin[2], pout[2]; + *ppIn = 0; + *ppOut = 0; + *pChildPid = 0; + + if( pipe(pin)<0 ){ + return 1; + } + if( pipe(pout)<0 ){ + close(pin[0]); + close(pin[1]); + return 1; + } + *pChildPid = fork(); + if( *pChildPid<0 ){ + close(pin[0]); + close(pin[1]); + close(pout[0]); + close(pout[1]); + *pChildPid = 0; + return 1; + } + signal(SIGPIPE,SIG_IGN); + if( *pChildPid==0 ){ + int fd; + /* This is the child process */ + close(0); + fd = dup(pout[0]); + if( fd!=0 ) { + fprintf(stderr,"popen2() failed to open file descriptor 0"); + exit(1); + } + close(pout[0]); + close(pout[1]); + close(1); + fd = dup(pin[1]); + if( fd!=1 ){ + fprintf(stderr,"popen() failed to open file descriptor 1"); + exit(1); + } + close(pin[0]); + close(pin[1]); + if( bDirect ){ + execl(zCmd, zCmd, (char*)0); + }else{ + execl("/bin/sh", "/bin/sh", "-c", zCmd, (char*)0); + } + return 1; + }else{ + /* This is the parent process */ + close(pin[1]); + *ppIn = fdopen(pin[0], "r"); + close(pout[0]); + *ppOut = fdopen(pout[1], "w"); + return 0; + } +#endif +} + +/* +** Close the connection to a child process previously created using +** popen2(). +*/ +static void pclose2(FILE *pIn, FILE *pOut, int childPid){ +#ifdef _WIN32 + /* Not implemented, yet */ + fclose(pIn); + fclose(pOut); +#else + fclose(pIn); + fclose(pOut); + while( waitpid(0, 0, WNOHANG)>0 ) {} +#endif +} +/***************************************************************************** +** End of the popen2() implementation copied from Fossil ********************* +*****************************************************************************/ + +/***************************************************************************** +** Beginning of the append_escaped_arg() routine, adapted from the Fossil ** +** subroutine nameed blob_append_escaped_arg() ** +*****************************************************************************/ +/* +** ASCII (for reference): +** x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xa xb xc xd xe xf +** 0x ^` ^a ^b ^c ^d ^e ^f ^g \b \t \n () \f \r ^n ^o +** 1x ^p ^q ^r ^s ^t ^u ^v ^w ^x ^y ^z ^{ ^| ^} ^~ ^ +** 2x () ! " # $ % & ' ( ) * + , - . / +** 3x 0 1 2 3 4 5 6 7 8 9 : ; < = > ? +** 4x @ A B C D E F G H I J K L M N O +** 5x P Q R S T U V W X Y Z [ \ ] ^ _ +** 6x ` a b c d e f g h i j k l m n o +** 7x p q r s t u v w x y z { | } ~ ^_ +*/ + +/* +** Meanings for bytes in a filename: +** +** 0 Ordinary character. No encoding required +** 1 Needs to be escaped +** 2 Illegal character. Do not allow in a filename +** 3 First byte of a 2-byte UTF-8 +** 4 First byte of a 3-byte UTF-8 +** 5 First byte of a 4-byte UTF-8 +*/ +static const char aSafeChar[256] = { +#ifdef _WIN32 +/* Windows +** Prohibit: all control characters, including tab, \r and \n. +** Escape: (space) " # $ % & ' ( ) * ; < > ? [ ] ^ ` { | } +*/ +/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xa xb xc xd xe xf */ + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, /* 0x */ + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, /* 1x */ + 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 2x */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, /* 3x */ + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 4x */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, /* 5x */ + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 6x */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 1, /* 7x */ +#else +/* Unix +** Prohibit: all control characters, including tab, \r and \n +** Escape: (space) ! " # $ % & ' ( ) * ; < > ? [ \ ] ^ ` { | } +*/ +/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xa xb xc xd xe xf */ + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, /* 0x */ + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, /* 1x */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 2x */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, /* 3x */ + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 4x */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, /* 5x */ + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 6x */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 1, /* 7x */ +#endif + /* all bytes 0x80 through 0xbf are unescaped, being secondary + ** bytes to UTF8 characters. Bytes 0xc0 through 0xff are the + ** first byte of a UTF8 character and do get escaped */ + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, /* 8x */ + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, /* 9x */ + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, /* ax */ + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, /* bx */ + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, /* cx */ + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, /* dx */ + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, /* ex */ + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5 /* fx */ +}; + +/* +** pStr is a shell command under construction. This routine safely +** appends filename argument zIn. It returns 0 on success or non-zero +** on any error. +** +** The argument is escaped if it contains white space or other characters +** that need to be escaped for the shell. If zIn contains characters +** that cannot be safely escaped, then throw a fatal error. +** +** If the isFilename argument is true, then the argument is expected +** to be a filename. As shell commands commonly have command-line +** options that begin with "-" and since we do not want an attacker +** to be able to invoke these switches using filenames that begin +** with "-", if zIn begins with "-", prepend an additional "./" +** (or ".\\" on Windows). +*/ +int append_escaped_arg(sqlite3_str *pStr, const char *zIn, int isFilename){ + int i; + unsigned char c; + int needEscape = 0; + int n = sqlite3_str_length(pStr); + char *z = sqlite3_str_value(pStr); + + /* Look for illegal byte-sequences and byte-sequences that require + ** escaping. No control-characters are allowed. All spaces and + ** non-ASCII unicode characters and some punctuation characters require + ** escaping. */ + for(i=0; (c = (unsigned char)zIn[i])!=0; i++){ + if( aSafeChar[c] ){ + unsigned char x = aSafeChar[c]; + needEscape = 1; + if( x==2 ){ + /* Bad ASCII character */ + return 1; + }else if( x>2 ){ + if( (zIn[i+1]&0xc0)!=0x80 + || (x>=4 && (zIn[i+2]&0xc0)!=0x80) + || (x==5 && (zIn[i+3]&0xc0)!=0x80) + ){ + /* Bad UTF8 character */ + return 1; + } + i += x-2; + } + } + } + + /* Separate from the previous argument by a space */ + if( n>0 && !isspace(z[n-1]) ){ + sqlite3_str_appendchar(pStr, 1, ' '); + } + + /* Check for characters that need quoting */ + if( !needEscape ){ + if( isFilename && zIn[0]=='-' ){ + sqlite3_str_appendchar(pStr, 1, '.'); +#if defined(_WIN32) + sqlite3_str_appendchar(pStr, 1, '\\'); +#else + sqlite3_str_appendchar(pStr, 1, '/'); +#endif + } + sqlite3_str_appendall(pStr, zIn); + }else{ +#if defined(_WIN32) + /* Quoting strategy for windows: + ** Put the entire name inside of "...". Any " characters within + ** the name get doubled. + */ + sqlite3_str_appendchar(pStr, 1, '"'); + if( isFilename && zIn[0]=='-' ){ + sqlite3_str_appendchar(pStr, 1, '.'); + sqlite3_str_appendchar(pStr, 1, '\\'); + }else if( zIn[0]=='/' ){ + sqlite3_str_appendchar(pStr, 1, '.'); + } + for(i=0; (c = (unsigned char)zIn[i])!=0; i++){ + sqlite3_str_appendchar(pStr, 1, (char)c); + if( c=='"' ) sqlite3_str_appendchar(pStr, 1, '"'); + if( c=='\\' ) sqlite3_str_appendchar(pStr, 1, '\\'); + if( c=='%' && isFilename ) sqlite3_str_append(pStr, "%cd:~,%", 7); + } + sqlite3_str_appendchar(pStr, 1, '"'); +#else + /* Quoting strategy for unix: + ** If the name does not contain ', then surround the whole thing + ** with '...'. If there is one or more ' characters within the + ** name, then put \ before each special character. + */ + if( strchr(zIn,'\'') ){ + if( isFilename && zIn[0]=='-' ){ + sqlite3_str_appendchar(pStr, 1, '.'); + sqlite3_str_appendchar(pStr, 1, '/'); + } + for(i=0; (c = (unsigned char)zIn[i])!=0; i++){ + if( aSafeChar[c] && aSafeChar[c]!=2 ){ + sqlite3_str_appendchar(pStr, 1, '\\'); + } + sqlite3_str_appendchar(pStr, 1, (char)c); + } + }else{ + sqlite3_str_appendchar(pStr, 1, '\''); + if( isFilename && zIn[0]=='-' ){ + sqlite3_str_appendchar(pStr, 1, '.'); + sqlite3_str_appendchar(pStr, 1, '/'); + } + sqlite3_str_appendall(pStr, zIn); + sqlite3_str_appendchar(pStr, 1, '\''); + } +#endif + } + return 0; +} +/***************************************************************************** +** End of the append_escaped_arg() routine, adapted from the Fossil ** +*****************************************************************************/ + + +/* For Debugging, specifically for --commcheck: +** +** Read a single line of text from p->pIn. Write this to standard +** output if and only if p->eVerbose>0. +*/ +static void echoOneLine(SQLiteRsync *p){ + char zLine[1000]; + if( fgets(zLine, sizeof(zLine), p->pIn) ){ + if( p->eVerbose ) printf("GOT: %s", zLine); + } +} + +/* +** Run the origin-side protocol. +** +** 1. Send the origin-begin message +** 2. Receive replica-begin message +** - Error check and abort if necessary +** 3. Receive replica-hash messages +** 4. BEGIN +** 5. Send changed pages +** 6. COMMIT +** 7. Send origin-end message +*/ +static void originSide(SQLiteRsync *p){ + if( p->bCommCheck ){ + fprintf(p->pOut, "sqlite3-rsync origin-begin %s\n", p->zOrigin); + fflush(p->pOut); + echoOneLine(p); + fprintf(p->pOut, "origin-end\n"); + fflush(p->pOut); + echoOneLine(p); + return; + } +} + +/* +** Run the replica-side protocol. +** +** 1. Receive the origin-begin message +** - Error check. If unable to continue, send replica-error and quit +** 2. BEGIN IMMEDIATE +** 3. Send replica-begin message +** 4. Send replica-hash messages +** 5. Receive changed pages and apply them +** 6. Receive origin-end message +** 7. COMMIT +*/ +static void replicaSide(SQLiteRsync *p){ + if( p->bCommCheck ){ + echoOneLine(p); + fprintf(p->pOut, "replica-begin %s\n", p->zReplica); + fflush(p->pOut); + echoOneLine(p); + fprintf(p->pOut, "replica-end\n"); + fflush(p->pOut); + return; + } +} + + +/* +** Parse command-line arguments. Dispatch subroutines to do the +** requested work. +** +** Input formats: +** +** (1) sqlite3-rsync FILENAME1 USER@HOST:FILENAME2 +** +** (2) sqlite3-rsync USER@HOST:FILENAME1 FILENAME2 +** +** (3) sqlite3-rsync --origin FILENAME1 +** +** (4) sqlite3-rsync --replica FILENAME2 +** +** The user types (1) or (2). SSH launches (3) or (4). +** +** If (1) is seen then popen2 is used launch (4) on the remote and +** originSide() is called locally. +** +** If (2) is seen, then popen2() is used to launch (3) on the remote +** and replicaSide() is run locally. +** +** If (3) is seen, call originSide() on stdin and stdout. +** +** If (4) is seen, call replicaSide() on stdin and stdout. +*/ +int main(int argc, char **argv){ + int isOrigin = 0; + int isReplica = 0; + int i; + SQLiteRsync ctx; + char *zDiv; + FILE *pIn = 0; + FILE *pOut = 0; + int childPid = 0; + const char *zSsh = "ssh"; + const char *zExe = argv[0]; + char *zCmd = 0; + + memset(&ctx, 0, sizeof(ctx)); + for(i=1; i Date: Tue, 10 Sep 2024 22:14:18 +0000 Subject: [PATCH 03/24] Improved SSH infrastructure. The foundation is now in place to begin working on the actual sync protocol. Still experimental. Still a work in progress. FossilOrigin-Name: 9a1a95f523a96303aad57e2422c2b51ea7e125f5490f32f7a2929d49b6c69ef8 --- Makefile.in | 2 +- manifest | 17 +++++++--------- manifest.uuid | 2 +- tool/sqlite3-rsync.c | 46 +++++++++++++++++++++++++++++++++++++++++--- 4 files changed, 52 insertions(+), 15 deletions(-) diff --git a/Makefile.in b/Makefile.in index f57fb9fa10..5a36dca9d1 100644 --- a/Makefile.in +++ b/Makefile.in @@ -700,7 +700,7 @@ sqldiff$(TEXE): $(TOP)/tool/sqldiff.c sqlite3.lo sqlite3.h dbhash$(TEXE): $(TOP)/tool/dbhash.c sqlite3.lo sqlite3.h $(LTLINK) -o $@ $(TOP)/tool/dbhash.c sqlite3.lo $(TLIBS) -sqlite3-rsync(TEXE): $(TOP)/tool/sqlite3-rsync.c sqlite3.lo sqlite3.h +sqlite3-rsync$(TEXE): $(TOP)/tool/sqlite3-rsync.c sqlite3.lo sqlite3.h $(LTLINK) -o $@ $(TOP)/tool/sqlite3-rsync.c sqlite3.lo $(TLIBS) scrub$(TEXE): $(TOP)/ext/misc/scrub.c sqlite3.lo diff --git a/manifest b/manifest index be390eef5f..c58c5ffe3d 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C Initial\sinfrastructure\sfor\sthe\ssqlite3-rsync\sutility.\s\sPrototype\sonly.\nDoes\snot\swork. -D 2024-09-10T17:05:12.819 +C Improved\sSSH\sinfrastructure.\s\sThe\sfoundation\sis\snow\sin\splace\sto\sbegin\sworking\non\sthe\sactual\ssync\sprotocol.\s\sStill\sexperimental.\s\sStill\sa\swork\sin\sprogress. +D 2024-09-10T22:14:18.799 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 -F Makefile.in 496dab6c49377c3acdb69cc8869227eddb0614b2bee01300e4c8ed9fabcb2a63 +F Makefile.in c9a51ee844a471c950881748f21699fdbf42ef540bf5e78d269f99003f510256 F Makefile.linux-gcc f3842a0b1efbfbb74ac0ef60e56b301836d05b4d867d014f714fa750048f1ab6 F Makefile.msc 4ecdd8ec6bb3264cc2f6c4b154cf9ddd2647e4c6fcb2a294c9725a1483cb2862 F README.md c3c0f19532ce28f6297a71870f3c7b424729f0e6d9ab889616d3587dd2332159 @@ -2174,7 +2174,7 @@ F tool/speedtest8inst1.c 7ce07da76b5e745783e703a834417d725b7d45fd F tool/spellsift.tcl 52b4b04dc4333c7ab024f09d9d66ed6b6f7c6eb00b38497a09f338fa55d40618 x F tool/split-sqlite3c.tcl 5aa60643afca558bc732b1444ae81a522326f91e1dc5665b369c54f09e20de60 F tool/sqldiff.c 847fc8fcfddf5ce4797b7394cad6372f2f5dc17d8186e2ef8fb44d50fae4f44a -F tool/sqlite3-rsync.c 951956172494a5091868126f36772d9c732f748e301429609638a682ffd0619f +F tool/sqlite3-rsync.c d9fd25997c34d9a63e7afdd99b467aaa69440e3ce4d4f85cf47da3e182f4c7e9 F tool/sqlite3_analyzer.c.in 8da2b08f56eeac331a715036cf707cc20f879f231362be0c22efd682e2b89b4f F tool/sqltclsh.c.in 1bcc2e9da58fadf17b0bf6a50e68c1159e602ce057210b655d50bad5aaaef898 F tool/sqltclsh.tcl 862f4cf1418df5e1315b5db3b5ebe88969e2a784525af5fbf9596592f14ed848 @@ -2213,11 +2213,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P eb3c89ee2e4c5425be75deaf46a06a9cd8b210c695b918dd63a78f930c6e6b63 -R 603b2a7d0991fde4fb9573d941ee4937 -T *branch * sqlite3-rsync -T *sym-sqlite3-rsync * -T -sym-dbpage * +P 397b2d37b7a6619b0c1eee201065585d03496f94786b21540f613e4716d56612 +R 721e909ab77c2cd7295ba63619aaa44c U drh -Z e29851e41b6e21f917894502487616d4 +Z 2e6a3a2b31db87fda5c83fa80a99e1a6 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 0685ed49d1..a90d6d4a2b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -397b2d37b7a6619b0c1eee201065585d03496f94786b21540f613e4716d56612 +9a1a95f523a96303aad57e2422c2b51ea7e125f5490f32f7a2929d49b6c69ef8 diff --git a/tool/sqlite3-rsync.c b/tool/sqlite3-rsync.c index 2f3002cafc..02a59a92bd 100644 --- a/tool/sqlite3-rsync.c +++ b/tool/sqlite3-rsync.c @@ -556,7 +556,7 @@ int main(int argc, char **argv){ FILE *pOut = 0; int childPid = 0; const char *zSsh = "ssh"; - const char *zExe = argv[0]; + const char *zExe = "sqlite3-rsync"; char *zCmd = 0; memset(&ctx, 0, sizeof(ctx)); @@ -579,7 +579,7 @@ int main(int argc, char **argv){ continue; } if( strcmp(z, "--exe")==0 ){ - zSsh = argv[++i]; + zExe = argv[++i]; continue; } if( strcmp(z, "-help")==0 || strcmp(z, "--help")==0 @@ -659,14 +659,54 @@ int main(int argc, char **argv){ return 1; } /* Remote ORIGIN and local REPLICA */ + sqlite3_str *pStr = sqlite3_str_new(0); + append_escaped_arg(pStr, zSsh, 1); + sqlite3_str_appendf(pStr, " -e none"); + *(zDiv++) = 0; + append_escaped_arg(pStr, ctx.zOrigin, 0); + append_escaped_arg(pStr, zExe, 1); + append_escaped_arg(pStr, "--origin", 0); + if( ctx.bCommCheck ){ + append_escaped_arg(pStr, "--commcheck", 0); + if( ctx.eVerbose==0 ) ctx.eVerbose = 1; + } + append_escaped_arg(pStr, zDiv, 1); + zCmd = sqlite3_str_finish(pStr); + if( ctx.eVerbose ) printf("%s\n", zCmd); + if( popen2(zCmd, &ctx.pIn, &ctx.pOut, &childPid, 0) ){ + fprintf(stderr, "Could not start auxiliary process: %s\n", zCmd); + return 1; + } + originSide(&ctx); }else if( (zDiv = strchr(ctx.zReplica,':'))!=0 ){ /* Local ORIGIN and remote REPLICA */ - printf("%s\n", zSsh); + sqlite3_str *pStr = sqlite3_str_new(0); + append_escaped_arg(pStr, zSsh, 1); + sqlite3_str_appendf(pStr, " -e none"); + *(zDiv++) = 0; + append_escaped_arg(pStr, ctx.zReplica, 0); + append_escaped_arg(pStr, zExe, 1); + append_escaped_arg(pStr, "--replica", 0); + if( ctx.bCommCheck ){ + append_escaped_arg(pStr, "--commcheck", 0); + if( ctx.eVerbose==0 ) ctx.eVerbose = 1; + } + append_escaped_arg(pStr, zDiv, 1); + zCmd = sqlite3_str_finish(pStr); + if( ctx.eVerbose ) printf("%s\n", zCmd); + if( popen2(zCmd, &ctx.pIn, &ctx.pOut, &childPid, 0) ){ + fprintf(stderr, "Could not start auxiliary process: %s\n", zCmd); + return 1; + } + originSide(&ctx); }else{ /* Local ORIGIN and REPLICA */ sqlite3_str *pStr = sqlite3_str_new(0); append_escaped_arg(pStr, zExe, 1); append_escaped_arg(pStr, "--replica", 0); + if( ctx.bCommCheck ){ + append_escaped_arg(pStr, "--commcheck", 0); + } append_escaped_arg(pStr, ctx.zReplica, 1); zCmd = sqlite3_str_finish(pStr); if( ctx.eVerbose ) printf("%s\n", zCmd); From dc3bec34a6398334ec24af7879ae648da8e96e21 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 11 Sep 2024 17:02:44 +0000 Subject: [PATCH 04/24] Progress on the sqlite3-rsync utility. This is an incremental check-in. It does compile, but it does not work. FossilOrigin-Name: fa06977b6db7fa745720561ec0b10570cf7e71598dc7a7c5ee650640e5bdf6f5 --- Makefile.in | 14 +- Makefile.msc | 14 +- ext/misc/sha1.c | 15 +- main.mk | 15 +- manifest | 20 +-- manifest.uuid | 2 +- tool/sqlite3-rsync.c | 391 ++++++++++++++++++++++++++++++++++++++++++- 7 files changed, 448 insertions(+), 23 deletions(-) diff --git a/Makefile.in b/Makefile.in index 5a36dca9d1..0cfae780cc 100644 --- a/Makefile.in +++ b/Makefile.in @@ -700,8 +700,18 @@ sqldiff$(TEXE): $(TOP)/tool/sqldiff.c sqlite3.lo sqlite3.h dbhash$(TEXE): $(TOP)/tool/dbhash.c sqlite3.lo sqlite3.h $(LTLINK) -o $@ $(TOP)/tool/dbhash.c sqlite3.lo $(TLIBS) -sqlite3-rsync$(TEXE): $(TOP)/tool/sqlite3-rsync.c sqlite3.lo sqlite3.h - $(LTLINK) -o $@ $(TOP)/tool/sqlite3-rsync.c sqlite3.lo $(TLIBS) +RSYNC_SRC = \ + $(TOP)/tool/sqlite3-rsync.c \ + $(TOP)/ext/misc/sha1.c \ + sqlite3.c + +RSYNC_OPT = \ + -DSQLITE_THREADSAFE=0 \ + -DSQLITE_OMIT_LOAD_EXTENSION \ + -DSQLITE_OMIT_DEPRECATED + +sqlite3-rsync$(TEXE): $(RSYNC_SRC) + $(TCC) -o $@ $(RSYNC_SRC) $(TLIBS) scrub$(TEXE): $(TOP)/ext/misc/scrub.c sqlite3.lo $(LTLINK) -o $@ -I. -DSCRUB_STANDALONE \ diff --git a/Makefile.msc b/Makefile.msc index e60df047e6..461859d1f1 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1867,8 +1867,18 @@ sqldiff.exe: $(TOP)\tool\sqldiff.c $(TOP)\ext\consio\console_io.h $(TOP)\ext\con dbhash.exe: $(TOP)\tool\dbhash.c $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) $(TOP)\tool\dbhash.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) -sqlite3-rsync.exe: $(TOP)\tool\sqlite3-rsync.c $(SQLITE3C) $(SQLITE3H) $(LIBRESOBJS) - $(LTLINK) $(NO_WARN) $(TOP)\tool\sqlite3-rsync.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) $(LIBRESOBJS) +RSYNC_SRC = \ + $(TOP)\tool\sqlite3-rsync.c \ + $(TOP)\ext\misc\sha1.c \ + $(SQLITE3C) + +RSYNC_OPT = \ + -DSQLITE_THREADSAFE=0 \ + -DSQLITE_OMIT_LOAD_EXTENSION \ + -DSQLITE_OMIT_DEPRECATED + +sqlite3-rsync.exe: $(RSYNC_SRC) $(LIBRESOBJS) + $(LTLINK) $(RSYNC_OPT) $(NO_WARN) $(RSYNC_SRC) /link $(LDFLAGS) $(LTLINKOPTS) $(LIBRESOBJS) scrub.exe: $(TOP)\ext\misc\scrub.c $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) -DSCRUB_STANDALONE=1 $(TOP)\ext\misc\scrub.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) diff --git a/ext/misc/sha1.c b/ext/misc/sha1.c index 9790a1d877..dbdebbff87 100644 --- a/ext/misc/sha1.c +++ b/ext/misc/sha1.c @@ -196,7 +196,8 @@ static void hash_step_vformat( ** zOut[]. zOut[] must be at least 41 bytes long. */ static void hash_finish( SHA1Context *p, /* The SHA1 context to finish and render */ - char *zOut /* Store hexadecimal hash here */ + char *zOut, /* Store hex or binary hash here */ + int bAsBinary /* 1 for binary hash, 0 for hex hash */ ){ unsigned int i; unsigned char finalcount[8]; @@ -251,7 +252,7 @@ static void sha1Func( }else{ hash_step(&cx, sqlite3_value_text(argv[0]), nByte); } - hash_finish(&cx, zOut); + hash_finish(&cx, zOut, sqlite3_user_data(context)!=0); sqlite3_result_text(context, zOut, 40, SQLITE_TRANSIENT); } @@ -365,7 +366,7 @@ static void sha1QueryFunc( } sqlite3_finalize(pStmt); } - hash_finish(&cx, zOut); + hash_finish(&cx, zOut, 0); sqlite3_result_text(context, zOut, 40, SQLITE_TRANSIENT); } @@ -379,11 +380,17 @@ int sqlite3_sha_init( const sqlite3_api_routines *pApi ){ int rc = SQLITE_OK; + static int one = 1; SQLITE_EXTENSION_INIT2(pApi); (void)pzErrMsg; /* Unused parameter */ rc = sqlite3_create_function(db, "sha1", 1, SQLITE_UTF8 | SQLITE_INNOCUOUS | SQLITE_DETERMINISTIC, - 0, sha1Func, 0, 0); + 0, sha1Func, 0, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "sha1b", 1, + SQLITE_UTF8 | SQLITE_INNOCUOUS | SQLITE_DETERMINISTIC, + (void*)&one, sha1Func, 0, 0); + } if( rc==SQLITE_OK ){ rc = sqlite3_create_function(db, "sha1_query", 1, SQLITE_UTF8|SQLITE_DIRECTONLY, 0, diff --git a/main.mk b/main.mk index 4c395e6e61..7414e7488e 100644 --- a/main.mk +++ b/main.mk @@ -568,9 +568,18 @@ dbhash$(EXE): $(TOP)/tool/dbhash.c sqlite3.c sqlite3.h $(TCCX) -o dbhash$(EXE) -DSQLITE_THREADSAFE=0 \ $(TOP)/tool/dbhash.c sqlite3.c $(TLIBS) $(THREADLIB) -sqlite3-rsync$(EXE): $(TOP)/tool/sqlite3-rsync.c sqlite3.o - $(TCCX) -o sqlite3-rsync$(EXE) -DSQLITE_THREADSAFE=0 \ - $(TOP)/tool/sqlite3-rsync.c sqlite3.o $(TLIBS) $(THREADLIB) +RSYNC_SRC = \ + $(TOP)/tool/sqlite3-rsync.c \ + $(TOP)/ext/misc/sha1.c \ + sqlite3.c + +RSYNC_OPT = \ + -DSQLITE_THREADSAFE=0 \ + -DSQLITE_OMIT_LOAD_EXTENSION \ + -DSQLITE_OMIT_DEPRECATED + +sqlite3-rsync$(EXE): $(RSYNC_SRC) + $(TCC) -o $@ $(RSYNC_OPT) $(RSYNC_SRC) $(TLIBS) scrub$(EXE): $(TOP)/ext/misc/scrub.c sqlite3.o $(TCC) -I. -DSCRUB_STANDALONE -o scrub$(EXE) $(TOP)/ext/misc/scrub.c sqlite3.o $(THREADLIB) diff --git a/manifest b/manifest index c58c5ffe3d..c99d1cd05f 100644 --- a/manifest +++ b/manifest @@ -1,11 +1,11 @@ -C Improved\sSSH\sinfrastructure.\s\sThe\sfoundation\sis\snow\sin\splace\sto\sbegin\sworking\non\sthe\sactual\ssync\sprotocol.\s\sStill\sexperimental.\s\sStill\sa\swork\sin\sprogress. -D 2024-09-10T22:14:18.799 +C Progress\son\sthe\ssqlite3-rsync\sutility.\s\sThis\sis\san\sincremental\scheck-in.\s\sIt\ndoes\scompile,\sbut\sit\sdoes\snot\swork. +D 2024-09-11T17:02:44.010 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 -F Makefile.in c9a51ee844a471c950881748f21699fdbf42ef540bf5e78d269f99003f510256 +F Makefile.in 54e80b016b0e58db383c0f08d340ef795b8b709ffb6f53a51a8ba7bf0c5e288f F Makefile.linux-gcc f3842a0b1efbfbb74ac0ef60e56b301836d05b4d867d014f714fa750048f1ab6 -F Makefile.msc 4ecdd8ec6bb3264cc2f6c4b154cf9ddd2647e4c6fcb2a294c9725a1483cb2862 +F Makefile.msc a86e0f3fe5f807daa82d44b5056e3dbc311e569bd4748646a776a994124ec58b F README.md c3c0f19532ce28f6297a71870f3c7b424729f0e6d9ab889616d3587dd2332159 F VERSION 0db40f92c04378404eb45bff93e9e42c148c7e54fd3da99469ed21e22411f5a6 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -420,7 +420,7 @@ F ext/misc/remember.c add730f0f7e7436cd15ea3fd6a90fd83c3f706ab44169f7f048438b7d6 F ext/misc/rot13.c 51ac5f51e9d5fd811db58a9c23c628ad5f333c173f1fc53c8491a3603d38556c F ext/misc/scrub.c 2a44b0d44c69584c0580ad2553f6290a307a49df4668941d2812135bfb96a946 F ext/misc/series.c a6089b5e8e3002bd1e5d9877cee6aead0b9a6426e406c09a399817db9e9ae823 -F ext/misc/sha1.c 4011aef176616872b2a0d5bccf0ecfb1f7ce3fe5c3d107f3a8e949d8e1e3f08d +F ext/misc/sha1.c dfd26eb3437a88fe7349d1fe080b761549c456ae17cb11242441bf66031942bf F ext/misc/shathree.c 1821d90a0040c9accdbe3e3527d378d30569475d758aa70f6848924c0b430e8c F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52 F ext/misc/spellfix.c c0aa7b80d6df45f7da59d912b38752bcac1af53a5766966160e6c5cdd397dbea @@ -687,7 +687,7 @@ F ext/wasm/wasmfs.make 8a4955882aaa0783b3f60a9484a1f0f3d8b6f775c0fcd17c082f31966 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 5ade0bc977aa135e79e3faaea894d5671b26107cc91e70783aa7dc83f22f3ba0 -F main.mk 936f535d99e70cf7c1f51c485a3fad7c7c858abad34d41ce100729befc2b2afe +F main.mk 9ffe4a14bdb4a0b856217a5465ca6b1ef4bef66a3c45e2da3fdb6d9bfc8d583e F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 F mptest/crash01.test 61e61469e257df0850df4293d7d4d6c2af301421 @@ -2174,7 +2174,7 @@ F tool/speedtest8inst1.c 7ce07da76b5e745783e703a834417d725b7d45fd F tool/spellsift.tcl 52b4b04dc4333c7ab024f09d9d66ed6b6f7c6eb00b38497a09f338fa55d40618 x F tool/split-sqlite3c.tcl 5aa60643afca558bc732b1444ae81a522326f91e1dc5665b369c54f09e20de60 F tool/sqldiff.c 847fc8fcfddf5ce4797b7394cad6372f2f5dc17d8186e2ef8fb44d50fae4f44a -F tool/sqlite3-rsync.c d9fd25997c34d9a63e7afdd99b467aaa69440e3ce4d4f85cf47da3e182f4c7e9 +F tool/sqlite3-rsync.c eb75a24e3a47fe7b5a4d5cbd2eadd4f4b6b6d6038ec7c94eb98a879ccd1bf8c5 F tool/sqlite3_analyzer.c.in 8da2b08f56eeac331a715036cf707cc20f879f231362be0c22efd682e2b89b4f F tool/sqltclsh.c.in 1bcc2e9da58fadf17b0bf6a50e68c1159e602ce057210b655d50bad5aaaef898 F tool/sqltclsh.tcl 862f4cf1418df5e1315b5db3b5ebe88969e2a784525af5fbf9596592f14ed848 @@ -2213,8 +2213,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 397b2d37b7a6619b0c1eee201065585d03496f94786b21540f613e4716d56612 -R 721e909ab77c2cd7295ba63619aaa44c +P 9a1a95f523a96303aad57e2422c2b51ea7e125f5490f32f7a2929d49b6c69ef8 +R f1d49deccd29ff432816ec80f04a6271 U drh -Z 2e6a3a2b31db87fda5c83fa80a99e1a6 +Z f5a21c2123f254d15ff8790517b1651d # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a90d6d4a2b..6113137f18 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9a1a95f523a96303aad57e2422c2b51ea7e125f5490f32f7a2929d49b6c69ef8 +fa06977b6db7fa745720561ec0b10570cf7e71598dc7a7c5ee650640e5bdf6f5 diff --git a/tool/sqlite3-rsync.c b/tool/sqlite3-rsync.c index 02a59a92bd..68bfe5d668 100644 --- a/tool/sqlite3-rsync.c +++ b/tool/sqlite3-rsync.c @@ -17,6 +17,7 @@ #include #include #include +#include #include "sqlite3.h" static const char zUsage[] = @@ -37,10 +38,29 @@ struct SQLiteRsync { FILE *pIn; /* Receive from the other side */ sqlite3_uint64 nOut; /* Bytes transmitted */ sqlite3_uint64 nIn; /* Bytes received */ + sqlite3 *db; /* Database connection */ + int nErr; /* Number of errors encountered */ int eVerbose; /* Bigger for more output. 0 means none. */ int bCommCheck; /* True to debug the communication protocol */ + int isRemote; /* On the remote side of a connection */ }; + +/* Magic numbers to identify particular messages sent over the wire. +*/ +#define ORIGIN_BEGIN 0x41 /* Initial message */ +#define ORIGIN_END 0x42 /* Time to quit */ +#define ORIGIN_ERROR 0x43 /* Error message from the remote */ +#define ORIGIN_PAGE 0x44 /* New page data */ +#define ORIGIN_TXN 0x45 /* Transaction commit */ + +#define REPLICA_BEGIN 0x61 /* Welcome message */ +#define REPLICA_ERROR 0x62 /* Error. Report and quit. */ +#define REPLICA_END 0x63 /* Replica wants to stop */ +#define REPLICA_HASH 0x64 /* One or more pages hashes to report */ +#define REPLICA_READY 0x65 /* Read to receive page content */ + + /**************************************************************************** ** Beginning of the popen2() implementation copied from Fossil ************* ****************************************************************************/ @@ -87,6 +107,12 @@ static void win32_fatal_error(const char *zMsg){ # define PTR_TO_INT(X) ((int)(X)) #endif +/* Register SQL functions provided by ext/misc/sha1.c */ +extern int sqlite3_sha_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +); #ifdef _WIN32 /* @@ -471,6 +497,238 @@ static void echoOneLine(SQLiteRsync *p){ } } +/* Read a single big-endian 32-bit unsigned integer from the input +** stream. Return 0 on success and 1 if there are any errors. +*/ +static int readUint32(SQLiteRsync *p, unsigned int *pU){ + unsigned char buf[4]; + if( fread(buf, sizeof(buf), 1, p->pIn)==1 ){ + *pU = (buf[0]<<24) | (buf[1]<<16) | (buf[2]<<8) | buf[3]; + return 0; + }else{ + p->nErr++; + return 1; + } +} + +/* Write a single big-endian 32-bit unsigned integer to the output stream. +** Return 0 on success and 1 if there are any errors. +*/ +static int writeUint32(SQLiteRsync *p, unsigned int x){ + unsigned char buf[4]; + buf[3] = x & 0xff; + x >>= 8; + buf[2] = x & 0xff; + x >>= 8; + buf[1] = x & 0xff; + x >>= 8; + buf[0] = x; + if( fwrite(buf, sizeof(buf), 1, p->pOut)!=1 ){ + p->nErr++; + return 1; + } + return 0; +} + +/* Report an error. +** +** If this happens on the remote side, we send back a REMOTE_ERROR +** message. On the local side, the error message goes to stderr. +*/ +static void reportError(SQLiteRsync *p, const char *zFormat, ...){ + va_list ap; + char *zMsg; + unsigned int nMsg; + va_start(ap, zFormat); + zMsg = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + nMsg = zMsg ? (unsigned int)strlen(zMsg) : 0; + if( p->isRemote ){ + if( p->zReplica ){ + putc(REPLICA_ERROR, p->pOut); + }else{ + putc(ORIGIN_ERROR, p->pOut); + } + writeUint32(p, nMsg); + fwrite(zMsg, nMsg, 1, p->pOut); + fflush(p->pOut); + }else{ + fprintf(stderr, "%s\n", zMsg); + } + sqlite3_free(zMsg); + p->nErr++; +} + +/* Receive and report an error message coming from the other side. +*/ +static void readAndDisplayError(SQLiteRsync *p){ + unsigned int n = 0; + char *zMsg; + (void)readUint32(p, &n); + if( n==0 ){ + fprintf(stderr,"ERROR: unknown (possibly out-of-memory)\n"); + }else{ + zMsg = sqlite3_malloc64( n+1 ); + if( zMsg==0 ){ + fprintf(stderr, "ERROR: out-of-memory\n"); + return; + } + memset(zMsg, 0, n+1); + fread(zMsg, 1, n, p->pIn); + fprintf(stderr,"ERROR: %s\n", zMsg); + sqlite3_free(zMsg); + } + p->nErr++; +} + +/* Construct a new prepared statement. Report an error and return NULL +** if anything goes wrong. +*/ +static sqlite3_stmt *prepareStmtVA( + SQLiteRsync *p, + char *zFormat, + va_list ap +){ + sqlite3_stmt *pStmt = 0; + char *zSql; + char *zToFree = 0; + int rc; + + if( strchr(zFormat,'%') ){ + zSql = sqlite3_vmprintf(zFormat, ap); + if( zSql==0 ){ + reportError(p, "out-of-memory"); + return 0; + }else{ + zToFree = zSql; + } + }else{ + zSql = zFormat; + } + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + if( rc || pStmt==0 ){ + reportError(p, "unable to prepare SQL [%s]: %s", zSql, + sqlite3_errmsg(p->db)); + sqlite3_finalize(pStmt); + pStmt = 0; + } + if( zToFree ) sqlite3_free(zToFree); + return pStmt; +} +static sqlite3_stmt *prepareStmt( + SQLiteRsync *p, + char *zFormat, + ... +){ + sqlite3_stmt *pStmt; + va_list ap; + va_start(ap, zFormat); + pStmt = prepareStmtVA(p, zFormat, ap); + va_end(ap); + return pStmt; +} + +/* Run a single SQL statement +*/ +static void runSql(SQLiteRsync *p, char *zSql, ...){ + sqlite3_stmt *pStmt; + va_list ap; + + va_start(ap, zSql); + pStmt = prepareStmtVA(p, zSql, ap); + va_end(ap); + if( pStmt ){ + int rc = sqlite3_step(pStmt); + if( rc!=SQLITE_OK && rc!=SQLITE_DONE ){ + reportError(p, "SQL statement [%s] failed: %s", zSql, + sqlite3_errmsg(p->db)); + } + sqlite3_finalize(pStmt); + } +} + +/* Run an SQL statement that returns a single unsigned 32-bit integer result +*/ +static int runSqlReturnUInt( + SQLiteRsync *p, + unsigned int *pRes, + char *zSql, + ... +){ + sqlite3_stmt *pStmt; + int res = 0; + va_list ap; + + va_start(ap, zSql); + pStmt = prepareStmtVA(p, zSql, ap); + va_end(ap); + if( pStmt==0 ){ + res = 1; + }else{ + int rc = sqlite3_step(pStmt); + if( rc==SQLITE_ROW ){ + *pRes = (unsigned int)(sqlite3_column_int64(pStmt, 0)&0xffffffff); + }else{ + reportError(p, "SQL statement [%s] failed: %s", zSql, + sqlite3_errmsg(p->db)); + res = 1; + } + sqlite3_finalize(pStmt); + } + return res; +} + +/* Run an SQL statement that returns a single TEXT value that is no more +** than 99 bytes in length. +*/ +static int runSqlReturnText( + SQLiteRsync *p, + char *pRes, + char *zSql, + ... +){ + sqlite3_stmt *pStmt; + int res = 0; + va_list ap; + + va_start(ap, zSql); + pStmt = prepareStmtVA(p, zSql, ap); + va_end(ap); + pRes[0] = 0; + if( pStmt==0 ){ + res = 1; + }else{ + int rc = sqlite3_step(pStmt); + if( rc==SQLITE_ROW ){ + const unsigned char *a = sqlite3_column_text(pStmt, 0); + int n; + if( a==0 ){ + pRes[0] = 0; + }else{ + n = sqlite3_column_bytes(pStmt, 0); + if( n>99 ) n = 99; + memcpy(pRes, a, n); + pRes[n] = 0; + } + }else{ + reportError(p, "SQL statement [%s] failed: %s", zSql, + sqlite3_errmsg(p->db)); + res = 1; + } + sqlite3_finalize(pStmt); + } + return res; +} + +/* Close the database connection associated with p +*/ +static void closeDb(SQLiteRsync *p){ + if( p->db ){ + sqlite3_close(p->db); + p->db = 0; + } +} + /* ** Run the origin-side protocol. ** @@ -484,6 +742,12 @@ static void echoOneLine(SQLiteRsync *p){ ** 7. Send origin-end message */ static void originSide(SQLiteRsync *p){ + int rc = 0; + int c = 0; + unsigned int nPage = 0; + unsigned int szPg = 0; + char buf[100]; + if( p->bCommCheck ){ fprintf(p->pOut, "sqlite3-rsync origin-begin %s\n", p->zOrigin); fflush(p->pOut); @@ -493,6 +757,59 @@ static void originSide(SQLiteRsync *p){ echoOneLine(p); return; } + + /* Open the ORIGIN database. */ + rc = sqlite3_open_v2(p->zOrigin, &p->db, SQLITE_OPEN_READONLY, 0); + if( rc ){ + reportError(p, "unable to open origin database file \"%s\": %s", + sqlite3_errmsg(p->db)); + closeDb(p); + return; + } + sqlite3_sha_init(p->db, 0, 0); + runSql(p, "BEGIN"); + runSqlReturnText(p, buf, "PRAGMA journal_mode"); + if( sqlite3_stricmp(buf,"wal")!=0 ){ + reportError(p, "Origin database is not in WAL mode"); + } + runSqlReturnUInt(p, &nPage, "PRAGMA page_count"); + runSqlReturnUInt(p, &szPg, "PRAGMA page_size"); + + if( p->nErr==0 ){ + /* Send the ORIGIN_BEGIN message */ + fputc(ORIGIN_BEGIN, p->pOut); + writeUint32(p, nPage); + writeUint32(p, szPg); + fflush(p->pOut); + } + + /* Respond to message from the replica */ + while( p->nErr==0 && (c = fgetc(p->pIn))!=EOF ){ + switch( c ){ + case REPLICA_ERROR: { + readAndDisplayError(p); + break; + } + case REPLICA_BEGIN: { + break; + } + case REPLICA_END: { + break; + } + case REPLICA_HASH: { + break; + } + case REPLICA_READY: { + break; + } + default: { + reportError(p, "Origin side received unknown message: 0x%02x", c); + break; + } + } + } + + closeDb(p); } /* @@ -508,6 +825,8 @@ static void originSide(SQLiteRsync *p){ ** 7. COMMIT */ static void replicaSide(SQLiteRsync *p){ + int c; + char buf[100]; if( p->bCommCheck ){ echoOneLine(p); fprintf(p->pOut, "replica-begin %s\n", p->zReplica); @@ -517,6 +836,74 @@ static void replicaSide(SQLiteRsync *p){ fflush(p->pOut); return; } + + /* Respond to message from the origin. The origin will initiate the + ** the conversation with an ORIGIN_BEGIN message. + */ + while( p->nErr==0 && (c = fgetc(p->pIn))!=EOF ){ + switch( c ){ + case ORIGIN_ERROR: { + readAndDisplayError(p); + break; + } + case ORIGIN_BEGIN: { + unsigned int nOPage = 0, szOPage = 0; + unsigned int nRPage = 0, szRPage = 0; + int rc = 0; + sqlite3_stmt *pStmt = 0; + + closeDb(p); + readUint32(p, &nOPage); + readUint32(p, &szOPage); + if( p->nErr ) break; + rc = sqlite3_open(p->zReplica, &p->db); + if( rc ){ + reportError(p, "cannot open replica database \"%s\": %s", + p->zReplica, sqlite3_errmsg(p->db)); + closeDb(p); + break; + } + sqlite3_sha_init(p->db, 0, 0); + if( runSqlReturnUInt(p, &nRPage, "PRAGMA page_count") ){ + break; + } + if( nRPage==0 ){ + runSql(p, "PRAGMA page_size=%u", szOPage); + runSql(p, "PRAGMA journal_mode=WAL"); + } + runSql(p, "BEGIN IMMEDIATE"); + runSqlReturnText(p, buf, "PRAGMA journal_mode"); + if( strcmp(buf, "wal")!=0 ){ + reportError(p, "replica is not in WAL mode"); + break; + } + runSqlReturnUInt(p, &nRPage, "PRAGMA page_count"); + runSqlReturnUInt(p, &szRPage, "PRAGMA page_size"); + if( szRPage!=szOPage ){ + reportError(p, "page size mismatch; origin is %d bytes and " + "replica is %d bytes", szOPage, szRPage); + break; + } + pStmt = prepareStmt(p, + "SELECT pgno, sha1(data) FROM sqlite_dbpage" + " WHERE pgno<=min(%d,%d)", nRPage, nOPage); + sqlite3_finalize(pStmt); + break; + } + case ORIGIN_END: { + break; + } + case ORIGIN_PAGE: { + break; + } + default: { + reportError(p, "Replica side received unknown message: 0x%02x", c); + break; + } + } + } + + closeDb(p); } @@ -544,7 +931,7 @@ static void replicaSide(SQLiteRsync *p){ ** ** If (3) is seen, call originSide() on stdin and stdout. ** -** If (4) is seen, call replicaSide() on stdin and stdout. +q** If (4) is seen, call replicaSide() on stdin and stdout. */ int main(int argc, char **argv){ int isOrigin = 0; @@ -635,6 +1022,7 @@ int main(int argc, char **argv){ if( isOrigin ){ ctx.pIn = stdin; ctx.pOut = stdout; + ctx.isRemote = 1; originSide(&ctx); return 0; } @@ -643,6 +1031,7 @@ int main(int argc, char **argv){ ctx.zOrigin = 0; ctx.pIn = stdin; ctx.pOut = stdout; + ctx.isRemote = 1; replicaSide(&ctx); return 0; } From f1fb5ce10f4d320fc9b012df8e6b7119f508ef42 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 12 Sep 2024 12:04:53 +0000 Subject: [PATCH 05/24] Improved debugging output. FossilOrigin-Name: 80461e0d724963aaf2646005298f1194c5f1c4c9ae41c1085d4d137ed485bd9f --- Makefile.in | 1 + Makefile.msc | 1 + main.mk | 1 + manifest | 18 +-- manifest.uuid | 2 +- tool/sqlite3-rsync.c | 283 ++++++++++++++++++++++++++++++++++++------- 6 files changed, 251 insertions(+), 55 deletions(-) diff --git a/Makefile.in b/Makefile.in index 0cfae780cc..8a070ebda8 100644 --- a/Makefile.in +++ b/Makefile.in @@ -706,6 +706,7 @@ RSYNC_SRC = \ sqlite3.c RSYNC_OPT = \ + -DSQLITE_ENABLE_DBPAGE_VTAB \ -DSQLITE_THREADSAFE=0 \ -DSQLITE_OMIT_LOAD_EXTENSION \ -DSQLITE_OMIT_DEPRECATED diff --git a/Makefile.msc b/Makefile.msc index 461859d1f1..40f8dc0f82 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1873,6 +1873,7 @@ RSYNC_SRC = \ $(SQLITE3C) RSYNC_OPT = \ + -DSQLITE_ENABLE_DBPAGE_VTAB \ -DSQLITE_THREADSAFE=0 \ -DSQLITE_OMIT_LOAD_EXTENSION \ -DSQLITE_OMIT_DEPRECATED diff --git a/main.mk b/main.mk index 7414e7488e..3ab5047b5a 100644 --- a/main.mk +++ b/main.mk @@ -574,6 +574,7 @@ RSYNC_SRC = \ sqlite3.c RSYNC_OPT = \ + -DSQLITE_ENABLE_DBPAGE_VTAB \ -DSQLITE_THREADSAFE=0 \ -DSQLITE_OMIT_LOAD_EXTENSION \ -DSQLITE_OMIT_DEPRECATED diff --git a/manifest b/manifest index c99d1cd05f..bb7f3a7267 100644 --- a/manifest +++ b/manifest @@ -1,11 +1,11 @@ -C Progress\son\sthe\ssqlite3-rsync\sutility.\s\sThis\sis\san\sincremental\scheck-in.\s\sIt\ndoes\scompile,\sbut\sit\sdoes\snot\swork. -D 2024-09-11T17:02:44.010 +C Improved\sdebugging\soutput. +D 2024-09-12T12:04:53.132 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 -F Makefile.in 54e80b016b0e58db383c0f08d340ef795b8b709ffb6f53a51a8ba7bf0c5e288f +F Makefile.in 167583cd37df435b3cd7e87de7a04247d341db83ffd363bd0240ddcc776c55d6 F Makefile.linux-gcc f3842a0b1efbfbb74ac0ef60e56b301836d05b4d867d014f714fa750048f1ab6 -F Makefile.msc a86e0f3fe5f807daa82d44b5056e3dbc311e569bd4748646a776a994124ec58b +F Makefile.msc 4af481bae608f19f869f7709d93ba04876480844044e14ce97f89e5ee2e51759 F README.md c3c0f19532ce28f6297a71870f3c7b424729f0e6d9ab889616d3587dd2332159 F VERSION 0db40f92c04378404eb45bff93e9e42c148c7e54fd3da99469ed21e22411f5a6 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -687,7 +687,7 @@ F ext/wasm/wasmfs.make 8a4955882aaa0783b3f60a9484a1f0f3d8b6f775c0fcd17c082f31966 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 5ade0bc977aa135e79e3faaea894d5671b26107cc91e70783aa7dc83f22f3ba0 -F main.mk 9ffe4a14bdb4a0b856217a5465ca6b1ef4bef66a3c45e2da3fdb6d9bfc8d583e +F main.mk f6424b8011c62b707fca5153a71a5d5a373f36ea6458908cc8858f7c5118c9f1 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 F mptest/crash01.test 61e61469e257df0850df4293d7d4d6c2af301421 @@ -2174,7 +2174,7 @@ F tool/speedtest8inst1.c 7ce07da76b5e745783e703a834417d725b7d45fd F tool/spellsift.tcl 52b4b04dc4333c7ab024f09d9d66ed6b6f7c6eb00b38497a09f338fa55d40618 x F tool/split-sqlite3c.tcl 5aa60643afca558bc732b1444ae81a522326f91e1dc5665b369c54f09e20de60 F tool/sqldiff.c 847fc8fcfddf5ce4797b7394cad6372f2f5dc17d8186e2ef8fb44d50fae4f44a -F tool/sqlite3-rsync.c eb75a24e3a47fe7b5a4d5cbd2eadd4f4b6b6d6038ec7c94eb98a879ccd1bf8c5 +F tool/sqlite3-rsync.c d9f8803f79c66dbc213761a345e24ae22c7de14fd334150086519c611ff1a705 F tool/sqlite3_analyzer.c.in 8da2b08f56eeac331a715036cf707cc20f879f231362be0c22efd682e2b89b4f F tool/sqltclsh.c.in 1bcc2e9da58fadf17b0bf6a50e68c1159e602ce057210b655d50bad5aaaef898 F tool/sqltclsh.tcl 862f4cf1418df5e1315b5db3b5ebe88969e2a784525af5fbf9596592f14ed848 @@ -2213,8 +2213,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9a1a95f523a96303aad57e2422c2b51ea7e125f5490f32f7a2929d49b6c69ef8 -R f1d49deccd29ff432816ec80f04a6271 +P fa06977b6db7fa745720561ec0b10570cf7e71598dc7a7c5ee650640e5bdf6f5 +R 7d94be6de6803646ba12e0250789553f U drh -Z f5a21c2123f254d15ff8790517b1651d +Z 2b03e42bffc925c056d94789b2d4ed35 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 6113137f18..af581bf8ac 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fa06977b6db7fa745720561ec0b10570cf7e71598dc7a7c5ee650640e5bdf6f5 +80461e0d724963aaf2646005298f1194c5f1c4c9ae41c1085d4d137ed485bd9f diff --git a/tool/sqlite3-rsync.c b/tool/sqlite3-rsync.c index 68bfe5d668..276b9af90f 100644 --- a/tool/sqlite3-rsync.c +++ b/tool/sqlite3-rsync.c @@ -29,6 +29,8 @@ static const char zUsage[] = "copy of ORIGIN\n" ; +typedef unsigned char u8; + /* Context for the run */ typedef struct SQLiteRsync SQLiteRsync; struct SQLiteRsync { @@ -36,13 +38,17 @@ struct SQLiteRsync { const char *zReplica; /* Name of the replica */ FILE *pOut; /* Transmit to the other side */ FILE *pIn; /* Receive from the other side */ - sqlite3_uint64 nOut; /* Bytes transmitted */ - sqlite3_uint64 nIn; /* Bytes received */ sqlite3 *db; /* Database connection */ int nErr; /* Number of errors encountered */ - int eVerbose; /* Bigger for more output. 0 means none. */ - int bCommCheck; /* True to debug the communication protocol */ - int isRemote; /* On the remote side of a connection */ + u8 eVerbose; /* Bigger for more output. 0 means none. */ + u8 bCommCheck; /* True to debug the communication protocol */ + u8 isRemote; /* On the remote side of a connection */ + sqlite3_uint64 nOut; /* Bytes transmitted */ + sqlite3_uint64 nIn; /* Bytes received */ + unsigned int nPage; /* Total number of pages in the database */ + unsigned int szPage; /* Database page size */ + unsigned int nHashSent; /* Hashes sent (replica to origin) */ + unsigned int nPageSent; /* Page contents sent (origin to replica) */ }; @@ -504,6 +510,7 @@ static int readUint32(SQLiteRsync *p, unsigned int *pU){ unsigned char buf[4]; if( fread(buf, sizeof(buf), 1, p->pIn)==1 ){ *pU = (buf[0]<<24) | (buf[1]<<16) | (buf[2]<<8) | buf[3]; + p->nIn += 4; return 0; }else{ p->nErr++; @@ -527,9 +534,45 @@ static int writeUint32(SQLiteRsync *p, unsigned int x){ p->nErr++; return 1; } + p->nOut += 4; return 0; } +/* Read a single byte from the wire. +*/ +int readByte(SQLiteRsync *p){ + int c = fgetc(p->pIn); + if( c!=EOF ) p->nIn++; + return c; +} + +/* Write a single byte into the wire. +*/ +void writeByte(SQLiteRsync *p, int c){ + fputc(c, p->pOut); + p->nOut++; +} + +/* Read an array of bytes from the wire. +*/ +void readBytes(SQLiteRsync *p, int nByte, void *pData){ + if( fread(pData, 1, nByte, p->pIn)==nByte ){ + p->nIn += nByte; + }else{ + p->nErr++; + } +} + +/* Write an array of bytes onto the wire. +*/ +void writeBytes(SQLiteRsync *p, int nByte, const void *pData){ + if( fwrite(pData, 1, nByte, p->pOut)==nByte ){ + p->nOut += nByte; + }else{ + p->nErr++; + } +} + /* Report an error. ** ** If this happens on the remote side, we send back a REMOTE_ERROR @@ -550,7 +593,7 @@ static void reportError(SQLiteRsync *p, const char *zFormat, ...){ putc(ORIGIN_ERROR, p->pOut); } writeUint32(p, nMsg); - fwrite(zMsg, nMsg, 1, p->pOut); + writeBytes(p, nMsg, zMsg); fflush(p->pOut); }else{ fprintf(stderr, "%s\n", zMsg); @@ -574,7 +617,7 @@ static void readAndDisplayError(SQLiteRsync *p){ return; } memset(zMsg, 0, n+1); - fread(zMsg, 1, n, p->pIn); + readBytes(p, n, zMsg); fprintf(stderr,"ERROR: %s\n", zMsg); sqlite3_free(zMsg); } @@ -639,6 +682,7 @@ static void runSql(SQLiteRsync *p, char *zSql, ...){ va_end(ap); if( pStmt ){ int rc = sqlite3_step(pStmt); + if( rc==SQLITE_ROW ) rc = sqlite3_step(pStmt); if( rc!=SQLITE_OK && rc!=SQLITE_DONE ){ reportError(p, "SQL statement [%s] failed: %s", zSql, sqlite3_errmsg(p->db)); @@ -732,21 +776,37 @@ static void closeDb(SQLiteRsync *p){ /* ** Run the origin-side protocol. ** -** 1. Send the origin-begin message -** 2. Receive replica-begin message -** - Error check and abort if necessary -** 3. Receive replica-hash messages -** 4. BEGIN -** 5. Send changed pages -** 6. COMMIT -** 7. Send origin-end message +** Begin by sending the ORIGIN_BEGIN message with two arguments, +** nPage, and szPage. Then enter a loop responding to message from +** the replica: +** +** REPLICA_ERROR size text +** +** Report an error from the replica and quit +** +** REPLICA_END +** +** The replica is terminating. Stop processing now. +** +** REPLICA_HASH hash +** +** The argument is the 20-byte SHA1 hash for the next page +** page hashes appear in sequential order with no gaps. +** +** REPLICA_READY +** +** The replica has sent all the hashes that it intends to send. +** This side (the origin) can now start responding with page +** content for pages that do not have a matching hash. */ static void originSide(SQLiteRsync *p){ int rc = 0; int c = 0; unsigned int nPage = 0; + unsigned int iPage = 0; unsigned int szPg = 0; - char buf[100]; + sqlite3_stmt *pCkHash = 0; + char buf[200]; if( p->bCommCheck ){ fprintf(p->pOut, "sqlite3-rsync origin-begin %s\n", p->zOrigin); @@ -759,7 +819,7 @@ static void originSide(SQLiteRsync *p){ } /* Open the ORIGIN database. */ - rc = sqlite3_open_v2(p->zOrigin, &p->db, SQLITE_OPEN_READONLY, 0); + rc = sqlite3_open_v2(p->zOrigin, &p->db, SQLITE_OPEN_READWRITE, 0); if( rc ){ reportError(p, "unable to open origin database file \"%s\": %s", sqlite3_errmsg(p->db)); @@ -777,30 +837,68 @@ static void originSide(SQLiteRsync *p){ if( p->nErr==0 ){ /* Send the ORIGIN_BEGIN message */ - fputc(ORIGIN_BEGIN, p->pOut); + writeByte(p, ORIGIN_BEGIN); writeUint32(p, nPage); writeUint32(p, szPg); fflush(p->pOut); + p->nPage = nPage; + p->szPage = szPg; } /* Respond to message from the replica */ - while( p->nErr==0 && (c = fgetc(p->pIn))!=EOF ){ + while( p->nErr==0 && (c = readByte(p))!=EOF && c!=REPLICA_END ){ switch( c ){ case REPLICA_ERROR: { readAndDisplayError(p); break; } - case REPLICA_BEGIN: { - break; - } - case REPLICA_END: { - break; - } case REPLICA_HASH: { + if( pCkHash==0 ){ + runSql(p, "CREATE TEMP TABLE badHash(pgno INTEGER PRIMARY KEY)"); + pCkHash = prepareStmt(p, + "INSERT INTO badHash SELECT pgno FROM sqlite_dbpage('main')" + " WHERE pgno=?1 AND sha1b(data)!=?2" + ); + if( pCkHash==0 ) break; + } + p->nHashSent++; + iPage++; + sqlite3_bind_int64(pCkHash, 1, iPage); + readBytes(p, 20, buf); + sqlite3_bind_blob(pCkHash, 2, buf, 20, SQLITE_STATIC); + rc = sqlite3_step(pCkHash); + if( rc!=SQLITE_DONE ){ + reportError(p, "SQL statement [%s] failed: %s", + sqlite3_sql(pCkHash), sqlite3_errmsg(p->db)); + } + sqlite3_reset(pCkHash); break; } case REPLICA_READY: { - break; + sqlite3_stmt *pStmt; + sqlite3_finalize(pCkHash); + pCkHash = 0; + pStmt = prepareStmt(p, + "SELECT pgno, data" + " FROM badHash JOIN sqlite_dbpage('main') USING(pgno) " + "UNION ALL " + "SELECT pgno, data" + " FROM sqlite_dbpage('main')" + " WHERE pgno>%d", + iPage); + if( pStmt==0 ) break; + while( sqlite3_step(pStmt)==SQLITE_ROW && p->nErr==0 ){ + const void *pContent = sqlite3_column_blob(pStmt, 1); + writeByte(p, ORIGIN_PAGE); + writeUint32(p, (unsigned int)sqlite3_column_int64(pStmt, 0)); + writeBytes(p, szPg, pContent); + p->nPageSent++; + } + sqlite3_finalize(pStmt); + writeByte(p, ORIGIN_TXN); + writeUint32(p, nPage); + writeByte(p, ORIGIN_END); + goto origin_end; } default: { reportError(p, "Origin side received unknown message: 0x%02x", c); @@ -809,24 +907,43 @@ static void originSide(SQLiteRsync *p){ } } +origin_end: + if( pCkHash ) sqlite3_finalize(pCkHash); closeDb(p); } /* -** Run the replica-side protocol. +** Run the replica-side protocol. The protocol is passive in the sense +** that it only response to message from the origin side. ** -** 1. Receive the origin-begin message -** - Error check. If unable to continue, send replica-error and quit -** 2. BEGIN IMMEDIATE -** 3. Send replica-begin message -** 4. Send replica-hash messages -** 5. Receive changed pages and apply them -** 6. Receive origin-end message -** 7. COMMIT +** ORIGIN_BEGIN nPage szPage +** +** The origin is reporting the number of pages and the size of each +** pages. This procedure checks compatibility, and if everything is +** ok, it sends hash for all its extant pages. +** +** ORIGIN_ERROR size text +** +** Report the received error and quit. +** +** ORIGIN_PAGE pgno content +** +** Update the content of the given page. +** +** ORIGIN_TXN pgno +** +** Close the update transaction. The total database size is pgno +** pages. +** +** ORIGIN_END +** +** Expect no more transmissions from the origin. */ static void replicaSide(SQLiteRsync *p){ int c; - char buf[100]; + sqlite3_stmt *pIns = 0; + unsigned int szOPage = 0; + char buf[65536]; if( p->bCommCheck ){ echoOneLine(p); fprintf(p->pOut, "replica-begin %s\n", p->zReplica); @@ -840,14 +957,14 @@ static void replicaSide(SQLiteRsync *p){ /* Respond to message from the origin. The origin will initiate the ** the conversation with an ORIGIN_BEGIN message. */ - while( p->nErr==0 && (c = fgetc(p->pIn))!=EOF ){ + while( p->nErr==0 && (c = readByte(p))!=EOF && c!=ORIGIN_END ){ switch( c ){ case ORIGIN_ERROR: { readAndDisplayError(p); break; } case ORIGIN_BEGIN: { - unsigned int nOPage = 0, szOPage = 0; + unsigned int nOPage = 0; unsigned int nRPage = 0, szRPage = 0; int rc = 0; sqlite3_stmt *pStmt = 0; @@ -856,6 +973,8 @@ static void replicaSide(SQLiteRsync *p){ readUint32(p, &nOPage); readUint32(p, &szOPage); if( p->nErr ) break; + p->nPage = nOPage; + p->szPage = szOPage; rc = sqlite3_open(p->zReplica, &p->db); if( rc ){ reportError(p, "cannot open replica database \"%s\": %s", @@ -885,15 +1004,64 @@ static void replicaSide(SQLiteRsync *p){ break; } pStmt = prepareStmt(p, - "SELECT pgno, sha1(data) FROM sqlite_dbpage" - " WHERE pgno<=min(%d,%d)", nRPage, nOPage); + "SELECT sha1b(data) FROM sqlite_dbpage" + " WHERE pgno<=min(%d,%d)" + " ORDER BY pgno", nRPage, nOPage); + while( sqlite3_step(pStmt)==SQLITE_ROW && p->nErr==0 ){ + const unsigned char *a = sqlite3_column_blob(pStmt, 0); + writeByte(p, REPLICA_HASH); + writeBytes(p, 20, a); + p->nHashSent++; + } sqlite3_finalize(pStmt); + writeByte(p, REPLICA_READY); + fflush(p->pOut); break; } - case ORIGIN_END: { + case ORIGIN_TXN: { + unsigned int nOPage = 0; + readUint32(p, &nOPage); + if( pIns==0 ){ + /* Nothing has changed */ + runSql(p, "COMMIT"); + }else if( p->nErr ){ + runSql(p, "ROLLBACK"); + }else{ + int rc; + sqlite3_bind_int64(pIns, 1, nOPage); + sqlite3_bind_null(pIns, 2); + rc = sqlite3_step(pIns); + if( rc!=SQLITE_DONE ){ + reportError(p, "SQL statement [%s] failed: %s", + sqlite3_sql(pIns), sqlite3_errmsg(p->db)); + } + sqlite3_reset(pIns); + runSql(p, "COMMIT"); + } break; } case ORIGIN_PAGE: { + unsigned int pgno = 0; + int rc; + readUint32(p, &pgno); + if( p->nErr ) break; + if( pIns==0 ){ + pIns = prepareStmt(p, + "INSERT INTO sqlite_dbpage(pgno,data,schema) VALUES(?1,?2,'main')" + ); + if( pIns==0 ) break; + } + readBytes(p, szOPage, buf); + if( p->nErr ) break; + p->nPageSent++; + sqlite3_bind_int64(pIns, 1, pgno); + sqlite3_bind_blob(pIns, 2, buf, szOPage, SQLITE_STATIC); + rc = sqlite3_step(pIns); + if( rc!=SQLITE_DONE ){ + reportError(p, "SQL statement [%s] failed: %s", + sqlite3_sql(pIns), sqlite3_errmsg(p->db)); + } + sqlite3_reset(pIns); break; } default: { @@ -903,9 +1071,24 @@ static void replicaSide(SQLiteRsync *p){ } } + if( pIns ) sqlite3_finalize(pIns); closeDb(p); } +/* +** The argument might be -vvv...vv with any number of "v"s. Return +** the number of "v"s. Return 0 if the argument is not a -vvv...v. +*/ +static int numVs(const char *z){ + int n = 0; + if( z[0]!='-' ) return 0; + z++; + if( z[0]=='-' ) z++; + while( z[0]=='v' ){ n++; z++; } + if( z[0]==0 ) return n; + return 0; +} + /* ** Parse command-line arguments. Dispatch subroutines to do the @@ -957,8 +1140,8 @@ int main(int argc, char **argv){ isReplica = 1; continue; } - if( strcmp(z, "-v")==0 ){ - ctx.eVerbose++; + if( numVs(z) ){ + ctx.eVerbose += numVs(z); continue; } if( strcmp(z, "--ssh")==0 ){ @@ -1066,7 +1249,7 @@ int main(int argc, char **argv){ fprintf(stderr, "Could not start auxiliary process: %s\n", zCmd); return 1; } - originSide(&ctx); + replicaSide(&ctx); }else if( (zDiv = strchr(ctx.zReplica,':'))!=0 ){ /* Local ORIGIN and remote REPLICA */ sqlite3_str *pStr = sqlite3_str_new(0); @@ -1091,7 +1274,7 @@ int main(int argc, char **argv){ }else{ /* Local ORIGIN and REPLICA */ sqlite3_str *pStr = sqlite3_str_new(0); - append_escaped_arg(pStr, zExe, 1); + append_escaped_arg(pStr, argv[0], 1); append_escaped_arg(pStr, "--replica", 0); if( ctx.bCommCheck ){ append_escaped_arg(pStr, "--commcheck", 0); @@ -1105,6 +1288,16 @@ int main(int argc, char **argv){ } originSide(&ctx); } + if( ctx.eVerbose ){ + if( ctx.nErr ) printf("%d errors, ", ctx.nErr); + printf("%lld bytes sent, %lld bytes received\n", ctx.nOut, ctx.nIn); + if( ctx.eVerbose>=2 ){ + printf("Database is %u pages of %u bytes each.\n", + ctx.nPage, ctx.szPage); + printf("Sent %u hashes, %u page contents\n", + ctx.nHashSent, ctx.nPageSent); + } + } sqlite3_free(zCmd); if( pIn!=0 && pOut!=0 ){ pclose2(pIn, pOut, childPid); From 165daef0432a72212997af12d29c04dd93607d7a Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 12 Sep 2024 14:43:05 +0000 Subject: [PATCH 06/24] Add sha1() functions to the CLI. Fix sha1b() such that it actually returns a BLOB. FossilOrigin-Name: fe65821a3b912f061026e6fd7174be26897010e6b474e2780350cac60faebaad --- Makefile.in | 1 + Makefile.msc | 1 + ext/misc/sha1.c | 21 +++++++++++++++------ main.mk | 1 + manifest | 20 ++++++++++---------- manifest.uuid | 2 +- src/shell.c.in | 2 ++ 7 files changed, 31 insertions(+), 17 deletions(-) diff --git a/Makefile.in b/Makefile.in index 8a070ebda8..10b5db15a5 100644 --- a/Makefile.in +++ b/Makefile.in @@ -1205,6 +1205,7 @@ SHELL_DEP = \ $(TOP)/ext/misc/percentile.c \ $(TOP)/ext/misc/regexp.c \ $(TOP)/ext/misc/series.c \ + $(TOP)/ext/misc/sha1.c \ $(TOP)/ext/misc/shathree.c \ $(TOP)/ext/misc/sqlar.c \ $(TOP)/ext/misc/uint.c \ diff --git a/Makefile.msc b/Makefile.msc index 40f8dc0f82..e19156c612 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -2331,6 +2331,7 @@ SHELL_DEP = \ $(TOP)\ext\misc\percentile.c \ $(TOP)\ext\misc\regexp.c \ $(TOP)\ext\misc\series.c \ + $(TOP)\ext\misc\sha1.c \ $(TOP)\ext\misc\shathree.c \ $(TOP)\ext\misc\sqlar.c \ $(TOP)\ext\misc\uint.c \ diff --git a/ext/misc/sha1.c b/ext/misc/sha1.c index dbdebbff87..07d7970609 100644 --- a/ext/misc/sha1.c +++ b/ext/misc/sha1.c @@ -216,11 +216,15 @@ static void hash_finish( for (i = 0; i < 20; i++){ digest[i] = (unsigned char)((p->state[i>>2] >> ((3-(i & 3)) * 8) ) & 255); } - for(i=0; i<20; i++){ - zOut[i*2] = zEncode[(digest[i]>>4)&0xf]; - zOut[i*2+1] = zEncode[digest[i] & 0xf]; + if( bAsBinary ){ + memcpy(zOut, digest, 20); + }else{ + for(i=0; i<20; i++){ + zOut[i*2] = zEncode[(digest[i]>>4)&0xf]; + zOut[i*2+1] = zEncode[digest[i] & 0xf]; + } + zOut[i*2]= 0; } - zOut[i*2]= 0; } /* End of the hashing logic *****************************************************************************/ @@ -252,8 +256,13 @@ static void sha1Func( }else{ hash_step(&cx, sqlite3_value_text(argv[0]), nByte); } - hash_finish(&cx, zOut, sqlite3_user_data(context)!=0); - sqlite3_result_text(context, zOut, 40, SQLITE_TRANSIENT); + if( sqlite3_user_data(context)!=0 ){ + hash_finish(&cx, zOut, 1); + sqlite3_result_blob(context, zOut, 20, SQLITE_TRANSIENT); + }else{ + hash_finish(&cx, zOut, 0); + sqlite3_result_blob(context, zOut, 40, SQLITE_TRANSIENT); + } } /* diff --git a/main.mk b/main.mk index 3ab5047b5a..1742b90707 100644 --- a/main.mk +++ b/main.mk @@ -782,6 +782,7 @@ SHELL_DEP = \ $(TOP)/ext/misc/percentile.c \ $(TOP)/ext/misc/regexp.c \ $(TOP)/ext/misc/series.c \ + $(TOP)/ext/misc/sha1.c \ $(TOP)/ext/misc/shathree.c \ $(TOP)/ext/misc/sqlar.c \ $(TOP)/ext/misc/uint.c \ diff --git a/manifest b/manifest index bb7f3a7267..a0d9e1b120 100644 --- a/manifest +++ b/manifest @@ -1,11 +1,11 @@ -C Improved\sdebugging\soutput. -D 2024-09-12T12:04:53.132 +C Add\ssha1()\sfunctions\sto\sthe\sCLI.\s\sFix\ssha1b()\ssuch\sthat\sit\sactually\sreturns\na\sBLOB. +D 2024-09-12T14:43:05.090 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 -F Makefile.in 167583cd37df435b3cd7e87de7a04247d341db83ffd363bd0240ddcc776c55d6 +F Makefile.in 4ac84300552e9ab8245288dec9941b6b820b48e15cbcd7c87db2ce45998c62f8 F Makefile.linux-gcc f3842a0b1efbfbb74ac0ef60e56b301836d05b4d867d014f714fa750048f1ab6 -F Makefile.msc 4af481bae608f19f869f7709d93ba04876480844044e14ce97f89e5ee2e51759 +F Makefile.msc f25c476f22a25f23af8dc7260e9f95896250e7fceb621388ca8ae4f096bd3fda F README.md c3c0f19532ce28f6297a71870f3c7b424729f0e6d9ab889616d3587dd2332159 F VERSION 0db40f92c04378404eb45bff93e9e42c148c7e54fd3da99469ed21e22411f5a6 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -420,7 +420,7 @@ F ext/misc/remember.c add730f0f7e7436cd15ea3fd6a90fd83c3f706ab44169f7f048438b7d6 F ext/misc/rot13.c 51ac5f51e9d5fd811db58a9c23c628ad5f333c173f1fc53c8491a3603d38556c F ext/misc/scrub.c 2a44b0d44c69584c0580ad2553f6290a307a49df4668941d2812135bfb96a946 F ext/misc/series.c a6089b5e8e3002bd1e5d9877cee6aead0b9a6426e406c09a399817db9e9ae823 -F ext/misc/sha1.c dfd26eb3437a88fe7349d1fe080b761549c456ae17cb11242441bf66031942bf +F ext/misc/sha1.c cb5002148c2661b5946f34561701e9105e9d339b713ec8ac057fd888b196dcb9 F ext/misc/shathree.c 1821d90a0040c9accdbe3e3527d378d30569475d758aa70f6848924c0b430e8c F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52 F ext/misc/spellfix.c c0aa7b80d6df45f7da59d912b38752bcac1af53a5766966160e6c5cdd397dbea @@ -687,7 +687,7 @@ F ext/wasm/wasmfs.make 8a4955882aaa0783b3f60a9484a1f0f3d8b6f775c0fcd17c082f31966 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 5ade0bc977aa135e79e3faaea894d5671b26107cc91e70783aa7dc83f22f3ba0 -F main.mk f6424b8011c62b707fca5153a71a5d5a373f36ea6458908cc8858f7c5118c9f1 +F main.mk 91d391457000574faaaa34a833564b98f4a3dae6c4a5681e8e6b5ff1cf5c4b57 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 F mptest/crash01.test 61e61469e257df0850df4293d7d4d6c2af301421 @@ -768,7 +768,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 2c127880c0634962837f16f2f48a295e514357af959330cc038de73015d5b5e8 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 F src/select.c 4b14337a2742f0c0beeba490e9a05507e9b4b12184b9cd12773501d08d48e3fe -F src/shell.c.in 470db843788d74234cc1e6873ac51c0ae6529994a52146fefe2e77c0754cbf96 +F src/shell.c.in 375f8a183126be96ec73db4e42c57917ff10a0900846b1b722dd4f8cef537812 F src/sqlite.h.in 77f55bd1978a04a14db211732f0a609077cf60ba4ccf9baf39988f508945419c F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54 @@ -2213,8 +2213,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P fa06977b6db7fa745720561ec0b10570cf7e71598dc7a7c5ee650640e5bdf6f5 -R 7d94be6de6803646ba12e0250789553f +P 80461e0d724963aaf2646005298f1194c5f1c4c9ae41c1085d4d137ed485bd9f +R 40763ccf45039aac90ad4e181b463c0b U drh -Z 2b03e42bffc925c056d94789b2d4ed35 +Z 969215ecb0944de74c8a9b9fbf319407 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index af581bf8ac..86f93a2219 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -80461e0d724963aaf2646005298f1194c5f1c4c9ae41c1085d4d137ed485bd9f +fe65821a3b912f061026e6fd7174be26897010e6b474e2780350cac60faebaad diff --git a/src/shell.c.in b/src/shell.c.in index 9bc6c2566f..2b0e506ed8 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -1209,6 +1209,7 @@ INCLUDE test_windirent.c INCLUDE ../ext/misc/memtrace.c INCLUDE ../ext/misc/pcachetrace.c INCLUDE ../ext/misc/shathree.c +INCLUDE ../ext/misc/sha1.c INCLUDE ../ext/misc/uint.c INCLUDE ../ext/misc/decimal.c INCLUDE ../ext/misc/percentile.c @@ -5392,6 +5393,7 @@ static void open_db(ShellState *p, int openFlags){ #ifndef SQLITE_OMIT_LOAD_EXTENSION sqlite3_enable_load_extension(p->db, 1); #endif + sqlite3_sha_init(p->db, 0, 0); sqlite3_shathree_init(p->db, 0, 0); sqlite3_uint_init(p->db, 0, 0); sqlite3_stmtrand_init(p->db, 0, 0); From f7c96ee98b47915bfd014a29162360886964a276 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 12 Sep 2024 15:36:34 +0000 Subject: [PATCH 07/24] Add a protocol version number to the first message, and give the two sides an opportunity to negotiate a suitable protocol number, for future compatibility. Send the page size as a power-of-two. FossilOrigin-Name: df0623aae1154281157409f62d6d3fb3ce41829281d53bc55868ce44b3d36883 --- manifest | 12 +++---- manifest.uuid | 2 +- tool/sqlite3-rsync.c | 77 ++++++++++++++++++++++++++++++++++++++++---- 3 files changed, 77 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index a0d9e1b120..aea0407ab4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\ssha1()\sfunctions\sto\sthe\sCLI.\s\sFix\ssha1b()\ssuch\sthat\sit\sactually\sreturns\na\sBLOB. -D 2024-09-12T14:43:05.090 +C Add\sa\sprotocol\sversion\snumber\sto\sthe\sfirst\smessage,\sand\sgive\sthe\stwo\ssides\nan\sopportunity\sto\snegotiate\sa\ssuitable\sprotocol\snumber,\sfor\sfuture\ncompatibility.\s\sSend\sthe\spage\ssize\sas\sa\spower-of-two. +D 2024-09-12T15:36:34.506 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -2174,7 +2174,7 @@ F tool/speedtest8inst1.c 7ce07da76b5e745783e703a834417d725b7d45fd F tool/spellsift.tcl 52b4b04dc4333c7ab024f09d9d66ed6b6f7c6eb00b38497a09f338fa55d40618 x F tool/split-sqlite3c.tcl 5aa60643afca558bc732b1444ae81a522326f91e1dc5665b369c54f09e20de60 F tool/sqldiff.c 847fc8fcfddf5ce4797b7394cad6372f2f5dc17d8186e2ef8fb44d50fae4f44a -F tool/sqlite3-rsync.c d9f8803f79c66dbc213761a345e24ae22c7de14fd334150086519c611ff1a705 +F tool/sqlite3-rsync.c f3283380beddab3cf876be781fbd2d00308249f08d6e4438411e64f27e7b67bd F tool/sqlite3_analyzer.c.in 8da2b08f56eeac331a715036cf707cc20f879f231362be0c22efd682e2b89b4f F tool/sqltclsh.c.in 1bcc2e9da58fadf17b0bf6a50e68c1159e602ce057210b655d50bad5aaaef898 F tool/sqltclsh.tcl 862f4cf1418df5e1315b5db3b5ebe88969e2a784525af5fbf9596592f14ed848 @@ -2213,8 +2213,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 80461e0d724963aaf2646005298f1194c5f1c4c9ae41c1085d4d137ed485bd9f -R 40763ccf45039aac90ad4e181b463c0b +P fe65821a3b912f061026e6fd7174be26897010e6b474e2780350cac60faebaad +R 70c38e114e917b5cf93c46e6c1a47f24 U drh -Z 969215ecb0944de74c8a9b9fbf319407 +Z 9a7103765746c5b1023f192fb95a127f # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 86f93a2219..01d7fcfa05 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fe65821a3b912f061026e6fd7174be26897010e6b474e2780350cac60faebaad +df0623aae1154281157409f62d6d3fb3ce41829281d53bc55868ce44b3d36883 diff --git a/tool/sqlite3-rsync.c b/tool/sqlite3-rsync.c index 276b9af90f..315c7198a4 100644 --- a/tool/sqlite3-rsync.c +++ b/tool/sqlite3-rsync.c @@ -21,12 +21,19 @@ #include "sqlite3.h" static const char zUsage[] = - "sqlite3-rsync ORIGIN REPLICA\n" + "sqlite3-rsync ORIGIN REPLICA ?OPTIONS?\n" "\n" "One of ORIGIN or REPLICA is a pathname to a database on the local\n" "machine and the other is of the form \"USER@HOST:PATH\" describing\n" "a database on a remote machine. This utility makes REPLICA into a\n" "copy of ORIGIN\n" + "\n" + "OPTIONS:\n" + "\n" + " --exe PATH Name of the sqlite3-rsync program on the remote side\n" + " --help Show this help screen\n" + " --ssh PATH Name of the SSH program used to reach the remote side\n" + " -v Verbose. Multiple v's for increasing output\n" ; typedef unsigned char u8; @@ -43,6 +50,7 @@ struct SQLiteRsync { u8 eVerbose; /* Bigger for more output. 0 means none. */ u8 bCommCheck; /* True to debug the communication protocol */ u8 isRemote; /* On the remote side of a connection */ + u8 iProtocol; /* Protocol version number */ sqlite3_uint64 nOut; /* Bytes transmitted */ sqlite3_uint64 nIn; /* Bytes received */ unsigned int nPage; /* Total number of pages in the database */ @@ -51,6 +59,11 @@ struct SQLiteRsync { unsigned int nPageSent; /* Page contents sent (origin to replica) */ }; +/* The version number of the protocol. Sent in the *_BEGIN message +** to verify that both sides speak the same dialect. +*/ +#define PROTOCOL_VERSION 1 + /* Magic numbers to identify particular messages sent over the wire. */ @@ -553,6 +566,28 @@ void writeByte(SQLiteRsync *p, int c){ p->nOut++; } +/* Read a power of two encoded as a single byte. +*/ +int readPow2(SQLiteRsync *p){ + int x = readByte(p); + if( x>=32 ){ + p->nErr++; + return 0; + } + return 1<nErr++; + } + for(n=0; c>1; n++){ c /= 2; } + writeByte(p, n); +} + /* Read an array of bytes from the wire. */ void readBytes(SQLiteRsync *p, int nByte, void *pData){ @@ -838,16 +873,30 @@ static void originSide(SQLiteRsync *p){ if( p->nErr==0 ){ /* Send the ORIGIN_BEGIN message */ writeByte(p, ORIGIN_BEGIN); + writeByte(p, PROTOCOL_VERSION); + writePow2(p, szPg); writeUint32(p, nPage); - writeUint32(p, szPg); fflush(p->pOut); p->nPage = nPage; p->szPage = szPg; + p->iProtocol = PROTOCOL_VERSION; } /* Respond to message from the replica */ while( p->nErr==0 && (c = readByte(p))!=EOF && c!=REPLICA_END ){ switch( c ){ + case REPLICA_BEGIN: { + /* This message is only sent if the replica received an origin-protocol + ** that is larger than what it knows about. The replica sends back + ** a counter-proposal of an earlier protocol which the origin can + ** accept by resending a new ORIGIN_BEGIN. */ + p->iProtocol = readByte(p); + writeByte(p, ORIGIN_BEGIN); + writeByte(p, p->iProtocol); + writePow2(p, p->szPage); + writeUint32(p, p->nPage); + break; + } case REPLICA_ERROR: { readAndDisplayError(p); break; @@ -916,11 +965,13 @@ origin_end: ** Run the replica-side protocol. The protocol is passive in the sense ** that it only response to message from the origin side. ** -** ORIGIN_BEGIN nPage szPage +** ORIGIN_BEGIN idProtocol szPage nPage ** -** The origin is reporting the number of pages and the size of each -** pages. This procedure checks compatibility, and if everything is -** ok, it sends hash for all its extant pages. +** The origin is reporting the protocol version number, the size of +** each page in the origin database (sent as a single-byte power-of-2), +** and the number of pages in the origin database. +** This procedure checks compatibility, and if everything is ok, +** it starts sending hashes of pages already present back to the origin. ** ** ORIGIN_ERROR size text ** @@ -970,9 +1021,19 @@ static void replicaSide(SQLiteRsync *p){ sqlite3_stmt *pStmt = 0; closeDb(p); + p->iProtocol = readByte(p); + szOPage = readPow2(p); readUint32(p, &nOPage); - readUint32(p, &szOPage); if( p->nErr ) break; + if( p->iProtocol>PROTOCOL_VERSION ){ + /* If the protocol version on the origin side is larger, send back + ** a REPLICA_BEGIN message with the protocol version number of the + ** replica side. This gives the origin an opportunity to resend + ** a new ORIGIN_BEGIN with a reduced protocol version. */ + writeByte(p, REPLICA_BEGIN); + writeByte(p, PROTOCOL_VERSION); + break; + } p->nPage = nOPage; p->szPage = szOPage; rc = sqlite3_open(p->zReplica, &p->db); @@ -1003,6 +1064,8 @@ static void replicaSide(SQLiteRsync *p){ "replica is %d bytes", szOPage, szRPage); break; } + + pStmt = prepareStmt(p, "SELECT sha1b(data) FROM sqlite_dbpage" " WHERE pgno<=min(%d,%d)" From 0b8c8c861bd55b766fbdaa7cadb799b5193368a9 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 12 Sep 2024 15:51:56 +0000 Subject: [PATCH 08/24] Pass the names of both the origin and the replica databases to the remote side, so that if the remote is the replica, it will have access to the origin database name in case the replica name is really a directory. FossilOrigin-Name: 435c30171d3c6073b7aaf5cc11cc4813f6a2d225ae6dce1b0e478f0cd5a0b532 --- manifest | 12 ++++++------ manifest.uuid | 2 +- tool/sqlite3-rsync.c | 29 ++++++++++++++++++++++------- 3 files changed, 29 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index aea0407ab4..502f45b59c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\sprotocol\sversion\snumber\sto\sthe\sfirst\smessage,\sand\sgive\sthe\stwo\ssides\nan\sopportunity\sto\snegotiate\sa\ssuitable\sprotocol\snumber,\sfor\sfuture\ncompatibility.\s\sSend\sthe\spage\ssize\sas\sa\spower-of-two. -D 2024-09-12T15:36:34.506 +C Pass\sthe\snames\sof\sboth\sthe\sorigin\sand\sthe\sreplica\sdatabases\sto\sthe\sremote\nside,\sso\sthat\sif\sthe\sremote\sis\sthe\sreplica,\sit\swill\shave\saccess\sto\sthe\norigin\sdatabase\sname\sin\scase\sthe\sreplica\sname\sis\sreally\sa\sdirectory. +D 2024-09-12T15:51:56.339 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -2174,7 +2174,7 @@ F tool/speedtest8inst1.c 7ce07da76b5e745783e703a834417d725b7d45fd F tool/spellsift.tcl 52b4b04dc4333c7ab024f09d9d66ed6b6f7c6eb00b38497a09f338fa55d40618 x F tool/split-sqlite3c.tcl 5aa60643afca558bc732b1444ae81a522326f91e1dc5665b369c54f09e20de60 F tool/sqldiff.c 847fc8fcfddf5ce4797b7394cad6372f2f5dc17d8186e2ef8fb44d50fae4f44a -F tool/sqlite3-rsync.c f3283380beddab3cf876be781fbd2d00308249f08d6e4438411e64f27e7b67bd +F tool/sqlite3-rsync.c 7b7d24177b27f42dfcd7e962be4a526e775b073f2044cd1f7d3414581810baea F tool/sqlite3_analyzer.c.in 8da2b08f56eeac331a715036cf707cc20f879f231362be0c22efd682e2b89b4f F tool/sqltclsh.c.in 1bcc2e9da58fadf17b0bf6a50e68c1159e602ce057210b655d50bad5aaaef898 F tool/sqltclsh.tcl 862f4cf1418df5e1315b5db3b5ebe88969e2a784525af5fbf9596592f14ed848 @@ -2213,8 +2213,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P fe65821a3b912f061026e6fd7174be26897010e6b474e2780350cac60faebaad -R 70c38e114e917b5cf93c46e6c1a47f24 +P df0623aae1154281157409f62d6d3fb3ce41829281d53bc55868ce44b3d36883 +R 137e53956a972f25c62e4e4b74ab6074 U drh -Z 9a7103765746c5b1023f192fb95a127f +Z 061042360cfb45012e27693425840440 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 01d7fcfa05..c4e68cc607 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -df0623aae1154281157409f62d6d3fb3ce41829281d53bc55868ce44b3d36883 +435c30171d3c6073b7aaf5cc11cc4813f6a2d225ae6dce1b0e478f0cd5a0b532 diff --git a/tool/sqlite3-rsync.c b/tool/sqlite3-rsync.c index 315c7198a4..60777c40c6 100644 --- a/tool/sqlite3-rsync.c +++ b/tool/sqlite3-rsync.c @@ -516,6 +516,21 @@ static void echoOneLine(SQLiteRsync *p){ } } +/* +** Return the tail of a file pathname. The tail is the last component +** of the path. For example, the tail of "/a/b/c.d" is "c.d". +*/ +const char *file_tail(const char *z){ + const char *zTail = z; + if( !zTail ) return 0; + while( z[0] ){ + if( z[0]=='/' ) zTail = &z[1]; + z++; + } + return zTail; +} + + /* Read a single big-endian 32-bit unsigned integer from the input ** stream. Return 0 on success and 1 if there are any errors. */ @@ -1064,7 +1079,6 @@ static void replicaSide(SQLiteRsync *p){ "replica is %d bytes", szOPage, szRPage); break; } - pStmt = prepareStmt(p, "SELECT sha1b(data) FROM sqlite_dbpage" @@ -1257,12 +1271,12 @@ int main(int argc, char **argv){ fprintf(stderr, "missing ORIGIN database filename\n"); return 1; } - if( isOrigin && isReplica ){ - fprintf(stderr, "bad option combination\n"); + if( ctx.zReplica==0 ){ + fprintf(stderr, "missing REPLICA database filename\n"); return 1; } - if( (isOrigin || isReplica) && ctx.zReplica!=0 ){ - fprintf(stderr, "Unknown argument: \"%s\"\n", ctx.zReplica); + if( isOrigin && isReplica ){ + fprintf(stderr, "bad option combination\n"); return 1; } if( isOrigin ){ @@ -1273,8 +1287,6 @@ int main(int argc, char **argv){ return 0; } if( isReplica ){ - ctx.zReplica = ctx.zOrigin; - ctx.zOrigin = 0; ctx.pIn = stdin; ctx.pOut = stdout; ctx.isRemote = 1; @@ -1306,6 +1318,7 @@ int main(int argc, char **argv){ if( ctx.eVerbose==0 ) ctx.eVerbose = 1; } append_escaped_arg(pStr, zDiv, 1); + append_escaped_arg(pStr, file_tail(ctx.zReplica), 1); zCmd = sqlite3_str_finish(pStr); if( ctx.eVerbose ) printf("%s\n", zCmd); if( popen2(zCmd, &ctx.pIn, &ctx.pOut, &childPid, 0) ){ @@ -1326,6 +1339,7 @@ int main(int argc, char **argv){ append_escaped_arg(pStr, "--commcheck", 0); if( ctx.eVerbose==0 ) ctx.eVerbose = 1; } + append_escaped_arg(pStr, file_tail(ctx.zOrigin), 1); append_escaped_arg(pStr, zDiv, 1); zCmd = sqlite3_str_finish(pStr); if( ctx.eVerbose ) printf("%s\n", zCmd); @@ -1342,6 +1356,7 @@ int main(int argc, char **argv){ if( ctx.bCommCheck ){ append_escaped_arg(pStr, "--commcheck", 0); } + append_escaped_arg(pStr, ctx.zOrigin, 1); append_escaped_arg(pStr, ctx.zReplica, 1); zCmd = sqlite3_str_finish(pStr); if( ctx.eVerbose ) printf("%s\n", zCmd); From 6df1de1c73f6ad93e8d3c1eb8480f0cf27f75c9c Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 12 Sep 2024 16:54:34 +0000 Subject: [PATCH 09/24] Improved --commcheck. Add the infoMsg() function which is useful for debugging. FossilOrigin-Name: b979d02ffd1370d8840328bce06c76c224f0fc1fb54b47d6c904547580a820a1 --- manifest | 12 ++-- manifest.uuid | 2 +- tool/sqlite3-rsync.c | 149 +++++++++++++++++++++++++------------------ 3 files changed, 93 insertions(+), 70 deletions(-) diff --git a/manifest b/manifest index 502f45b59c..58cda58374 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Pass\sthe\snames\sof\sboth\sthe\sorigin\sand\sthe\sreplica\sdatabases\sto\sthe\sremote\nside,\sso\sthat\sif\sthe\sremote\sis\sthe\sreplica,\sit\swill\shave\saccess\sto\sthe\norigin\sdatabase\sname\sin\scase\sthe\sreplica\sname\sis\sreally\sa\sdirectory. -D 2024-09-12T15:51:56.339 +C Improved\s--commcheck.\s\sAdd\sthe\sinfoMsg()\sfunction\swhich\sis\suseful\sfor\ndebugging. +D 2024-09-12T16:54:34.620 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -2174,7 +2174,7 @@ F tool/speedtest8inst1.c 7ce07da76b5e745783e703a834417d725b7d45fd F tool/spellsift.tcl 52b4b04dc4333c7ab024f09d9d66ed6b6f7c6eb00b38497a09f338fa55d40618 x F tool/split-sqlite3c.tcl 5aa60643afca558bc732b1444ae81a522326f91e1dc5665b369c54f09e20de60 F tool/sqldiff.c 847fc8fcfddf5ce4797b7394cad6372f2f5dc17d8186e2ef8fb44d50fae4f44a -F tool/sqlite3-rsync.c 7b7d24177b27f42dfcd7e962be4a526e775b073f2044cd1f7d3414581810baea +F tool/sqlite3-rsync.c 72632a237c91c97a7320cf064b8f2d6662721b61fc5f123e40e6d272207b4a96 F tool/sqlite3_analyzer.c.in 8da2b08f56eeac331a715036cf707cc20f879f231362be0c22efd682e2b89b4f F tool/sqltclsh.c.in 1bcc2e9da58fadf17b0bf6a50e68c1159e602ce057210b655d50bad5aaaef898 F tool/sqltclsh.tcl 862f4cf1418df5e1315b5db3b5ebe88969e2a784525af5fbf9596592f14ed848 @@ -2213,8 +2213,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P df0623aae1154281157409f62d6d3fb3ce41829281d53bc55868ce44b3d36883 -R 137e53956a972f25c62e4e4b74ab6074 +P 435c30171d3c6073b7aaf5cc11cc4813f6a2d225ae6dce1b0e478f0cd5a0b532 +R aafd9793078a7056ec93ac4f3aad0430 U drh -Z 061042360cfb45012e27693425840440 +Z 9f40e243e2ff00f83272b62111b602fb # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index c4e68cc607..869057fb88 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -435c30171d3c6073b7aaf5cc11cc4813f6a2d225ae6dce1b0e478f0cd5a0b532 +b979d02ffd1370d8840328bce06c76c224f0fc1fb54b47d6c904547580a820a1 diff --git a/tool/sqlite3-rsync.c b/tool/sqlite3-rsync.c index 60777c40c6..4c0208fbcb 100644 --- a/tool/sqlite3-rsync.c +++ b/tool/sqlite3-rsync.c @@ -50,6 +50,7 @@ struct SQLiteRsync { u8 eVerbose; /* Bigger for more output. 0 means none. */ u8 bCommCheck; /* True to debug the communication protocol */ u8 isRemote; /* On the remote side of a connection */ + u8 isReplica; /* True if running on the replica side */ u8 iProtocol; /* Protocol version number */ sqlite3_uint64 nOut; /* Bytes transmitted */ sqlite3_uint64 nIn; /* Bytes received */ @@ -72,12 +73,14 @@ struct SQLiteRsync { #define ORIGIN_ERROR 0x43 /* Error message from the remote */ #define ORIGIN_PAGE 0x44 /* New page data */ #define ORIGIN_TXN 0x45 /* Transaction commit */ +#define ORIGIN_MSG 0x46 /* Informational message */ #define REPLICA_BEGIN 0x61 /* Welcome message */ #define REPLICA_ERROR 0x62 /* Error. Report and quit. */ #define REPLICA_END 0x63 /* Replica wants to stop */ #define REPLICA_HASH 0x64 /* One or more pages hashes to report */ #define REPLICA_READY 0x65 /* Read to receive page content */ +#define REPLICA_MSG 0x66 /* Informational message */ /**************************************************************************** @@ -503,19 +506,6 @@ int append_escaped_arg(sqlite3_str *pStr, const char *zIn, int isFilename){ ** End of the append_escaped_arg() routine, adapted from the Fossil ** *****************************************************************************/ - -/* For Debugging, specifically for --commcheck: -** -** Read a single line of text from p->pIn. Write this to standard -** output if and only if p->eVerbose>0. -*/ -static void echoOneLine(SQLiteRsync *p){ - char zLine[1000]; - if( fgets(zLine, sizeof(zLine), p->pIn) ){ - if( p->eVerbose ) printf("GOT: %s", zLine); - } -} - /* ** Return the tail of a file pathname. The tail is the last component ** of the path. For example, the tail of "/a/b/c.d" is "c.d". @@ -625,7 +615,7 @@ void writeBytes(SQLiteRsync *p, int nByte, const void *pData){ /* Report an error. ** -** If this happens on the remote side, we send back a REMOTE_ERROR +** If this happens on the remote side, we send back a *_ERROR ** message. On the local side, the error message goes to stderr. */ static void reportError(SQLiteRsync *p, const char *zFormat, ...){ @@ -637,7 +627,7 @@ static void reportError(SQLiteRsync *p, const char *zFormat, ...){ va_end(ap); nMsg = zMsg ? (unsigned int)strlen(zMsg) : 0; if( p->isRemote ){ - if( p->zReplica ){ + if( p->isReplica ){ putc(REPLICA_ERROR, p->pOut); }else{ putc(ORIGIN_ERROR, p->pOut); @@ -652,12 +642,47 @@ static void reportError(SQLiteRsync *p, const char *zFormat, ...){ p->nErr++; } +/* Send an informational message. +** +** If this happens on the remote side, we send back a *_MSG +** message. On the local side, the message goes to stdout. +*/ +static void infoMsg(SQLiteRsync *p, const char *zFormat, ...){ + va_list ap; + char *zMsg; + unsigned int nMsg; + va_start(ap, zFormat); + zMsg = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + nMsg = zMsg ? (unsigned int)strlen(zMsg) : 0; + if( p->isRemote ){ + if( p->isReplica ){ + putc(REPLICA_MSG, p->pOut); + }else{ + putc(ORIGIN_MSG, p->pOut); + } + writeUint32(p, nMsg); + writeBytes(p, nMsg, zMsg); + fflush(p->pOut); + }else{ + printf("%s\n", zMsg); + } + sqlite3_free(zMsg); +} + /* Receive and report an error message coming from the other side. */ -static void readAndDisplayError(SQLiteRsync *p){ +static void readAndDisplayMessage(SQLiteRsync *p, int c){ unsigned int n = 0; char *zMsg; - (void)readUint32(p, &n); + const char *zPrefix; + if( c==ORIGIN_ERROR || c==REPLICA_ERROR ){ + zPrefix = "ERROR: "; + p->nErr++; + }else{ + zPrefix = ""; + } + readUint32(p, &n); if( n==0 ){ fprintf(stderr,"ERROR: unknown (possibly out-of-memory)\n"); }else{ @@ -668,10 +693,9 @@ static void readAndDisplayError(SQLiteRsync *p){ } memset(zMsg, 0, n+1); readBytes(p, n, zMsg); - fprintf(stderr,"ERROR: %s\n", zMsg); + fprintf(stderr,"%s%s\n", zPrefix, zMsg); sqlite3_free(zMsg); } - p->nErr++; } /* Construct a new prepared statement. Report an error and return NULL @@ -858,45 +882,43 @@ static void originSide(SQLiteRsync *p){ sqlite3_stmt *pCkHash = 0; char buf[200]; + p->isReplica = 0; if( p->bCommCheck ){ - fprintf(p->pOut, "sqlite3-rsync origin-begin %s\n", p->zOrigin); + infoMsg(p, "origin zOrigin=%Q zReplica=%Q isRemote=%d protocol=%d", + p->zOrigin, p->zReplica, p->isRemote, PROTOCOL_VERSION); + writeByte(p, ORIGIN_END); fflush(p->pOut); - echoOneLine(p); - fprintf(p->pOut, "origin-end\n"); - fflush(p->pOut); - echoOneLine(p); - return; + }else{ + /* Open the ORIGIN database. */ + rc = sqlite3_open_v2(p->zOrigin, &p->db, SQLITE_OPEN_READWRITE, 0); + if( rc ){ + reportError(p, "unable to open origin database file \"%s\": %s", + sqlite3_errmsg(p->db)); + closeDb(p); + return; + } + sqlite3_sha_init(p->db, 0, 0); + runSql(p, "BEGIN"); + runSqlReturnText(p, buf, "PRAGMA journal_mode"); + if( sqlite3_stricmp(buf,"wal")!=0 ){ + reportError(p, "Origin database is not in WAL mode"); + } + runSqlReturnUInt(p, &nPage, "PRAGMA page_count"); + runSqlReturnUInt(p, &szPg, "PRAGMA page_size"); + + if( p->nErr==0 ){ + /* Send the ORIGIN_BEGIN message */ + writeByte(p, ORIGIN_BEGIN); + writeByte(p, PROTOCOL_VERSION); + writePow2(p, szPg); + writeUint32(p, nPage); + fflush(p->pOut); + p->nPage = nPage; + p->szPage = szPg; + p->iProtocol = PROTOCOL_VERSION; + } } - - /* Open the ORIGIN database. */ - rc = sqlite3_open_v2(p->zOrigin, &p->db, SQLITE_OPEN_READWRITE, 0); - if( rc ){ - reportError(p, "unable to open origin database file \"%s\": %s", - sqlite3_errmsg(p->db)); - closeDb(p); - return; - } - sqlite3_sha_init(p->db, 0, 0); - runSql(p, "BEGIN"); - runSqlReturnText(p, buf, "PRAGMA journal_mode"); - if( sqlite3_stricmp(buf,"wal")!=0 ){ - reportError(p, "Origin database is not in WAL mode"); - } - runSqlReturnUInt(p, &nPage, "PRAGMA page_count"); - runSqlReturnUInt(p, &szPg, "PRAGMA page_size"); - - if( p->nErr==0 ){ - /* Send the ORIGIN_BEGIN message */ - writeByte(p, ORIGIN_BEGIN); - writeByte(p, PROTOCOL_VERSION); - writePow2(p, szPg); - writeUint32(p, nPage); - fflush(p->pOut); - p->nPage = nPage; - p->szPage = szPg; - p->iProtocol = PROTOCOL_VERSION; - } - + /* Respond to message from the replica */ while( p->nErr==0 && (c = readByte(p))!=EOF && c!=REPLICA_END ){ switch( c ){ @@ -912,8 +934,9 @@ static void originSide(SQLiteRsync *p){ writeUint32(p, p->nPage); break; } + case REPLICA_MSG: case REPLICA_ERROR: { - readAndDisplayError(p); + readAndDisplayMessage(p, c); break; } case REPLICA_HASH: { @@ -1010,14 +1033,13 @@ static void replicaSide(SQLiteRsync *p){ sqlite3_stmt *pIns = 0; unsigned int szOPage = 0; char buf[65536]; + + p->isReplica = 1; if( p->bCommCheck ){ - echoOneLine(p); - fprintf(p->pOut, "replica-begin %s\n", p->zReplica); + infoMsg(p, "replica zOrigin=%Q zReplica=%Q isRemote=%d protocol=%d", + p->zOrigin, p->zReplica, p->isRemote, PROTOCOL_VERSION); + writeByte(p, REPLICA_END); fflush(p->pOut); - echoOneLine(p); - fprintf(p->pOut, "replica-end\n"); - fflush(p->pOut); - return; } /* Respond to message from the origin. The origin will initiate the @@ -1025,8 +1047,9 @@ static void replicaSide(SQLiteRsync *p){ */ while( p->nErr==0 && (c = readByte(p))!=EOF && c!=ORIGIN_END ){ switch( c ){ + case ORIGIN_MSG: case ORIGIN_ERROR: { - readAndDisplayError(p); + readAndDisplayMessage(p, c); break; } case ORIGIN_BEGIN: { From 40b831851e21f3808da0eeed835d8f36be87a4c6 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 12 Sep 2024 17:06:02 +0000 Subject: [PATCH 10/24] Replica must be in writable_schema mode. FossilOrigin-Name: e385525793c7d74ce8ee139c9d6cfc1248834754598f3fd45b22b9426ff106ee --- manifest | 12 ++++++------ manifest.uuid | 2 +- tool/sqlite3-rsync.c | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 58cda58374..fabf4f8ef1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\s--commcheck.\s\sAdd\sthe\sinfoMsg()\sfunction\swhich\sis\suseful\sfor\ndebugging. -D 2024-09-12T16:54:34.620 +C Replica\smust\sbe\sin\swritable_schema\smode. +D 2024-09-12T17:06:02.316 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -2174,7 +2174,7 @@ F tool/speedtest8inst1.c 7ce07da76b5e745783e703a834417d725b7d45fd F tool/spellsift.tcl 52b4b04dc4333c7ab024f09d9d66ed6b6f7c6eb00b38497a09f338fa55d40618 x F tool/split-sqlite3c.tcl 5aa60643afca558bc732b1444ae81a522326f91e1dc5665b369c54f09e20de60 F tool/sqldiff.c 847fc8fcfddf5ce4797b7394cad6372f2f5dc17d8186e2ef8fb44d50fae4f44a -F tool/sqlite3-rsync.c 72632a237c91c97a7320cf064b8f2d6662721b61fc5f123e40e6d272207b4a96 +F tool/sqlite3-rsync.c a183370f4b837ac9e6dc2d347217b1df3b47c7300151a2e8207cea228df88f39 F tool/sqlite3_analyzer.c.in 8da2b08f56eeac331a715036cf707cc20f879f231362be0c22efd682e2b89b4f F tool/sqltclsh.c.in 1bcc2e9da58fadf17b0bf6a50e68c1159e602ce057210b655d50bad5aaaef898 F tool/sqltclsh.tcl 862f4cf1418df5e1315b5db3b5ebe88969e2a784525af5fbf9596592f14ed848 @@ -2213,8 +2213,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 435c30171d3c6073b7aaf5cc11cc4813f6a2d225ae6dce1b0e478f0cd5a0b532 -R aafd9793078a7056ec93ac4f3aad0430 +P b979d02ffd1370d8840328bce06c76c224f0fc1fb54b47d6c904547580a820a1 +R 726e2548d9f75b6be59271aa0b3ef896 U drh -Z 9f40e243e2ff00f83272b62111b602fb +Z f4621da70e1b6c4b3102731ca73be41a # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 869057fb88..ba2db44c52 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b979d02ffd1370d8840328bce06c76c224f0fc1fb54b47d6c904547580a820a1 +e385525793c7d74ce8ee139c9d6cfc1248834754598f3fd45b22b9426ff106ee diff --git a/tool/sqlite3-rsync.c b/tool/sqlite3-rsync.c index 4c0208fbcb..e7d2e49551 100644 --- a/tool/sqlite3-rsync.c +++ b/tool/sqlite3-rsync.c @@ -1116,6 +1116,7 @@ static void replicaSide(SQLiteRsync *p){ sqlite3_finalize(pStmt); writeByte(p, REPLICA_READY); fflush(p->pOut); + runSql(p, "PRAGMA writable_schema=ON"); break; } case ORIGIN_TXN: { From 6677cfeda7f9fcecc4b0f638bd7e0a0058ec14a8 Mon Sep 17 00:00:00 2001 From: stephan Date: Thu, 12 Sep 2024 18:15:28 +0000 Subject: [PATCH 11/24] Port some of the shell's CLI flag-handling code into sqlite3-rsync to make the arg handling more robust. FossilOrigin-Name: 53fb9b11807ff7accd8cd41f9cb6516d2503f161ea976940437a1d3aae868665 --- manifest | 14 +++++++------- manifest.uuid | 2 +- tool/sqlite3-rsync.c | 25 ++++++++++++++++++++----- 3 files changed, 28 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index fabf4f8ef1..9ee27c15a8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Replica\smust\sbe\sin\swritable_schema\smode. -D 2024-09-12T17:06:02.316 +C Port\ssome\sof\sthe\sshell's\sCLI\sflag-handling\scode\sinto\ssqlite3-rsync\sto\smake\sthe\sarg\shandling\smore\srobust. +D 2024-09-12T18:15:28.846 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -2174,7 +2174,7 @@ F tool/speedtest8inst1.c 7ce07da76b5e745783e703a834417d725b7d45fd F tool/spellsift.tcl 52b4b04dc4333c7ab024f09d9d66ed6b6f7c6eb00b38497a09f338fa55d40618 x F tool/split-sqlite3c.tcl 5aa60643afca558bc732b1444ae81a522326f91e1dc5665b369c54f09e20de60 F tool/sqldiff.c 847fc8fcfddf5ce4797b7394cad6372f2f5dc17d8186e2ef8fb44d50fae4f44a -F tool/sqlite3-rsync.c a183370f4b837ac9e6dc2d347217b1df3b47c7300151a2e8207cea228df88f39 +F tool/sqlite3-rsync.c 20783cd979df8c32d33eab3c38c44a79343be63a73b6cbd1ee56a0a14d73b3fe F tool/sqlite3_analyzer.c.in 8da2b08f56eeac331a715036cf707cc20f879f231362be0c22efd682e2b89b4f F tool/sqltclsh.c.in 1bcc2e9da58fadf17b0bf6a50e68c1159e602ce057210b655d50bad5aaaef898 F tool/sqltclsh.tcl 862f4cf1418df5e1315b5db3b5ebe88969e2a784525af5fbf9596592f14ed848 @@ -2213,8 +2213,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b979d02ffd1370d8840328bce06c76c224f0fc1fb54b47d6c904547580a820a1 -R 726e2548d9f75b6be59271aa0b3ef896 -U drh -Z f4621da70e1b6c4b3102731ca73be41a +P e385525793c7d74ce8ee139c9d6cfc1248834754598f3fd45b22b9426ff106ee +R 2634868962b675f42ebfd90b6ccf2615 +U stephan +Z ca092fee4c8dab92860037cbb2c82076 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index ba2db44c52..67577b0f49 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e385525793c7d74ce8ee139c9d6cfc1248834754598f3fd45b22b9426ff106ee +53fb9b11807ff7accd8cd41f9cb6516d2503f161ea976940437a1d3aae868665 diff --git a/tool/sqlite3-rsync.c b/tool/sqlite3-rsync.c index e7d2e49551..350ed84afd 100644 --- a/tool/sqlite3-rsync.c +++ b/tool/sqlite3-rsync.c @@ -1190,6 +1190,20 @@ static int numVs(const char *z){ return 0; } +/* +** Get the argument to an --option. Throw an error and die if no argument +** is available. +*/ +static const char *cmdline_option_value(int argc, const char * const*argv, + int i){ + if( i==argc ){ + fprintf(stderr,"%s: Error: missing argument to %s\n", + argv[0], argv[argc-1]); + exit(1); + } + return argv[i]; +} + /* ** Parse command-line arguments. Dispatch subroutines to do the @@ -1207,7 +1221,7 @@ static int numVs(const char *z){ ** ** The user types (1) or (2). SSH launches (3) or (4). ** -** If (1) is seen then popen2 is used launch (4) on the remote and +** If (1) is seen then popen2 is used launch (4) on the remote and ** originSide() is called locally. ** ** If (2) is seen, then popen2() is used to launch (3) on the remote @@ -1217,7 +1231,7 @@ static int numVs(const char *z){ ** q** If (4) is seen, call replicaSide() on stdin and stdout. */ -int main(int argc, char **argv){ +int main(int argc, char const * const *argv){ int isOrigin = 0; int isReplica = 0; int i; @@ -1230,6 +1244,7 @@ int main(int argc, char **argv){ const char *zExe = "sqlite3-rsync"; char *zCmd = 0; +#define cli_opt_val cmdline_option_value(argc, argv, ++i) memset(&ctx, 0, sizeof(ctx)); for(i=1; i Date: Thu, 12 Sep 2024 21:03:11 +0000 Subject: [PATCH 12/24] Add RSYNC_OPT to the sqlite3-rsync build flags. FossilOrigin-Name: b2a3497e5525dd33faf70961107a0529f476735fef756953c66e105747271c6d --- Makefile.in | 3 ++- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/Makefile.in b/Makefile.in index 10b5db15a5..ebe8f3c59a 100644 --- a/Makefile.in +++ b/Makefile.in @@ -707,12 +707,13 @@ RSYNC_SRC = \ RSYNC_OPT = \ -DSQLITE_ENABLE_DBPAGE_VTAB \ + -USQLITE_THREADSAFE \ -DSQLITE_THREADSAFE=0 \ -DSQLITE_OMIT_LOAD_EXTENSION \ -DSQLITE_OMIT_DEPRECATED sqlite3-rsync$(TEXE): $(RSYNC_SRC) - $(TCC) -o $@ $(RSYNC_SRC) $(TLIBS) + $(TCC) -o $@ $(RSYNC_OPT) $(RSYNC_SRC) $(TLIBS) scrub$(TEXE): $(TOP)/ext/misc/scrub.c sqlite3.lo $(LTLINK) -o $@ -I. -DSCRUB_STANDALONE \ diff --git a/manifest b/manifest index 9ee27c15a8..adaa2851cc 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C Port\ssome\sof\sthe\sshell's\sCLI\sflag-handling\scode\sinto\ssqlite3-rsync\sto\smake\sthe\sarg\shandling\smore\srobust. -D 2024-09-12T18:15:28.846 +C Add\sRSYNC_OPT\sto\sthe\ssqlite3-rsync\sbuild\sflags. +D 2024-09-12T21:03:11.083 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 -F Makefile.in 4ac84300552e9ab8245288dec9941b6b820b48e15cbcd7c87db2ce45998c62f8 +F Makefile.in 086a7ca4e5223da3990e1a6f0cdc082af74c3ca0e175b0af955a44115832ba85 F Makefile.linux-gcc f3842a0b1efbfbb74ac0ef60e56b301836d05b4d867d014f714fa750048f1ab6 F Makefile.msc f25c476f22a25f23af8dc7260e9f95896250e7fceb621388ca8ae4f096bd3fda F README.md c3c0f19532ce28f6297a71870f3c7b424729f0e6d9ab889616d3587dd2332159 @@ -2213,8 +2213,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e385525793c7d74ce8ee139c9d6cfc1248834754598f3fd45b22b9426ff106ee -R 2634868962b675f42ebfd90b6ccf2615 +P 53fb9b11807ff7accd8cd41f9cb6516d2503f161ea976940437a1d3aae868665 +R 81dd8418b79d70f2c41a3d45559c0c93 U stephan -Z ca092fee4c8dab92860037cbb2c82076 +Z c1b8ca8f0b0849e8a689568eca20e4f4 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 67577b0f49..929ef187b8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -53fb9b11807ff7accd8cd41f9cb6516d2503f161ea976940437a1d3aae868665 +b2a3497e5525dd33faf70961107a0529f476735fef756953c66e105747271c6d From 5c6dc45a10478cc16607170314efb48a361cc5ef Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 12 Sep 2024 23:30:29 +0000 Subject: [PATCH 13/24] Better initialization of new database files in sqlite3-rsync. FossilOrigin-Name: 75d5a8eb3d4ece06900109ad4022ba2a3e82de2f0acb012e3a02bfb4326bfa6d --- manifest | 14 +++++++------- manifest.uuid | 2 +- tool/sqlite3-rsync.c | 1 + 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index adaa2851cc..4ae4d1787e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sRSYNC_OPT\sto\sthe\ssqlite3-rsync\sbuild\sflags. -D 2024-09-12T21:03:11.083 +C Better\sinitialization\sof\snew\sdatabase\sfiles\sin\ssqlite3-rsync. +D 2024-09-12T23:30:29.682 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -2174,7 +2174,7 @@ F tool/speedtest8inst1.c 7ce07da76b5e745783e703a834417d725b7d45fd F tool/spellsift.tcl 52b4b04dc4333c7ab024f09d9d66ed6b6f7c6eb00b38497a09f338fa55d40618 x F tool/split-sqlite3c.tcl 5aa60643afca558bc732b1444ae81a522326f91e1dc5665b369c54f09e20de60 F tool/sqldiff.c 847fc8fcfddf5ce4797b7394cad6372f2f5dc17d8186e2ef8fb44d50fae4f44a -F tool/sqlite3-rsync.c 20783cd979df8c32d33eab3c38c44a79343be63a73b6cbd1ee56a0a14d73b3fe +F tool/sqlite3-rsync.c 4a2bfb77e633b6a6edbd88d7bfc3e6e03a0c8662d0c3ae815e6c1ff7eb7df94e F tool/sqlite3_analyzer.c.in 8da2b08f56eeac331a715036cf707cc20f879f231362be0c22efd682e2b89b4f F tool/sqltclsh.c.in 1bcc2e9da58fadf17b0bf6a50e68c1159e602ce057210b655d50bad5aaaef898 F tool/sqltclsh.tcl 862f4cf1418df5e1315b5db3b5ebe88969e2a784525af5fbf9596592f14ed848 @@ -2213,8 +2213,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 53fb9b11807ff7accd8cd41f9cb6516d2503f161ea976940437a1d3aae868665 -R 81dd8418b79d70f2c41a3d45559c0c93 -U stephan -Z c1b8ca8f0b0849e8a689568eca20e4f4 +P b2a3497e5525dd33faf70961107a0529f476735fef756953c66e105747271c6d +R 3def4dc608d40602e340858854fc17d3 +U drh +Z 3eea4133291b7d423d6d8c3c9c75fcb1 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 929ef187b8..f5d4b54d7d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b2a3497e5525dd33faf70961107a0529f476735fef756953c66e105747271c6d +75d5a8eb3d4ece06900109ad4022ba2a3e82de2f0acb012e3a02bfb4326bfa6d diff --git a/tool/sqlite3-rsync.c b/tool/sqlite3-rsync.c index 350ed84afd..3c7ae0e70a 100644 --- a/tool/sqlite3-rsync.c +++ b/tool/sqlite3-rsync.c @@ -1088,6 +1088,7 @@ static void replicaSide(SQLiteRsync *p){ if( nRPage==0 ){ runSql(p, "PRAGMA page_size=%u", szOPage); runSql(p, "PRAGMA journal_mode=WAL"); + runSql(p, "SELECT * FROM sqlite_schema"); } runSql(p, "BEGIN IMMEDIATE"); runSqlReturnText(p, buf, "PRAGMA journal_mode"); From d0d3182fa8755f8d8df80168db5432219ea092dd Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 13 Sep 2024 12:28:11 +0000 Subject: [PATCH 14/24] Change the hash algorithm from SHA1 to a 6-round KeccakF1600 with a rate of 160. This uses about 1/3rd fewer CPU cycles. FossilOrigin-Name: 96c7f47a8f59e5078bd296979421c1b57fbcb7be261f8a7a0b1d22a4b5914db0 --- Makefile.in | 1 - Makefile.msc | 1 - main.mk | 1 - manifest | 18 +- manifest.uuid | 2 +- tool/sqlite3-rsync.c | 500 ++++++++++++++++++++++++++++++++++++++++++- 6 files changed, 506 insertions(+), 17 deletions(-) diff --git a/Makefile.in b/Makefile.in index ebe8f3c59a..c9e27a4cc1 100644 --- a/Makefile.in +++ b/Makefile.in @@ -702,7 +702,6 @@ dbhash$(TEXE): $(TOP)/tool/dbhash.c sqlite3.lo sqlite3.h RSYNC_SRC = \ $(TOP)/tool/sqlite3-rsync.c \ - $(TOP)/ext/misc/sha1.c \ sqlite3.c RSYNC_OPT = \ diff --git a/Makefile.msc b/Makefile.msc index e19156c612..295e07b075 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1869,7 +1869,6 @@ dbhash.exe: $(TOP)\tool\dbhash.c $(SQLITE3C) $(SQLITE3H) RSYNC_SRC = \ $(TOP)\tool\sqlite3-rsync.c \ - $(TOP)\ext\misc\sha1.c \ $(SQLITE3C) RSYNC_OPT = \ diff --git a/main.mk b/main.mk index 1742b90707..ba1ef2484a 100644 --- a/main.mk +++ b/main.mk @@ -570,7 +570,6 @@ dbhash$(EXE): $(TOP)/tool/dbhash.c sqlite3.c sqlite3.h RSYNC_SRC = \ $(TOP)/tool/sqlite3-rsync.c \ - $(TOP)/ext/misc/sha1.c \ sqlite3.c RSYNC_OPT = \ diff --git a/manifest b/manifest index 4ae4d1787e..0356a80fe4 100644 --- a/manifest +++ b/manifest @@ -1,11 +1,11 @@ -C Better\sinitialization\sof\snew\sdatabase\sfiles\sin\ssqlite3-rsync. -D 2024-09-12T23:30:29.682 +C Change\sthe\shash\salgorithm\sfrom\sSHA1\sto\sa\s6-round\sKeccakF1600\swith\sa\srate\nof\s160.\s\sThis\suses\sabout\s1/3rd\sfewer\sCPU\scycles. +D 2024-09-13T12:28:11.247 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 -F Makefile.in 086a7ca4e5223da3990e1a6f0cdc082af74c3ca0e175b0af955a44115832ba85 +F Makefile.in 31368ad3e1800bb5f311adede543ee456ca7d2595403c7f131797ae65a7d415c F Makefile.linux-gcc f3842a0b1efbfbb74ac0ef60e56b301836d05b4d867d014f714fa750048f1ab6 -F Makefile.msc f25c476f22a25f23af8dc7260e9f95896250e7fceb621388ca8ae4f096bd3fda +F Makefile.msc 62ace0005c53b52f189c20c1d6d8fa4dbd2a37c90d9c1362b60f4fb4c841fa15 F README.md c3c0f19532ce28f6297a71870f3c7b424729f0e6d9ab889616d3587dd2332159 F VERSION 0db40f92c04378404eb45bff93e9e42c148c7e54fd3da99469ed21e22411f5a6 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -687,7 +687,7 @@ F ext/wasm/wasmfs.make 8a4955882aaa0783b3f60a9484a1f0f3d8b6f775c0fcd17c082f31966 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 5ade0bc977aa135e79e3faaea894d5671b26107cc91e70783aa7dc83f22f3ba0 -F main.mk 91d391457000574faaaa34a833564b98f4a3dae6c4a5681e8e6b5ff1cf5c4b57 +F main.mk 8b9c0252aef57b5b2a10f34b8b46e89f9ed06bdccef1df98673a12f34e9b3e79 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 F mptest/crash01.test 61e61469e257df0850df4293d7d4d6c2af301421 @@ -2174,7 +2174,7 @@ F tool/speedtest8inst1.c 7ce07da76b5e745783e703a834417d725b7d45fd F tool/spellsift.tcl 52b4b04dc4333c7ab024f09d9d66ed6b6f7c6eb00b38497a09f338fa55d40618 x F tool/split-sqlite3c.tcl 5aa60643afca558bc732b1444ae81a522326f91e1dc5665b369c54f09e20de60 F tool/sqldiff.c 847fc8fcfddf5ce4797b7394cad6372f2f5dc17d8186e2ef8fb44d50fae4f44a -F tool/sqlite3-rsync.c 4a2bfb77e633b6a6edbd88d7bfc3e6e03a0c8662d0c3ae815e6c1ff7eb7df94e +F tool/sqlite3-rsync.c 10fa0e40cc637195dc86c84390f1d7bae3be0830a0e50990df9289234097889c F tool/sqlite3_analyzer.c.in 8da2b08f56eeac331a715036cf707cc20f879f231362be0c22efd682e2b89b4f F tool/sqltclsh.c.in 1bcc2e9da58fadf17b0bf6a50e68c1159e602ce057210b655d50bad5aaaef898 F tool/sqltclsh.tcl 862f4cf1418df5e1315b5db3b5ebe88969e2a784525af5fbf9596592f14ed848 @@ -2213,8 +2213,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b2a3497e5525dd33faf70961107a0529f476735fef756953c66e105747271c6d -R 3def4dc608d40602e340858854fc17d3 +P 75d5a8eb3d4ece06900109ad4022ba2a3e82de2f0acb012e3a02bfb4326bfa6d +R 6d0c1116cd9e3a065842817267acb222 U drh -Z 3eea4133291b7d423d6d8c3c9c75fcb1 +Z 4ed0549ad93aad61732b14753f0892c6 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index f5d4b54d7d..d64d091567 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -75d5a8eb3d4ece06900109ad4022ba2a3e82de2f0acb012e3a02bfb4326bfa6d +96c7f47a8f59e5078bd296979421c1b57fbcb7be261f8a7a0b1d22a4b5914db0 diff --git a/tool/sqlite3-rsync.c b/tool/sqlite3-rsync.c index 3c7ae0e70a..4193afec89 100644 --- a/tool/sqlite3-rsync.c +++ b/tool/sqlite3-rsync.c @@ -506,6 +506,498 @@ int append_escaped_arg(sqlite3_str *pStr, const char *zIn, int isFilename){ ** End of the append_escaped_arg() routine, adapted from the Fossil ** *****************************************************************************/ +/***************************************************************************** +** The Hash Engine +** +** This is basically SHA3, though with a 160-bit hash, and reducing the +** number of rounds in the KeccakF1600 step function from 24 to 6. +*/ +/* +** Macros to determine whether the machine is big or little endian, +** and whether or not that determination is run-time or compile-time. +** +** For best performance, an attempt is made to guess at the byte-order +** using C-preprocessor macros. If that is unsuccessful, or if +** -DHash_BYTEORDER=0 is set, then byte-order is determined +** at run-time. +*/ +#ifndef Hash_BYTEORDER +# if defined(i386) || defined(__i386__) || defined(_M_IX86) || \ + defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || \ + defined(_M_AMD64) || defined(_M_ARM) || defined(__x86) || \ + defined(__arm__) +# define Hash_BYTEORDER 1234 +# elif defined(sparc) || defined(__ppc__) +# define Hash_BYTEORDER 4321 +# else +# define Hash_BYTEORDER 0 +# endif +#endif + +typedef sqlite3_uint64 u64; + +/* +** State structure for a Hash hash in progress +*/ +typedef struct HashContext HashContext; +struct HashContext { + union { + u64 s[25]; /* Keccak state. 5x5 lines of 64 bits each */ + unsigned char x[1600]; /* ... or 1600 bytes */ + } u; + unsigned nRate; /* Bytes of input accepted per Keccak iteration */ + unsigned nLoaded; /* Input bytes loaded into u.x[] so far this cycle */ + unsigned ixMask; /* Insert next input into u.x[nLoaded^ixMask]. */ + unsigned iSize; /* 224, 256, 358, or 512 */ +}; + +/* +** A single step of the Keccak mixing function for a 1600-bit state +*/ +static void KeccakF1600Step(HashContext *p){ + int i; + u64 b0, b1, b2, b3, b4; + u64 c0, c1, c2, c3, c4; + u64 d0, d1, d2, d3, d4; + static const u64 RC[] = { + 0x0000000000000001ULL, 0x0000000000008082ULL, + 0x800000000000808aULL, 0x8000000080008000ULL, + 0x000000000000808bULL, 0x0000000080000001ULL, + 0x8000000080008081ULL, 0x8000000000008009ULL, + 0x000000000000008aULL, 0x0000000000000088ULL, + 0x0000000080008009ULL, 0x000000008000000aULL, + 0x000000008000808bULL, 0x800000000000008bULL, + 0x8000000000008089ULL, 0x8000000000008003ULL, + 0x8000000000008002ULL, 0x8000000000000080ULL, + 0x000000000000800aULL, 0x800000008000000aULL, + 0x8000000080008081ULL, 0x8000000000008080ULL, + 0x0000000080000001ULL, 0x8000000080008008ULL + }; +# define a00 (p->u.s[0]) +# define a01 (p->u.s[1]) +# define a02 (p->u.s[2]) +# define a03 (p->u.s[3]) +# define a04 (p->u.s[4]) +# define a10 (p->u.s[5]) +# define a11 (p->u.s[6]) +# define a12 (p->u.s[7]) +# define a13 (p->u.s[8]) +# define a14 (p->u.s[9]) +# define a20 (p->u.s[10]) +# define a21 (p->u.s[11]) +# define a22 (p->u.s[12]) +# define a23 (p->u.s[13]) +# define a24 (p->u.s[14]) +# define a30 (p->u.s[15]) +# define a31 (p->u.s[16]) +# define a32 (p->u.s[17]) +# define a33 (p->u.s[18]) +# define a34 (p->u.s[19]) +# define a40 (p->u.s[20]) +# define a41 (p->u.s[21]) +# define a42 (p->u.s[22]) +# define a43 (p->u.s[23]) +# define a44 (p->u.s[24]) +# define ROL64(a,x) ((a<>(64-x))) + + /* v---- Number of rounds. SHA3 has 24 here. */ + for(i=0; i<6; i+=4){ + c0 = a00^a10^a20^a30^a40; + c1 = a01^a11^a21^a31^a41; + c2 = a02^a12^a22^a32^a42; + c3 = a03^a13^a23^a33^a43; + c4 = a04^a14^a24^a34^a44; + d0 = c4^ROL64(c1, 1); + d1 = c0^ROL64(c2, 1); + d2 = c1^ROL64(c3, 1); + d3 = c2^ROL64(c4, 1); + d4 = c3^ROL64(c0, 1); + + b0 = (a00^d0); + b1 = ROL64((a11^d1), 44); + b2 = ROL64((a22^d2), 43); + b3 = ROL64((a33^d3), 21); + b4 = ROL64((a44^d4), 14); + a00 = b0 ^((~b1)& b2 ); + a00 ^= RC[i]; + a11 = b1 ^((~b2)& b3 ); + a22 = b2 ^((~b3)& b4 ); + a33 = b3 ^((~b4)& b0 ); + a44 = b4 ^((~b0)& b1 ); + + b2 = ROL64((a20^d0), 3); + b3 = ROL64((a31^d1), 45); + b4 = ROL64((a42^d2), 61); + b0 = ROL64((a03^d3), 28); + b1 = ROL64((a14^d4), 20); + a20 = b0 ^((~b1)& b2 ); + a31 = b1 ^((~b2)& b3 ); + a42 = b2 ^((~b3)& b4 ); + a03 = b3 ^((~b4)& b0 ); + a14 = b4 ^((~b0)& b1 ); + + b4 = ROL64((a40^d0), 18); + b0 = ROL64((a01^d1), 1); + b1 = ROL64((a12^d2), 6); + b2 = ROL64((a23^d3), 25); + b3 = ROL64((a34^d4), 8); + a40 = b0 ^((~b1)& b2 ); + a01 = b1 ^((~b2)& b3 ); + a12 = b2 ^((~b3)& b4 ); + a23 = b3 ^((~b4)& b0 ); + a34 = b4 ^((~b0)& b1 ); + + b1 = ROL64((a10^d0), 36); + b2 = ROL64((a21^d1), 10); + b3 = ROL64((a32^d2), 15); + b4 = ROL64((a43^d3), 56); + b0 = ROL64((a04^d4), 27); + a10 = b0 ^((~b1)& b2 ); + a21 = b1 ^((~b2)& b3 ); + a32 = b2 ^((~b3)& b4 ); + a43 = b3 ^((~b4)& b0 ); + a04 = b4 ^((~b0)& b1 ); + + b3 = ROL64((a30^d0), 41); + b4 = ROL64((a41^d1), 2); + b0 = ROL64((a02^d2), 62); + b1 = ROL64((a13^d3), 55); + b2 = ROL64((a24^d4), 39); + a30 = b0 ^((~b1)& b2 ); + a41 = b1 ^((~b2)& b3 ); + a02 = b2 ^((~b3)& b4 ); + a13 = b3 ^((~b4)& b0 ); + a24 = b4 ^((~b0)& b1 ); + + c0 = a00^a20^a40^a10^a30; + c1 = a11^a31^a01^a21^a41; + c2 = a22^a42^a12^a32^a02; + c3 = a33^a03^a23^a43^a13; + c4 = a44^a14^a34^a04^a24; + d0 = c4^ROL64(c1, 1); + d1 = c0^ROL64(c2, 1); + d2 = c1^ROL64(c3, 1); + d3 = c2^ROL64(c4, 1); + d4 = c3^ROL64(c0, 1); + + b0 = (a00^d0); + b1 = ROL64((a31^d1), 44); + b2 = ROL64((a12^d2), 43); + b3 = ROL64((a43^d3), 21); + b4 = ROL64((a24^d4), 14); + a00 = b0 ^((~b1)& b2 ); + a00 ^= RC[i+1]; + a31 = b1 ^((~b2)& b3 ); + a12 = b2 ^((~b3)& b4 ); + a43 = b3 ^((~b4)& b0 ); + a24 = b4 ^((~b0)& b1 ); + + b2 = ROL64((a40^d0), 3); + b3 = ROL64((a21^d1), 45); + b4 = ROL64((a02^d2), 61); + b0 = ROL64((a33^d3), 28); + b1 = ROL64((a14^d4), 20); + a40 = b0 ^((~b1)& b2 ); + a21 = b1 ^((~b2)& b3 ); + a02 = b2 ^((~b3)& b4 ); + a33 = b3 ^((~b4)& b0 ); + a14 = b4 ^((~b0)& b1 ); + + b4 = ROL64((a30^d0), 18); + b0 = ROL64((a11^d1), 1); + b1 = ROL64((a42^d2), 6); + b2 = ROL64((a23^d3), 25); + b3 = ROL64((a04^d4), 8); + a30 = b0 ^((~b1)& b2 ); + a11 = b1 ^((~b2)& b3 ); + a42 = b2 ^((~b3)& b4 ); + a23 = b3 ^((~b4)& b0 ); + a04 = b4 ^((~b0)& b1 ); + + b1 = ROL64((a20^d0), 36); + b2 = ROL64((a01^d1), 10); + b3 = ROL64((a32^d2), 15); + b4 = ROL64((a13^d3), 56); + b0 = ROL64((a44^d4), 27); + a20 = b0 ^((~b1)& b2 ); + a01 = b1 ^((~b2)& b3 ); + a32 = b2 ^((~b3)& b4 ); + a13 = b3 ^((~b4)& b0 ); + a44 = b4 ^((~b0)& b1 ); + + b3 = ROL64((a10^d0), 41); + b4 = ROL64((a41^d1), 2); + b0 = ROL64((a22^d2), 62); + b1 = ROL64((a03^d3), 55); + b2 = ROL64((a34^d4), 39); + a10 = b0 ^((~b1)& b2 ); + a41 = b1 ^((~b2)& b3 ); + a22 = b2 ^((~b3)& b4 ); + a03 = b3 ^((~b4)& b0 ); + a34 = b4 ^((~b0)& b1 ); + + c0 = a00^a40^a30^a20^a10; + c1 = a31^a21^a11^a01^a41; + c2 = a12^a02^a42^a32^a22; + c3 = a43^a33^a23^a13^a03; + c4 = a24^a14^a04^a44^a34; + d0 = c4^ROL64(c1, 1); + d1 = c0^ROL64(c2, 1); + d2 = c1^ROL64(c3, 1); + d3 = c2^ROL64(c4, 1); + d4 = c3^ROL64(c0, 1); + + b0 = (a00^d0); + b1 = ROL64((a21^d1), 44); + b2 = ROL64((a42^d2), 43); + b3 = ROL64((a13^d3), 21); + b4 = ROL64((a34^d4), 14); + a00 = b0 ^((~b1)& b2 ); + a00 ^= RC[i+2]; + a21 = b1 ^((~b2)& b3 ); + a42 = b2 ^((~b3)& b4 ); + a13 = b3 ^((~b4)& b0 ); + a34 = b4 ^((~b0)& b1 ); + + b2 = ROL64((a30^d0), 3); + b3 = ROL64((a01^d1), 45); + b4 = ROL64((a22^d2), 61); + b0 = ROL64((a43^d3), 28); + b1 = ROL64((a14^d4), 20); + a30 = b0 ^((~b1)& b2 ); + a01 = b1 ^((~b2)& b3 ); + a22 = b2 ^((~b3)& b4 ); + a43 = b3 ^((~b4)& b0 ); + a14 = b4 ^((~b0)& b1 ); + + b4 = ROL64((a10^d0), 18); + b0 = ROL64((a31^d1), 1); + b1 = ROL64((a02^d2), 6); + b2 = ROL64((a23^d3), 25); + b3 = ROL64((a44^d4), 8); + a10 = b0 ^((~b1)& b2 ); + a31 = b1 ^((~b2)& b3 ); + a02 = b2 ^((~b3)& b4 ); + a23 = b3 ^((~b4)& b0 ); + a44 = b4 ^((~b0)& b1 ); + + b1 = ROL64((a40^d0), 36); + b2 = ROL64((a11^d1), 10); + b3 = ROL64((a32^d2), 15); + b4 = ROL64((a03^d3), 56); + b0 = ROL64((a24^d4), 27); + a40 = b0 ^((~b1)& b2 ); + a11 = b1 ^((~b2)& b3 ); + a32 = b2 ^((~b3)& b4 ); + a03 = b3 ^((~b4)& b0 ); + a24 = b4 ^((~b0)& b1 ); + + b3 = ROL64((a20^d0), 41); + b4 = ROL64((a41^d1), 2); + b0 = ROL64((a12^d2), 62); + b1 = ROL64((a33^d3), 55); + b2 = ROL64((a04^d4), 39); + a20 = b0 ^((~b1)& b2 ); + a41 = b1 ^((~b2)& b3 ); + a12 = b2 ^((~b3)& b4 ); + a33 = b3 ^((~b4)& b0 ); + a04 = b4 ^((~b0)& b1 ); + + c0 = a00^a30^a10^a40^a20; + c1 = a21^a01^a31^a11^a41; + c2 = a42^a22^a02^a32^a12; + c3 = a13^a43^a23^a03^a33; + c4 = a34^a14^a44^a24^a04; + d0 = c4^ROL64(c1, 1); + d1 = c0^ROL64(c2, 1); + d2 = c1^ROL64(c3, 1); + d3 = c2^ROL64(c4, 1); + d4 = c3^ROL64(c0, 1); + + b0 = (a00^d0); + b1 = ROL64((a01^d1), 44); + b2 = ROL64((a02^d2), 43); + b3 = ROL64((a03^d3), 21); + b4 = ROL64((a04^d4), 14); + a00 = b0 ^((~b1)& b2 ); + a00 ^= RC[i+3]; + a01 = b1 ^((~b2)& b3 ); + a02 = b2 ^((~b3)& b4 ); + a03 = b3 ^((~b4)& b0 ); + a04 = b4 ^((~b0)& b1 ); + + b2 = ROL64((a10^d0), 3); + b3 = ROL64((a11^d1), 45); + b4 = ROL64((a12^d2), 61); + b0 = ROL64((a13^d3), 28); + b1 = ROL64((a14^d4), 20); + a10 = b0 ^((~b1)& b2 ); + a11 = b1 ^((~b2)& b3 ); + a12 = b2 ^((~b3)& b4 ); + a13 = b3 ^((~b4)& b0 ); + a14 = b4 ^((~b0)& b1 ); + + b4 = ROL64((a20^d0), 18); + b0 = ROL64((a21^d1), 1); + b1 = ROL64((a22^d2), 6); + b2 = ROL64((a23^d3), 25); + b3 = ROL64((a24^d4), 8); + a20 = b0 ^((~b1)& b2 ); + a21 = b1 ^((~b2)& b3 ); + a22 = b2 ^((~b3)& b4 ); + a23 = b3 ^((~b4)& b0 ); + a24 = b4 ^((~b0)& b1 ); + + b1 = ROL64((a30^d0), 36); + b2 = ROL64((a31^d1), 10); + b3 = ROL64((a32^d2), 15); + b4 = ROL64((a33^d3), 56); + b0 = ROL64((a34^d4), 27); + a30 = b0 ^((~b1)& b2 ); + a31 = b1 ^((~b2)& b3 ); + a32 = b2 ^((~b3)& b4 ); + a33 = b3 ^((~b4)& b0 ); + a34 = b4 ^((~b0)& b1 ); + + b3 = ROL64((a40^d0), 41); + b4 = ROL64((a41^d1), 2); + b0 = ROL64((a42^d2), 62); + b1 = ROL64((a43^d3), 55); + b2 = ROL64((a44^d4), 39); + a40 = b0 ^((~b1)& b2 ); + a41 = b1 ^((~b2)& b3 ); + a42 = b2 ^((~b3)& b4 ); + a43 = b3 ^((~b4)& b0 ); + a44 = b4 ^((~b0)& b1 ); + } +} + +/* +** Initialize a new hash. iSize determines the size of the hash +** in bits and should be one of 224, 256, 384, or 512. Or iSize +** can be zero to use the default hash size of 256 bits. +*/ +static void HashInit(HashContext *p, int iSize){ + memset(p, 0, sizeof(*p)); + p->iSize = iSize; + if( iSize>=128 && iSize<=512 ){ + p->nRate = (1600 - ((iSize + 31)&~31)*2)/8; + }else{ + p->nRate = (1600 - 2*256)/8; + } +#if Hash_BYTEORDER==1234 + /* Known to be little-endian at compile-time. No-op */ +#elif Hash_BYTEORDER==4321 + p->ixMask = 7; /* Big-endian */ +#else + { + static unsigned int one = 1; + if( 1==*(unsigned char*)&one ){ + /* Little endian. No byte swapping. */ + p->ixMask = 0; + }else{ + /* Big endian. Byte swap. */ + p->ixMask = 7; + } + } +#endif +} + +/* +** Make consecutive calls to the HashUpdate function to add new content +** to the hash +*/ +static void HashUpdate( + HashContext *p, + const unsigned char *aData, + unsigned int nData +){ + unsigned int i = 0; + if( aData==0 ) return; +#if Hash_BYTEORDER==1234 + if( (p->nLoaded % 8)==0 && ((aData - (const unsigned char*)0)&7)==0 ){ + for(; i+7u.s[p->nLoaded/8] ^= *(u64*)&aData[i]; + p->nLoaded += 8; + if( p->nLoaded>=p->nRate ){ + KeccakF1600Step(p); + p->nLoaded = 0; + } + } + } +#endif + for(; iu.x[p->nLoaded] ^= aData[i]; +#elif Hash_BYTEORDER==4321 + p->u.x[p->nLoaded^0x07] ^= aData[i]; +#else + p->u.x[p->nLoaded^p->ixMask] ^= aData[i]; +#endif + p->nLoaded++; + if( p->nLoaded==p->nRate ){ + KeccakF1600Step(p); + p->nLoaded = 0; + } + } +} + +/* +** After all content has been added, invoke HashFinal() to compute +** the final hash. The function returns a pointer to the binary +** hash value. +*/ +static unsigned char *HashFinal(HashContext *p){ + unsigned int i; + if( p->nLoaded==p->nRate-1 ){ + const unsigned char c1 = 0x86; + HashUpdate(p, &c1, 1); + }else{ + const unsigned char c2 = 0x06; + const unsigned char c3 = 0x80; + HashUpdate(p, &c2, 1); + p->nLoaded = p->nRate - 1; + HashUpdate(p, &c3, 1); + } + for(i=0; inRate; i++){ + p->u.x[i+p->nRate] = p->u.x[i^p->ixMask]; + } + return &p->u.x[p->nRate]; +} + +/* +** Implementation of the hash(X) function. +** +** Return a 160-bit BLOB which is the hash of X. +*/ +static void hashFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + HashContext cx; + int eType = sqlite3_value_type(argv[0]); + int nByte = sqlite3_value_bytes(argv[0]); + if( eType==SQLITE_NULL ) return; + HashInit(&cx, 160); + if( eType==SQLITE_BLOB ){ + HashUpdate(&cx, sqlite3_value_blob(argv[0]), nByte); + }else{ + HashUpdate(&cx, sqlite3_value_text(argv[0]), nByte); + } + sqlite3_result_blob(context, HashFinal(&cx), 160/8, SQLITE_TRANSIENT); +} + +/* Register the hash function */ +static int hashRegister(sqlite3 *db){ + return sqlite3_create_function(db, "hash", 1, + SQLITE_UTF8 | SQLITE_INNOCUOUS | SQLITE_DETERMINISTIC, + 0, hashFunc, 0, 0); +} + +/* End of the hashing logic +*****************************************************************************/ + /* ** Return the tail of a file pathname. The tail is the last component ** of the path. For example, the tail of "/a/b/c.d" is "c.d". @@ -897,7 +1389,7 @@ static void originSide(SQLiteRsync *p){ closeDb(p); return; } - sqlite3_sha_init(p->db, 0, 0); + hashRegister(p->db); runSql(p, "BEGIN"); runSqlReturnText(p, buf, "PRAGMA journal_mode"); if( sqlite3_stricmp(buf,"wal")!=0 ){ @@ -944,7 +1436,7 @@ static void originSide(SQLiteRsync *p){ runSql(p, "CREATE TEMP TABLE badHash(pgno INTEGER PRIMARY KEY)"); pCkHash = prepareStmt(p, "INSERT INTO badHash SELECT pgno FROM sqlite_dbpage('main')" - " WHERE pgno=?1 AND sha1b(data)!=?2" + " WHERE pgno=?1 AND hash(data)!=?2" ); if( pCkHash==0 ) break; } @@ -1081,7 +1573,7 @@ static void replicaSide(SQLiteRsync *p){ closeDb(p); break; } - sqlite3_sha_init(p->db, 0, 0); + hashRegister(p->db); if( runSqlReturnUInt(p, &nRPage, "PRAGMA page_count") ){ break; } @@ -1105,7 +1597,7 @@ static void replicaSide(SQLiteRsync *p){ } pStmt = prepareStmt(p, - "SELECT sha1b(data) FROM sqlite_dbpage" + "SELECT hash(data) FROM sqlite_dbpage" " WHERE pgno<=min(%d,%d)" " ORDER BY pgno", nRPage, nOPage); while( sqlite3_step(pStmt)==SQLITE_ROW && p->nErr==0 ){ From d61d934f9bad67f5104239c8f1d7b994554acced Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 13 Sep 2024 12:35:04 +0000 Subject: [PATCH 15/24] Preveious check-in was actually doing an 8-round KeccakF1600. This one corrects that to just 6 rounds. FossilOrigin-Name: 3c36f5814f25483586c4fd49ef2fe5c7c0ff8c59672b1622c92061ec0ba8547a --- manifest | 12 +-- manifest.uuid | 2 +- tool/sqlite3-rsync.c | 203 +------------------------------------------ 3 files changed, 8 insertions(+), 209 deletions(-) diff --git a/manifest b/manifest index 0356a80fe4..bd0fbd3263 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\shash\salgorithm\sfrom\sSHA1\sto\sa\s6-round\sKeccakF1600\swith\sa\srate\nof\s160.\s\sThis\suses\sabout\s1/3rd\sfewer\sCPU\scycles. -D 2024-09-13T12:28:11.247 +C Preveious\scheck-in\swas\sactually\sdoing\san\s8-round\sKeccakF1600.\s\sThis\sone\ncorrects\sthat\sto\sjust\s6\srounds. +D 2024-09-13T12:35:04.073 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -2174,7 +2174,7 @@ F tool/speedtest8inst1.c 7ce07da76b5e745783e703a834417d725b7d45fd F tool/spellsift.tcl 52b4b04dc4333c7ab024f09d9d66ed6b6f7c6eb00b38497a09f338fa55d40618 x F tool/split-sqlite3c.tcl 5aa60643afca558bc732b1444ae81a522326f91e1dc5665b369c54f09e20de60 F tool/sqldiff.c 847fc8fcfddf5ce4797b7394cad6372f2f5dc17d8186e2ef8fb44d50fae4f44a -F tool/sqlite3-rsync.c 10fa0e40cc637195dc86c84390f1d7bae3be0830a0e50990df9289234097889c +F tool/sqlite3-rsync.c a8ae2390e6b78efda268ae97d2be0bf5a26d33777088a88b4e9e105e789f3141 F tool/sqlite3_analyzer.c.in 8da2b08f56eeac331a715036cf707cc20f879f231362be0c22efd682e2b89b4f F tool/sqltclsh.c.in 1bcc2e9da58fadf17b0bf6a50e68c1159e602ce057210b655d50bad5aaaef898 F tool/sqltclsh.tcl 862f4cf1418df5e1315b5db3b5ebe88969e2a784525af5fbf9596592f14ed848 @@ -2213,8 +2213,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 75d5a8eb3d4ece06900109ad4022ba2a3e82de2f0acb012e3a02bfb4326bfa6d -R 6d0c1116cd9e3a065842817267acb222 +P 96c7f47a8f59e5078bd296979421c1b57fbcb7be261f8a7a0b1d22a4b5914db0 +R 642fdf901a5bf4deb3070404be191395 U drh -Z 4ed0549ad93aad61732b14753f0892c6 +Z b81ca59094e5256917da50f121832a87 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d64d091567..4ea2ee2a9b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -96c7f47a8f59e5078bd296979421c1b57fbcb7be261f8a7a0b1d22a4b5914db0 +3c36f5814f25483586c4fd49ef2fe5c7c0ff8c59672b1622c92061ec0ba8547a diff --git a/tool/sqlite3-rsync.c b/tool/sqlite3-rsync.c index 4193afec89..305dc1519d 100644 --- a/tool/sqlite3-rsync.c +++ b/tool/sqlite3-rsync.c @@ -601,7 +601,7 @@ static void KeccakF1600Step(HashContext *p){ # define ROL64(a,x) ((a<>(64-x))) /* v---- Number of rounds. SHA3 has 24 here. */ - for(i=0; i<6; i+=4){ + for(i=0; i<6; i++){ c0 = a00^a10^a20^a30^a40; c1 = a01^a11^a21^a31^a41; c2 = a02^a12^a22^a32^a42; @@ -668,207 +668,6 @@ static void KeccakF1600Step(HashContext *p){ a02 = b2 ^((~b3)& b4 ); a13 = b3 ^((~b4)& b0 ); a24 = b4 ^((~b0)& b1 ); - - c0 = a00^a20^a40^a10^a30; - c1 = a11^a31^a01^a21^a41; - c2 = a22^a42^a12^a32^a02; - c3 = a33^a03^a23^a43^a13; - c4 = a44^a14^a34^a04^a24; - d0 = c4^ROL64(c1, 1); - d1 = c0^ROL64(c2, 1); - d2 = c1^ROL64(c3, 1); - d3 = c2^ROL64(c4, 1); - d4 = c3^ROL64(c0, 1); - - b0 = (a00^d0); - b1 = ROL64((a31^d1), 44); - b2 = ROL64((a12^d2), 43); - b3 = ROL64((a43^d3), 21); - b4 = ROL64((a24^d4), 14); - a00 = b0 ^((~b1)& b2 ); - a00 ^= RC[i+1]; - a31 = b1 ^((~b2)& b3 ); - a12 = b2 ^((~b3)& b4 ); - a43 = b3 ^((~b4)& b0 ); - a24 = b4 ^((~b0)& b1 ); - - b2 = ROL64((a40^d0), 3); - b3 = ROL64((a21^d1), 45); - b4 = ROL64((a02^d2), 61); - b0 = ROL64((a33^d3), 28); - b1 = ROL64((a14^d4), 20); - a40 = b0 ^((~b1)& b2 ); - a21 = b1 ^((~b2)& b3 ); - a02 = b2 ^((~b3)& b4 ); - a33 = b3 ^((~b4)& b0 ); - a14 = b4 ^((~b0)& b1 ); - - b4 = ROL64((a30^d0), 18); - b0 = ROL64((a11^d1), 1); - b1 = ROL64((a42^d2), 6); - b2 = ROL64((a23^d3), 25); - b3 = ROL64((a04^d4), 8); - a30 = b0 ^((~b1)& b2 ); - a11 = b1 ^((~b2)& b3 ); - a42 = b2 ^((~b3)& b4 ); - a23 = b3 ^((~b4)& b0 ); - a04 = b4 ^((~b0)& b1 ); - - b1 = ROL64((a20^d0), 36); - b2 = ROL64((a01^d1), 10); - b3 = ROL64((a32^d2), 15); - b4 = ROL64((a13^d3), 56); - b0 = ROL64((a44^d4), 27); - a20 = b0 ^((~b1)& b2 ); - a01 = b1 ^((~b2)& b3 ); - a32 = b2 ^((~b3)& b4 ); - a13 = b3 ^((~b4)& b0 ); - a44 = b4 ^((~b0)& b1 ); - - b3 = ROL64((a10^d0), 41); - b4 = ROL64((a41^d1), 2); - b0 = ROL64((a22^d2), 62); - b1 = ROL64((a03^d3), 55); - b2 = ROL64((a34^d4), 39); - a10 = b0 ^((~b1)& b2 ); - a41 = b1 ^((~b2)& b3 ); - a22 = b2 ^((~b3)& b4 ); - a03 = b3 ^((~b4)& b0 ); - a34 = b4 ^((~b0)& b1 ); - - c0 = a00^a40^a30^a20^a10; - c1 = a31^a21^a11^a01^a41; - c2 = a12^a02^a42^a32^a22; - c3 = a43^a33^a23^a13^a03; - c4 = a24^a14^a04^a44^a34; - d0 = c4^ROL64(c1, 1); - d1 = c0^ROL64(c2, 1); - d2 = c1^ROL64(c3, 1); - d3 = c2^ROL64(c4, 1); - d4 = c3^ROL64(c0, 1); - - b0 = (a00^d0); - b1 = ROL64((a21^d1), 44); - b2 = ROL64((a42^d2), 43); - b3 = ROL64((a13^d3), 21); - b4 = ROL64((a34^d4), 14); - a00 = b0 ^((~b1)& b2 ); - a00 ^= RC[i+2]; - a21 = b1 ^((~b2)& b3 ); - a42 = b2 ^((~b3)& b4 ); - a13 = b3 ^((~b4)& b0 ); - a34 = b4 ^((~b0)& b1 ); - - b2 = ROL64((a30^d0), 3); - b3 = ROL64((a01^d1), 45); - b4 = ROL64((a22^d2), 61); - b0 = ROL64((a43^d3), 28); - b1 = ROL64((a14^d4), 20); - a30 = b0 ^((~b1)& b2 ); - a01 = b1 ^((~b2)& b3 ); - a22 = b2 ^((~b3)& b4 ); - a43 = b3 ^((~b4)& b0 ); - a14 = b4 ^((~b0)& b1 ); - - b4 = ROL64((a10^d0), 18); - b0 = ROL64((a31^d1), 1); - b1 = ROL64((a02^d2), 6); - b2 = ROL64((a23^d3), 25); - b3 = ROL64((a44^d4), 8); - a10 = b0 ^((~b1)& b2 ); - a31 = b1 ^((~b2)& b3 ); - a02 = b2 ^((~b3)& b4 ); - a23 = b3 ^((~b4)& b0 ); - a44 = b4 ^((~b0)& b1 ); - - b1 = ROL64((a40^d0), 36); - b2 = ROL64((a11^d1), 10); - b3 = ROL64((a32^d2), 15); - b4 = ROL64((a03^d3), 56); - b0 = ROL64((a24^d4), 27); - a40 = b0 ^((~b1)& b2 ); - a11 = b1 ^((~b2)& b3 ); - a32 = b2 ^((~b3)& b4 ); - a03 = b3 ^((~b4)& b0 ); - a24 = b4 ^((~b0)& b1 ); - - b3 = ROL64((a20^d0), 41); - b4 = ROL64((a41^d1), 2); - b0 = ROL64((a12^d2), 62); - b1 = ROL64((a33^d3), 55); - b2 = ROL64((a04^d4), 39); - a20 = b0 ^((~b1)& b2 ); - a41 = b1 ^((~b2)& b3 ); - a12 = b2 ^((~b3)& b4 ); - a33 = b3 ^((~b4)& b0 ); - a04 = b4 ^((~b0)& b1 ); - - c0 = a00^a30^a10^a40^a20; - c1 = a21^a01^a31^a11^a41; - c2 = a42^a22^a02^a32^a12; - c3 = a13^a43^a23^a03^a33; - c4 = a34^a14^a44^a24^a04; - d0 = c4^ROL64(c1, 1); - d1 = c0^ROL64(c2, 1); - d2 = c1^ROL64(c3, 1); - d3 = c2^ROL64(c4, 1); - d4 = c3^ROL64(c0, 1); - - b0 = (a00^d0); - b1 = ROL64((a01^d1), 44); - b2 = ROL64((a02^d2), 43); - b3 = ROL64((a03^d3), 21); - b4 = ROL64((a04^d4), 14); - a00 = b0 ^((~b1)& b2 ); - a00 ^= RC[i+3]; - a01 = b1 ^((~b2)& b3 ); - a02 = b2 ^((~b3)& b4 ); - a03 = b3 ^((~b4)& b0 ); - a04 = b4 ^((~b0)& b1 ); - - b2 = ROL64((a10^d0), 3); - b3 = ROL64((a11^d1), 45); - b4 = ROL64((a12^d2), 61); - b0 = ROL64((a13^d3), 28); - b1 = ROL64((a14^d4), 20); - a10 = b0 ^((~b1)& b2 ); - a11 = b1 ^((~b2)& b3 ); - a12 = b2 ^((~b3)& b4 ); - a13 = b3 ^((~b4)& b0 ); - a14 = b4 ^((~b0)& b1 ); - - b4 = ROL64((a20^d0), 18); - b0 = ROL64((a21^d1), 1); - b1 = ROL64((a22^d2), 6); - b2 = ROL64((a23^d3), 25); - b3 = ROL64((a24^d4), 8); - a20 = b0 ^((~b1)& b2 ); - a21 = b1 ^((~b2)& b3 ); - a22 = b2 ^((~b3)& b4 ); - a23 = b3 ^((~b4)& b0 ); - a24 = b4 ^((~b0)& b1 ); - - b1 = ROL64((a30^d0), 36); - b2 = ROL64((a31^d1), 10); - b3 = ROL64((a32^d2), 15); - b4 = ROL64((a33^d3), 56); - b0 = ROL64((a34^d4), 27); - a30 = b0 ^((~b1)& b2 ); - a31 = b1 ^((~b2)& b3 ); - a32 = b2 ^((~b3)& b4 ); - a33 = b3 ^((~b4)& b0 ); - a34 = b4 ^((~b0)& b1 ); - - b3 = ROL64((a40^d0), 41); - b4 = ROL64((a41^d1), 2); - b0 = ROL64((a42^d2), 62); - b1 = ROL64((a43^d3), 55); - b2 = ROL64((a44^d4), 39); - a40 = b0 ^((~b1)& b2 ); - a41 = b1 ^((~b2)& b3 ); - a42 = b2 ^((~b3)& b4 ); - a43 = b3 ^((~b4)& b0 ); - a44 = b4 ^((~b0)& b1 ); } } From 7316b6c0726e3fd4b41e70e4bf96d5950f46d045 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 13 Sep 2024 13:53:20 +0000 Subject: [PATCH 16/24] Improved informational output from sqlite3-rsync. FossilOrigin-Name: e55e3e8ec2fe3a9190872d999cee55c85bde92667040cc166233faaa2fa34266 --- manifest | 12 ++++++------ manifest.uuid | 2 +- tool/sqlite3-rsync.c | 11 +++++++++-- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index bd0fbd3263..d2e70dbbfc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Preveious\scheck-in\swas\sactually\sdoing\san\s8-round\sKeccakF1600.\s\sThis\sone\ncorrects\sthat\sto\sjust\s6\srounds. -D 2024-09-13T12:35:04.073 +C Improved\sinformational\soutput\sfrom\ssqlite3-rsync. +D 2024-09-13T13:53:20.033 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -2174,7 +2174,7 @@ F tool/speedtest8inst1.c 7ce07da76b5e745783e703a834417d725b7d45fd F tool/spellsift.tcl 52b4b04dc4333c7ab024f09d9d66ed6b6f7c6eb00b38497a09f338fa55d40618 x F tool/split-sqlite3c.tcl 5aa60643afca558bc732b1444ae81a522326f91e1dc5665b369c54f09e20de60 F tool/sqldiff.c 847fc8fcfddf5ce4797b7394cad6372f2f5dc17d8186e2ef8fb44d50fae4f44a -F tool/sqlite3-rsync.c a8ae2390e6b78efda268ae97d2be0bf5a26d33777088a88b4e9e105e789f3141 +F tool/sqlite3-rsync.c 12b5d0f062496451c9e2a9c9d898ccb04fd90f3c2d309ebdd4e183bc516ad4c6 F tool/sqlite3_analyzer.c.in 8da2b08f56eeac331a715036cf707cc20f879f231362be0c22efd682e2b89b4f F tool/sqltclsh.c.in 1bcc2e9da58fadf17b0bf6a50e68c1159e602ce057210b655d50bad5aaaef898 F tool/sqltclsh.tcl 862f4cf1418df5e1315b5db3b5ebe88969e2a784525af5fbf9596592f14ed848 @@ -2213,8 +2213,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 96c7f47a8f59e5078bd296979421c1b57fbcb7be261f8a7a0b1d22a4b5914db0 -R 642fdf901a5bf4deb3070404be191395 +P 3c36f5814f25483586c4fd49ef2fe5c7c0ff8c59672b1622c92061ec0ba8547a +R 8cd1f423985f9cdd2c7d3914f9dc50f0 U drh -Z b81ca59094e5256917da50f121832a87 +Z 1cb8b10009d5db561201d72aab0227e3 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 4ea2ee2a9b..3ce2c13b08 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3c36f5814f25483586c4fd49ef2fe5c7c0ff8c59672b1622c92061ec0ba8547a +e55e3e8ec2fe3a9190872d999cee55c85bde92667040cc166233faaa2fa34266 diff --git a/tool/sqlite3-rsync.c b/tool/sqlite3-rsync.c index 305dc1519d..3b52da01e3 100644 --- a/tool/sqlite3-rsync.c +++ b/tool/sqlite3-rsync.c @@ -1697,7 +1697,14 @@ int main(int argc, char const * const *argv){ } originSide(&ctx); } - if( ctx.eVerbose ){ + if( ctx.nErr ){ + printf("Databases where not synced due to errors\n"); + } + if( ctx.eVerbose==1 ){ + printf("Network traffic is %.1f%% of database size\n", + (100.0*(double)(ctx.nIn+ctx.nOut))/(ctx.szPage*(double)ctx.nPage)); + } + if( ctx.eVerbose>=2 ){ if( ctx.nErr ) printf("%d errors, ", ctx.nErr); printf("%lld bytes sent, %lld bytes received\n", ctx.nOut, ctx.nIn); if( ctx.eVerbose>=2 ){ @@ -1711,5 +1718,5 @@ int main(int argc, char const * const *argv){ if( pIn!=0 && pOut!=0 ){ pclose2(pIn, pOut, childPid); } - return 0; + return ctx.nErr; } From 8550e4a9f80663b443379f55028df013bd51053e Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 13 Sep 2024 16:12:54 +0000 Subject: [PATCH 17/24] Fix harmless compiler warning on Windows. FossilOrigin-Name: 86e0219d977c493ac19d00c3ddcf560eb317d506c7cf6e4ef17e92daa91e1762 --- manifest | 12 ++++++------ manifest.uuid | 2 +- tool/sqlite3-rsync.c | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index d2e70dbbfc..447d4988e5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\sinformational\soutput\sfrom\ssqlite3-rsync. -D 2024-09-13T13:53:20.033 +C Fix\sharmless\scompiler\swarning\son\sWindows. +D 2024-09-13T16:12:54.397 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -2174,7 +2174,7 @@ F tool/speedtest8inst1.c 7ce07da76b5e745783e703a834417d725b7d45fd F tool/spellsift.tcl 52b4b04dc4333c7ab024f09d9d66ed6b6f7c6eb00b38497a09f338fa55d40618 x F tool/split-sqlite3c.tcl 5aa60643afca558bc732b1444ae81a522326f91e1dc5665b369c54f09e20de60 F tool/sqldiff.c 847fc8fcfddf5ce4797b7394cad6372f2f5dc17d8186e2ef8fb44d50fae4f44a -F tool/sqlite3-rsync.c 12b5d0f062496451c9e2a9c9d898ccb04fd90f3c2d309ebdd4e183bc516ad4c6 +F tool/sqlite3-rsync.c 3097aa389c41780bc52ad59d5205dafabb229f7d38ea8fbcd9f47d431ce6c0b1 F tool/sqlite3_analyzer.c.in 8da2b08f56eeac331a715036cf707cc20f879f231362be0c22efd682e2b89b4f F tool/sqltclsh.c.in 1bcc2e9da58fadf17b0bf6a50e68c1159e602ce057210b655d50bad5aaaef898 F tool/sqltclsh.tcl 862f4cf1418df5e1315b5db3b5ebe88969e2a784525af5fbf9596592f14ed848 @@ -2213,8 +2213,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3c36f5814f25483586c4fd49ef2fe5c7c0ff8c59672b1622c92061ec0ba8547a -R 8cd1f423985f9cdd2c7d3914f9dc50f0 +P e55e3e8ec2fe3a9190872d999cee55c85bde92667040cc166233faaa2fa34266 +R 6b032d30c7c32f4ed5f1d114c751e049 U drh -Z 1cb8b10009d5db561201d72aab0227e3 +Z c5bd48112d303e775518f5a9a4f43e77 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 3ce2c13b08..dc2a576f42 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e55e3e8ec2fe3a9190872d999cee55c85bde92667040cc166233faaa2fa34266 +86e0219d977c493ac19d00c3ddcf560eb317d506c7cf6e4ef17e92daa91e1762 diff --git a/tool/sqlite3-rsync.c b/tool/sqlite3-rsync.c index 3b52da01e3..2f9af55b7f 100644 --- a/tool/sqlite3-rsync.c +++ b/tool/sqlite3-rsync.c @@ -96,6 +96,7 @@ static void win32_fatal_error(const char *zMsg){ fprintf(stderr, "%s", zMsg); exit(1); } +extern int _open_osfhandle(intptr_t,int); #else #include #include From 762946b236eacb00f3c545adca5a6aa5beea736f Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 13 Sep 2024 21:47:57 +0000 Subject: [PATCH 18/24] Bug fix in the enhanced sqlite_dbpage for when truncating two or more ATTACH-ed databases within the same transaction. FossilOrigin-Name: 6aa9c8e79b440c6419e65990d9ceba8f00a6f975455138cf2aa82b113daec825 --- manifest | 15 ++++++--------- manifest.uuid | 2 +- src/dbpage.c | 5 ++++- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index fc47d2846a..191df77f06 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sability\sfor\ssqlite_dbpage\sto\struncate\sthe\sdatabase\sfile\sby\swriting\na\sNULL\spage.\s\sExperimental. -D 2024-09-10T12:09:03.839 +C Bug\sfix\sin\sthe\senhanced\ssqlite_dbpage\sfor\swhen\struncating\stwo\sor\smore\nATTACH-ed\sdatabases\swithin\sthe\ssame\stransaction. +D 2024-09-13T21:47:57.423 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -714,7 +714,7 @@ F src/callback.c db3a45e376deff6a16c0058163fe0ae2b73a2945f3f408ca32cf74960b28d49 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c b224d3db0f28c4a5f1407c50107a0a8133bd244ff3c7f6f8cedeb896a8cf1b64 F src/date.c 89ce1ff20512a7fa5070ba6e7dd5c171148ca7d580955795bf97c79c2456144a -F src/dbpage.c 3c437630c2933b9eefca915d191f8dea9da135195593bb17f553be58ffcd3634 +F src/dbpage.c 12e49515d67d4a59625d71f9aa42499556cfdc2e4f1ea49086e674a7f47f46e5 F src/dbstat.c 73362c0df0f40ad5523a6f5501224959d0976757b511299bf892313e79d14f5c F src/delete.c 444c4d1eaac40103461e3b6f0881846dd3aafc1cec1dd169d3482fa331667da7 F src/expr.c 6d5f2c38fe3ec06a7eac599dac822788b36064124e20112a844e9cd5156cb239 @@ -2212,11 +2212,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P fe0d67e72d4228661c021f227bfc0d5ddb1b726db0f36c7221ead8dd8bd1dc73 -R fb6f94f4d85e2b2f81d969e85d52cc7e -T *branch * dbpage -T *sym-dbpage * -T -sym-trunk * +P eb3c89ee2e4c5425be75deaf46a06a9cd8b210c695b918dd63a78f930c6e6b63 +R c19f64698097fbdbb0ae983d8267bb4c U drh -Z 44b34a478a24f6c5d7a75b5688f34391 +Z b211655571bcb6e66e7014318509d82f # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 653a750d52..e4fde79e08 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -eb3c89ee2e4c5425be75deaf46a06a9cd8b210c695b918dd63a78f930c6e6b63 +6aa9c8e79b440c6419e65990d9ceba8f00a6f975455138cf2aa82b113daec825 diff --git a/src/dbpage.c b/src/dbpage.c index 4da81f8ca1..42b24f9b8d 100644 --- a/src/dbpage.c +++ b/src/dbpage.c @@ -360,7 +360,7 @@ static int dbpageUpdate( iDb = 0; }else{ const char *zSchema = (const char*)sqlite3_value_text(argv[4]); - iDb = zSchema ? sqlite3FindDbName(pTab->db, zSchema) : -1; + iDb = sqlite3FindDbName(pTab->db, zSchema); if( iDb<0 ){ zErr = "no such schema"; goto update_fail; @@ -377,8 +377,11 @@ static int dbpageUpdate( ){ if( sqlite3_value_type(argv[3])==SQLITE_NULL && isInsert ){ if( iDb>=pTab->nTrunc ){ + testcase( pTab->aTrunc!=0 ); pTab->aTrunc = sqlite3_realloc(pTab->aTrunc, (iDb+1)*sizeof(Pgno)); if( pTab->aTrunc ){ + int j; + for(j=pTab->nTrunc; jaTrunc[j] = 0; pTab->nTrunc = iDb+1; }else{ return SQLITE_NOMEM; From 2b305188043b8a242c2b8419b8331141d89b40c3 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 14 Sep 2024 10:48:05 +0000 Subject: [PATCH 19/24] Make the output from sqlite3-rsync with a single -v option the same as it is with regular rsync. Only show the SSH command with two or more -v options, or if there is an error in popen2(). FossilOrigin-Name: 105ec44b470318fc9ff1773027c4064343f224068c9b6e71c5618f18f7dfcc3f --- manifest | 12 +++++----- manifest.uuid | 2 +- tool/sqlite3-rsync.c | 56 +++++++++++++++++++++++++++++++++----------- 3 files changed, 49 insertions(+), 21 deletions(-) diff --git a/manifest b/manifest index 33a282bb46..718c5d7e29 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\ssqlite_dbpage\sfixes\sinto\sthe\ssqlite3-rsync\sbranch. -D 2024-09-13T23:41:16.081 +C Make\sthe\soutput\sfrom\ssqlite3-rsync\swith\sa\ssingle\s-v\soption\sthe\ssame\sas\sit\nis\swith\sregular\srsync.\s\sOnly\sshow\sthe\sSSH\scommand\swith\stwo\sor\smore\s-v\soptions,\nor\sif\sthere\sis\san\serror\sin\spopen2(). +D 2024-09-14T10:48:05.083 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -2174,7 +2174,7 @@ F tool/speedtest8inst1.c 7ce07da76b5e745783e703a834417d725b7d45fd F tool/spellsift.tcl 52b4b04dc4333c7ab024f09d9d66ed6b6f7c6eb00b38497a09f338fa55d40618 x F tool/split-sqlite3c.tcl 5aa60643afca558bc732b1444ae81a522326f91e1dc5665b369c54f09e20de60 F tool/sqldiff.c 847fc8fcfddf5ce4797b7394cad6372f2f5dc17d8186e2ef8fb44d50fae4f44a -F tool/sqlite3-rsync.c 3097aa389c41780bc52ad59d5205dafabb229f7d38ea8fbcd9f47d431ce6c0b1 +F tool/sqlite3-rsync.c bb2e19ff46b9679b2c605c474f16a237d81784a819ea6a9265c4d405f323c6e9 F tool/sqlite3_analyzer.c.in 8da2b08f56eeac331a715036cf707cc20f879f231362be0c22efd682e2b89b4f F tool/sqltclsh.c.in 1bcc2e9da58fadf17b0bf6a50e68c1159e602ce057210b655d50bad5aaaef898 F tool/sqltclsh.tcl 862f4cf1418df5e1315b5db3b5ebe88969e2a784525af5fbf9596592f14ed848 @@ -2213,8 +2213,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 86e0219d977c493ac19d00c3ddcf560eb317d506c7cf6e4ef17e92daa91e1762 6aa9c8e79b440c6419e65990d9ceba8f00a6f975455138cf2aa82b113daec825 -R 9005fc31425a0253e67d368b9122f599 +P dff76b7a3436031ea5a61b8a44ddfa1d40ea20c983f3d34a8501cd7074db68b8 +R 95aaf9a89250b8760572431cb6032176 U drh -Z 4975b1813e58aa71fc8abcda1ea8ca48 +Z abc0757084d259b46638ccae491d778a # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b895d5424d..ff3b207760 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -dff76b7a3436031ea5a61b8a44ddfa1d40ea20c983f3d34a8501cd7074db68b8 +105ec44b470318fc9ff1773027c4064343f224068c9b6e71c5618f18f7dfcc3f diff --git a/tool/sqlite3-rsync.c b/tool/sqlite3-rsync.c index 2f9af55b7f..9554b87c92 100644 --- a/tool/sqlite3-rsync.c +++ b/tool/sqlite3-rsync.c @@ -1497,6 +1497,17 @@ static const char *cmdline_option_value(int argc, const char * const*argv, return argv[i]; } +/* +** Return the current time in milliseconds since the Julian epoch. +*/ +sqlite3_int64 currentTime(void){ + sqlite3_int64 now = 0; + sqlite3_vfs *pVfs = sqlite3_vfs_find(0); + if( pVfs && pVfs->iVersion>=2 && pVfs->xCurrentTimeInt64!=0 ){ + pVfs->xCurrentTimeInt64(pVfs, &now); + } + return now; +} /* ** Parse command-line arguments. Dispatch subroutines to do the @@ -1536,6 +1547,9 @@ int main(int argc, char const * const *argv){ const char *zSsh = "ssh"; const char *zExe = "sqlite3-rsync"; char *zCmd = 0; + sqlite3_int64 tmStart; + sqlite3_int64 tmEnd; + sqlite3_int64 tmElapse; #define cli_opt_val cmdline_option_value(argc, argv, ++i) memset(&ctx, 0, sizeof(ctx)); @@ -1629,6 +1643,7 @@ int main(int argc, char const * const *argv){ fprintf(stderr, "missing REPLICA database filename\n"); return 1; } + tmStart = currentTime(); zDiv = strchr(ctx.zOrigin,':'); if( zDiv ){ if( strchr(ctx.zReplica,':')!=0 ){ @@ -1652,7 +1667,7 @@ int main(int argc, char const * const *argv){ append_escaped_arg(pStr, zDiv, 1); append_escaped_arg(pStr, file_tail(ctx.zReplica), 1); zCmd = sqlite3_str_finish(pStr); - if( ctx.eVerbose ) printf("%s\n", zCmd); + if( ctx.eVerbose>=2 ) printf("%s\n", zCmd); if( popen2(zCmd, &ctx.pIn, &ctx.pOut, &childPid, 0) ){ fprintf(stderr, "Could not start auxiliary process: %s\n", zCmd); return 1; @@ -1674,7 +1689,7 @@ int main(int argc, char const * const *argv){ append_escaped_arg(pStr, file_tail(ctx.zOrigin), 1); append_escaped_arg(pStr, zDiv, 1); zCmd = sqlite3_str_finish(pStr); - if( ctx.eVerbose ) printf("%s\n", zCmd); + if( ctx.eVerbose>=2 ) printf("%s\n", zCmd); if( popen2(zCmd, &ctx.pIn, &ctx.pOut, &childPid, 0) ){ fprintf(stderr, "Could not start auxiliary process: %s\n", zCmd); return 1; @@ -1691,29 +1706,42 @@ int main(int argc, char const * const *argv){ append_escaped_arg(pStr, ctx.zOrigin, 1); append_escaped_arg(pStr, ctx.zReplica, 1); zCmd = sqlite3_str_finish(pStr); - if( ctx.eVerbose ) printf("%s\n", zCmd); + if( ctx.eVerbose>=2 ) printf("%s\n", zCmd); if( popen2(zCmd, &ctx.pIn, &ctx.pOut, &childPid, 0) ){ fprintf(stderr, "Could not start auxiliary process: %s\n", zCmd); return 1; } originSide(&ctx); } + tmEnd = currentTime(); + tmElapse = tmEnd - tmStart; /* Elapse time in milliseconds */ if( ctx.nErr ){ printf("Databases where not synced due to errors\n"); } if( ctx.eVerbose==1 ){ - printf("Network traffic is %.1f%% of database size\n", - (100.0*(double)(ctx.nIn+ctx.nOut))/(ctx.szPage*(double)ctx.nPage)); - } - if( ctx.eVerbose>=2 ){ - if( ctx.nErr ) printf("%d errors, ", ctx.nErr); - printf("%lld bytes sent, %lld bytes received\n", ctx.nOut, ctx.nIn); - if( ctx.eVerbose>=2 ){ - printf("Database is %u pages of %u bytes each.\n", - ctx.nPage, ctx.szPage); - printf("Sent %u hashes, %u page contents\n", - ctx.nHashSent, ctx.nPageSent); + char *zMsg; + sqlite3_int64 szTotal = (sqlite3_int64)ctx.nPage*(sqlite3_int64)ctx.szPage; + sqlite3_int64 nIO = ctx.nOut +ctx.nIn; + zMsg = sqlite3_mprintf("sent %,lld bytes, received %,lld bytes", + ctx.nOut, ctx.nIn); + printf("%s", zMsg); + sqlite3_free(zMsg); + if( tmElapse>0 ){ + zMsg = sqlite3_mprintf(", %,.2f bytes/sec", + 1000.0*(double)nIO/(double)tmElapse); + printf("%s\n", zMsg); + sqlite3_free(zMsg); + }else{ + printf("\n"); } + if( nIO<=szTotal && nIO>0 ){ + zMsg = sqlite3_mprintf("total size %,lld speedup is %.2f", + szTotal, (double)szTotal/(double)nIO); + }else{ + zMsg = sqlite3_mprintf("total size %,lld", szTotal); + } + printf("%s\n", zMsg); + sqlite3_free(zMsg); } sqlite3_free(zCmd); if( pIn!=0 && pOut!=0 ){ From 9d3047bd0d66964ff007ba5bfdccb04f4d1cf688 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 14 Sep 2024 10:59:32 +0000 Subject: [PATCH 20/24] Improved error messages coming out of sqlite3-rsync. FossilOrigin-Name: 452fb6de3984c3cb10d30b51dcdb2574578ca128a0c519b2bd43df0bdd343083 --- manifest | 12 ++++++------ manifest.uuid | 2 +- tool/sqlite3-rsync.c | 23 +++++++++++++---------- 3 files changed, 20 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index 718c5d7e29..a7fa243946 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\sthe\soutput\sfrom\ssqlite3-rsync\swith\sa\ssingle\s-v\soption\sthe\ssame\sas\sit\nis\swith\sregular\srsync.\s\sOnly\sshow\sthe\sSSH\scommand\swith\stwo\sor\smore\s-v\soptions,\nor\sif\sthere\sis\san\serror\sin\spopen2(). -D 2024-09-14T10:48:05.083 +C Improved\serror\smessages\scoming\sout\sof\ssqlite3-rsync. +D 2024-09-14T10:59:32.119 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -2174,7 +2174,7 @@ F tool/speedtest8inst1.c 7ce07da76b5e745783e703a834417d725b7d45fd F tool/spellsift.tcl 52b4b04dc4333c7ab024f09d9d66ed6b6f7c6eb00b38497a09f338fa55d40618 x F tool/split-sqlite3c.tcl 5aa60643afca558bc732b1444ae81a522326f91e1dc5665b369c54f09e20de60 F tool/sqldiff.c 847fc8fcfddf5ce4797b7394cad6372f2f5dc17d8186e2ef8fb44d50fae4f44a -F tool/sqlite3-rsync.c bb2e19ff46b9679b2c605c474f16a237d81784a819ea6a9265c4d405f323c6e9 +F tool/sqlite3-rsync.c a92a4fa8752690546aada08fa04250c2c0c9a600156193a862ad27ebe4bbf34e F tool/sqlite3_analyzer.c.in 8da2b08f56eeac331a715036cf707cc20f879f231362be0c22efd682e2b89b4f F tool/sqltclsh.c.in 1bcc2e9da58fadf17b0bf6a50e68c1159e602ce057210b655d50bad5aaaef898 F tool/sqltclsh.tcl 862f4cf1418df5e1315b5db3b5ebe88969e2a784525af5fbf9596592f14ed848 @@ -2213,8 +2213,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P dff76b7a3436031ea5a61b8a44ddfa1d40ea20c983f3d34a8501cd7074db68b8 -R 95aaf9a89250b8760572431cb6032176 +P 105ec44b470318fc9ff1773027c4064343f224068c9b6e71c5618f18f7dfcc3f +R 11b249479d28ab8ab57cb1dd31e2a2f3 U drh -Z abc0757084d259b46638ccae491d778a +Z 8a831d8281ba8f908e1ad29b156358da # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index ff3b207760..c542164d90 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -105ec44b470318fc9ff1773027c4064343f224068c9b6e71c5618f18f7dfcc3f +452fb6de3984c3cb10d30b51dcdb2574578ca128a0c519b2bd43df0bdd343083 diff --git a/tool/sqlite3-rsync.c b/tool/sqlite3-rsync.c index 9554b87c92..d3898b49a0 100644 --- a/tool/sqlite3-rsync.c +++ b/tool/sqlite3-rsync.c @@ -1184,8 +1184,8 @@ static void originSide(SQLiteRsync *p){ /* Open the ORIGIN database. */ rc = sqlite3_open_v2(p->zOrigin, &p->db, SQLITE_OPEN_READWRITE, 0); if( rc ){ - reportError(p, "unable to open origin database file \"%s\": %s", - sqlite3_errmsg(p->db)); + reportError(p, "cannot open origin \"%s\": %s", + p->zOrigin, sqlite3_errmsg(p->db)); closeDb(p); return; } @@ -1368,7 +1368,7 @@ static void replicaSide(SQLiteRsync *p){ p->szPage = szOPage; rc = sqlite3_open(p->zReplica, &p->db); if( rc ){ - reportError(p, "cannot open replica database \"%s\": %s", + reportError(p, "cannot open replica \"%s\": %s", p->zReplica, sqlite3_errmsg(p->db)); closeDb(p); break; @@ -1430,6 +1430,7 @@ static void replicaSide(SQLiteRsync *p){ sqlite3_sql(pIns), sqlite3_errmsg(p->db)); } sqlite3_reset(pIns); + p->nPage = nOPage; runSql(p, "COMMIT"); } break; @@ -1734,14 +1735,16 @@ int main(int argc, char const * const *argv){ }else{ printf("\n"); } - if( nIO<=szTotal && nIO>0 ){ - zMsg = sqlite3_mprintf("total size %,lld speedup is %.2f", - szTotal, (double)szTotal/(double)nIO); - }else{ - zMsg = sqlite3_mprintf("total size %,lld", szTotal); + if( ctx.nErr==0 ){ + if( nIO<=szTotal && nIO>0 ){ + zMsg = sqlite3_mprintf("total size %,lld speedup is %.2f", + szTotal, (double)szTotal/(double)nIO); + }else{ + zMsg = sqlite3_mprintf("total size %,lld", szTotal); + } + printf("%s\n", zMsg); + sqlite3_free(zMsg); } - printf("%s\n", zMsg); - sqlite3_free(zMsg); } sqlite3_free(zCmd); if( pIn!=0 && pOut!=0 ){ From 15e05eab32a257172e031c7c426ebda063351801 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 14 Sep 2024 11:23:57 +0000 Subject: [PATCH 21/24] Add the --logfile FILE option for debugging. FossilOrigin-Name: 30e1b92d5663e24d2f325f2bab35f81b55848ef39d15688e40b9005269626303 --- manifest | 12 ++++++------ manifest.uuid | 2 +- tool/sqlite3-rsync.c | 14 ++++++++++++++ 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index a7fa243946..fc99c17e78 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\serror\smessages\scoming\sout\sof\ssqlite3-rsync. -D 2024-09-14T10:59:32.119 +C Add\sthe\s--logfile\sFILE\soption\sfor\sdebugging. +D 2024-09-14T11:23:57.579 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -2174,7 +2174,7 @@ F tool/speedtest8inst1.c 7ce07da76b5e745783e703a834417d725b7d45fd F tool/spellsift.tcl 52b4b04dc4333c7ab024f09d9d66ed6b6f7c6eb00b38497a09f338fa55d40618 x F tool/split-sqlite3c.tcl 5aa60643afca558bc732b1444ae81a522326f91e1dc5665b369c54f09e20de60 F tool/sqldiff.c 847fc8fcfddf5ce4797b7394cad6372f2f5dc17d8186e2ef8fb44d50fae4f44a -F tool/sqlite3-rsync.c a92a4fa8752690546aada08fa04250c2c0c9a600156193a862ad27ebe4bbf34e +F tool/sqlite3-rsync.c 8cef3550aa9eb94e243c833c53e66cfe534e62d33e5490b2f884d4c73a184fb2 F tool/sqlite3_analyzer.c.in 8da2b08f56eeac331a715036cf707cc20f879f231362be0c22efd682e2b89b4f F tool/sqltclsh.c.in 1bcc2e9da58fadf17b0bf6a50e68c1159e602ce057210b655d50bad5aaaef898 F tool/sqltclsh.tcl 862f4cf1418df5e1315b5db3b5ebe88969e2a784525af5fbf9596592f14ed848 @@ -2213,8 +2213,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 105ec44b470318fc9ff1773027c4064343f224068c9b6e71c5618f18f7dfcc3f -R 11b249479d28ab8ab57cb1dd31e2a2f3 +P 452fb6de3984c3cb10d30b51dcdb2574578ca128a0c519b2bd43df0bdd343083 +R a57a86421474386b5ed9caf031db8013 U drh -Z 8a831d8281ba8f908e1ad29b156358da +Z 4ff8584d3a3a5f66a03fe1e9c7d688fd # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index c542164d90..0c1a83e195 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -452fb6de3984c3cb10d30b51dcdb2574578ca128a0c519b2bd43df0bdd343083 +30e1b92d5663e24d2f325f2bab35f81b55848ef39d15688e40b9005269626303 diff --git a/tool/sqlite3-rsync.c b/tool/sqlite3-rsync.c index d3898b49a0..fd2cb068e6 100644 --- a/tool/sqlite3-rsync.c +++ b/tool/sqlite3-rsync.c @@ -45,6 +45,7 @@ struct SQLiteRsync { const char *zReplica; /* Name of the replica */ FILE *pOut; /* Transmit to the other side */ FILE *pIn; /* Receive from the other side */ + FILE *pLog; /* Duplicate output here if not NULL */ sqlite3 *db; /* Database connection */ int nErr; /* Number of errors encountered */ u8 eVerbose; /* Bigger for more output. 0 means none. */ @@ -840,6 +841,7 @@ static int writeUint32(SQLiteRsync *p, unsigned int x){ buf[1] = x & 0xff; x >>= 8; buf[0] = x; + if( p->pLog ) fwrite(buf, sizeof(buf), 1, p->pLog); if( fwrite(buf, sizeof(buf), 1, p->pOut)!=1 ){ p->nErr++; return 1; @@ -859,6 +861,7 @@ int readByte(SQLiteRsync *p){ /* Write a single byte into the wire. */ void writeByte(SQLiteRsync *p, int c){ + if( p->pLog ) fputc(c, p->pLog); fputc(c, p->pOut); p->nOut++; } @@ -898,6 +901,7 @@ void readBytes(SQLiteRsync *p, int nByte, void *pData){ /* Write an array of bytes onto the wire. */ void writeBytes(SQLiteRsync *p, int nByte, const void *pData){ + if( p->pLog ) fwrite(pData, 1, nByte, p->pLog); if( fwrite(pData, 1, nByte, p->pOut)==nByte ){ p->nOut += nByte; }else{ @@ -1576,6 +1580,15 @@ int main(int argc, char const * const *argv){ zExe = cli_opt_val; continue; } + if( strcmp(z, "--logfile")==0 ){ + if( ctx.pLog ) fclose(ctx.pLog); + ctx.pLog = fopen(argv[++i],"wb"); + if( ctx.pLog==0 ){ + fprintf(stderr, "cannot open \"%s\" for writing\n", argv[i]); + return 1; + } + continue; + } if( strcmp(z, "-help")==0 || strcmp(z, "--help")==0 || strcmp(z, "-?")==0 ){ @@ -1714,6 +1727,7 @@ int main(int argc, char const * const *argv){ } originSide(&ctx); } + if( ctx.pLog ) fclose(ctx.pLog); tmEnd = currentTime(); tmElapse = tmEnd - tmStart; /* Elapse time in milliseconds */ if( ctx.nErr ){ From df9b4887991e94a2414c3f0b5b4263801f22deed Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 14 Sep 2024 11:38:46 +0000 Subject: [PATCH 22/24] Always show the -v traffic message even if there are multiple -v options. FossilOrigin-Name: fc05a5b7f77cdbfcc659d49eb09569a64a172362cb90199e2861028085178f10 --- manifest | 12 ++++++------ manifest.uuid | 2 +- tool/sqlite3-rsync.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index fc99c17e78..a69024423f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s--logfile\sFILE\soption\sfor\sdebugging. -D 2024-09-14T11:23:57.579 +C Always\sshow\sthe\s-v\straffic\smessage\seven\sif\sthere\sare\smultiple\s-v\soptions. +D 2024-09-14T11:38:46.736 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -2174,7 +2174,7 @@ F tool/speedtest8inst1.c 7ce07da76b5e745783e703a834417d725b7d45fd F tool/spellsift.tcl 52b4b04dc4333c7ab024f09d9d66ed6b6f7c6eb00b38497a09f338fa55d40618 x F tool/split-sqlite3c.tcl 5aa60643afca558bc732b1444ae81a522326f91e1dc5665b369c54f09e20de60 F tool/sqldiff.c 847fc8fcfddf5ce4797b7394cad6372f2f5dc17d8186e2ef8fb44d50fae4f44a -F tool/sqlite3-rsync.c 8cef3550aa9eb94e243c833c53e66cfe534e62d33e5490b2f884d4c73a184fb2 +F tool/sqlite3-rsync.c ac2160892a2baf0317d1232c4678db148b90285c422ca3bc3ec3a96194cf4129 F tool/sqlite3_analyzer.c.in 8da2b08f56eeac331a715036cf707cc20f879f231362be0c22efd682e2b89b4f F tool/sqltclsh.c.in 1bcc2e9da58fadf17b0bf6a50e68c1159e602ce057210b655d50bad5aaaef898 F tool/sqltclsh.tcl 862f4cf1418df5e1315b5db3b5ebe88969e2a784525af5fbf9596592f14ed848 @@ -2213,8 +2213,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 452fb6de3984c3cb10d30b51dcdb2574578ca128a0c519b2bd43df0bdd343083 -R a57a86421474386b5ed9caf031db8013 +P 30e1b92d5663e24d2f325f2bab35f81b55848ef39d15688e40b9005269626303 +R 3a4c9e53edbc74f8aebbd484815d1c77 U drh -Z 4ff8584d3a3a5f66a03fe1e9c7d688fd +Z 7d402ed7dd081ccd3ef9c52fe661c558 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 0c1a83e195..7ae6c7119a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -30e1b92d5663e24d2f325f2bab35f81b55848ef39d15688e40b9005269626303 +fc05a5b7f77cdbfcc659d49eb09569a64a172362cb90199e2861028085178f10 diff --git a/tool/sqlite3-rsync.c b/tool/sqlite3-rsync.c index fd2cb068e6..47f34b5132 100644 --- a/tool/sqlite3-rsync.c +++ b/tool/sqlite3-rsync.c @@ -1733,7 +1733,7 @@ int main(int argc, char const * const *argv){ if( ctx.nErr ){ printf("Databases where not synced due to errors\n"); } - if( ctx.eVerbose==1 ){ + if( ctx.eVerbose>=1 ){ char *zMsg; sqlite3_int64 szTotal = (sqlite3_int64)ctx.nPage*(sqlite3_int64)ctx.szPage; sqlite3_int64 nIO = ctx.nOut +ctx.nIn; From 7154803d4ceb57c0c5fcde14fa37811ef67c2ec6 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 14 Sep 2024 16:03:16 +0000 Subject: [PATCH 23/24] New debugging options: --logfile, --errorfile, --remote-errorfile. If page 1 changes, always send it last. FossilOrigin-Name: 2d8cd76691554578e987ce682cf0c42c083711dd1511a178148978182ef43ba2 --- manifest | 12 ++--- manifest.uuid | 2 +- tool/sqlite3-rsync.c | 116 +++++++++++++++++++++++++++++++++++-------- 3 files changed, 101 insertions(+), 29 deletions(-) diff --git a/manifest b/manifest index a69024423f..311c60f135 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Always\sshow\sthe\s-v\straffic\smessage\seven\sif\sthere\sare\smultiple\s-v\soptions. -D 2024-09-14T11:38:46.736 +C New\sdebugging\soptions:\s\s--logfile,\s--errorfile,\s--remote-errorfile.\nIf\spage\s1\schanges,\salways\ssend\sit\slast. +D 2024-09-14T16:03:16.556 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -2174,7 +2174,7 @@ F tool/speedtest8inst1.c 7ce07da76b5e745783e703a834417d725b7d45fd F tool/spellsift.tcl 52b4b04dc4333c7ab024f09d9d66ed6b6f7c6eb00b38497a09f338fa55d40618 x F tool/split-sqlite3c.tcl 5aa60643afca558bc732b1444ae81a522326f91e1dc5665b369c54f09e20de60 F tool/sqldiff.c 847fc8fcfddf5ce4797b7394cad6372f2f5dc17d8186e2ef8fb44d50fae4f44a -F tool/sqlite3-rsync.c ac2160892a2baf0317d1232c4678db148b90285c422ca3bc3ec3a96194cf4129 +F tool/sqlite3-rsync.c e73e4c7bbaf0a9c41e1ae8e81db45550d6722276ba7c1eda2771faf59ba1ac4f F tool/sqlite3_analyzer.c.in 8da2b08f56eeac331a715036cf707cc20f879f231362be0c22efd682e2b89b4f F tool/sqltclsh.c.in 1bcc2e9da58fadf17b0bf6a50e68c1159e602ce057210b655d50bad5aaaef898 F tool/sqltclsh.tcl 862f4cf1418df5e1315b5db3b5ebe88969e2a784525af5fbf9596592f14ed848 @@ -2213,8 +2213,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 30e1b92d5663e24d2f325f2bab35f81b55848ef39d15688e40b9005269626303 -R 3a4c9e53edbc74f8aebbd484815d1c77 +P fc05a5b7f77cdbfcc659d49eb09569a64a172362cb90199e2861028085178f10 +R 0679514ffeda4dcc5dfb35cbac8e373b U drh -Z 7d402ed7dd081ccd3ef9c52fe661c558 +Z e74e05952b36b86ecc8174c3233b0df1 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 7ae6c7119a..57d9739c3f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fc05a5b7f77cdbfcc659d49eb09569a64a172362cb90199e2861028085178f10 +2d8cd76691554578e987ce682cf0c42c083711dd1511a178148978182ef43ba2 diff --git a/tool/sqlite3-rsync.c b/tool/sqlite3-rsync.c index 47f34b5132..f581ba84bf 100644 --- a/tool/sqlite3-rsync.c +++ b/tool/sqlite3-rsync.c @@ -43,6 +43,7 @@ typedef struct SQLiteRsync SQLiteRsync; struct SQLiteRsync { const char *zOrigin; /* Name of the origin */ const char *zReplica; /* Name of the replica */ + const char *zErrFile; /* Append error messages to this file */ FILE *pOut; /* Transmit to the other side */ FILE *pIn; /* Receive from the other side */ FILE *pLog; /* Duplicate output here if not NULL */ @@ -813,6 +814,24 @@ const char *file_tail(const char *z){ return zTail; } +/* +** Append error message text to the error file, if an error file is +** specified. In any case, increment the error count. +*/ +static void logError(SQLiteRsync *p, const char *zFormat, ...){ + if( p->zErrFile ){ + FILE *pErr = fopen(p->zErrFile, "a"); + if( pErr ){ + va_list ap; + va_start(ap, zFormat); + vfprintf(pErr, zFormat, ap); + va_end(ap); + fclose(pErr); + } + } + p->nErr++; +} + /* Read a single big-endian 32-bit unsigned integer from the input ** stream. Return 0 on success and 1 if there are any errors. @@ -843,7 +862,7 @@ static int writeUint32(SQLiteRsync *p, unsigned int x){ buf[0] = x; if( p->pLog ) fwrite(buf, sizeof(buf), 1, p->pLog); if( fwrite(buf, sizeof(buf), 1, p->pOut)!=1 ){ - p->nErr++; + logError(p, "failed to write 32-bit integer 0x%x", x); return 1; } p->nOut += 4; @@ -871,7 +890,7 @@ void writeByte(SQLiteRsync *p, int c){ int readPow2(SQLiteRsync *p){ int x = readByte(p); if( x>=32 ){ - p->nErr++; + logError(p, "read invalid page size %d\n", x); return 0; } return 1<nErr++; + logError(p, "trying to read invalid page size %d\n", c); } for(n=0; c>1; n++){ c /= 2; } writeByte(p, n); @@ -894,7 +913,7 @@ void readBytes(SQLiteRsync *p, int nByte, void *pData){ if( fread(pData, 1, nByte, p->pIn)==nByte ){ p->nIn += nByte; }else{ - p->nErr++; + logError(p, "failed to read %d bytes", nByte); } } @@ -905,7 +924,7 @@ void writeBytes(SQLiteRsync *p, int nByte, const void *pData){ if( fwrite(pData, 1, nByte, p->pOut)==nByte ){ p->nOut += nByte; }else{ - p->nErr++; + logError(p, "failed to write %d bytes", nByte); } } @@ -934,8 +953,8 @@ static void reportError(SQLiteRsync *p, const char *zFormat, ...){ }else{ fprintf(stderr, "%s\n", zMsg); } + logError(p, "%s\n", zMsg); sqlite3_free(zMsg); - p->nErr++; } /* Send an informational message. @@ -974,7 +993,6 @@ static void readAndDisplayMessage(SQLiteRsync *p, int c){ const char *zPrefix; if( c==ORIGIN_ERROR || c==REPLICA_ERROR ){ zPrefix = "ERROR: "; - p->nErr++; }else{ zPrefix = ""; } @@ -990,6 +1008,7 @@ static void readAndDisplayMessage(SQLiteRsync *p, int c){ memset(zMsg, 0, n+1); readBytes(p, n, zMsg); fprintf(stderr,"%s%s\n", zPrefix, zMsg); + if( zPrefix[0] ) logError(p, "%s%s\n", zPrefix, zMsg); sqlite3_free(zMsg); } } @@ -1259,32 +1278,56 @@ static void originSide(SQLiteRsync *p){ } case REPLICA_READY: { sqlite3_stmt *pStmt; + int needPageOne = 0; sqlite3_finalize(pCkHash); pCkHash = 0; + if( iPage+1nPage ){ + runSql(p, "WITH RECURSIVE c(n) AS" + " (VALUES(%d) UNION ALL SELECT n+1 FROM c WHERE n<%d)" + " INSERT INTO badHash SELECT n FROM c", + iPage+1, p->nPage); + } pStmt = prepareStmt(p, "SELECT pgno, data" - " FROM badHash JOIN sqlite_dbpage('main') USING(pgno) " - "UNION ALL " - "SELECT pgno, data" - " FROM sqlite_dbpage('main')" - " WHERE pgno>%d", - iPage); + " FROM badHash JOIN sqlite_dbpage('main') USING(pgno)"); if( pStmt==0 ) break; while( sqlite3_step(pStmt)==SQLITE_ROW && p->nErr==0 ){ + unsigned int pgno = (unsigned int)sqlite3_column_int64(pStmt,0); const void *pContent = sqlite3_column_blob(pStmt, 1); - writeByte(p, ORIGIN_PAGE); - writeUint32(p, (unsigned int)sqlite3_column_int64(pStmt, 0)); - writeBytes(p, szPg, pContent); - p->nPageSent++; + if( pgno==1 ){ + needPageOne = 1; + }else{ + writeByte(p, ORIGIN_PAGE); + writeUint32(p, (unsigned int)sqlite3_column_int64(pStmt, 0)); + writeBytes(p, szPg, pContent); + p->nPageSent++; + } } sqlite3_finalize(pStmt); + if( needPageOne ){ + pStmt = prepareStmt(p, + "SELECT data" + " FROM sqlite_dbpage('main')" + " WHERE pgno=1" + ); + if( pStmt==0 ) break; + while( sqlite3_step(pStmt)==SQLITE_ROW && p->nErr==0 ){ + const void *pContent = sqlite3_column_blob(pStmt, 0); + writeByte(p, ORIGIN_PAGE); + writeUint32(p, 1); + writeBytes(p, szPg, pContent); + p->nPageSent++; + } + sqlite3_finalize(pStmt); + } writeByte(p, ORIGIN_TXN); writeUint32(p, nPage); writeByte(p, ORIGIN_END); goto origin_end; } default: { - reportError(p, "Origin side received unknown message: 0x%02x", c); + reportError(p, "Unknown message 0x%02x %lld bytes into conversation", + c, p->nIn); break; } } @@ -1446,7 +1489,7 @@ static void replicaSide(SQLiteRsync *p){ if( p->nErr ) break; if( pIns==0 ){ pIns = prepareStmt(p, - "INSERT INTO sqlite_dbpage(pgno,data,schema) VALUES(?1,?2,'main')" + "INSERT INTO sqlite_dbpage(pgno,data) VALUES(?1,?2)" ); if( pIns==0 ) break; } @@ -1457,14 +1500,15 @@ static void replicaSide(SQLiteRsync *p){ sqlite3_bind_blob(pIns, 2, buf, szOPage, SQLITE_STATIC); rc = sqlite3_step(pIns); if( rc!=SQLITE_DONE ){ - reportError(p, "SQL statement [%s] failed: %s", - sqlite3_sql(pIns), sqlite3_errmsg(p->db)); + reportError(p, "SQL statement [%s] failed (pgno=%u): %s", + sqlite3_sql(pIns), pgno, sqlite3_errmsg(p->db)); } sqlite3_reset(pIns); break; } default: { - reportError(p, "Replica side received unknown message: 0x%02x", c); + reportError(p, "Unknown message 0x%02x %lld bytes into conversation", + c, p->nIn); break; } } @@ -1555,6 +1599,7 @@ int main(int argc, char const * const *argv){ sqlite3_int64 tmStart; sqlite3_int64 tmEnd; sqlite3_int64 tmElapse; + const char *zRemoteErrFile = 0; #define cli_opt_val cmdline_option_value(argc, argv, ++i) memset(&ctx, 0, sizeof(ctx)); @@ -1581,6 +1626,8 @@ int main(int argc, char const * const *argv){ continue; } if( strcmp(z, "--logfile")==0 ){ + /* DEBUG OPTION: --logfile FILENAME + ** Cause all local output traffic to be duplicated in FILENAME */ if( ctx.pLog ) fclose(ctx.pLog); ctx.pLog = fopen(argv[++i],"wb"); if( ctx.pLog==0 ){ @@ -1589,6 +1636,19 @@ int main(int argc, char const * const *argv){ } continue; } + if( strcmp(z, "--errorfile")==0 ){ + /* DEBUG OPTION: --errorfile FILENAME + ** Error messages on the local side are written into FILENAME */ + ctx.zErrFile = argv[++i]; + continue; + } + if( strcmp(z, "--remote-errorfile")==0 ){ + /* DEBUG OPTION: --remote-errorfile FILENAME + ** Error messages on the remote side are written into FILENAME on + ** the remote side. */ + zRemoteErrFile = argv[++i]; + continue; + } if( strcmp(z, "-help")==0 || strcmp(z, "--help")==0 || strcmp(z, "-?")==0 ){ @@ -1678,6 +1738,10 @@ int main(int argc, char const * const *argv){ append_escaped_arg(pStr, "--commcheck", 0); if( ctx.eVerbose==0 ) ctx.eVerbose = 1; } + if( zRemoteErrFile ){ + append_escaped_arg(pStr, "--errorfile", 0); + append_escaped_arg(pStr, zRemoteErrFile, 1); + } append_escaped_arg(pStr, zDiv, 1); append_escaped_arg(pStr, file_tail(ctx.zReplica), 1); zCmd = sqlite3_str_finish(pStr); @@ -1700,6 +1764,10 @@ int main(int argc, char const * const *argv){ append_escaped_arg(pStr, "--commcheck", 0); if( ctx.eVerbose==0 ) ctx.eVerbose = 1; } + if( zRemoteErrFile ){ + append_escaped_arg(pStr, "--errorfile", 0); + append_escaped_arg(pStr, zRemoteErrFile, 1); + } append_escaped_arg(pStr, file_tail(ctx.zOrigin), 1); append_escaped_arg(pStr, zDiv, 1); zCmd = sqlite3_str_finish(pStr); @@ -1717,6 +1785,10 @@ int main(int argc, char const * const *argv){ if( ctx.bCommCheck ){ append_escaped_arg(pStr, "--commcheck", 0); } + if( zRemoteErrFile ){ + append_escaped_arg(pStr, "--errorfile", 0); + append_escaped_arg(pStr, zRemoteErrFile, 1); + } append_escaped_arg(pStr, ctx.zOrigin, 1); append_escaped_arg(pStr, ctx.zReplica, 1); zCmd = sqlite3_str_finish(pStr); From 3d56d59adc1e0c60ca752e2dd062e78d3927102e Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 14 Sep 2024 16:28:38 +0000 Subject: [PATCH 24/24] Add the --version option. FossilOrigin-Name: 9961334c8007e7cb6ae55885075b74acddc4fa701b359cf67e0f3c237d7eba4a --- manifest | 12 ++++++------ manifest.uuid | 2 +- tool/sqlite3-rsync.c | 5 +++++ 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 311c60f135..119071b4bc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C New\sdebugging\soptions:\s\s--logfile,\s--errorfile,\s--remote-errorfile.\nIf\spage\s1\schanges,\salways\ssend\sit\slast. -D 2024-09-14T16:03:16.556 +C Add\sthe\s--version\soption. +D 2024-09-14T16:28:38.248 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -2174,7 +2174,7 @@ F tool/speedtest8inst1.c 7ce07da76b5e745783e703a834417d725b7d45fd F tool/spellsift.tcl 52b4b04dc4333c7ab024f09d9d66ed6b6f7c6eb00b38497a09f338fa55d40618 x F tool/split-sqlite3c.tcl 5aa60643afca558bc732b1444ae81a522326f91e1dc5665b369c54f09e20de60 F tool/sqldiff.c 847fc8fcfddf5ce4797b7394cad6372f2f5dc17d8186e2ef8fb44d50fae4f44a -F tool/sqlite3-rsync.c e73e4c7bbaf0a9c41e1ae8e81db45550d6722276ba7c1eda2771faf59ba1ac4f +F tool/sqlite3-rsync.c f34c43d88141dff62fb83067059bca9ec79d85fbc5be92bfe0d95db95ce21107 F tool/sqlite3_analyzer.c.in 8da2b08f56eeac331a715036cf707cc20f879f231362be0c22efd682e2b89b4f F tool/sqltclsh.c.in 1bcc2e9da58fadf17b0bf6a50e68c1159e602ce057210b655d50bad5aaaef898 F tool/sqltclsh.tcl 862f4cf1418df5e1315b5db3b5ebe88969e2a784525af5fbf9596592f14ed848 @@ -2213,8 +2213,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P fc05a5b7f77cdbfcc659d49eb09569a64a172362cb90199e2861028085178f10 -R 0679514ffeda4dcc5dfb35cbac8e373b +P 2d8cd76691554578e987ce682cf0c42c083711dd1511a178148978182ef43ba2 +R 599973ba253a67fa628460fcbcaa6551 U drh -Z e74e05952b36b86ecc8174c3233b0df1 +Z 274e3943ab699374e1863c8c0827d301 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 57d9739c3f..35f0a7c5ec 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2d8cd76691554578e987ce682cf0c42c083711dd1511a178148978182ef43ba2 +9961334c8007e7cb6ae55885075b74acddc4fa701b359cf67e0f3c237d7eba4a diff --git a/tool/sqlite3-rsync.c b/tool/sqlite3-rsync.c index f581ba84bf..2a58125aa0 100644 --- a/tool/sqlite3-rsync.c +++ b/tool/sqlite3-rsync.c @@ -34,6 +34,7 @@ static const char zUsage[] = " --help Show this help screen\n" " --ssh PATH Name of the SSH program used to reach the remote side\n" " -v Verbose. Multiple v's for increasing output\n" + " --version Show detailed version information\n" ; typedef unsigned char u8; @@ -1655,6 +1656,10 @@ int main(int argc, char const * const *argv){ printf("%s", zUsage); return 0; } + if( strcmp(z, "--version")==0 ){ + printf("%s\n", sqlite3_sourceid()); + return 0; + } if( z[0]=='-' ){ if( strcmp(z,"--commcheck")==0 ){ /* DEBUG ONLY */ /* Run a communication check with the remote side. Do not attempt