From cda185d3a268f17faf85968b036070d45bd030fb Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 16 Nov 2017 20:48:47 +0000 Subject: [PATCH] Ensure that empty space on pages is zeroed before they are written to disk. This helps with compressed databases, and stops valgrind complaining about uninitialized bytes and write(). FossilOrigin-Name: 5ffec5db33137251090d45e2ca7e3e7823e3215ae89822d905923424ceba2ac2 --- ext/lsm1/lsm_sorted.c | 66 +++++++++++++++++++++++++++---------------- manifest | 14 ++++----- manifest.uuid | 2 +- 3 files changed, 50 insertions(+), 32 deletions(-) diff --git a/ext/lsm1/lsm_sorted.c b/ext/lsm1/lsm_sorted.c index f479f4ce8c..9eff2d08cc 100644 --- a/ext/lsm1/lsm_sorted.c +++ b/ext/lsm1/lsm_sorted.c @@ -92,7 +92,7 @@ #define SEGMENT_POINTER_OFFSET(pgsz) ((pgsz) - 2 - 2 - 8) #define SEGMENT_CELLPTR_OFFSET(pgsz, iCell) ((pgsz) - 2 - 2 - 8 - 2 - (iCell)*2) -#define SEGMENT_EOF(pgsz, nEntry) SEGMENT_CELLPTR_OFFSET(pgsz, nEntry) +#define SEGMENT_EOF(pgsz, nEntry) SEGMENT_CELLPTR_OFFSET(pgsz, nEntry-1) #define SEGMENT_BTREE_FLAG 0x0001 #define PGFTR_SKIP_NEXT_FLAG 0x0002 @@ -3999,6 +3999,11 @@ static int mergeWorkerWrite( ** marked read-only, advance to the next page of the output run. */ iOff = pMerge->iOutputOff; if( iOff<0 || pPg==0 || iOff+nHdr > SEGMENT_EOF(nData, nRec+1) ){ + if( iOff>=0 && pPg ){ + /* Zero any free space on the page */ + assert( aData ); + memset(&aData[iOff], 0, SEGMENT_EOF(nData, nRec)-iOff); + } iFPtr = (int)*pMW->pCsr->pPrevMergePtr; iRPtr = iPtr - iFPtr; iOff = 0; @@ -4069,36 +4074,49 @@ static void mergeWorkerShutdown(MergeWorker *pMW, int *pRc){ /* Unless the merge has finished, save the cursor position in the ** Merge.aInput[] array. See function mergeWorkerInit() for the ** code to restore a cursor position based on aInput[]. */ - if( rc==LSM_OK && pCsr && lsmMCursorValid(pCsr) ){ + if( rc==LSM_OK && pCsr ){ Merge *pMerge = pMW->pLevel->pMerge; - int bBtree = (pCsr->pBtCsr!=0); - int iPtr; + if( lsmMCursorValid(pCsr) ){ + int bBtree = (pCsr->pBtCsr!=0); + int iPtr; - /* pMerge->nInput==0 indicates that this is a FlushTree() operation. */ - assert( pMerge->nInput==0 || pMW->pLevel->nRight>0 ); - assert( pMerge->nInput==0 || pMerge->nInput==(pCsr->nPtr+bBtree) ); + /* pMerge->nInput==0 indicates that this is a FlushTree() operation. */ + assert( pMerge->nInput==0 || pMW->pLevel->nRight>0 ); + assert( pMerge->nInput==0 || pMerge->nInput==(pCsr->nPtr+bBtree) ); - for(i=0; i<(pMerge->nInput-bBtree); i++){ - SegmentPtr *pPtr = &pCsr->aPtr[i]; - if( pPtr->pPg ){ - pMerge->aInput[i].iPg = lsmFsPageNumber(pPtr->pPg); - pMerge->aInput[i].iCell = pPtr->iCell; + for(i=0; i<(pMerge->nInput-bBtree); i++){ + SegmentPtr *pPtr = &pCsr->aPtr[i]; + if( pPtr->pPg ){ + pMerge->aInput[i].iPg = lsmFsPageNumber(pPtr->pPg); + pMerge->aInput[i].iCell = pPtr->iCell; + }else{ + pMerge->aInput[i].iPg = 0; + pMerge->aInput[i].iCell = 0; + } + } + if( bBtree && pMerge->nInput ){ + assert( i==pCsr->nPtr ); + btreeCursorPosition(pCsr->pBtCsr, &pMerge->aInput[i]); + } + + /* Store the location of the split-key */ + iPtr = pCsr->aTree[1] - CURSOR_DATA_SEGMENT; + if( iPtrnPtr ){ + pMerge->splitkey = pMerge->aInput[iPtr]; }else{ - pMerge->aInput[i].iPg = 0; - pMerge->aInput[i].iCell = 0; + btreeCursorSplitkey(pCsr->pBtCsr, &pMerge->splitkey); } } - if( bBtree && pMerge->nInput ){ - assert( i==pCsr->nPtr ); - btreeCursorPosition(pCsr->pBtCsr, &pMerge->aInput[i]); - } - /* Store the location of the split-key */ - iPtr = pCsr->aTree[1] - CURSOR_DATA_SEGMENT; - if( iPtrnPtr ){ - pMerge->splitkey = pMerge->aInput[iPtr]; - }else{ - btreeCursorSplitkey(pCsr->pBtCsr, &pMerge->splitkey); + /* Zero any free space left on the final page. This helps with + ** compression if using a compression hook. And prevents valgrind + ** from complaining about uninitialized byte passed to write(). */ + if( pMW->pPage ){ + int nData; + u8 *aData = fsPageData(pMW->pPage, &nData); + int iOff = pMerge->iOutputOff; + int iEof = SEGMENT_EOF(nData, pageGetNRec(aData, nData)); + memset(&aData[iOff], 0, iEof - iOff); } pMerge->iOutputOff = -1; diff --git a/manifest b/manifest index df822c015e..d601661e29 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\smissing\s"finish_test"\sto\sthe\send\sof\sthe\sstmtvtab1.test\sscript. -D 2017-11-16T19:04:33.787 +C Ensure\sthat\sempty\sspace\son\spages\sis\szeroed\sbefore\sthey\sare\swritten\sto\sdisk.\nThis\shelps\swith\scompressed\sdatabases,\sand\sstops\svalgrind\scomplaining\sabout\nuninitialized\sbytes\sand\swrite(). +D 2017-11-16T20:48:47.980 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b @@ -246,7 +246,7 @@ F ext/lsm1/lsm_main.c 15e73ccdafdd44ddeefc29e332079d88ba8f00c12c797b3c2b63d3171b F ext/lsm1/lsm_mem.c 4c51ea9fa285ee6e35301b33491642d071740a0a F ext/lsm1/lsm_mutex.c 378edf0a2b142b4f7640ee982df06d50b98788ea F ext/lsm1/lsm_shared.c 76adfc1ed9ffebaf92746dde4b370ccc48143ca8b05b563816eadd2aadf1c525 -F ext/lsm1/lsm_sorted.c a04518dfbfff0171fafb152a46e9fe9f45e1edbf3570e4533dd58ddb6567f0c9 +F ext/lsm1/lsm_sorted.c 8f899fb64a4c736ff3c27d5126c7ce181129ddffde947fe5fb657a7a413f470b F ext/lsm1/lsm_str.c 65e361b488c87b10bf3e5c0070b14ffc602cf84f094880bece77bbf6678bca82 F ext/lsm1/lsm_tree.c 682679d7ef2b8b6f2fe77aeb532c8d29695bca671c220b0abac77069de5fb9fb F ext/lsm1/lsm_unix.c 57361bcf5b1a1a028f5d66571ee490e9064d2cfb145a2cc9e5ddade467bb551b @@ -1677,7 +1677,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1ca3b8cce93e83fabe35c22cb726b8ce0883e0448afa3301ad0b53073055fa8d -R 43f6376592b4d26341e49684bbb727c1 -U drh -Z 7e47c726c44355ca56f3a93b39a4aaed +P e0b5c0585e8530bd516b340093a46d8e358a731facb78da22d3208633c852804 +R a54da776e28f1875d13b67b86731a30b +U dan +Z 4bb1ae38bdfadd3dc2e110f3406150e8 diff --git a/manifest.uuid b/manifest.uuid index 4216e3c295..39825a308f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e0b5c0585e8530bd516b340093a46d8e358a731facb78da22d3208633c852804 \ No newline at end of file +5ffec5db33137251090d45e2ca7e3e7823e3215ae89822d905923424ceba2ac2 \ No newline at end of file