database/postgres
1
0
Fork 0
mirror of https://github.com/postgres/postgres.git synced 2025-08-31 17:02:12 +03:00
Code Activity

Report progress of CREATE INDEX operations

Browse Source 
This uses the progress reporting infrastructure added by c16dc1aca5,
adding support for CREATE INDEX and CREATE INDEX CONCURRENTLY.

There are two pieces to this: one is index-AM-agnostic, and the other is
AM-specific.  The latter is fairly elaborate for btrees, including
reportage for parallel index builds and the separate phases that btree
index creation uses; other index AMs, which are much simpler in their
building procedures, have simplistic reporting only, but that seems
sufficient, at least for non-concurrent builds.

The index-AM-agnostic part is fairly complete, providing insight into
the CONCURRENTLY wait phases as well as block-based progress during the
index validation table scan.  (The index validation index scan requires
patching each AM, which has not been included here.)

Reviewers: Rahila Syed, Pavan Deolasee, Tatsuro Yamada
Discussion: https://postgr.es/m/20181220220022.mg63bhk26zdpvmcj@alvherre.pgsql
This commit is contained in:
Alvaro Herrera
2019-04-02 15:18:08 -03:00
parent 4d0e994eed
commit ab0dfc961b
37 changed files with 768 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
src/backend/access/nbtree/nbtree.c
View File

@@ -22,6 +22,7 @@
#include "access/nbtxlog.h"
#include "access/relscan.h"
#include "access/xlog.h"
#include "commands/progress.h"
#include "commands/vacuum.h"
#include "miscadmin.h"
#include "nodes/execnodes.h"
@@ -133,6 +134,7 @@ bthandler(PG_FUNCTION_ARGS)
amroutine->amcostestimate = btcostestimate;
amroutine->amoptions = btoptions;
amroutine->amproperty = btproperty;
amroutine->ambuildphasename = btbuildphasename;
amroutine->amvalidate = btvalidate;
amroutine->ambeginscan = btbeginscan;
amroutine->amrescan = btrescan;
@@ -1021,6 +1023,10 @@ btvacuumscan(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
if (needLock)
UnlockRelationForExtension(rel, ExclusiveLock);
if (info->report_progress)
pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_TOTAL,
num_pages);
/* Quit if we've scanned the whole relation */
if (blkno >= num_pages)
break;
@@ -1028,6 +1034,9 @@ btvacuumscan(IndexVacuumInfo *info, IndexBulkDeleteResult *stats,
for (; blkno < num_pages; blkno++)
{
btvacuumpage(&vstate, blkno, blkno);
if (info->report_progress)
pgstat_progress_update_param(PROGRESS_SCAN_BLOCKS_DONE,
blkno);
}
}

56
src/backend/access/nbtree/nbtsort.c
View File

@@ -66,6 +66,7 @@
#include "access/xlog.h"
#include "access/xloginsert.h"
#include "catalog/index.h"
#include "commands/progress.h"
#include "miscadmin.h"
#include "pgstat.h"
#include "storage/smgr.h"
@@ -298,7 +299,8 @@ static double _bt_parallel_heapscan(BTBuildState *buildstate,
static void _bt_leader_participate_as_worker(BTBuildState *buildstate);
static void _bt_parallel_scan_and_sort(BTSpool *btspool, BTSpool *btspool2,
BTShared *btshared, Sharedsort *sharedsort,
Sharedsort *sharedsort2, int sortmem);
Sharedsort *sharedsort2, int sortmem,
bool progress);
/*
@@ -394,6 +396,10 @@ _bt_spools_heapscan(Relation heap, Relation index, BTBuildState *buildstate,
/* Save as primary spool */
buildstate->spool = btspool;
/* Report table scan phase started */
pgstat_progress_update_param(PROGRESS_CREATEIDX_SUBPHASE,
PROGRESS_BTREE_PHASE_INDEXBUILD_TABLESCAN);
/* Attempt to launch parallel worker scan when required */
if (indexInfo->ii_ParallelWorkers > 0)
_bt_begin_parallel(buildstate, indexInfo->ii_Concurrent,
@@ -480,13 +486,31 @@ _bt_spools_heapscan(Relation heap, Relation index, BTBuildState *buildstate,
/* Fill spool using either serial or parallel heap scan */
if (!buildstate->btleader)
reltuples = table_index_build_scan(heap, index, indexInfo, true,
reltuples = table_index_build_scan(heap, index, indexInfo, true, true,
_bt_build_callback, (void *) buildstate,
NULL);
else
reltuples = _bt_parallel_heapscan(buildstate,
&indexInfo->ii_BrokenHotChain);
/*
* Set the progress target for the next phase. Reset the block number
* values set by table_index_build_scan
*/
{
const int index[] = {
PROGRESS_CREATEIDX_TUPLES_TOTAL,
PROGRESS_SCAN_BLOCKS_TOTAL,
PROGRESS_SCAN_BLOCKS_DONE
};
const int64 val[] = {
buildstate->indtuples,
0, 0
};
pgstat_progress_update_multi_param(3, index, val);
}
/* okay, all heap tuples are spooled */
if (buildstate->spool2 && !buildstate->havedead)
{
@@ -535,9 +559,15 @@ _bt_leafbuild(BTSpool *btspool, BTSpool *btspool2)
}
#endif /* BTREE_BUILD_STATS */
pgstat_progress_update_param(PROGRESS_CREATEIDX_SUBPHASE,
PROGRESS_BTREE_PHASE_PERFORMSORT_1);
tuplesort_performsort(btspool->sortstate);
if (btspool2)
{
pgstat_progress_update_param(PROGRESS_CREATEIDX_SUBPHASE,
PROGRESS_BTREE_PHASE_PERFORMSORT_2);
tuplesort_performsort(btspool2->sortstate);
}
wstate.heap = btspool->heap;
wstate.index = btspool->index;
@@ -554,6 +584,8 @@ _bt_leafbuild(BTSpool *btspool, BTSpool *btspool2)
wstate.btws_pages_written = 0;
wstate.btws_zeropage = NULL; /* until needed */
pgstat_progress_update_param(PROGRESS_CREATEIDX_SUBPHASE,
PROGRESS_BTREE_PHASE_LEAF_LOAD);
_bt_load(&wstate, btspool, btspool2);
}
@@ -1098,6 +1130,7 @@ _bt_load(BTWriteState *wstate, BTSpool *btspool, BTSpool *btspool2)
int i,
keysz = IndexRelationGetNumberOfKeyAttributes(wstate->index);
SortSupport sortKeys;
long tuples_done = 0;
if (merge)
{
@@ -1202,6 +1235,10 @@ _bt_load(BTWriteState *wstate, BTSpool *btspool, BTSpool *btspool2)
_bt_buildadd(wstate, state, itup2);
itup2 = tuplesort_getindextuple(btspool2->sortstate, true);
}
/* Report progress */
pgstat_progress_update_param(PROGRESS_CREATEIDX_TUPLES_DONE,
++tuples_done);
}
pfree(sortKeys);
}
@@ -1216,6 +1253,10 @@ _bt_load(BTWriteState *wstate, BTSpool *btspool, BTSpool *btspool2)
state = _bt_pagestate(wstate, 0);
_bt_buildadd(wstate, state, itup);
/* Report progress */
pgstat_progress_update_param(PROGRESS_CREATEIDX_TUPLES_DONE,
++tuples_done);
}
}
@@ -1528,7 +1569,7 @@ _bt_leader_participate_as_worker(BTBuildState *buildstate)
/* Perform work common to all participants */
_bt_parallel_scan_and_sort(leaderworker, leaderworker2, btleader->btshared,
btleader->sharedsort, btleader->sharedsort2,
sortmem);
sortmem, true);
#ifdef BTREE_BUILD_STATS
if (log_btree_build_stats)
@@ -1619,7 +1660,7 @@ _bt_parallel_build_main(dsm_segment *seg, shm_toc *toc)
/* Perform sorting of spool, and possibly a spool2 */
sortmem = maintenance_work_mem / btshared->scantuplesortstates;
_bt_parallel_scan_and_sort(btspool, btspool2, btshared, sharedsort,
sharedsort2, sortmem);
sharedsort2, sortmem, false);
#ifdef BTREE_BUILD_STATS
if (log_btree_build_stats)
@@ -1648,7 +1689,7 @@ _bt_parallel_build_main(dsm_segment *seg, shm_toc *toc)
static void
_bt_parallel_scan_and_sort(BTSpool *btspool, BTSpool *btspool2,
BTShared *btshared, Sharedsort *sharedsort,
Sharedsort *sharedsort2, int sortmem)
Sharedsort *sharedsort2, int sortmem, bool progress)
{
SortCoordinate coordinate;
BTBuildState buildstate;
@@ -1705,9 +1746,10 @@ _bt_parallel_scan_and_sort(BTSpool *btspool, BTSpool *btspool2,
/* Join parallel scan */
indexInfo = BuildIndexInfo(btspool->index);
indexInfo->ii_Concurrent = btshared->isconcurrent;
scan = table_beginscan_parallel(btspool->heap, ParallelTableScanFromBTShared(btshared));
scan = table_beginscan_parallel(btspool->heap,
ParallelTableScanFromBTShared(btshared));
reltuples = table_index_build_scan(btspool->heap, btspool->index, indexInfo,
true, _bt_build_callback,
true, progress, _bt_build_callback,
(void *) &buildstate, scan);
/*

24
src/backend/access/nbtree/nbtutils.c
View File

@@ -20,6 +20,7 @@
#include "access/nbtree.h"
#include "access/reloptions.h"
#include "access/relscan.h"
#include "commands/progress.h"
#include "miscadmin.h"
#include "utils/array.h"
#include "utils/datum.h"
@@ -2051,6 +2052,29 @@ btproperty(Oid index_oid, int attno,
}
}
/*
* btbuildphasename() -- Return name of index build phase.
*/
char *
btbuildphasename(int64 phasenum)
{
switch (phasenum)
{
case PROGRESS_CREATEIDX_SUBPHASE_INITIALIZE:
return "initializing";
case PROGRESS_BTREE_PHASE_INDEXBUILD_TABLESCAN:
return "scanning table";
case PROGRESS_BTREE_PHASE_PERFORMSORT_1:
return "sorting live tuples";
case PROGRESS_BTREE_PHASE_PERFORMSORT_2:
return "sorting dead tuples";
case PROGRESS_BTREE_PHASE_LEAF_LOAD:
return "loading tuples in tree";
default:
return NULL;
}
}
/*
* _bt_truncate() -- create tuple without unneeded suffix attributes.
*