From a3d80139d89019820455aea196ade7c814317ece Mon Sep 17 00:00:00 2001 From: amit Date: Mon, 7 Mar 2016 14:38:34 +0900 Subject: [PATCH 2/2] Implement progress reporting for VACUUM command. This basically utilizes the pgstat_progress* API to report a handful of paramters to indicate its progress. lazy_vacuum_rel() and lazy_scan_heap() have been altered to report command start, command target table, and the following parameters: processing phase, number of heap blocks, number of index blocks (all indexes), current block number in the main scan loop (whenever changes), index blocks vacuumed (once per finished index vacuum), and number of index vacuum passes (every time when all indexes are vacuumed). Following processing phases are identified and reported whenever it changes: 'scanning heap', 'vacuuming indexes', 'vacuuming heap', 'cleanup', 'done'. The last value is really a misnomer but maybe clearer when someone is staring at the progress view being polled. A view named pg_stat_vacuum_progress has been added that shows these values. --- doc/src/sgml/monitoring.sgml | 114 ++++++++++++++++++++++++++++++++++ src/backend/catalog/system_views.sql | 23 +++++++ src/backend/commands/vacuumlazy.c | 82 ++++++++++++++++++++++++- src/include/pgstat.h | 1 + src/test/regress/expected/rules.out | 20 ++++++ 5 files changed, 239 insertions(+), 1 deletions(-) diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 45d9ed7..e4361ad 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -507,6 +507,13 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser yet included in pg_stat_user_functions). + + pg_stat_vacuum_progresspg_stat_vacuum_progress + One row for each backend (including autovacuum worker processes) running + VACUUM, showing current progress in terms of heap pages it + has finished processing. Note that the backends running + VACUUM FULL are not included. + @@ -1822,6 +1829,113 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser controls exactly which functions are tracked. + + + + + + Column + Type + Description + + + + + + pid + integer + Process ID of backend + + + relid + oid + OID of a table + + + processing_phase + integer + Current phase of vacuum. + Possible values are: + + + + scanning heap + + + + + vacuuming indexes + + + + + vacuuming heap + + + + + cleanup + + + + + done + + + + + + + total_heap_blks + integer + Total number of heap blocks in the table + + + current_heap_blkno + integer + Current heap block being processed + + + total_index_blks + integer + Total number of index blocks to be processed + + + index_blks_done + integer + Number of index blocks processed + + + index_scan_count + integer + Number of times index scans has been performed so far + + + percent_done + numeric + + Amount of work finished in percent in terms of table blocks processed + + + + + + + + The pg_stat_vacuum_progress view will contain + one row for each backend (including autovacuum worker processes), showing + progress of VACUUM running in it. Note that the backends + running VACUUM FULL are not shown. + + + + When interpreting the value of the percent_done column, also + note the value of processing_phase. It's possible for the + former to be 100.00, while the VACUUM still + has not returned. In that case, wait for the latter to turn to the value + done. + + diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index abf9a70..156d379 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -971,3 +971,26 @@ RETURNS jsonb LANGUAGE INTERNAL STRICT IMMUTABLE AS 'jsonb_set'; + +CREATE VIEW pg_stat_vacuum_progress AS + SELECT + S.pid AS pid, + S.param1 AS relid, + CASE S.param2 + WHEN 1 THEN 'Scanning Heap' + WHEN 2 THEN 'Vacuuming Index' + WHEN 3 THEN 'Vacuuming Heap' + WHEN 4 THEN 'Cleanup' + WHEN 5 THEN 'Done' + ELSE 'Unknown' + END AS processing_phase, + S.param3 AS total_heap_blks, + S.param4 AS current_heap_blkno, + S.param5 AS total_index_blks, + S.param6 AS index_blks_done, + S.param7 AS index_scan_count, + CASE S.param3 + WHEN 0 THEN 100::numeric(5, 2) + ELSE ((S.param4 + 1)::numeric / S.param3 * 100)::numeric(5, 2) + END AS percent_done + FROM pg_stat_get_progress_info(1) AS S; diff --git a/src/backend/commands/vacuumlazy.c b/src/backend/commands/vacuumlazy.c index 8f7b248..73ccd53 100644 --- a/src/backend/commands/vacuumlazy.c +++ b/src/backend/commands/vacuumlazy.c @@ -97,6 +97,29 @@ */ #define SKIP_PAGES_THRESHOLD ((BlockNumber) 32) +/* + * Follwing progress parameters for lazy vacuum are reported to pgstat + */ +#define PROG_PAR_VAC_RELID 0 +#define PROG_PAR_VAC_PHASE_ID 1 +#define PROG_PAR_VAC_HEAP_BLKS 2 +#define PROG_PAR_VAC_CUR_HEAP_BLK 3 +#define PROG_PAR_VAC_IDX_BLKS 4 +#define PROG_PAR_VAC_IDX_BLKS_DONE 5 +#define PROG_PAR_VAC_N_IDX_SCAN 6 + +/* + * Following distinct phases of lazy vacuum are identified. Although #1, #2 + * and #3 run in a cyclical manner due to possibly limited memory to work + * with, wherein #1 is periodically interrupted to run #2 followed by #3 + * and back, until all the blocks of the relations have been covered by #1. + */ +#define LV_PHASE_SCAN_HEAP 1 +#define LV_PHASE_VACUUM_INDEX 2 +#define LV_PHASE_VACUUM_HEAP 3 +#define LV_PHASE_CLEANUP 4 +#define LV_PHASE_DONE 5 + typedef struct LVRelStats { /* hasindex = true means two-pass strategy; false means one-pass */ @@ -195,6 +218,10 @@ lazy_vacuum_rel(Relation onerel, int options, VacuumParams *params, Assert(params != NULL); + /* initialize pgstat progress info */ + pgstat_progress_start_command(COMMAND_LAZY_VACUUM); + pgstat_progress_update_param(PROG_PAR_VAC_RELID, RelationGetRelid(onerel)); + /* measure elapsed time iff autovacuum logging requires it */ if (IsAutoVacuumWorkerProcess() && params->log_min_duration >= 0) { @@ -270,6 +297,9 @@ lazy_vacuum_rel(Relation onerel, int options, VacuumParams *params, /* Vacuum the Free Space Map */ FreeSpaceMapVacuum(onerel); + /* We're done doing any heavy handling, so report */ + pgstat_progress_update_param(PROG_PAR_VAC_PHASE_ID, LV_PHASE_DONE); + /* * Update statistics in pg_class. * @@ -433,7 +463,9 @@ lazy_scan_heap(Relation onerel, LVRelStats *vacrelstats, Relation *Irel, int nindexes, bool scan_all) { BlockNumber nblocks, - blkno; + blkno, + total_index_blks, + *current_index_blks; HeapTupleData tuple; char *relname; BlockNumber empty_pages, @@ -474,6 +506,24 @@ lazy_scan_heap(Relation onerel, LVRelStats *vacrelstats, lazy_space_alloc(vacrelstats, nblocks); frozen = palloc(sizeof(xl_heap_freeze_tuple) * MaxHeapTuplesPerPage); + /* about to begin heap scan */ + pgstat_progress_update_param(PROG_PAR_VAC_PHASE_ID, LV_PHASE_SCAN_HEAP); + + /* total_heap_blks */ + pgstat_progress_update_param(PROG_PAR_VAC_HEAP_BLKS, nblocks); + + /* total_index_blks */ + current_index_blks = (BlockNumber *) palloc(nindexes * sizeof(BlockNumber)); + total_index_blks = 0; + for (i = 0; i < nindexes; i++) + { + BlockNumber nblocks = RelationGetNumberOfBlocks(Irel[i]); + + current_index_blks[i] = nblocks; + total_index_blks += nblocks; + } + pgstat_progress_update_param(PROG_PAR_VAC_IDX_BLKS, total_index_blks); + /* * We want to skip pages that don't require vacuuming according to the * visibility map, but only when we can skip at least SKIP_PAGES_THRESHOLD @@ -581,6 +631,9 @@ lazy_scan_heap(Relation onerel, LVRelStats *vacrelstats, vacuum_delay_point(); + /* current_heap_blkno: 0..nblocks-1 */ + pgstat_progress_update_param(PROG_PAR_VAC_CUR_HEAP_BLK, blkno); + /* * If we are close to overrunning the available space for dead-tuple * TIDs, pause and do a cycle of vacuuming before we tackle this page. @@ -604,11 +657,22 @@ lazy_scan_heap(Relation onerel, LVRelStats *vacrelstats, vacuum_log_cleanup_info(onerel, vacrelstats); /* Remove index entries */ + pgstat_progress_update_param(PROG_PAR_VAC_PHASE_ID, LV_PHASE_VACUUM_INDEX); for (i = 0; i < nindexes; i++) + { lazy_vacuum_index(Irel[i], &indstats[i], vacrelstats); + + pgstat_progress_update_param(PROG_PAR_VAC_IDX_BLKS_DONE, + current_index_blks[i]); + } + + pgstat_progress_update_param(PROG_PAR_VAC_N_IDX_SCAN, + vacrelstats->num_index_scans+1); + /* Remove tuples from heap */ + pgstat_progress_update_param(PROG_PAR_VAC_PHASE_ID, LV_PHASE_VACUUM_HEAP); lazy_vacuum_heap(onerel, vacrelstats); /* @@ -618,6 +682,10 @@ lazy_scan_heap(Relation onerel, LVRelStats *vacrelstats, */ vacrelstats->num_dead_tuples = 0; vacrelstats->num_index_scans++; + + /* go back to scanning the heap */ + pgstat_progress_update_param(PROG_PAR_VAC_PHASE_ID, + LV_PHASE_SCAN_HEAP); } /* @@ -1153,17 +1221,29 @@ lazy_scan_heap(Relation onerel, LVRelStats *vacrelstats, /* Log cleanup info before we touch indexes */ vacuum_log_cleanup_info(onerel, vacrelstats); + pgstat_progress_update_param(PROG_PAR_VAC_PHASE_ID, LV_PHASE_VACUUM_INDEX); + /* Remove index entries */ for (i = 0; i < nindexes; i++) + { lazy_vacuum_index(Irel[i], &indstats[i], vacrelstats); + + pgstat_progress_update_param(PROG_PAR_VAC_IDX_BLKS_DONE, + current_index_blks[i]); + } + pgstat_progress_update_param(PROG_PAR_VAC_N_IDX_SCAN, + vacrelstats->num_index_scans + 1); + /* Remove tuples from heap */ + pgstat_progress_update_param(PROG_PAR_VAC_PHASE_ID, LV_PHASE_VACUUM_HEAP); lazy_vacuum_heap(onerel, vacrelstats); vacrelstats->num_index_scans++; } /* Do post-vacuum cleanup and statistics update for each index */ + pgstat_progress_update_param(PROG_PAR_VAC_PHASE_ID, LV_PHASE_CLEANUP); for (i = 0; i < nindexes; i++) lazy_cleanup_index(Irel[i], indstats[i], vacrelstats); diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 17fae7d..eca451c 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -702,6 +702,7 @@ typedef enum BackendState typedef enum BackendCommandType { COMMAND_INVALID = 0, + COMMAND_LAZY_VACUUM } BackendCommandType; #define N_PROGRESS_PARAM 10 diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index 81bc5c9..1d14272 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -1851,6 +1851,26 @@ pg_stat_user_tables| SELECT pg_stat_all_tables.relid, pg_stat_all_tables.autoanalyze_count FROM pg_stat_all_tables WHERE ((pg_stat_all_tables.schemaname <> ALL (ARRAY['pg_catalog'::name, 'information_schema'::name])) AND (pg_stat_all_tables.schemaname !~ '^pg_toast'::text)); +pg_stat_vacuum_progress| SELECT s.pid, + s.param1 AS relid, + CASE s.param2 + WHEN 1 THEN 'Scanning Heap'::text + WHEN 2 THEN 'Vacuuming Index'::text + WHEN 3 THEN 'Vacuuming Heap'::text + WHEN 4 THEN 'Cleanup'::text + WHEN 5 THEN 'Done'::text + ELSE 'Unknown'::text + END AS processing_phase, + s.param3 AS total_heap_blks, + s.param4 AS current_heap_blkno, + s.param5 AS total_index_blks, + s.param6 AS index_blks_done, + s.param7 AS index_scan_count, + CASE s.param2 + WHEN 0 THEN 0.0 + ELSE (((((s.param3 + 1))::numeric / (s.param2)::numeric) * (100)::numeric))::numeric(5,2) + END AS percent_done + FROM pg_stat_get_progress_info(1) s(pid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10); pg_stat_wal_receiver| SELECT s.pid, s.status, s.receive_start_lsn, -- 1.7.1