From 5a89ddc0132fc4660cabcf1acc0f614d01db2d92 Mon Sep 17 00:00:00 2001 From: Melanie Plageman Date: Fri, 8 Dec 2023 16:30:59 -0500 Subject: [PATCH v2 08/10] Track VM sets by vacuum Earlier commits adding tracking for when the all visible and all frozen bits were cleared in the visibility map. Add tracking for when they are set by vacuum so that we can observe how many frozen pages are unfrozen. --- src/backend/access/heap/vacuumlazy.c | 71 ++++++++++++++++++-- src/backend/utils/activity/pgstat_relation.c | 26 +++++++ src/include/pgstat.h | 19 ++++++ src/tools/pgindent/typedefs.list | 1 + 4 files changed, 111 insertions(+), 6 deletions(-) diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index 6bd64b1599..c37d57d154 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -211,6 +211,9 @@ typedef struct LVRelState int64 recently_dead_tuples; /* # dead, but not yet removable */ int64 missed_dead_tuples; /* # removable, but not removed */ + /* VM updates by vacuum. Used by stats */ + PgStat_VMSet vmsets; + /* * The youngest page we predict will stay unmodified for * target_freeze_duration. We will not opportunistically freeze pages @@ -501,6 +504,7 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, vacrel->relname))); } + vacrel->vmsets = (PgStat_VMSet) {0}; vacrel->frz_threshold_min = pgstat_refresh_frz_stats(RelationGetRelid(rel), rel->rd_rel->relisshared); @@ -613,6 +617,11 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, Max(vacrel->new_live_tuples, 0), vacrel->recently_dead_tuples + vacrel->missed_dead_tuples); + + pgstat_report_heap_vacfrz(RelationGetRelid(rel), + rel->rd_rel->relisshared, + &vacrel->vmsets); + pgstat_progress_end_command(); if (instrument) @@ -1114,6 +1123,7 @@ lazy_scan_heap(LVRelState *vacrel) if (!all_visible_according_to_vm && prunestate.all_visible) { uint8 flags = VISIBILITYMAP_ALL_VISIBLE; + uint8 previous_flags; if (prunestate.all_frozen) { @@ -1136,9 +1146,23 @@ lazy_scan_heap(LVRelState *vacrel) */ PageSetAllVisible(page); MarkBufferDirty(buf); - visibilitymap_set(vacrel->rel, blkno, buf, InvalidXLogRecPtr, - vmbuffer, prunestate.visibility_cutoff_xid, - flags); + previous_flags = visibilitymap_set(vacrel->rel, blkno, buf, InvalidXLogRecPtr, + vmbuffer, prunestate.visibility_cutoff_xid, + flags); + + /* + * If the page isn't empty after vacuuming and we newly marked it + * all visible and all frozen, record that in stats. + */ + if (!PageIsEmpty(page)) + { + if (!(previous_flags & VISIBILITYMAP_ALL_VISIBLE)) + vacrel->vmsets.vis++; + + if (prunestate.all_frozen && + !(previous_flags & VISIBILITYMAP_ALL_FROZEN)) + vacrel->vmsets.vm_freezes++; + } } /* @@ -1190,6 +1214,8 @@ lazy_scan_heap(LVRelState *vacrel) prunestate.all_frozen && !VM_ALL_FROZEN(vacrel->rel, blkno, &vmbuffer)) { + uint8 previous_flags; + /* * Avoid relying on all_visible_according_to_vm as a proxy for the * page-level PD_ALL_VISIBLE bit being set, since it might have @@ -1209,10 +1235,21 @@ lazy_scan_heap(LVRelState *vacrel) * safe for REDO was logged when the page's tuples were frozen. */ Assert(!TransactionIdIsValid(prunestate.visibility_cutoff_xid)); - visibilitymap_set(vacrel->rel, blkno, buf, InvalidXLogRecPtr, + previous_flags = visibilitymap_set(vacrel->rel, blkno, buf, InvalidXLogRecPtr, vmbuffer, InvalidTransactionId, VISIBILITYMAP_ALL_VISIBLE | VISIBILITYMAP_ALL_FROZEN); + + /* + * If the page isn't empty after vacuuming and we newly marked it + * all visible and all frozen, record that in stats. + */ + if (!PageIsEmpty(page)) + { + vacrel->vmsets.vm_freezes++; + if (!(previous_flags & VISIBILITYMAP_ALL_VISIBLE)) + vacrel->vmsets.vis++; + } } /* @@ -1834,6 +1871,10 @@ lazy_scan_prune(LVRelState *vacrel, TransactionId snapshotConflictHorizon; vacrel->frozen_pages++; + fpi_before = pgWalUsage.wal_fpi; + + if (!PageIsEmpty(page)) + vacrel->vmsets.page_freezes++; /* * We can use visibility_cutoff_xid as our cutoff for conflicts @@ -1858,6 +1899,9 @@ lazy_scan_prune(LVRelState *vacrel, heap_freeze_execute_prepared(vacrel->rel, buf, snapshotConflictHorizon, frozen, tuples_frozen); + + if (pgWalUsage.wal_fpi > fpi_before) + vacrel->vmsets.freeze_fpis++; } } else @@ -2610,6 +2654,7 @@ lazy_vacuum_heap_page(LVRelState *vacrel, BlockNumber blkno, Buffer buffer, if (heap_page_is_all_visible(vacrel, buffer, &visibility_cutoff_xid, &all_frozen)) { + uint8 previous_flags; uint8 flags = VISIBILITYMAP_ALL_VISIBLE; if (all_frozen) @@ -2619,8 +2664,22 @@ lazy_vacuum_heap_page(LVRelState *vacrel, BlockNumber blkno, Buffer buffer, } PageSetAllVisible(page); - visibilitymap_set(vacrel->rel, blkno, buffer, InvalidXLogRecPtr, - vmbuffer, visibility_cutoff_xid, flags); + previous_flags = visibilitymap_set(vacrel->rel, blkno, buffer, InvalidXLogRecPtr, + vmbuffer, visibility_cutoff_xid, flags); + + /* + * If the page isn't empty after vacuuming and we newly marked it all + * visible and all frozen, record that in stats. + */ + if (!PageIsEmpty(page)) + { + if (!(previous_flags & VISIBILITYMAP_ALL_VISIBLE)) + vacrel->vmsets.vis++; + + if (all_frozen && + !(previous_flags & VISIBILITYMAP_ALL_FROZEN)) + vacrel->vmsets.vm_freezes++; + } } /* Revert to the previous phase information for error traceback */ diff --git a/src/backend/utils/activity/pgstat_relation.c b/src/backend/utils/activity/pgstat_relation.c index 52cd37f91d..d30ba0530e 100644 --- a/src/backend/utils/activity/pgstat_relation.c +++ b/src/backend/utils/activity/pgstat_relation.c @@ -494,6 +494,32 @@ pgstat_report_vacuum(Oid tableoid, bool shared, pgstat_flush_io(false); } +void +pgstat_report_heap_vacfrz(Oid tableoid, bool shared, PgStat_VMSet *vmsets) +{ + PgStat_EntryRef *entry_ref; + PgStat_StatTabEntry *tabentry; + Oid dboid = (shared ? InvalidOid : MyDatabaseId); + + if (!pgstat_track_counts) + return; + + + /* block acquiring lock for the same reason as pgstat_report_autovac() */ + entry_ref = pgstat_get_entry_ref_locked(PGSTAT_KIND_RELATION, + dboid, tableoid, false); + + tabentry = &((PgStatShared_Relation *) entry_ref->shared_stats)->stats; + + tabentry->vm_set.vis += vmsets->vis; + tabentry->vm_set.page_freezes += vmsets->page_freezes; + tabentry->vm_set.vm_freezes += vmsets->vm_freezes; + tabentry->vm_set.freeze_fpis += vmsets->freeze_fpis; + + pgstat_unlock_entry(entry_ref); +} + + /* * Report that the table was just analyzed and flush IO statistics. * diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 034a596f99..0384afef9d 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -499,6 +499,21 @@ typedef struct PgStat_StatSubEntry TimestampTz stat_reset_timestamp; } PgStat_StatSubEntry; + +typedef struct PgStat_VMSet +{ + /* number of pages set all visible in the VM */ + int64 vis; + /* number of pages newly marked frozen in the visibility map by vacuum */ + int64 vm_freezes; + /* Number of pages with newly frozen tuples */ + int64 page_freezes; + /* number of freeze records emitted by vacuum containing FPIs */ + int64 freeze_fpis; +} PgStat_VMSet; + + + typedef struct PgStat_StatTabEntry { PgStat_Counter numscans; @@ -534,6 +549,8 @@ typedef struct PgStat_StatTabEntry XLogRecPtr target_frz_dur_lsns; /* updated upon VM unset */ PgStat_VMUnset vm_unset; + /* updated during vacuum and used in stats */ + PgStat_VMSet vm_set; } PgStat_StatTabEntry; /* A time and the insert_lsn recorded at that time. */ @@ -744,6 +761,8 @@ extern void pgstat_report_analyze(Relation rel, extern XLogRecPtr pgstat_refresh_frz_stats(Oid tableoid, bool shared); +extern void pgstat_report_heap_vacfrz(Oid tableoid, bool shared, PgStat_VMSet *vmsets); + extern void pgstat_count_vm_unset(Relation relation, XLogRecPtr page_lsn, XLogRecPtr current_lsn, uint8 old_vmbits); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index ff184db48b..c268bc9338 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2087,6 +2087,7 @@ PgStat_SubXactStatus PgStat_TableCounts PgStat_TableStatus PgStat_TableXactStatus +PgStat_VMSet PgStat_VMUnset PgStat_WalStats PgXmlErrorContext -- 2.37.2