From ac5c40ce460ca7fd70a4b57a25435839eaac9aa8 Mon Sep 17 00:00:00 2001 From: Zhang Mingli Date: Thu, 3 Apr 2025 10:17:11 +0800 Subject: [PATCH] Optimize MV invalidation overhead using reference counting. This commit optimizes the materialized view (MV) invalidation mechanism to reduce overhead in OLTP workloads. The AQUMV feature introduced `SetRelativeMatviewAuxStatus()` to invalidate MVs at appropriate times, which GP7 lacks. However, TPC-B testing showed this function adds ~8.72% OLTP overhead due to mandatory metadata operations, even when no MVs are present. To address this, we added `relmvrefcount` to track the number of MVs dependent on a table. When no MVs reference a table, the system now skips `SetRelativeMatviewAuxStatus()` entirely, avoiding unnecessary catalog access. Benchmark results demonstrate significant improvements: under 20 concurrent connections, transactions increased from 373,511 to 404,976, latency decreased from 16.06ms to 14.813ms, and TPS improved from 1,245.3 to 1,350.2. This optimization restores performance parity with GP7 in MV-free scenarios while maintaining correctness for tables with MVs, delivering measurable throughput gains and reduced latency in high-concurrency OLTP workloads. Authored-by: Zhang Mingli avamingli@gmail.com --- src/backend/catalog/gp_matview_aux.c | 40 +++++++++++++++++++ src/backend/catalog/heap.c | 1 + src/backend/catalog/index.c | 1 + src/backend/utils/cache/lsyscache.c | 24 +++++++++++ src/backend/utils/cache/relcache.c | 1 + src/include/catalog/pg_class.h | 3 ++ src/include/utils/lsyscache.h | 1 + src/include/utils/rel.h | 2 + .../parallel_retrieve_cursor/explain.source | 10 ++--- 9 files changed, 78 insertions(+), 5 deletions(-) diff --git a/src/backend/catalog/gp_matview_aux.c b/src/backend/catalog/gp_matview_aux.c index fb728e666e2..190a668883a 100644 --- a/src/backend/catalog/gp_matview_aux.c +++ b/src/backend/catalog/gp_matview_aux.c @@ -58,6 +58,8 @@ static void RemoveMatviewTablesEntries(Oid mvoid); static void SetMatviewAuxStatus_guts(Oid mvoid, char status); +static void addRelationMVRefCount(Oid relid, int32 mvrefcount); + /* * GetViewBaseRelids * Get all base tables's oid of a query tree. @@ -230,6 +232,9 @@ InsertMatviewTablesEntries(Oid mvoid, List *relids) values[Anum_gp_matview_tables_mvoid - 1] = ObjectIdGetDatum(mvoid); tup = heap_form_tuple(RelationGetDescr(mtRel), values, nulls); CatalogTupleInsert(mtRel, tup); + + /* update relation's pg_class entry */ + addRelationMVRefCount(relid, 1); } table_close(mtRel, RowExclusiveLock); @@ -278,6 +283,7 @@ RemoveMatviewTablesEntries(Oid mvoid) Relation mtRel; CatCList *catlist; int i; + Oid relid; mtRel = table_open(GpMatviewTablesId, RowExclusiveLock); @@ -290,6 +296,10 @@ RemoveMatviewTablesEntries(Oid mvoid) if (!HeapTupleIsValid(tuple)) continue; + /* update relation's pg_class entry */ + relid = ((Form_gp_matview_tables) GETSTRUCT(tuple))->relid; + addRelationMVRefCount(relid, -1); + CatalogTupleDelete(mtRel, &tuple->t_self); } @@ -315,6 +325,12 @@ SetRelativeMatviewAuxStatus(Oid relid, char status, char direction) List *base_oids; ListCell *cell; + /* + * Do a quick check if relation has relative materialized views. + */ + if (get_rel_relmvrefcount(relid) <= 0) + return; + mvauxRel = table_open(GpMatviewAuxId, RowExclusiveLock); /* Find all mvoids have relid */ @@ -555,3 +571,27 @@ MatviewIsUpToDate(Oid mvoid) Form_gp_matview_aux auxform = (Form_gp_matview_aux) GETSTRUCT(mvauxtup); return (auxform->datastatus == MV_DATA_STATUS_UP_TO_DATE); } + +static void +addRelationMVRefCount(Oid relid, int32 mvrefcount) +{ + Relation pgrel; + HeapTuple tuple; + + pgrel = table_open(RelationRelationId, RowExclusiveLock); + /* + * Update relation's pg_class entry. + */ + tuple = SearchSysCacheCopy1(RELOID, + ObjectIdGetDatum(relid)); + if (!HeapTupleIsValid(tuple)) + elog(ERROR, "cache lookup failed for relation %u", relid); + + ((Form_pg_class) GETSTRUCT(tuple))->relmvrefcount += mvrefcount; + + CatalogTupleUpdate(pgrel, &tuple->t_self, tuple); + + heap_freetuple(tuple); + + table_close(pgrel, RowExclusiveLock); +} \ No newline at end of file diff --git a/src/backend/catalog/heap.c b/src/backend/catalog/heap.c index 1e03200a735..7e9291e2bc4 100644 --- a/src/backend/catalog/heap.c +++ b/src/backend/catalog/heap.c @@ -1328,6 +1328,7 @@ InsertPgClassTuple(Relation pg_class_desc, values[Anum_pg_class_relminmxid - 1] = MultiXactIdGetDatum(rd_rel->relminmxid); values[Anum_pg_class_relisivm - 1] = BoolGetDatum(rd_rel->relisivm); values[Anum_pg_class_relisdynamic - 1] = BoolGetDatum(rd_rel->relisdynamic); + values[Anum_pg_class_relmvrefcount - 1] = Int32GetDatum(rd_rel->relmvrefcount); if (relacl != (Datum) 0) values[Anum_pg_class_relacl - 1] = relacl; else diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c index 4349b1eb899..d754d97c781 100644 --- a/src/backend/catalog/index.c +++ b/src/backend/catalog/index.c @@ -1074,6 +1074,7 @@ index_create_internal(Relation heapRelation, indexRelation->rd_rel->relispartition = OidIsValid(parentIndexRelid); indexRelation->rd_rel->relisivm = false; indexRelation->rd_rel->relisdynamic = false; + indexRelation->rd_rel->relmvrefcount = 0; /* * store index's pg_class entry diff --git a/src/backend/utils/cache/lsyscache.c b/src/backend/utils/cache/lsyscache.c index e2053385f2d..25cefe8bc91 100644 --- a/src/backend/utils/cache/lsyscache.c +++ b/src/backend/utils/cache/lsyscache.c @@ -2142,6 +2142,30 @@ get_rel_relisdynamic(Oid relid) return false; } +/* + * get_rel_relmvrefcount + * + * Returns the count of materialzed views associated with a given relation. + */ +int32 +get_rel_relmvrefcount(Oid relid) +{ + HeapTuple tp; + + tp = SearchSysCache1(RELOID, ObjectIdGetDatum(relid)); + if (HeapTupleIsValid(tp)) + { + Form_pg_class reltup = (Form_pg_class) GETSTRUCT(tp); + int32 result; + + result = reltup->relmvrefcount; + ReleaseSysCache(tp); + return result; + } + else + return 0; +} + /* * get_rel_tablespace * diff --git a/src/backend/utils/cache/relcache.c b/src/backend/utils/cache/relcache.c index 6f419e4428e..a7ba7f5a4ca 100644 --- a/src/backend/utils/cache/relcache.c +++ b/src/backend/utils/cache/relcache.c @@ -1987,6 +1987,7 @@ formrdesc(const char *relationName, Oid relationReltype, relation->rd_rel->relisivm = false; /* ... and they're always not dynamic, too */ relation->rd_rel->relisdynamic = false; + relation->rd_rel->relmvrefcount = 0; relation->rd_rel->relreplident = REPLICA_IDENTITY_NOTHING; relation->rd_rel->relpages = 0; diff --git a/src/include/catalog/pg_class.h b/src/include/catalog/pg_class.h index 6b29f0a73f2..5d60f3fa336 100644 --- a/src/include/catalog/pg_class.h +++ b/src/include/catalog/pg_class.h @@ -125,6 +125,9 @@ CATALOG(pg_class,1259,RelationRelationId) BKI_BOOTSTRAP BKI_ROWTYPE_OID(83,Relat /* is relation a dynamic table? */ bool relisdynamic BKI_DEFAULT(f); + /* count of materialized views referred to the relation */ + int32 relmvrefcount BKI_DEFAULT(0); + /* link to original rel during table rewrite; otherwise 0 */ Oid relrewrite BKI_DEFAULT(0) BKI_LOOKUP_OPT(pg_class); diff --git a/src/include/utils/lsyscache.h b/src/include/utils/lsyscache.h index 0822b1ae499..51224853cef 100644 --- a/src/include/utils/lsyscache.h +++ b/src/include/utils/lsyscache.h @@ -173,6 +173,7 @@ extern char get_rel_relkind(Oid relid); extern bool get_rel_relispartition(Oid relid); extern bool get_rel_relisivm(Oid relid); extern bool get_rel_relisdynamic(Oid relid); +extern int32 get_rel_relmvrefcount(Oid relid); extern Oid get_rel_tablespace(Oid relid); extern char get_rel_persistence(Oid relid); extern Oid get_transform_fromsql(Oid typid, Oid langid, List *trftypes); diff --git a/src/include/utils/rel.h b/src/include/utils/rel.h index 12b89808f73..d0b8c5cb473 100644 --- a/src/include/utils/rel.h +++ b/src/include/utils/rel.h @@ -783,6 +783,8 @@ typedef struct ViewOptions #define RelationIsIVM(relation) ((relation)->rd_rel->relisivm) +#define RelationHasRelativeMV(relation) (((relation)->rd_rel->relmvrefcount) > 0) + /* * RelationIsAccessibleInLogicalDecoding * True if we need to log enough information to have access via diff --git a/src/test/isolation2/output/parallel_retrieve_cursor/explain.source b/src/test/isolation2/output/parallel_retrieve_cursor/explain.source index 3b4fe8a62fe..e26fe7bd9fa 100644 --- a/src/test/isolation2/output/parallel_retrieve_cursor/explain.source +++ b/src/test/isolation2/output/parallel_retrieve_cursor/explain.source @@ -113,7 +113,7 @@ EXPLAIN (VERBOSE, COSTS false) DECLARE c1 CURSOR FOR SELECT * FROM pg_class; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ Seq Scan on pg_catalog.pg_class - Output: oid, relname, relnamespace, reltype, reloftype, relowner, relam, relfilenode, reltablespace, relpages, reltuples, relallvisible, reltoastrelid, relhasindex, relisshared, relpersistence, relkind, relnatts, relchecks, relhasrules, relhastriggers, relhassubclass, relrowsecurity, relforcerowsecurity, relispopulated, relreplident, relispartition, relisivm, relisdynamic, relrewrite, relfrozenxid, relminmxid, relacl, reloptions, relpartbound + Output: oid, relname, relnamespace, reltype, reloftype, relowner, relam, relfilenode, reltablespace, relpages, reltuples, relallvisible, reltoastrelid, relhasindex, relisshared, relpersistence, relkind, relnatts, relchecks, relhasrules, relhastriggers, relhassubclass, relrowsecurity, relforcerowsecurity, relispopulated, relreplident, relispartition, relisivm, relisdynamic, relmvrefcount, relrewrite, relfrozenxid, relminmxid, relacl, reloptions, relpartbound Settings: enable_incremental_sort = 'on' Optimizer: Postgres query optimizer (4 rows) @@ -123,7 +123,7 @@ EXPLAIN (VERBOSE, COSTS false) DECLARE c1 PARALLEL RETRIEVE CURSOR FOR SELECT * QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ Seq Scan on pg_catalog.pg_class - Output: oid, relname, relnamespace, reltype, reloftype, relowner, relam, relfilenode, reltablespace, relpages, reltuples, relallvisible, reltoastrelid, relhasindex, relisshared, relpersistence, relkind, relnatts, relchecks, relhasrules, relhastriggers, relhassubclass, relrowsecurity, relforcerowsecurity, relispopulated, relreplident, relispartition, relisivm, relisdynamic, relrewrite, relfrozenxid, relminmxid, relacl, reloptions, relpartbound + Output: oid, relname, relnamespace, reltype, reloftype, relowner, relam, relfilenode, reltablespace, relpages, reltuples, relallvisible, reltoastrelid, relhasindex, relisshared, relpersistence, relkind, relnatts, relchecks, relhasrules, relhastriggers, relhassubclass, relrowsecurity, relforcerowsecurity, relispopulated, relreplident, relispartition, relisivm, relisdynamic, relmvrefcount, relrewrite, relfrozenxid, relminmxid, relacl, reloptions, relpartbound Settings: enable_incremental_sort = 'on' Endpoint: "on coordinator" Optimizer: Postgres query optimizer @@ -132,7 +132,7 @@ EXPLAIN (VERBOSE, COSTS false) DECLARE c1 PARALLEL RETRIEVE CURSOR FOR SELECT * QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ Index Scan using pg_class_relname_nsp_index on pg_catalog.pg_class - Output: oid, relname, relnamespace, reltype, reloftype, relowner, relam, relfilenode, reltablespace, relpages, reltuples, relallvisible, reltoastrelid, relhasindex, relisshared, relpersistence, relkind, relnatts, relchecks, relhasrules, relhastriggers, relhassubclass, relrowsecurity, relforcerowsecurity, relispopulated, relreplident, relispartition, relisivm, relisdynamic, relrewrite, relfrozenxid, relminmxid, relacl, reloptions, relpartbound + Output: oid, relname, relnamespace, reltype, reloftype, relowner, relam, relfilenode, reltablespace, relpages, reltuples, relallvisible, reltoastrelid, relhasindex, relisshared, relpersistence, relkind, relnatts, relchecks, relhasrules, relhastriggers, relhassubclass, relrowsecurity, relforcerowsecurity, relispopulated, relreplident, relispartition, relisivm, relisdynamic, relmvrefcount, relrewrite, relfrozenxid, relminmxid, relacl, reloptions, relpartbound Settings: enable_incremental_sort = 'on' Endpoint: "on coordinator" Optimizer: Postgres query optimizer @@ -141,7 +141,7 @@ EXPLAIN (VERBOSE, COSTS false) DECLARE c1 PARALLEL RETRIEVE CURSOR FOR SELECT * QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ Seq Scan on pg_catalog.pg_class - Output: oid, relname, relnamespace, reltype, reloftype, relowner, relam, relfilenode, reltablespace, relpages, reltuples, relallvisible, reltoastrelid, relhasindex, relisshared, relpersistence, relkind, relnatts, relchecks, relhasrules, relhastriggers, relhassubclass, relrowsecurity, relforcerowsecurity, relispopulated, relreplident, relispartition, relisivm, relisdynamic, relrewrite, relfrozenxid, relminmxid, relacl, reloptions, relpartbound + Output: oid, relname, relnamespace, reltype, reloftype, relowner, relam, relfilenode, reltablespace, relpages, reltuples, relallvisible, reltoastrelid, relhasindex, relisshared, relpersistence, relkind, relnatts, relchecks, relhasrules, relhastriggers, relhassubclass, relrowsecurity, relforcerowsecurity, relispopulated, relreplident, relispartition, relisivm, relisdynamic, relmvrefcount, relrewrite, relfrozenxid, relminmxid, relacl, reloptions, relpartbound Filter: (pg_class.gp_segment_id = 1) Settings: enable_incremental_sort = 'on' Endpoint: "on coordinator" @@ -151,7 +151,7 @@ EXPLAIN (VERBOSE, COSTS false) DECLARE c1 PARALLEL RETRIEVE CURSOR FOR SELECT * QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ Seq Scan on pg_catalog.pg_class - Output: oid, relname, relnamespace, reltype, reloftype, relowner, relam, relfilenode, reltablespace, relpages, reltuples, relallvisible, reltoastrelid, relhasindex, relisshared, relpersistence, relkind, relnatts, relchecks, relhasrules, relhastriggers, relhassubclass, relrowsecurity, relforcerowsecurity, relispopulated, relreplident, relispartition, relisivm, relisdynamic, relrewrite, relfrozenxid, relminmxid, relacl, reloptions, relpartbound + Output: oid, relname, relnamespace, reltype, reloftype, relowner, relam, relfilenode, reltablespace, relpages, reltuples, relallvisible, reltoastrelid, relhasindex, relisshared, relpersistence, relkind, relnatts, relchecks, relhasrules, relhastriggers, relhassubclass, relrowsecurity, relforcerowsecurity, relispopulated, relreplident, relispartition, relisivm, relisdynamic, relmvrefcount, relrewrite, relfrozenxid, relminmxid, relacl, reloptions, relpartbound Filter: ((pg_class.gp_segment_id = 1) OR (pg_class.gp_segment_id = 2)) Settings: enable_incremental_sort = 'on' Endpoint: "on coordinator"