From 83a01e0f99ddeeb96efaa2b8a66fab55d1eecf57 Mon Sep 17 00:00:00 2001 From: roseduan Date: Fri, 28 Jul 2023 10:33:05 +0800 Subject: [PATCH 01/15] init support warehouse --- gpcontrib/gp_inject_fault/gp_inject_fault.c | 10 ++- src/backend/catalog/catalog.c | 4 +- src/backend/cdb/cdbutil.c | 48 ++++++++++-- src/backend/fts/fts.c | 12 ++- src/backend/parser/gram.y | 46 +++++++++++- src/backend/tcop/utility.c | 4 + src/backend/utils/gp/segadmin.c | 73 ++++++++++++++----- src/backend/utils/misc/guc_gp.c | 20 +++++ .../catalog/gp_segment_configuration.h | 1 + .../gp_segment_configuration_indexing.h | 8 +- src/include/nodes/nodes.h | 1 + src/include/nodes/parsenodes.h | 11 +++ src/include/parser/kwlist.h | 2 + src/include/postmaster/fts_comm.h | 1 + src/include/tcop/cmdtaglist.h | 1 + src/include/utils/guc.h | 2 + src/include/utils/unsync_guc_name.h | 1 + 17 files changed, 205 insertions(+), 40 deletions(-) diff --git a/gpcontrib/gp_inject_fault/gp_inject_fault.c b/gpcontrib/gp_inject_fault/gp_inject_fault.c index 22c31cf72a0..3ca55ddabb8 100644 --- a/gpcontrib/gp_inject_fault/gp_inject_fault.c +++ b/gpcontrib/gp_inject_fault/gp_inject_fault.c @@ -115,7 +115,7 @@ get_segment_configuration(int dbid, char **hostname, int *port, int *content) #else HeapTuple tuple; Relation configrel; - ScanKeyData scankey[1]; + ScanKeyData scankey[2]; SysScanDesc scan; Datum attr; bool isNull; @@ -125,8 +125,12 @@ get_segment_configuration(int dbid, char **hostname, int *port, int *content) Anum_gp_segment_configuration_dbid, BTEqualStrategyNumber, F_INT2EQ, Int16GetDatum(dbid)); - scan = systable_beginscan(configrel, GpSegmentConfigDbidIndexId, true, - NULL, 1, scankey); + ScanKeyInit(&scankey[1], + Anum_gp_segment_configuration_warehouse_name, + BTEqualStrategyNumber, F_TEXTEQ, + CStringGetTextDatum(current_warehouse)); + scan = systable_beginscan(configrel, GpSegmentConfigDbidWarehouseIndexId, true, + NULL, 2, scankey); tuple = systable_getnext(scan); diff --git a/src/backend/catalog/catalog.c b/src/backend/catalog/catalog.c index 52112416628..79cbab92f62 100644 --- a/src/backend/catalog/catalog.c +++ b/src/backend/catalog/catalog.c @@ -459,8 +459,8 @@ IsSharedRelation(Oid relationId) relationId == AuthIdRolResQueueIndexId || relationId == AuthIdRolResGroupIndexId || #ifdef USE_INTERNAL_FTS - relationId == GpSegmentConfigContentPreferred_roleIndexId || - relationId == GpSegmentConfigDbidIndexId || + relationId == GpSegmentConfigContentPreferred_roleWarehouseIndexId || + relationId == GpSegmentConfigDbidWarehouseIndexId || #endif relationId == AuthTimeConstraintAuthIdIndexId) { diff --git a/src/backend/cdb/cdbutil.c b/src/backend/cdb/cdbutil.c index 09d77d2cde4..b41b4a78bca 100644 --- a/src/backend/cdb/cdbutil.c +++ b/src/backend/cdb/cdbutil.c @@ -256,6 +256,11 @@ readGpSegConfigFromCatalog(int *total_dbs) while (HeapTupleIsValid(gp_seg_config_tuple = systable_getnext(gp_seg_config_scan))) { + /* warehouse */ + attr = heap_getattr(gp_seg_config_tuple, Anum_gp_segment_configuration_warehouse_name, RelationGetDescr(gp_seg_config_rel), &isNull); + if (!isNull && strcmp(TextDatumGetCString(attr), current_warehouse) != 0) + continue; + config = &configs[idx]; /* dbid */ @@ -1492,7 +1497,8 @@ dbid_get_dbinfo(int16 dbid) { HeapTuple tuple; Relation rel; - ScanKeyData scankey; + ScanKeyData scankey[2]; + int nkeys = 1; SysScanDesc scan; GpSegConfigEntry *i = NULL; @@ -1507,12 +1513,20 @@ dbid_get_dbinfo(int16 dbid) rel = heap_open(GpSegmentConfigRelationId, AccessShareLock); /* SELECT * FROM gp_segment_configuration WHERE dbid = :1 */ - ScanKeyInit(&scankey, + ScanKeyInit(&scankey[0], Anum_gp_segment_configuration_dbid, BTEqualStrategyNumber, F_INT2EQ, Int16GetDatum(dbid)); - scan = systable_beginscan(rel, GpSegmentConfigDbidIndexId, true, - NULL, 1, &scankey); + if (dbid != 1) + { + nkeys++; + ScanKeyInit(&scankey[1], + Anum_gp_segment_configuration_warehouse_name, + BTEqualStrategyNumber, F_TEXTEQ, + CStringGetTextDatum(current_warehouse)); + } + scan = systable_beginscan(rel, GpSegmentConfigDbidWarehouseIndexId, true, + NULL, nkeys, scankey); tuple = systable_getnext(scan); if (HeapTupleIsValid(tuple)) @@ -1619,7 +1633,8 @@ contentid_get_dbid(int16 contentid, char role, bool getPreferredRoleNotCurrentRo { int16 dbid = 0; Relation rel; - ScanKeyData scankey[2]; + ScanKeyData scankey[3]; + int nkeys = 2; SysScanDesc scan; HeapTuple tup; @@ -1648,8 +1663,17 @@ contentid_get_dbid(int16 contentid, char role, bool getPreferredRoleNotCurrentRo Anum_gp_segment_configuration_preferred_role, BTEqualStrategyNumber, F_CHAREQ, CharGetDatum(role)); - scan = systable_beginscan(rel, GpSegmentConfigContentPreferred_roleIndexId, true, - NULL, 2, scankey); + if (contentid != MASTER_CONTENT_ID) + { + nkeys++; + ScanKeyInit(&scankey[2], + Anum_gp_segment_configuration_warehouse_name, + BTEqualStrategyNumber, F_TEXTEQ, + CStringGetTextDatum(current_warehouse)); + } + + scan = systable_beginscan(rel, GpSegmentConfigContentPreferred_roleWarehouseIndexId, true, + NULL, nkeys, scankey); } else { @@ -1665,9 +1689,17 @@ contentid_get_dbid(int16 contentid, char role, bool getPreferredRoleNotCurrentRo Anum_gp_segment_configuration_role, BTEqualStrategyNumber, F_CHAREQ, CharGetDatum(role)); + if (contentid != MASTER_CONTENT_ID) + { + nkeys++; + ScanKeyInit(&scankey[2], + Anum_gp_segment_configuration_warehouse_name, + BTEqualStrategyNumber, F_TEXTEQ, + CStringGetTextDatum(current_warehouse)); + } /* no index */ scan = systable_beginscan(rel, InvalidOid, false, - NULL, 2, scankey); + NULL, nkeys, scankey); } tup = systable_getnext(scan); diff --git a/src/backend/fts/fts.c b/src/backend/fts/fts.c index cdec9dcf7aa..732c8969478 100644 --- a/src/backend/fts/fts.c +++ b/src/backend/fts/fts.c @@ -228,18 +228,22 @@ probeWalRepUpdateConfig(int16 dbid, int16 segindex, char role, bool confignulls[Natts_gp_segment_configuration] = { false }; bool repls[Natts_gp_segment_configuration] = { false }; - ScanKeyData scankey; + ScanKeyData scankey[2]; SysScanDesc sscan; configrel = table_open(GpSegmentConfigRelationId, RowExclusiveLock); - ScanKeyInit(&scankey, + ScanKeyInit(&scankey[0], Anum_gp_segment_configuration_dbid, BTEqualStrategyNumber, F_INT2EQ, Int16GetDatum(dbid)); - sscan = systable_beginscan(configrel, GpSegmentConfigDbidIndexId, - true, NULL, 1, &scankey); + ScanKeyInit(&scankey[1], + Anum_gp_segment_configuration_warehouse_name, + BTEqualStrategyNumber, F_TEXTEQ, + CStringGetTextDatum(current_warehouse)); + sscan = systable_beginscan(configrel, GpSegmentConfigDbidWarehouseIndexId, + true, NULL, 2, scankey); configtuple = systable_getnext(sscan); diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index 4f286ad13a4..267f1a68602 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -294,7 +294,7 @@ static void check_expressions_in_partition_key(PartitionSpec *spec, core_yyscan_ CreateFdwStmt CreateForeignServerStmt CreateForeignTableStmt CreateAssertionStmt CreateTransformStmt CreateTrigStmt CreateEventTrigStmt CreateUserStmt CreateUserMappingStmt CreateRoleStmt CreatePolicyStmt - CreatedbStmt DeclareCursorStmt DefineStmt DeleteStmt DiscardStmt DoStmt + CreatedbStmt CreateWarehouseStmt DeclareCursorStmt DefineStmt DeleteStmt DiscardStmt DoStmt DropOpClassStmt DropOpFamilyStmt DropStmt DropCastStmt DropRoleStmt DropdbStmt DropTableSpaceStmt @@ -705,6 +705,8 @@ static void check_expressions_in_partition_key(PartitionSpec *spec, core_yyscan_ %type hash_partbound %type hash_partbound_elem +%type OptWarehouseOptList WarehouseOptList +%type WarehouseOptElem /* * Non-keyword token types. These are hard-wired into the "flex" lexer. @@ -872,6 +874,10 @@ static void check_expressions_in_partition_key(PartitionSpec *spec, core_yyscan_ VALIDATION + WAREHOUSE + + WAREHOUSE_SIZE + WEB WRITABLE /* @@ -1420,6 +1426,7 @@ stmt: | CreateOpClassStmt | CreateOpFamilyStmt | CreatePublicationStmt + | CreateWarehouseStmt | AlterOpFamilyStmt | CreatePolicyStmt | CreatePLangStmt @@ -12333,6 +12340,39 @@ publication_for_tables: ; +/***************************************************************************** + * + * QUERY: + * CREATE WAREHOUSE name + * [WAREHOUSE_SIZE ] + * + *****************************************************************************/ + +CreateWarehouseStmt: CREATE WAREHOUSE name OptWarehouseOptList + { + CreateWarehouseStmt *n = makeNode(CreateWarehouseStmt); + n->whname = $3; + n->options = $4; + $$ = (Node *) n; + } + ; + +OptWarehouseOptList: WarehouseOptList { $$ = $1; } + | /*EMPTY*/ { $$ = NIL; } + ; + +WarehouseOptList: WarehouseOptElem { $$ = list_make1($1); } + | WarehouseOptList WarehouseOptElem { $$ = lappend($1, $2); } + ; + +WarehouseOptElem: + WAREHOUSE_SIZE NumericOnly + { + $$ = makeDefElem("warehouse_size", (Node *)$2, @1); + } + ; + + /***************************************************************************** * * ALTER PUBLICATION name SET ( options ) @@ -18795,6 +18835,8 @@ unreserved_keyword: | VIEW | VIEWS | VOLATILE + | WAREHOUSE + | WAREHOUSE_SIZE | WEB /* gp */ | WHITESPACE_P | WITHIN @@ -19783,6 +19825,8 @@ bare_label_keyword: | VIEW | VIEWS | VOLATILE + | WAREHOUSE + | WAREHOUSE_SIZE | WEB | WHEN | WHITESPACE_P diff --git a/src/backend/tcop/utility.c b/src/backend/tcop/utility.c index 966b355e98e..68679d4bac0 100644 --- a/src/backend/tcop/utility.c +++ b/src/backend/tcop/utility.c @@ -3729,6 +3729,10 @@ CreateCommandTag(Node *parsetree) tag = CMDTAG_RETRIEVE; break; + case T_CreateWarehouseStmt: + tag = CMDTAG_CREATE_WAREHOUSE; + break; + default: elog(WARNING, "unrecognized node type: %d", (int) nodeTag(parsetree)); diff --git a/src/backend/utils/gp/segadmin.c b/src/backend/utils/gp/segadmin.c index 31baed7394e..e81f30c5e8b 100644 --- a/src/backend/utils/gp/segadmin.c +++ b/src/backend/utils/gp/segadmin.c @@ -183,20 +183,28 @@ remove_segment_config(int16 dbid) { #ifdef USE_INTERNAL_FTS int numDel = 0; - ScanKeyData scankey; + ScanKeyData scankey[2]; + int nkeys = 1; SysScanDesc sscan; HeapTuple tuple; Relation rel; rel = table_open(GpSegmentConfigRelationId, RowExclusiveLock); - ScanKeyInit(&scankey, + ScanKeyInit(&scankey[0], Anum_gp_segment_configuration_dbid, BTEqualStrategyNumber, F_INT2EQ, Int16GetDatum(dbid)); - - sscan = systable_beginscan(rel, GpSegmentConfigDbidIndexId, true, - NULL, 1, &scankey); + if (dbid != 1) + { + nkeys++; + ScanKeyInit(&scankey[1], + Anum_gp_segment_configuration_warehouse_name, + BTEqualStrategyNumber, F_TEXTEQ, + CStringGetTextDatum(current_warehouse)); + } + sscan = systable_beginscan(rel, GpSegmentConfigDbidWarehouseIndexId, true, + NULL, nkeys, scankey); while ((tuple = systable_getnext(sscan)) != NULL) { CatalogTupleDelete(rel, &tuple->t_self); @@ -240,6 +248,11 @@ add_segment_config_entry(GpSegConfigEntry *i) CStringGetTextDatum(i->address); values[Anum_gp_segment_configuration_datadir - 1] = CStringGetTextDatum(i->datadir); + if (i->warehousename != NULL) + values[Anum_gp_segment_configuration_warehouse_name - 1] = + CStringGetTextDatum(i->warehousename); + else + nulls[Anum_gp_segment_configuration_warehouse_name - 1] = true; tuple = heap_form_tuple(RelationGetDescr(rel), values, nulls); @@ -393,6 +406,11 @@ gp_add_segment(PG_FUNCTION_ARGS) elog(ERROR, "datadir cannot be NULL"); new.datadir = TextDatumGetCString(PG_GETARG_DATUM(9)); + if (new.segindex == MASTER_CONTENT_ID) + new.warehousename = NULL; + else + new.warehousename = current_warehouse; + mirroring_sanity_check(MASTER_ONLY | SUPERUSER, "gp_add_segment"); new.mode = GP_SEGMENT_CONFIGURATION_MODE_NOTINSYNC; @@ -472,7 +490,12 @@ gp_add_segment_mirror(PG_FUNCTION_ARGS) if (PG_ARGISNULL(4)) elog(ERROR, "datadir cannot be NULL"); new.datadir = TextDatumGetCString(PG_GETARG_DATUM(4)); - + + if (new.segindex == MASTER_CONTENT_ID) + new.warehousename = NULL; + else + new.warehousename = current_warehouse; + mirroring_sanity_check(MASTER_ONLY | SUPERUSER, "gp_add_segment_mirror"); new.dbid = get_availableDbId(); @@ -609,7 +632,9 @@ gp_add_master_standby(PG_FUNCTION_ARGS) config->address = TextDatumGetCString(PG_GETARG_TEXT_P(1)); config->datadir = TextDatumGetCString(PG_GETARG_TEXT_P(2)); - + + config->warehousename = NULL; + /* Use the new port number if specified */ if (PG_NARGS() > 3 && !PG_ARGISNULL(3)) config->port = PG_GETARG_INT32(3); @@ -656,17 +681,21 @@ catalog_activate_standby(int16 standby_dbid, int16 master_dbid) /* we use AccessExclusiveLock to prevent races */ Relation rel = table_open(GpSegmentConfigRelationId, AccessExclusiveLock); HeapTuple tuple; - ScanKeyData scankey; + ScanKeyData scankey[2]; SysScanDesc sscan; int numDel = 0; /* first, delete the old master */ - ScanKeyInit(&scankey, + ScanKeyInit(&scankey[0], Anum_gp_segment_configuration_dbid, BTEqualStrategyNumber, F_INT2EQ, Int16GetDatum(master_dbid)); - sscan = systable_beginscan(rel, GpSegmentConfigDbidIndexId, true, - NULL, 1, &scankey); + ScanKeyInit(&scankey[1], + Anum_gp_segment_configuration_warehouse_name, + BTEqualStrategyNumber, F_TEXTEQ, + CStringGetTextDatum(current_warehouse)); + sscan = systable_beginscan(rel, GpSegmentConfigDbidWarehouseIndexId, true, + NULL, 2, scankey); while ((tuple = systable_getnext(sscan)) != NULL) { CatalogTupleDelete(rel, &tuple->t_self); @@ -678,12 +707,16 @@ catalog_activate_standby(int16 standby_dbid, int16 master_dbid) elog(ERROR, "cannot find old master, dbid %i", master_dbid); /* now, set out rows for old standby. */ - ScanKeyInit(&scankey, + ScanKeyInit(&scankey[0], Anum_gp_segment_configuration_dbid, BTEqualStrategyNumber, F_INT2EQ, Int16GetDatum(standby_dbid)); - sscan = systable_beginscan(rel, GpSegmentConfigDbidIndexId, true, - NULL, 1, &scankey); + ScanKeyInit(&scankey[1], + Anum_gp_segment_configuration_warehouse_name, + BTEqualStrategyNumber, F_TEXTEQ, + CStringGetTextDatum(current_warehouse)); + sscan = systable_beginscan(rel, GpSegmentConfigDbidWarehouseIndexId, true, + NULL, 2, scankey); tuple = systable_getnext(sscan); @@ -830,16 +863,20 @@ gp_update_segment_configuration_mode_status(PG_FUNCTION_ARGS) /* we use AccessExclusiveLock to prevent races */ Relation rel = table_open(GpSegmentConfigRelationId, AccessExclusiveLock); HeapTuple tuple; - ScanKeyData scankey; + ScanKeyData scankey[2]; SysScanDesc sscan; /* now, set out rows for old standby. */ - ScanKeyInit(&scankey, + ScanKeyInit(&scankey[0], Anum_gp_segment_configuration_dbid, BTEqualStrategyNumber, F_INT2EQ, Int16GetDatum(dbid)); - sscan = systable_beginscan(rel, GpSegmentConfigDbidIndexId, true, - NULL, 1, &scankey); + ScanKeyInit(&scankey[1], + Anum_gp_segment_configuration_warehouse_name, + BTEqualStrategyNumber, F_TEXTEQ, + CStringGetTextDatum(current_warehouse)); + sscan = systable_beginscan(rel, GpSegmentConfigDbidWarehouseIndexId, true, + NULL, 2, scankey); tuple = systable_getnext(sscan); diff --git a/src/backend/utils/misc/guc_gp.c b/src/backend/utils/misc/guc_gp.c index 2e39cea261c..d72e73b45f2 100644 --- a/src/backend/utils/misc/guc_gp.c +++ b/src/backend/utils/misc/guc_gp.c @@ -101,6 +101,7 @@ static void assign_pljava_classpath_insecure(bool newval, void *extra); static bool check_gp_resource_group_bypass(bool *newval, void **extra, GucSource source); static int guc_array_compare(const void *a, const void *b); static bool check_max_running_tasks(int *newval, void **extra, GucSource source); +static void assign_current_warehouse(const char *newval, void *extra); int listenerBacklog = 128; @@ -430,6 +431,9 @@ bool gp_enable_global_deadlock_detector = false; bool gp_enable_predicate_pushdown; int gp_predicate_pushdown_sample_rows; +/* Current warehouse */ +char *current_warehouse = "default"; + static const struct config_enum_entry gp_log_format_options[] = { {"text", 0}, {"csv", 1}, @@ -4539,6 +4543,16 @@ struct config_string ConfigureNamesString_gp[] = check_gp_default_storage_options, assign_gp_default_storage_options, NULL }, + { + {"warehouse", PGC_USERSET, CUSTOM_OPTIONS, + gettext_noop("Sets the current warehouse."), + NULL, + GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE + }, + ¤t_warehouse, "default", + NULL, assign_current_warehouse, NULL + }, + { {"task_timezone", PGC_POSTMASTER, TASK_SCHEDULE_OPTIONS, gettext_noop("Specify timezone used for cron task schedule."), @@ -5226,6 +5240,12 @@ check_gp_workfile_compression(bool *newval, void **extra, GucSource source) return true; } +static void +assign_current_warehouse(const char *newval, void *extra) +{ + cdbcomponent_destroyCdbComponents(); +} + void DispatchSyncPGVariable(struct config_generic * gconfig) { diff --git a/src/include/catalog/gp_segment_configuration.h b/src/include/catalog/gp_segment_configuration.h index 46ac37e02c2..35119f18bac 100644 --- a/src/include/catalog/gp_segment_configuration.h +++ b/src/include/catalog/gp_segment_configuration.h @@ -58,6 +58,7 @@ CATALOG(gp_segment_configuration,7026,GpSegmentConfigRelationId) BKI_SHARED_RELA text address; text datadir; + text warehouse_name BKI_DEFAULT(_null_); #endif } FormData_gp_segment_configuration; diff --git a/src/include/catalog/gp_segment_configuration_indexing.h b/src/include/catalog/gp_segment_configuration_indexing.h index 4d530aa1e22..42c85fe001f 100644 --- a/src/include/catalog/gp_segment_configuration_indexing.h +++ b/src/include/catalog/gp_segment_configuration_indexing.h @@ -14,9 +14,9 @@ #include "catalog/genbki.h" -DECLARE_UNIQUE_INDEX(gp_segment_config_content_preferred_role_index, 7139, on gp_segment_configuration using btree(content int2_ops, preferred_role char_ops)); -#define GpSegmentConfigContentPreferred_roleIndexId 7139 -DECLARE_UNIQUE_INDEX(gp_segment_config_dbid_index, 7140, on gp_segment_configuration using btree(dbid int2_ops)); -#define GpSegmentConfigDbidIndexId 7140 +DECLARE_UNIQUE_INDEX(gp_segment_config_content_preferred_role_index, 7139, on gp_segment_configuration using btree(content int2_ops, preferred_role char_ops, warehouse_name text_ops)); +#define GpSegmentConfigContentPreferred_roleWarehouseIndexId 7139 +DECLARE_UNIQUE_INDEX(gp_segment_config_dbid_index, 7140, on gp_segment_configuration using btree(dbid int2_ops, warehouse_name text_ops)); +#define GpSegmentConfigDbidWarehouseIndexId 7140 #endif // GP_SEGMENT_CONFIGURATION_INDEXING_H \ No newline at end of file diff --git a/src/include/nodes/nodes.h b/src/include/nodes/nodes.h index 89469a8c316..ec574a99816 100644 --- a/src/include/nodes/nodes.h +++ b/src/include/nodes/nodes.h @@ -542,6 +542,7 @@ typedef enum NodeTag T_GpDropPartitionCmd, T_GpSplitPartitionCmd, T_GpAlterPartitionCmd, + T_CreateWarehouseStmt, /* * TAGS FOR PARSE TREE NODES (parsenodes.h) diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index 77e60a68765..7109e6ab56b 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -4260,4 +4260,15 @@ typedef struct RetrieveStmt bool is_all; } RetrieveStmt; +/* ---------------------- + * Warehouse Statement + * ---------------------- + */ +typedef struct CreateWarehouseStmt +{ + NodeTag type; + char *whname; + List *options; /* List of DefElem nodes */ +} CreateWarehouseStmt; + #endif /* PARSENODES_H */ diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h index 3df64f49ca1..a6e840fc4fe 100644 --- a/src/include/parser/kwlist.h +++ b/src/include/parser/kwlist.h @@ -520,6 +520,8 @@ PG_KEYWORD("version", VERSION_P, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("view", VIEW, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("views", VIEWS, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("volatile", VOLATILE, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("warehouse", WAREHOUSE, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("warehouse_size", WAREHOUSE_SIZE, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("web", WEB, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("when", WHEN, RESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("where", WHERE, RESERVED_KEYWORD, AS_LABEL) diff --git a/src/include/postmaster/fts_comm.h b/src/include/postmaster/fts_comm.h index 765ad45e994..feb3e0b7232 100644 --- a/src/include/postmaster/fts_comm.h +++ b/src/include/postmaster/fts_comm.h @@ -148,6 +148,7 @@ typedef struct GpSegConfigEntry char *hostname; /* name or ip address of host machine */ char *address; /* ip address of host machine */ char *datadir; /* absolute path to data directory on the host. */ + char *warehousename; /* additional cached info */ char *hostip; /* cached lookup of name */ diff --git a/src/include/tcop/cmdtaglist.h b/src/include/tcop/cmdtaglist.h index a6c36d31350..7f4868e9cef 100644 --- a/src/include/tcop/cmdtaglist.h +++ b/src/include/tcop/cmdtaglist.h @@ -135,6 +135,7 @@ PG_CMDTAG(CMDTAG_CREATE_TYPE, "CREATE TYPE", true, false, false) PG_CMDTAG(CMDTAG_CREATE_TASK, "CREATE TASK", true, false, false) PG_CMDTAG(CMDTAG_CREATE_USER_MAPPING, "CREATE USER MAPPING", true, false, false) PG_CMDTAG(CMDTAG_CREATE_VIEW, "CREATE VIEW", true, false, false) +PG_CMDTAG(CMDTAG_CREATE_WAREHOUSE, "CREATE WAREHOUSE", true, false, false) PG_CMDTAG(CMDTAG_DEALLOCATE, "DEALLOCATE", false, false, false) PG_CMDTAG(CMDTAG_DEALLOCATE_ALL, "DEALLOCATE ALL", false, false, false) PG_CMDTAG(CMDTAG_DECLARE_CURSOR, "DECLARE CURSOR", false, false, false) diff --git a/src/include/utils/guc.h b/src/include/utils/guc.h index 0e1c15bcf64..fd35f21cbe0 100644 --- a/src/include/utils/guc.h +++ b/src/include/utils/guc.h @@ -625,6 +625,8 @@ extern bool gp_enable_global_deadlock_detector; extern bool gp_enable_predicate_pushdown; extern int gp_predicate_pushdown_sample_rows; +extern char *current_warehouse; + typedef enum { INDEX_CHECK_NONE, diff --git a/src/include/utils/unsync_guc_name.h b/src/include/utils/unsync_guc_name.h index d35e7f72320..c439773a3e9 100644 --- a/src/include/utils/unsync_guc_name.h +++ b/src/include/utils/unsync_guc_name.h @@ -609,6 +609,7 @@ "vacuum_freeze_table_age", "vacuum_multixact_freeze_min_age", "vacuum_multixact_freeze_table_age", + "warehouse", "wait_for_replication_threshold", "wal_block_size", "wal_buffers", From df282fffb55ddc6bb27a4e6df1a6e00ae6bb59c9 Mon Sep 17 00:00:00 2001 From: roseduan Date: Tue, 1 Aug 2023 16:24:05 +0800 Subject: [PATCH 02/15] Feature: add warehouse grammar and GUC variable Since we are going to support storage/compute/catalog separation, we need to add a warehouse logic to the system. Then we can use the warehouse clause to specify the warehouse for a SQL query. A warehouse is a group of segments, and we can create/alter/delete a warehouse. This commit does the following things: 1. add the grammar for the warehouse clause. 2. add the GUC variable for the current warehouse. 3. add some fileter conditions for gp_segment_configuration table, since we need to filter out the segments that are not in the current warehouse. --- src/backend/cdb/cdbutil.c | 52 +++++++++++++++++++++++++++++++++ src/backend/parser/gram.y | 4 +-- src/backend/utils/misc/guc_gp.c | 11 ++----- 3 files changed, 57 insertions(+), 10 deletions(-) diff --git a/src/backend/cdb/cdbutil.c b/src/backend/cdb/cdbutil.c index b41b4a78bca..d7901c48506 100644 --- a/src/backend/cdb/cdbutil.c +++ b/src/backend/cdb/cdbutil.c @@ -103,6 +103,12 @@ static HTAB *segment_ip_cache_htab = NULL; int numsegmentsFromQD = -1; +/* + * Helper method that verifies setting of default priority guc. + */ +bool check_current_warehouse(char **newval, void **extra, GucSource source); +void assign_current_warehouse(const char *newval, void *extra); + typedef struct SegIpEntry { char key[NAMEDATALEN]; @@ -957,6 +963,52 @@ cdbcomponent_activeQEsExist(void) return !cdb_component_dbs ? false : cdb_component_dbs->numActiveQEs > 0; } +bool +check_current_warehouse(char **newval, void **extra, GucSource source) +{ + if (Gp_role == GP_ROLE_EXECUTE) + return true; + if (*newval == NULL || strcmp(*newval, "default") == 0) + return true; + + Relation rel = table_open(GpSegmentConfigRelationId, AccessShareLock); + HeapTuple tuple; + SysScanDesc sscan; + bool warehouse_exist = false; + bool isNull; + Datum attr; + + sscan = systable_beginscan(rel, InvalidOid, false, NULL, 0, NULL); + while ((tuple = systable_getnext(sscan)) != NULL) + { + char *warehouse_name = NULL; + attr = heap_getattr(tuple, Anum_gp_segment_configuration_warehouse_name, + RelationGetDescr(rel), &isNull); + if (!isNull) + warehouse_name = TextDatumGetCString(attr); + + if (warehouse_name && strcmp(warehouse_name, *newval) == 0) + { + warehouse_exist = true; + break; + } + } + systable_endscan(sscan); + table_close(rel, AccessShareLock); + + if (!warehouse_exist) + GUC_check_errmsg("warehouse %s does not exist", *newval); + + return warehouse_exist; +} + +void +assign_current_warehouse(const char *newval, void *extra) +{ + /* clear current cdb components cache */ + cdbcomponent_destroyCdbComponents(); +} + /* * Find CdbComponentDatabaseInfo in the array by segment index. */ diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index 267f1a68602..44353235c72 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -12366,9 +12366,9 @@ WarehouseOptList: WarehouseOptElem { $$ = list_make1($1); } ; WarehouseOptElem: - WAREHOUSE_SIZE NumericOnly + WAREHOUSE_SIZE SignedIconst { - $$ = makeDefElem("warehouse_size", (Node *)$2, @1); + $$ = makeDefElem("warehouse_size", (Node *)makeInteger($2), @1); } ; diff --git a/src/backend/utils/misc/guc_gp.c b/src/backend/utils/misc/guc_gp.c index d72e73b45f2..e456e65fb97 100644 --- a/src/backend/utils/misc/guc_gp.c +++ b/src/backend/utils/misc/guc_gp.c @@ -101,7 +101,8 @@ static void assign_pljava_classpath_insecure(bool newval, void *extra); static bool check_gp_resource_group_bypass(bool *newval, void **extra, GucSource source); static int guc_array_compare(const void *a, const void *b); static bool check_max_running_tasks(int *newval, void **extra, GucSource source); -static void assign_current_warehouse(const char *newval, void *extra); +bool check_current_warehouse(char **newval, void **extra, GucSource source); +void assign_current_warehouse(const char *newval, void *extra); int listenerBacklog = 128; @@ -4550,7 +4551,7 @@ struct config_string ConfigureNamesString_gp[] = GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE }, ¤t_warehouse, "default", - NULL, assign_current_warehouse, NULL + check_current_warehouse, assign_current_warehouse, NULL }, { @@ -5240,12 +5241,6 @@ check_gp_workfile_compression(bool *newval, void **extra, GucSource source) return true; } -static void -assign_current_warehouse(const char *newval, void *extra) -{ - cdbcomponent_destroyCdbComponents(); -} - void DispatchSyncPGVariable(struct config_generic * gconfig) { From f37e7cdfd1fdcd2af24a1c1bcb5e4e343e5223fa Mon Sep 17 00:00:00 2001 From: roseduan Date: Wed, 2 Aug 2023 15:23:06 +0800 Subject: [PATCH 03/15] fix GUC functions --- src/backend/cdb/cdbutil.c | 21 ++++----------------- src/backend/utils/misc/guc_gp.c | 15 +++++++++++++-- src/include/cdb/cdbutil.h | 1 + 3 files changed, 18 insertions(+), 19 deletions(-) diff --git a/src/backend/cdb/cdbutil.c b/src/backend/cdb/cdbutil.c index d7901c48506..cd14f12e8dd 100644 --- a/src/backend/cdb/cdbutil.c +++ b/src/backend/cdb/cdbutil.c @@ -103,12 +103,6 @@ static HTAB *segment_ip_cache_htab = NULL; int numsegmentsFromQD = -1; -/* - * Helper method that verifies setting of default priority guc. - */ -bool check_current_warehouse(char **newval, void **extra, GucSource source); -void assign_current_warehouse(const char *newval, void *extra); - typedef struct SegIpEntry { char key[NAMEDATALEN]; @@ -964,11 +958,11 @@ cdbcomponent_activeQEsExist(void) } bool -check_current_warehouse(char **newval, void **extra, GucSource source) +cdb_checkWarehouseName(char *new_name) { if (Gp_role == GP_ROLE_EXECUTE) return true; - if (*newval == NULL || strcmp(*newval, "default") == 0) + if (new_name == NULL || strcmp(new_name, "default") == 0) return true; Relation rel = table_open(GpSegmentConfigRelationId, AccessShareLock); @@ -987,7 +981,7 @@ check_current_warehouse(char **newval, void **extra, GucSource source) if (!isNull) warehouse_name = TextDatumGetCString(attr); - if (warehouse_name && strcmp(warehouse_name, *newval) == 0) + if (warehouse_name && strcmp(warehouse_name, new_name) == 0) { warehouse_exist = true; break; @@ -997,18 +991,11 @@ check_current_warehouse(char **newval, void **extra, GucSource source) table_close(rel, AccessShareLock); if (!warehouse_exist) - GUC_check_errmsg("warehouse %s does not exist", *newval); + GUC_check_errmsg("warehouse %s does not exist", new_name); return warehouse_exist; } -void -assign_current_warehouse(const char *newval, void *extra) -{ - /* clear current cdb components cache */ - cdbcomponent_destroyCdbComponents(); -} - /* * Find CdbComponentDatabaseInfo in the array by segment index. */ diff --git a/src/backend/utils/misc/guc_gp.c b/src/backend/utils/misc/guc_gp.c index e456e65fb97..cf62a392f32 100644 --- a/src/backend/utils/misc/guc_gp.c +++ b/src/backend/utils/misc/guc_gp.c @@ -101,8 +101,8 @@ static void assign_pljava_classpath_insecure(bool newval, void *extra); static bool check_gp_resource_group_bypass(bool *newval, void **extra, GucSource source); static int guc_array_compare(const void *a, const void *b); static bool check_max_running_tasks(int *newval, void **extra, GucSource source); -bool check_current_warehouse(char **newval, void **extra, GucSource source); -void assign_current_warehouse(const char *newval, void *extra); +static bool check_current_warehouse(char **newval, void **extra, GucSource source); +static void assign_current_warehouse(const char *newval, void *extra); int listenerBacklog = 128; @@ -5094,6 +5094,17 @@ check_max_running_tasks(int *newval, void **extra, GucSource source) return true; } +bool +check_current_warehouse(char **newval, void **extra, GucSource source) +{ + return cdb_checkWarehouseName(*newval); +} + +void assign_current_warehouse(const char *newval, void *extra) +{ + cdbcomponent_destroyCdbComponents(); +} + /* * Malloc a new string representing current storage_opts. */ diff --git a/src/include/cdb/cdbutil.h b/src/include/cdb/cdbutil.h index b2424b0efd6..b47da8d2077 100644 --- a/src/include/cdb/cdbutil.h +++ b/src/include/cdb/cdbutil.h @@ -172,6 +172,7 @@ extern int16 contentid_get_dbid(int16 contentid, char role, bool getPreferredRol extern int16 cdbcomponent_get_maxdbid(void); extern int16 cdbcomponent_get_availableDbId(void); extern int16 cdbcomponent_get_maxcontentid(void); +extern bool cdb_checkWarehouseName(char *new_name); extern int numsegmentsFromQD; /* From 77e6d23ac3429942b3a3e53bf2090ab7c3b4f1c0 Mon Sep 17 00:00:00 2001 From: roseduan Date: Thu, 3 Aug 2023 10:15:41 +0800 Subject: [PATCH 04/15] add a condition for checkWarehouseName --- src/backend/utils/misc/guc_gp.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/backend/utils/misc/guc_gp.c b/src/backend/utils/misc/guc_gp.c index cf62a392f32..58af55044e3 100644 --- a/src/backend/utils/misc/guc_gp.c +++ b/src/backend/utils/misc/guc_gp.c @@ -5097,7 +5097,11 @@ check_max_running_tasks(int *newval, void **extra, GucSource source) bool check_current_warehouse(char **newval, void **extra, GucSource source) { +#ifdef USE_INTERNAL_FTS return cdb_checkWarehouseName(*newval); +#else + return true; +#endif } void assign_current_warehouse(const char *newval, void *extra) From 6126645615bf5576fa3355c2db4f4d3330a983b2 Mon Sep 17 00:00:00 2001 From: roseduan Date: Thu, 3 Aug 2023 18:50:23 +0800 Subject: [PATCH 05/15] fix warehouse_name query and add sgml doc --- doc/src/sgml/ref/create_warehouse.sgml | 53 +++++++++ src/backend/cdb/cdbutil.c | 158 +++++++++++++------------ src/backend/utils/gp/segadmin.c | 34 +++--- src/backend/utils/misc/guc_gp.c | 1 - 4 files changed, 153 insertions(+), 93 deletions(-) create mode 100644 doc/src/sgml/ref/create_warehouse.sgml diff --git a/doc/src/sgml/ref/create_warehouse.sgml b/doc/src/sgml/ref/create_warehouse.sgml new file mode 100644 index 00000000000..65ba0cf272f --- /dev/null +++ b/doc/src/sgml/ref/create_warehouse.sgml @@ -0,0 +1,53 @@ + + + + + CREATE WAREHOUSE + + + + CREATE WAREHOUSE + 7 + SQL - Language Statements + + + + CREATE WAREHOUSE + define a new warehouse for query processing + + + + +CREATE WAREHOUSE name +[ WAREHOUSE_SIZE size ] + + + + + Description + + + CREATE WAREHOUSE Creates a new virtual warehouse in the system. + Initial creation of a virtual warehouse might take some time to provision the compute resources. + + + + + + Parameters + + + WAREHOUSE_SIZE + + + The size of the warehouse. + + + + + + + diff --git a/src/backend/cdb/cdbutil.c b/src/backend/cdb/cdbutil.c index cd14f12e8dd..a9aa3a2d6af 100644 --- a/src/backend/cdb/cdbutil.c +++ b/src/backend/cdb/cdbutil.c @@ -1536,8 +1536,7 @@ dbid_get_dbinfo(int16 dbid) { HeapTuple tuple; Relation rel; - ScanKeyData scankey[2]; - int nkeys = 1; + ScanKeyData scankey; SysScanDesc scan; GpSegConfigEntry *i = NULL; @@ -1552,105 +1551,108 @@ dbid_get_dbinfo(int16 dbid) rel = heap_open(GpSegmentConfigRelationId, AccessShareLock); /* SELECT * FROM gp_segment_configuration WHERE dbid = :1 */ - ScanKeyInit(&scankey[0], + ScanKeyInit(&scankey, Anum_gp_segment_configuration_dbid, BTEqualStrategyNumber, F_INT2EQ, Int16GetDatum(dbid)); - if (dbid != 1) - { - nkeys++; - ScanKeyInit(&scankey[1], - Anum_gp_segment_configuration_warehouse_name, - BTEqualStrategyNumber, F_TEXTEQ, - CStringGetTextDatum(current_warehouse)); - } scan = systable_beginscan(rel, GpSegmentConfigDbidWarehouseIndexId, true, - NULL, nkeys, scankey); + NULL, 1, &scankey); - tuple = systable_getnext(scan); - if (HeapTupleIsValid(tuple)) + while (HeapTupleIsValid(tuple = systable_getnext(scan))) { Datum attr; bool isNull; + char *warehouse_name = NULL; - i = palloc(sizeof(GpSegConfigEntry)); - - /* - * dbid - */ - attr = heap_getattr(tuple, Anum_gp_segment_configuration_dbid, + attr = heap_getattr(tuple, Anum_gp_segment_configuration_warehouse_name, RelationGetDescr(rel), &isNull); - Assert(!isNull); - i->dbid = DatumGetInt16(attr); + if (!isNull) + warehouse_name = TextDatumGetCString(attr); - /* - * content - */ attr = heap_getattr(tuple, Anum_gp_segment_configuration_content, RelationGetDescr(rel), &isNull); Assert(!isNull); - i->segindex = DatumGetInt16(attr); + if (DatumGetInt16(attr) == MASTER_CONTENT_ID || strcmp(warehouse_name, current_warehouse) == 0) + { + i = palloc(sizeof(GpSegConfigEntry)); - /* - * role - */ - attr = heap_getattr(tuple, Anum_gp_segment_configuration_role, - RelationGetDescr(rel), &isNull); - Assert(!isNull); - i->role = DatumGetChar(attr); + /* + * dbid + */ + attr = heap_getattr(tuple, Anum_gp_segment_configuration_dbid, + RelationGetDescr(rel), &isNull); + Assert(!isNull); + i->dbid = DatumGetInt16(attr); - /* - * preferred-role - */ - attr = heap_getattr(tuple, - Anum_gp_segment_configuration_preferred_role, - RelationGetDescr(rel), &isNull); - Assert(!isNull); - i->preferred_role = DatumGetChar(attr); + /* + * content + */ + attr = heap_getattr(tuple, Anum_gp_segment_configuration_content, + RelationGetDescr(rel), &isNull); + Assert(!isNull); + i->segindex = DatumGetInt16(attr); - /* - * mode - */ - attr = heap_getattr(tuple, Anum_gp_segment_configuration_mode, - RelationGetDescr(rel), &isNull); - Assert(!isNull); - i->mode = DatumGetChar(attr); + /* + * role + */ + attr = heap_getattr(tuple, Anum_gp_segment_configuration_role, + RelationGetDescr(rel), &isNull); + Assert(!isNull); + i->role = DatumGetChar(attr); - /* - * status - */ - attr = heap_getattr(tuple, Anum_gp_segment_configuration_status, - RelationGetDescr(rel), &isNull); - Assert(!isNull); - i->status = DatumGetChar(attr); + /* + * preferred-role + */ + attr = heap_getattr(tuple, + Anum_gp_segment_configuration_preferred_role, + RelationGetDescr(rel), &isNull); + Assert(!isNull); + i->preferred_role = DatumGetChar(attr); - /* - * hostname - */ - attr = heap_getattr(tuple, Anum_gp_segment_configuration_hostname, - RelationGetDescr(rel), &isNull); - Assert(!isNull); - i->hostname = TextDatumGetCString(attr); + /* + * mode + */ + attr = heap_getattr(tuple, Anum_gp_segment_configuration_mode, + RelationGetDescr(rel), &isNull); + Assert(!isNull); + i->mode = DatumGetChar(attr); - /* - * address - */ - attr = heap_getattr(tuple, Anum_gp_segment_configuration_address, - RelationGetDescr(rel), &isNull); - Assert(!isNull); - i->address = TextDatumGetCString(attr); + /* + * status + */ + attr = heap_getattr(tuple, Anum_gp_segment_configuration_status, + RelationGetDescr(rel), &isNull); + Assert(!isNull); + i->status = DatumGetChar(attr); - /* - * port - */ - attr = heap_getattr(tuple, Anum_gp_segment_configuration_port, - RelationGetDescr(rel), &isNull); - Assert(!isNull); - i->port = DatumGetInt32(attr); + /* + * hostname + */ + attr = heap_getattr(tuple, Anum_gp_segment_configuration_hostname, + RelationGetDescr(rel), &isNull); + Assert(!isNull); + i->hostname = TextDatumGetCString(attr); - Assert(systable_getnext(scan) == NULL); /* should be only 1 */ + /* + * address + */ + attr = heap_getattr(tuple, Anum_gp_segment_configuration_address, + RelationGetDescr(rel), &isNull); + Assert(!isNull); + i->address = TextDatumGetCString(attr); + + /* + * port + */ + attr = heap_getattr(tuple, Anum_gp_segment_configuration_port, + RelationGetDescr(rel), &isNull); + Assert(!isNull); + i->port = DatumGetInt32(attr); + + break; + } } - else + if (i == NULL) { elog(ERROR, "could not find configuration entry for dbid %i", dbid); } diff --git a/src/backend/utils/gp/segadmin.c b/src/backend/utils/gp/segadmin.c index e81f30c5e8b..a4d8ad7fe81 100644 --- a/src/backend/utils/gp/segadmin.c +++ b/src/backend/utils/gp/segadmin.c @@ -183,32 +183,38 @@ remove_segment_config(int16 dbid) { #ifdef USE_INTERNAL_FTS int numDel = 0; - ScanKeyData scankey[2]; - int nkeys = 1; + ScanKeyData scankey; SysScanDesc sscan; HeapTuple tuple; Relation rel; rel = table_open(GpSegmentConfigRelationId, RowExclusiveLock); - ScanKeyInit(&scankey[0], + ScanKeyInit(&scankey, Anum_gp_segment_configuration_dbid, BTEqualStrategyNumber, F_INT2EQ, Int16GetDatum(dbid)); - if (dbid != 1) - { - nkeys++; - ScanKeyInit(&scankey[1], - Anum_gp_segment_configuration_warehouse_name, - BTEqualStrategyNumber, F_TEXTEQ, - CStringGetTextDatum(current_warehouse)); - } sscan = systable_beginscan(rel, GpSegmentConfigDbidWarehouseIndexId, true, - NULL, nkeys, scankey); + NULL, 1, &scankey); while ((tuple = systable_getnext(sscan)) != NULL) { - CatalogTupleDelete(rel, &tuple->t_self); - numDel++; + Datum attr; + bool isNull; + char *warehouse_name = NULL; + + attr = heap_getattr(tuple, Anum_gp_segment_configuration_warehouse_name, + RelationGetDescr(rel), &isNull); + if (!isNull) + warehouse_name = TextDatumGetCString(attr); + + attr = heap_getattr(tuple, Anum_gp_segment_configuration_content, + RelationGetDescr(rel), &isNull); + Assert(!isNull); + if (DatumGetInt16(attr) == MASTER_CONTENT_ID || strcmp(warehouse_name, current_warehouse) == 0) + { + CatalogTupleDelete(rel, &tuple->t_self); + numDel++; + } } systable_endscan(sscan); diff --git a/src/backend/utils/misc/guc_gp.c b/src/backend/utils/misc/guc_gp.c index 58af55044e3..75179bb59ed 100644 --- a/src/backend/utils/misc/guc_gp.c +++ b/src/backend/utils/misc/guc_gp.c @@ -4548,7 +4548,6 @@ struct config_string ConfigureNamesString_gp[] = {"warehouse", PGC_USERSET, CUSTOM_OPTIONS, gettext_noop("Sets the current warehouse."), NULL, - GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE }, ¤t_warehouse, "default", check_current_warehouse, assign_current_warehouse, NULL From dd97dcb915815e0dbecc84556ad5925915c57fb4 Mon Sep 17 00:00:00 2001 From: roseduan Date: Fri, 4 Aug 2023 10:56:36 +0800 Subject: [PATCH 06/15] fix gp_seg_config table query in gp_inject_fault --- gpcontrib/gp_inject_fault/gp_inject_fault.c | 49 ++++++++++++--------- 1 file changed, 29 insertions(+), 20 deletions(-) diff --git a/gpcontrib/gp_inject_fault/gp_inject_fault.c b/gpcontrib/gp_inject_fault/gp_inject_fault.c index 3ca55ddabb8..183e88cb477 100644 --- a/gpcontrib/gp_inject_fault/gp_inject_fault.c +++ b/gpcontrib/gp_inject_fault/gp_inject_fault.c @@ -115,43 +115,52 @@ get_segment_configuration(int dbid, char **hostname, int *port, int *content) #else HeapTuple tuple; Relation configrel; - ScanKeyData scankey[2]; + ScanKeyData scankey[1]; SysScanDesc scan; Datum attr; bool isNull; + char *warehouse_name = NULL; + bool foundconfig = false; configrel = table_open(GpSegmentConfigRelationId, AccessShareLock); ScanKeyInit(&scankey[0], Anum_gp_segment_configuration_dbid, BTEqualStrategyNumber, F_INT2EQ, Int16GetDatum(dbid)); - ScanKeyInit(&scankey[1], - Anum_gp_segment_configuration_warehouse_name, - BTEqualStrategyNumber, F_TEXTEQ, - CStringGetTextDatum(current_warehouse)); scan = systable_beginscan(configrel, GpSegmentConfigDbidWarehouseIndexId, true, - NULL, 2, scankey); + NULL, 1, scankey); - tuple = systable_getnext(scan); - - if (HeapTupleIsValid(tuple)) + while (HeapTupleIsValid(tuple = systable_getnext(scan))) { - attr = heap_getattr(tuple, Anum_gp_segment_configuration_hostname, - RelationGetDescr(configrel), &isNull); - Assert(!isNull); - *hostname = TextDatumGetCString(attr); - - attr = heap_getattr(tuple, Anum_gp_segment_configuration_port, + attr = heap_getattr(tuple, Anum_gp_segment_configuration_warehouse_name, RelationGetDescr(configrel), &isNull); - Assert(!isNull); - *port = DatumGetInt32(attr); - + if (!isNull) + warehouse_name = TextDatumGetCString(attr); attr = heap_getattr(tuple, Anum_gp_segment_configuration_content, RelationGetDescr(configrel), &isNull); Assert(!isNull); - *content = DatumGetInt32(attr); + if (DatumGetInt16(attr) == MASTER_CONTENT_ID || strcmp(warehouse_name, current_warehouse) == 0) + { + attr = heap_getattr(tuple, Anum_gp_segment_configuration_hostname, + RelationGetDescr(configrel), &isNull); + Assert(!isNull); + *hostname = TextDatumGetCString(attr); + + attr = heap_getattr(tuple, Anum_gp_segment_configuration_port, + RelationGetDescr(configrel), &isNull); + Assert(!isNull); + *port = DatumGetInt32(attr); + + attr = heap_getattr(tuple, Anum_gp_segment_configuration_content, + RelationGetDescr(configrel), &isNull); + Assert(!isNull); + *content = DatumGetInt32(attr); + + foundconfig = true; + break; + } } - else + if (!foundconfig) elog(ERROR, "dbid %d not found", dbid); systable_endscan(scan); From f8491099e12a46e2299af86c7e19fd14ca155326 Mon Sep 17 00:00:00 2001 From: roseduan Date: Fri, 4 Aug 2023 14:05:35 +0800 Subject: [PATCH 07/15] fix dispatch test --- src/test/regress/output/dispatch.source | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/regress/output/dispatch.source b/src/test/regress/output/dispatch.source index 109f6a2e8fa..96d9e94c422 100644 --- a/src/test/regress/output/dispatch.source +++ b/src/test/regress/output/dispatch.source @@ -894,8 +894,8 @@ SELECT gp_inject_fault('cdbcomponent_recycle_idle_qe_error', 'interrupt', dbid, (1 row) select * from gp_segment_configuration a, t13393 ,gp_segment_configuration b where a.dbid = t13393.tc1 limit 0; - dbid | content | role | preferred_role | mode | status | port | hostname | address | datadir | tc1 | dbid | content | role | preferred_role | mode | status | port | hostname | address | datadir -------+---------+------+----------------+------+--------+------+----------+---------+---------+-----+------+---------+------+----------------+------+--------+------+----------+---------+--------- + dbid | content | role | preferred_role | mode | status | port | hostname | address | datadir | tc1 | dbid | content | role | preferred_role | mode | status | port | hostname | address | datadir | warehouse_name +------+---------+------+----------------+------+--------+------+----------+---------+---------+-----+------+---------+------+----------------+------+--------+------+----------+---------+---------+---------------- (0 rows) SELECT gp_inject_fault('cdbcomponent_recycle_idle_qe_error', 'reset', dbid, current_setting('gp_session_id')::int) From 6fccf1df351f2c4325e196aa0ce5a5c57a4f539f Mon Sep 17 00:00:00 2001 From: roseduan Date: Fri, 4 Aug 2023 14:48:04 +0800 Subject: [PATCH 08/15] fix dispatch test --- src/test/regress/output/dispatch.source | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/regress/output/dispatch.source b/src/test/regress/output/dispatch.source index 96d9e94c422..3b0f54967f4 100644 --- a/src/test/regress/output/dispatch.source +++ b/src/test/regress/output/dispatch.source @@ -894,8 +894,8 @@ SELECT gp_inject_fault('cdbcomponent_recycle_idle_qe_error', 'interrupt', dbid, (1 row) select * from gp_segment_configuration a, t13393 ,gp_segment_configuration b where a.dbid = t13393.tc1 limit 0; - dbid | content | role | preferred_role | mode | status | port | hostname | address | datadir | tc1 | dbid | content | role | preferred_role | mode | status | port | hostname | address | datadir | warehouse_name -------+---------+------+----------------+------+--------+------+----------+---------+---------+-----+------+---------+------+----------------+------+--------+------+----------+---------+---------+---------------- + dbid | content | role | preferred_role | mode | status | port | hostname | address | datadir | warehouse_name | tc1 | dbid | content | role | preferred_role | mode | status | port | hostname | address | datadir | warehouse_name +------+---------+------+----------------+------+--------+------+----------+---------+---------+----------------+-----+------+---------+------+----------------+------+--------+------+----------+---------+---------+---------------- (0 rows) SELECT gp_inject_fault('cdbcomponent_recycle_idle_qe_error', 'reset', dbid, current_setting('gp_session_id')::int) From 010085676ab252a89619a1d0f43b9def341d788d Mon Sep 17 00:00:00 2001 From: roseduan Date: Fri, 4 Aug 2023 18:50:38 +0800 Subject: [PATCH 09/15] fix catalog active standby --- src/backend/utils/gp/segadmin.c | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/src/backend/utils/gp/segadmin.c b/src/backend/utils/gp/segadmin.c index a4d8ad7fe81..ff0481df0a5 100644 --- a/src/backend/utils/gp/segadmin.c +++ b/src/backend/utils/gp/segadmin.c @@ -687,21 +687,17 @@ catalog_activate_standby(int16 standby_dbid, int16 master_dbid) /* we use AccessExclusiveLock to prevent races */ Relation rel = table_open(GpSegmentConfigRelationId, AccessExclusiveLock); HeapTuple tuple; - ScanKeyData scankey[2]; + ScanKeyData scankey; SysScanDesc sscan; int numDel = 0; /* first, delete the old master */ - ScanKeyInit(&scankey[0], + ScanKeyInit(&scankey, Anum_gp_segment_configuration_dbid, BTEqualStrategyNumber, F_INT2EQ, Int16GetDatum(master_dbid)); - ScanKeyInit(&scankey[1], - Anum_gp_segment_configuration_warehouse_name, - BTEqualStrategyNumber, F_TEXTEQ, - CStringGetTextDatum(current_warehouse)); sscan = systable_beginscan(rel, GpSegmentConfigDbidWarehouseIndexId, true, - NULL, 2, scankey); + NULL, 1, &scankey); while ((tuple = systable_getnext(sscan)) != NULL) { CatalogTupleDelete(rel, &tuple->t_self); @@ -713,16 +709,12 @@ catalog_activate_standby(int16 standby_dbid, int16 master_dbid) elog(ERROR, "cannot find old master, dbid %i", master_dbid); /* now, set out rows for old standby. */ - ScanKeyInit(&scankey[0], + ScanKeyInit(&scankey, Anum_gp_segment_configuration_dbid, BTEqualStrategyNumber, F_INT2EQ, Int16GetDatum(standby_dbid)); - ScanKeyInit(&scankey[1], - Anum_gp_segment_configuration_warehouse_name, - BTEqualStrategyNumber, F_TEXTEQ, - CStringGetTextDatum(current_warehouse)); sscan = systable_beginscan(rel, GpSegmentConfigDbidWarehouseIndexId, true, - NULL, 2, scankey); + NULL, 1, &scankey); tuple = systable_getnext(sscan); From f8e35592421913aed000c70432aa9e51a71c91a9 Mon Sep 17 00:00:00 2001 From: roseduan Date: Mon, 7 Aug 2023 10:58:22 +0800 Subject: [PATCH 10/15] add tab complete for warehouse --- src/bin/psql/tab-complete.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/bin/psql/tab-complete.c b/src/bin/psql/tab-complete.c index aea25234a23..e38a602d8a9 100644 --- a/src/bin/psql/tab-complete.c +++ b/src/bin/psql/tab-complete.c @@ -1142,6 +1142,7 @@ static const pgsql_thing_t words_after_create[] = { {"USER", Query_for_list_of_roles " UNION SELECT 'MAPPING FOR'"}, {"USER MAPPING FOR", NULL, NULL, NULL}, {"VIEW", NULL, NULL, &Query_for_list_of_views}, + {"WAREHOUSE", NULL}, {NULL} /* end of list */ }; @@ -2735,6 +2736,10 @@ psql_completion(const char *text, int start, int end) Matches("CREATE", "TASK", MatchAny, "SCHEDULE", MatchAny, "USER", MatchAny)) COMPLETE_WITH("AS"); +/* CREATE WAREHOUSE */ + else if (Matches("CREATE", "WAREHOUSE", MatchAny)) + COMPLETE_WITH("WAREHOUSE_SIZE"); + /* CREATE SERVER */ else if (Matches("CREATE", "SERVER", MatchAny)) COMPLETE_WITH("TYPE", "VERSION", "FOREIGN DATA WRAPPER"); From d69261287319fbac1e827b7ab294129b802897ab Mon Sep 17 00:00:00 2001 From: roseduan Date: Mon, 7 Aug 2023 11:15:47 +0800 Subject: [PATCH 11/15] fix variable name and add code commnet --- gpcontrib/gp_inject_fault/gp_inject_fault.c | 6 +++--- src/backend/utils/misc/guc_gp.c | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/gpcontrib/gp_inject_fault/gp_inject_fault.c b/gpcontrib/gp_inject_fault/gp_inject_fault.c index 183e88cb477..01646cd7c0a 100644 --- a/gpcontrib/gp_inject_fault/gp_inject_fault.c +++ b/gpcontrib/gp_inject_fault/gp_inject_fault.c @@ -120,7 +120,7 @@ get_segment_configuration(int dbid, char **hostname, int *port, int *content) Datum attr; bool isNull; char *warehouse_name = NULL; - bool foundconfig = false; + bool find_config = false; configrel = table_open(GpSegmentConfigRelationId, AccessShareLock); ScanKeyInit(&scankey[0], @@ -156,11 +156,11 @@ get_segment_configuration(int dbid, char **hostname, int *port, int *content) Assert(!isNull); *content = DatumGetInt32(attr); - foundconfig = true; + find_config = true; break; } } - if (!foundconfig) + if (!find_config) elog(ERROR, "dbid %d not found", dbid); systable_endscan(scan); diff --git a/src/backend/utils/misc/guc_gp.c b/src/backend/utils/misc/guc_gp.c index 75179bb59ed..66c8dca93c1 100644 --- a/src/backend/utils/misc/guc_gp.c +++ b/src/backend/utils/misc/guc_gp.c @@ -5105,6 +5105,7 @@ check_current_warehouse(char **newval, void **extra, GucSource source) void assign_current_warehouse(const char *newval, void *extra) { + /* clear cache after warehouse changed */ cdbcomponent_destroyCdbComponents(); } From fa096d79edb95153bf216256cd9daca5eff98f23 Mon Sep 17 00:00:00 2001 From: roseduan Date: Tue, 8 Aug 2023 18:53:09 +0800 Subject: [PATCH 12/15] lock wharehouse when using it --- src/backend/catalog/Makefile | 2 + src/backend/catalog/catalog.c | 9 ++++ src/backend/catalog/gp_warehouse.c | 73 +++++++++++++++++++++++++ src/backend/cdb/cdbutil.c | 85 +++++++++++++++++++++--------- src/backend/parser/gram.y | 12 ++++- src/backend/storage/lmgr/lmgr.c | 26 +++++++++ src/backend/tcop/utility.c | 13 +++++ src/backend/utils/adt/lockfuncs.c | 6 ++- src/backend/utils/misc/guc_gp.c | 7 ++- src/include/catalog/gp_warehouse.h | 26 +++++++++ src/include/cdb/cdbutil.h | 1 + src/include/nodes/nodes.h | 1 + src/include/nodes/parsenodes.h | 6 +++ src/include/storage/lmgr.h | 4 ++ src/include/storage/lock.h | 13 ++++- src/include/tcop/cmdtaglist.h | 1 + 16 files changed, 254 insertions(+), 31 deletions(-) create mode 100644 src/backend/catalog/gp_warehouse.c create mode 100644 src/include/catalog/gp_warehouse.h diff --git a/src/backend/catalog/Makefile b/src/backend/catalog/Makefile index 0fe440e8d98..1bd7017af80 100644 --- a/src/backend/catalog/Makefile +++ b/src/backend/catalog/Makefile @@ -49,6 +49,7 @@ OBJS += pg_extprotocol.o \ pg_proc_callback.o \ aoseg.o aoblkdir.o gp_fastsequence.o gp_segment_config.o \ pg_attribute_encoding.o pg_compression.o aovisimap.o \ + gp_warehouse.o \ pg_appendonly.o \ oid_dispatch.o aocatalog.o storage_tablespace.o storage_database.o \ storage_tablespace_twophase.o storage_tablespace_xact.o \ @@ -78,6 +79,7 @@ CATALOG_HEADERS := \ pg_resqueue.h pg_resqueuecapability.h pg_resourcetype.h \ pg_resgroup.h pg_resgroupcapability.h \ gp_configuration_history.h gp_id.h gp_distribution_policy.h gp_version_at_initdb.h \ + gp_warehouse.h \ pg_appendonly.h \ gp_fastsequence.h pg_extprotocol.h \ pg_attribute_encoding.h \ diff --git a/src/backend/catalog/catalog.c b/src/backend/catalog/catalog.c index 79cbab92f62..814188d1ade 100644 --- a/src/backend/catalog/catalog.c +++ b/src/backend/catalog/catalog.c @@ -59,6 +59,7 @@ #include "catalog/gp_configuration_history.h" #include "catalog/gp_id.h" #include "catalog/gp_version_at_initdb.h" +#include "catalog/gp_warehouse.h" #include "catalog/pg_event_trigger.h" #include "catalog/pg_largeobject_metadata.h" #include "catalog/pg_resourcetype.h" @@ -505,6 +506,14 @@ IsSharedRelation(Oid relationId) return true; } + /* warehouse table and its indexes */ + if (relationId == GpWarehouseRelationId || + relationId == GpWarehouseOidIndexId || + relationId == GpWarehouseNameIndexId) + { + return true; + } + return false; } diff --git a/src/backend/catalog/gp_warehouse.c b/src/backend/catalog/gp_warehouse.c new file mode 100644 index 00000000000..fa698947f83 --- /dev/null +++ b/src/backend/catalog/gp_warehouse.c @@ -0,0 +1,73 @@ +#include "postgres.h" + +#include "access/htup_details.h" +#include "access/genam.h" +#include "access/table.h" +#include "catalog/catalog.h" +#include "catalog/indexing.h" +#include "catalog/gp_warehouse.h" + +#include "utils/builtins.h" +#include "utils/rel.h" + +void +GpWarehouseCreate(char *warehouse_name) +{ + Relation gp_warehouse; + Oid warehouseid; + HeapTuple tup; + Datum values[Natts_gp_warehouse]; + bool nulls[Natts_gp_warehouse]; + + memset(values, 0, sizeof(values)); + memset(nulls, false, sizeof(nulls)); + + gp_warehouse = table_open(GpWarehouseRelationId, RowExclusiveLock); + + warehouseid = GetNewOidWithIndex(gp_warehouse, GpWarehouseOidIndexId, + Anum_gp_warehouse_oid); + values[Anum_gp_warehouse_oid - 1] = ObjectIdGetDatum(warehouseid); + values[Anum_gp_warehouse_warehouse_name - 1] = CStringGetTextDatum(warehouse_name); + + tup = heap_form_tuple(RelationGetDescr(gp_warehouse), values, nulls); + CatalogTupleInsert(gp_warehouse, tup); + heap_freetuple(tup); + + table_close(gp_warehouse, RowExclusiveLock); +} + +Oid +GetGpWarehouseOid(char *warehouse_name, bool missing_ok) +{ + Relation gp_warehouse; + ScanKeyData key[1]; + SysScanDesc scan; + HeapTuple tuple; + Oid warehouseid; + + gp_warehouse = table_open(GpWarehouseRelationId, AccessShareLock); + + ScanKeyInit(&key[0], + Anum_gp_warehouse_warehouse_name, + BTEqualStrategyNumber, F_TEXTEQ, + CStringGetTextDatum(warehouse_name)); + + scan = systable_beginscan(gp_warehouse, GpWarehouseNameIndexId, true, + NULL, 1, key); + + tuple = systable_getnext(scan); + if (HeapTupleIsValid(tuple)) + warehouseid = ((Form_gp_warehouse) GETSTRUCT(tuple))->oid; + else + warehouseid = InvalidOid; + + systable_endscan(scan); + table_close(gp_warehouse, AccessShareLock); + + if (!OidIsValid(warehouseid) && !missing_ok) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_OBJECT), + errmsg("warehouse \"%s\" does not exist", warehouse_name))); + + return warehouseid; +} diff --git a/src/backend/cdb/cdbutil.c b/src/backend/cdb/cdbutil.c index a9aa3a2d6af..66d678cdf34 100644 --- a/src/backend/cdb/cdbutil.c +++ b/src/backend/cdb/cdbutil.c @@ -51,6 +51,7 @@ #include "cdb/cdbconn.h" #include "cdb/cdbfts.h" #include "storage/ipc.h" +#include "storage/lmgr.h" #include "storage/proc.h" #include "postmaster/fts.h" #include "postmaster/postmaster.h" @@ -63,6 +64,7 @@ #include "common/etcdutils.h" #include "catalog/gp_indexing.h" +#include "catalog/gp_warehouse.h" #define MAX_CACHED_1_GANGS 1 #define INCR_COUNT(cdbinfo, arg) \ @@ -99,6 +101,11 @@ static HTAB *hostSegsHashTableInit(void); static int nextQEIdentifer(CdbComponentDatabases *cdbs); +static void unlock_warehouse_exit_callback(int code, Datum arg); +static void register_unlock_warehouse_handler(void); + +static Oid current_warehouse_oid = InvalidOid; + static HTAB *segment_ip_cache_htab = NULL; int numsegmentsFromQD = -1; @@ -965,35 +972,65 @@ cdb_checkWarehouseName(char *new_name) if (new_name == NULL || strcmp(new_name, "default") == 0) return true; - Relation rel = table_open(GpSegmentConfigRelationId, AccessShareLock); - HeapTuple tuple; - SysScanDesc sscan; - bool warehouse_exist = false; - bool isNull; - Datum attr; + Oid warehouseid = GetGpWarehouseOid(new_name, true); - sscan = systable_beginscan(rel, InvalidOid, false, NULL, 0, NULL); - while ((tuple = systable_getnext(sscan)) != NULL) - { - char *warehouse_name = NULL; - attr = heap_getattr(tuple, Anum_gp_segment_configuration_warehouse_name, - RelationGetDescr(rel), &isNull); - if (!isNull) - warehouse_name = TextDatumGetCString(attr); + if (!OidIsValid(warehouseid)) + GUC_check_errmsg("warehouse %s does not exist", new_name); - if (warehouse_name && strcmp(warehouse_name, new_name) == 0) - { - warehouse_exist = true; - break; - } + return true; +} + +void +cdb_assignCurrentWarehouse(char *old_name, char *new_name) +{ + Oid newwarehouse; + Oid oldwarehouse; + + if (Gp_role == GP_ROLE_EXECUTE) + return; + if (new_name == NULL || strcmp(new_name, "default") == 0) + return; + + register_unlock_warehouse_handler(); + + /* lock warehouse */ + if (strcmp(new_name, "default") != 0){ + newwarehouse = GetGpWarehouseOid(new_name, false); + LockWarehouse(newwarehouse, ShareLock); + current_warehouse_oid = newwarehouse; } - systable_endscan(sscan); - table_close(rel, AccessShareLock); - if (!warehouse_exist) - GUC_check_errmsg("warehouse %s does not exist", new_name); + /* unlock old warehouse */ + if (strcmp(old_name, "default") != 0) + { + oldwarehouse = GetGpWarehouseOid(old_name, false); + UnlockWarehouse(oldwarehouse, ShareLock); + } + + /* clear cache after warehouse name changed */ + cdbcomponent_destroyCdbComponents(); +} - return warehouse_exist; +static void +unlock_warehouse_exit_callback(int code, Datum arg) +{ + if (Gp_role == GP_ROLE_EXECUTE) + return; + + /* unlock current warehouse */ + if (strcmp(current_warehouse, "default") != 0) + UnlockWarehouse(current_warehouse_oid, ShareLock); +} + +static void +register_unlock_warehouse_handler(void) +{ + static bool already_done = false; + + if (already_done) + return; + before_shmem_exit(unlock_warehouse_exit_callback, (Datum) 0); + already_done = true; } /* diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index 44353235c72..45e22f74bab 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -295,7 +295,7 @@ static void check_expressions_in_partition_key(PartitionSpec *spec, core_yyscan_ CreateAssertionStmt CreateTransformStmt CreateTrigStmt CreateEventTrigStmt CreateUserStmt CreateUserMappingStmt CreateRoleStmt CreatePolicyStmt CreatedbStmt CreateWarehouseStmt DeclareCursorStmt DefineStmt DeleteStmt DiscardStmt DoStmt - DropOpClassStmt DropOpFamilyStmt DropStmt + DropOpClassStmt DropOpFamilyStmt DropStmt DropWarehouseStmt DropCastStmt DropRoleStmt DropdbStmt DropTableSpaceStmt DropTransformStmt @@ -1466,6 +1466,7 @@ stmt: | DropRoleStmt | DropUserMappingStmt | DropdbStmt + | DropWarehouseStmt | ExecuteStmt | ExplainStmt | FetchStmt @@ -12373,6 +12374,15 @@ WarehouseOptElem: ; +DropWarehouseStmt: DROP WAREHOUSE name + { + DropWarehouseStmt *n = makeNode(DropWarehouseStmt); + n->whname = $3; + $$ = (Node *) n; + } + ; + + /***************************************************************************** * * ALTER PUBLICATION name SET ( options ) diff --git a/src/backend/storage/lmgr/lmgr.c b/src/backend/storage/lmgr/lmgr.c index e4bc2f0c977..34c17a00db0 100644 --- a/src/backend/storage/lmgr/lmgr.c +++ b/src/backend/storage/lmgr/lmgr.c @@ -803,6 +803,26 @@ GxactLockTableWait(DistributedTransactionId gxid) LockRelease(&tag, ShareLock, false); } +void +LockWarehouse(Oid warehouseOid, LOCKMODE lockmode) +{ + LOCKTAG tag; + + SET_LOCKTAG_WAREHOUSE(tag, warehouseOid); + + (void) LockAcquire(&tag, lockmode, true, false); +} + +void +UnlockWarehouse(Oid warehouseOid, LOCKMODE lockmode) +{ + LOCKTAG tag; + + SET_LOCKTAG_WAREHOUSE(tag, warehouseOid); + + LockRelease(&tag, lockmode, true); +} + /* * ConditionalXactLockTableWait * @@ -1284,6 +1304,11 @@ DescribeLockTag(StringInfo buf, const LOCKTAG *tag) _("resource queue %u"), tag->locktag_field1); break; + case LOCKTAG_WAREHOUSE: + appendStringInfo(buf, + _("warehouse %u"), + tag->locktag_field1); + break; default: appendStringInfo(buf, _("unrecognized locktag type %d"), @@ -1330,6 +1355,7 @@ LockTagIsTemp(const LOCKTAG *tag) break; case LOCKTAG_USERLOCK: case LOCKTAG_ADVISORY: + case LOCKTAG_WAREHOUSE: /* assume these aren't temp */ break; } diff --git a/src/backend/tcop/utility.c b/src/backend/tcop/utility.c index 68679d4bac0..b558ae1db4d 100644 --- a/src/backend/tcop/utility.c +++ b/src/backend/tcop/utility.c @@ -241,8 +241,10 @@ ClassifyUtilityCommandAsReadOnly(Node *parsetree) case T_DropTaskStmt: case T_DropQueueStmt: case T_DropResourceGroupStmt: + case T_DropWarehouseStmt: case T_CreateExternalStmt: case T_RetrieveStmt: + case T_CreateWarehouseStmt: { /* DDL is not read-only, and neither is TRUNCATE. */ return COMMAND_IS_NOT_READ_ONLY; @@ -3732,6 +3734,9 @@ CreateCommandTag(Node *parsetree) case T_CreateWarehouseStmt: tag = CMDTAG_CREATE_WAREHOUSE; break; + case T_DropWarehouseStmt: + tag = CMDTAG_DROP_WAREHOUSE; + break; default: elog(WARNING, "unrecognized node type: %d", @@ -4210,6 +4215,14 @@ GetCommandLogLevel(Node *parsetree) case T_AlterCollationStmt: lev = LOGSTMT_DDL; break; + + case T_CreateWarehouseStmt: + lev = LOGSTMT_DDL; + break; + + case T_DropWarehouseStmt: + lev = LOGSTMT_DDL; + break; /* already-planned queries */ case T_PlannedStmt: diff --git a/src/backend/utils/adt/lockfuncs.c b/src/backend/utils/adt/lockfuncs.c index 7afbbd33817..bc0da11a729 100644 --- a/src/backend/utils/adt/lockfuncs.c +++ b/src/backend/utils/adt/lockfuncs.c @@ -44,10 +44,11 @@ const char *const LockTagTypeNames[] = { "resource queue", "distributed xid", "userlock", - "advisory" + "advisory", + "warehouse" }; -StaticAssertDecl(lengthof(LockTagTypeNames) == (LOCKTAG_ADVISORY + 1), +StaticAssertDecl(lengthof(LockTagTypeNames) == (LOCKTAG_WAREHOUSE + 1), "array length mismatch"); /* This must match enum PredicateLockTargetType (predicate_internals.h) */ @@ -459,6 +460,7 @@ pg_lock_status(PG_FUNCTION_ARGS) nulls[9] = true; break; case LOCKTAG_RESOURCE_QUEUE: + case LOCKTAG_WAREHOUSE: #if 0 values[1] = ObjectIdGetDatum(proc->databaseId); #endif diff --git a/src/backend/utils/misc/guc_gp.c b/src/backend/utils/misc/guc_gp.c index 66c8dca93c1..ccce56dcd5f 100644 --- a/src/backend/utils/misc/guc_gp.c +++ b/src/backend/utils/misc/guc_gp.c @@ -4548,6 +4548,7 @@ struct config_string ConfigureNamesString_gp[] = {"warehouse", PGC_USERSET, CUSTOM_OPTIONS, gettext_noop("Sets the current warehouse."), NULL, + GUC_NO_RESET_ALL }, ¤t_warehouse, "default", check_current_warehouse, assign_current_warehouse, NULL @@ -5105,8 +5106,10 @@ check_current_warehouse(char **newval, void **extra, GucSource source) void assign_current_warehouse(const char *newval, void *extra) { - /* clear cache after warehouse changed */ - cdbcomponent_destroyCdbComponents(); +#ifdef USE_INTERNAL_FTS + char *new_name = pstrdup(newval); + cdb_assignCurrentWarehouse(current_warehouse, new_name); +#endif } /* diff --git a/src/include/catalog/gp_warehouse.h b/src/include/catalog/gp_warehouse.h new file mode 100644 index 00000000000..6aeae42c544 --- /dev/null +++ b/src/include/catalog/gp_warehouse.h @@ -0,0 +1,26 @@ +#ifndef GP_WAREHOUSE_H +#define GP_WAREHOUSE_H + +#include "catalog/genbki.h" +#include "catalog/gp_warehouse_d.h" + +/* + * Defines for gp_version_at_initdb table + */ +CATALOG(gp_warehouse,8690,GpWarehouseRelationId) BKI_SHARED_RELATION +{ + Oid oid; /* oid */ + text warehouse_name; /* warehouse name */ +} FormData_gp_warehouse; + +typedef FormData_gp_warehouse *Form_gp_warehouse; + +DECLARE_UNIQUE_INDEX(gp_warehouse_oid_index, 8086, on gp_warehouse using btree(oid oid_ops)); +#define GpWarehouseOidIndexId 8086 +DECLARE_UNIQUE_INDEX(gp_warehouse_name_index, 8059, on gp_warehouse using btree(warehouse_name text_ops)); +#define GpWarehouseNameIndexId 8059 + +extern void GpWarehouseCreate(char *warehouse_name); +extern Oid GetGpWarehouseOid(char *warehouse_name, bool missing_ok); + +#endif /* GP_WAREHOUSE_H */ diff --git a/src/include/cdb/cdbutil.h b/src/include/cdb/cdbutil.h index b47da8d2077..27ba446cd2a 100644 --- a/src/include/cdb/cdbutil.h +++ b/src/include/cdb/cdbutil.h @@ -173,6 +173,7 @@ extern int16 cdbcomponent_get_maxdbid(void); extern int16 cdbcomponent_get_availableDbId(void); extern int16 cdbcomponent_get_maxcontentid(void); extern bool cdb_checkWarehouseName(char *new_name); +extern void cdb_assignCurrentWarehouse(char *old_name, char *new_name); extern int numsegmentsFromQD; /* diff --git a/src/include/nodes/nodes.h b/src/include/nodes/nodes.h index ec574a99816..0912902eb21 100644 --- a/src/include/nodes/nodes.h +++ b/src/include/nodes/nodes.h @@ -543,6 +543,7 @@ typedef enum NodeTag T_GpSplitPartitionCmd, T_GpAlterPartitionCmd, T_CreateWarehouseStmt, + T_DropWarehouseStmt, /* * TAGS FOR PARSE TREE NODES (parsenodes.h) diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index 7109e6ab56b..4af6cbc1f9f 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -4271,4 +4271,10 @@ typedef struct CreateWarehouseStmt List *options; /* List of DefElem nodes */ } CreateWarehouseStmt; +typedef struct DropWarehouseStmt +{ + NodeTag type; + char *whname; +} DropWarehouseStmt; + #endif /* PARSENODES_H */ diff --git a/src/include/storage/lmgr.h b/src/include/storage/lmgr.h index 42014da620f..a8806269fa6 100644 --- a/src/include/storage/lmgr.h +++ b/src/include/storage/lmgr.h @@ -120,4 +120,8 @@ extern bool LockTagIsTemp(const LOCKTAG *tag); extern void GxactLockTableInsert(DistributedTransactionId xid); extern void GxactLockTableWait(DistributedTransactionId xid); + +/* Lock a warehouse */ +extern void LockWarehouse(Oid warehouseOid, LOCKMODE lockmode); +extern void UnlockWarehouse(Oid warehouseOid, LOCKMODE lockmode); #endif /* LMGR_H */ diff --git a/src/include/storage/lock.h b/src/include/storage/lock.h index b70eea63d6e..5f77b5f942a 100644 --- a/src/include/storage/lock.h +++ b/src/include/storage/lock.h @@ -168,10 +168,11 @@ typedef enum LockTagType LOCKTAG_RESOURCE_QUEUE, /* ID info for resource queue is QUEUE ID */ LOCKTAG_DISTRIB_TRANSACTION,/* CDB: distributed transaction (for waiting for distributed xact done) */ LOCKTAG_USERLOCK, /* reserved for old contrib/userlock code */ - LOCKTAG_ADVISORY /* advisory user locks */ + LOCKTAG_ADVISORY, /* advisory user locks */ + LOCKTAG_WAREHOUSE /* warehouse locks */ } LockTagType; -#define LOCKTAG_LAST_TYPE LOCKTAG_ADVISORY +#define LOCKTAG_LAST_TYPE LOCKTAG_WAREHOUSE extern const char *const LockTagTypeNames[]; @@ -317,6 +318,14 @@ typedef struct LOCKTAG (locktag).locktag_type = LOCKTAG_RESOURCE_QUEUE, \ (locktag).locktag_lockmethodid = RESOURCE_LOCKMETHOD) +#define SET_LOCKTAG_WAREHOUSE(locktag,warehouseid) \ + ((locktag).locktag_field1 = (warehouseid), \ + (locktag).locktag_field2 = 0, \ + (locktag).locktag_field3 = 0, \ + (locktag).locktag_field4 = 0, \ + (locktag).locktag_type = LOCKTAG_WAREHOUSE, \ + (locktag).locktag_lockmethodid = DEFAULT_LOCKMETHOD) + /* * Per-locked-object lock information: * diff --git a/src/include/tcop/cmdtaglist.h b/src/include/tcop/cmdtaglist.h index 7f4868e9cef..61469142633 100644 --- a/src/include/tcop/cmdtaglist.h +++ b/src/include/tcop/cmdtaglist.h @@ -202,6 +202,7 @@ PG_CMDTAG(CMDTAG_DROP_TRIGGER, "DROP TRIGGER", true, false, false) PG_CMDTAG(CMDTAG_DROP_TYPE, "DROP TYPE", true, false, false) PG_CMDTAG(CMDTAG_DROP_USER_MAPPING, "DROP USER MAPPING", true, false, false) PG_CMDTAG(CMDTAG_DROP_VIEW, "DROP VIEW", true, false, false) +PG_CMDTAG(CMDTAG_DROP_WAREHOUSE, "DROP WAREHOUSE", true, false, false) From 8d74ebb268a96f8977a4378cc94a4dc725c0d582 Mon Sep 17 00:00:00 2001 From: roseduan Date: Wed, 9 Aug 2023 10:31:33 +0800 Subject: [PATCH 13/15] move GpWarehouseCreate to extension and fix tests --- src/backend/catalog/gp_warehouse.c | 27 ----------------------- src/backend/cdb/cdbutil.c | 9 +++++--- src/include/catalog/gp_warehouse.h | 1 - src/test/regress/expected/misc_sanity.out | 2 ++ 4 files changed, 8 insertions(+), 31 deletions(-) diff --git a/src/backend/catalog/gp_warehouse.c b/src/backend/catalog/gp_warehouse.c index fa698947f83..79f86796645 100644 --- a/src/backend/catalog/gp_warehouse.c +++ b/src/backend/catalog/gp_warehouse.c @@ -3,39 +3,12 @@ #include "access/htup_details.h" #include "access/genam.h" #include "access/table.h" -#include "catalog/catalog.h" #include "catalog/indexing.h" #include "catalog/gp_warehouse.h" #include "utils/builtins.h" #include "utils/rel.h" -void -GpWarehouseCreate(char *warehouse_name) -{ - Relation gp_warehouse; - Oid warehouseid; - HeapTuple tup; - Datum values[Natts_gp_warehouse]; - bool nulls[Natts_gp_warehouse]; - - memset(values, 0, sizeof(values)); - memset(nulls, false, sizeof(nulls)); - - gp_warehouse = table_open(GpWarehouseRelationId, RowExclusiveLock); - - warehouseid = GetNewOidWithIndex(gp_warehouse, GpWarehouseOidIndexId, - Anum_gp_warehouse_oid); - values[Anum_gp_warehouse_oid - 1] = ObjectIdGetDatum(warehouseid); - values[Anum_gp_warehouse_warehouse_name - 1] = CStringGetTextDatum(warehouse_name); - - tup = heap_form_tuple(RelationGetDescr(gp_warehouse), values, nulls); - CatalogTupleInsert(gp_warehouse, tup); - heap_freetuple(tup); - - table_close(gp_warehouse, RowExclusiveLock); -} - Oid GetGpWarehouseOid(char *warehouse_name, bool missing_ok) { diff --git a/src/backend/cdb/cdbutil.c b/src/backend/cdb/cdbutil.c index 66d678cdf34..5967437eebf 100644 --- a/src/backend/cdb/cdbutil.c +++ b/src/backend/cdb/cdbutil.c @@ -967,12 +967,14 @@ cdbcomponent_activeQEsExist(void) bool cdb_checkWarehouseName(char *new_name) { + Oid warehouseid; + if (Gp_role == GP_ROLE_EXECUTE) return true; if (new_name == NULL || strcmp(new_name, "default") == 0) return true; - Oid warehouseid = GetGpWarehouseOid(new_name, true); + warehouseid = GetGpWarehouseOid(new_name, true); if (!OidIsValid(warehouseid)) GUC_check_errmsg("warehouse %s does not exist", new_name); @@ -993,8 +995,9 @@ cdb_assignCurrentWarehouse(char *old_name, char *new_name) register_unlock_warehouse_handler(); - /* lock warehouse */ - if (strcmp(new_name, "default") != 0){ + /* lock new warehouse */ + if (strcmp(new_name, "default") != 0) + { newwarehouse = GetGpWarehouseOid(new_name, false); LockWarehouse(newwarehouse, ShareLock); current_warehouse_oid = newwarehouse; diff --git a/src/include/catalog/gp_warehouse.h b/src/include/catalog/gp_warehouse.h index 6aeae42c544..f637032ef41 100644 --- a/src/include/catalog/gp_warehouse.h +++ b/src/include/catalog/gp_warehouse.h @@ -20,7 +20,6 @@ DECLARE_UNIQUE_INDEX(gp_warehouse_oid_index, 8086, on gp_warehouse using btree(o DECLARE_UNIQUE_INDEX(gp_warehouse_name_index, 8059, on gp_warehouse using btree(warehouse_name text_ops)); #define GpWarehouseNameIndexId 8059 -extern void GpWarehouseCreate(char *warehouse_name); extern Oid GetGpWarehouseOid(char *warehouse_name, bool missing_ok); #endif /* GP_WAREHOUSE_H */ diff --git a/src/test/regress/expected/misc_sanity.out b/src/test/regress/expected/misc_sanity.out index 13947c53237..1ab8ae44ea2 100644 --- a/src/test/regress/expected/misc_sanity.out +++ b/src/test/regress/expected/misc_sanity.out @@ -105,6 +105,7 @@ ORDER BY 1, 2; --------------------------+--------------------+-------------- gp_configuration_history | desc | text gp_version_at_initdb | productversion | text + gp_warehouse | warehouse_name | text pg_attribute | attacl | aclitem[] pg_attribute | attfdwoptions | text[] pg_attribute | attmissingval | anyarray @@ -153,6 +154,7 @@ ORDER BY 1; gp_id gp_partition_template gp_version_at_initdb + gp_warehouse pg_appendonly pg_attribute_encoding pg_auth_time_constraint From e320b4a909eab630440f93ddd608886966715507 Mon Sep 17 00:00:00 2001 From: roseduan Date: Thu, 10 Aug 2023 10:46:07 +0800 Subject: [PATCH 14/15] fix GUC and lock --- gpcontrib/gp_inject_fault/gp_inject_fault.c | 12 +- src/backend/catalog/Makefile | 1 - src/backend/catalog/gp_warehouse.c | 46 ------- src/backend/cdb/cdbutil.c | 113 +++--------------- src/backend/fts/fts.c | 6 +- src/backend/utils/gp/segadmin.c | 36 +++--- src/backend/utils/init/miscinit.c | 14 +++ src/backend/utils/misc/guc_gp.c | 33 ----- .../catalog/gp_segment_configuration.h | 2 +- .../gp_segment_configuration_indexing.h | 4 +- src/include/catalog/gp_warehouse.h | 2 - src/include/cdb/cdbutil.h | 2 - src/include/miscadmin.h | 2 + src/include/postmaster/fts_comm.h | 2 +- src/include/utils/guc.h | 2 - src/include/utils/unsync_guc_name.h | 1 - 16 files changed, 60 insertions(+), 218 deletions(-) delete mode 100644 src/backend/catalog/gp_warehouse.c diff --git a/gpcontrib/gp_inject_fault/gp_inject_fault.c b/gpcontrib/gp_inject_fault/gp_inject_fault.c index 01646cd7c0a..009873941d7 100644 --- a/gpcontrib/gp_inject_fault/gp_inject_fault.c +++ b/gpcontrib/gp_inject_fault/gp_inject_fault.c @@ -119,7 +119,7 @@ get_segment_configuration(int dbid, char **hostname, int *port, int *content) SysScanDesc scan; Datum attr; bool isNull; - char *warehouse_name = NULL; + Oid warehouseid = InvalidOid; bool find_config = false; configrel = table_open(GpSegmentConfigRelationId, AccessShareLock); @@ -132,14 +132,12 @@ get_segment_configuration(int dbid, char **hostname, int *port, int *content) while (HeapTupleIsValid(tuple = systable_getnext(scan))) { - attr = heap_getattr(tuple, Anum_gp_segment_configuration_warehouse_name, - RelationGetDescr(configrel), &isNull); - if (!isNull) - warehouse_name = TextDatumGetCString(attr); - attr = heap_getattr(tuple, Anum_gp_segment_configuration_content, + attr = heap_getattr(tuple, Anum_gp_segment_configuration_warehouseid, RelationGetDescr(configrel), &isNull); Assert(!isNull); - if (DatumGetInt16(attr) == MASTER_CONTENT_ID || strcmp(warehouse_name, current_warehouse) == 0) + warehouseid = DatumGetObjectId(attr); + + if (!OidIsValid(warehouseid) || warehouseid == GetCurrentWarehouseId()) { attr = heap_getattr(tuple, Anum_gp_segment_configuration_hostname, RelationGetDescr(configrel), &isNull); diff --git a/src/backend/catalog/Makefile b/src/backend/catalog/Makefile index 1bd7017af80..6b800053526 100644 --- a/src/backend/catalog/Makefile +++ b/src/backend/catalog/Makefile @@ -49,7 +49,6 @@ OBJS += pg_extprotocol.o \ pg_proc_callback.o \ aoseg.o aoblkdir.o gp_fastsequence.o gp_segment_config.o \ pg_attribute_encoding.o pg_compression.o aovisimap.o \ - gp_warehouse.o \ pg_appendonly.o \ oid_dispatch.o aocatalog.o storage_tablespace.o storage_database.o \ storage_tablespace_twophase.o storage_tablespace_xact.o \ diff --git a/src/backend/catalog/gp_warehouse.c b/src/backend/catalog/gp_warehouse.c deleted file mode 100644 index 79f86796645..00000000000 --- a/src/backend/catalog/gp_warehouse.c +++ /dev/null @@ -1,46 +0,0 @@ -#include "postgres.h" - -#include "access/htup_details.h" -#include "access/genam.h" -#include "access/table.h" -#include "catalog/indexing.h" -#include "catalog/gp_warehouse.h" - -#include "utils/builtins.h" -#include "utils/rel.h" - -Oid -GetGpWarehouseOid(char *warehouse_name, bool missing_ok) -{ - Relation gp_warehouse; - ScanKeyData key[1]; - SysScanDesc scan; - HeapTuple tuple; - Oid warehouseid; - - gp_warehouse = table_open(GpWarehouseRelationId, AccessShareLock); - - ScanKeyInit(&key[0], - Anum_gp_warehouse_warehouse_name, - BTEqualStrategyNumber, F_TEXTEQ, - CStringGetTextDatum(warehouse_name)); - - scan = systable_beginscan(gp_warehouse, GpWarehouseNameIndexId, true, - NULL, 1, key); - - tuple = systable_getnext(scan); - if (HeapTupleIsValid(tuple)) - warehouseid = ((Form_gp_warehouse) GETSTRUCT(tuple))->oid; - else - warehouseid = InvalidOid; - - systable_endscan(scan); - table_close(gp_warehouse, AccessShareLock); - - if (!OidIsValid(warehouseid) && !missing_ok) - ereport(ERROR, - (errcode(ERRCODE_UNDEFINED_OBJECT), - errmsg("warehouse \"%s\" does not exist", warehouse_name))); - - return warehouseid; -} diff --git a/src/backend/cdb/cdbutil.c b/src/backend/cdb/cdbutil.c index 5967437eebf..0d6f7147d30 100644 --- a/src/backend/cdb/cdbutil.c +++ b/src/backend/cdb/cdbutil.c @@ -51,7 +51,6 @@ #include "cdb/cdbconn.h" #include "cdb/cdbfts.h" #include "storage/ipc.h" -#include "storage/lmgr.h" #include "storage/proc.h" #include "postmaster/fts.h" #include "postmaster/postmaster.h" @@ -64,7 +63,6 @@ #include "common/etcdutils.h" #include "catalog/gp_indexing.h" -#include "catalog/gp_warehouse.h" #define MAX_CACHED_1_GANGS 1 #define INCR_COUNT(cdbinfo, arg) \ @@ -101,11 +99,6 @@ static HTAB *hostSegsHashTableInit(void); static int nextQEIdentifer(CdbComponentDatabases *cdbs); -static void unlock_warehouse_exit_callback(int code, Datum arg); -static void register_unlock_warehouse_handler(void); - -static Oid current_warehouse_oid = InvalidOid; - static HTAB *segment_ip_cache_htab = NULL; int numsegmentsFromQD = -1; @@ -248,6 +241,7 @@ readGpSegConfigFromCatalog(int *total_dbs) int array_size; bool isNull; Datum attr; + Oid warehouseid = InvalidOid; Relation gp_seg_config_rel; HeapTuple gp_seg_config_tuple = NULL; SysScanDesc gp_seg_config_scan; @@ -263,9 +257,11 @@ readGpSegConfigFromCatalog(int *total_dbs) while (HeapTupleIsValid(gp_seg_config_tuple = systable_getnext(gp_seg_config_scan))) { - /* warehouse */ - attr = heap_getattr(gp_seg_config_tuple, Anum_gp_segment_configuration_warehouse_name, RelationGetDescr(gp_seg_config_rel), &isNull); - if (!isNull && strcmp(TextDatumGetCString(attr), current_warehouse) != 0) + /* warehouseid */ + attr = heap_getattr(gp_seg_config_tuple, Anum_gp_segment_configuration_warehouseid, RelationGetDescr(gp_seg_config_rel), &isNull); + Assert(!isNull); + warehouseid = DatumGetObjectId(attr); + if (OidIsValid(warehouseid) && warehouseid != GetCurrentWarehouseId()) continue; config = &configs[idx]; @@ -964,78 +960,6 @@ cdbcomponent_activeQEsExist(void) return !cdb_component_dbs ? false : cdb_component_dbs->numActiveQEs > 0; } -bool -cdb_checkWarehouseName(char *new_name) -{ - Oid warehouseid; - - if (Gp_role == GP_ROLE_EXECUTE) - return true; - if (new_name == NULL || strcmp(new_name, "default") == 0) - return true; - - warehouseid = GetGpWarehouseOid(new_name, true); - - if (!OidIsValid(warehouseid)) - GUC_check_errmsg("warehouse %s does not exist", new_name); - - return true; -} - -void -cdb_assignCurrentWarehouse(char *old_name, char *new_name) -{ - Oid newwarehouse; - Oid oldwarehouse; - - if (Gp_role == GP_ROLE_EXECUTE) - return; - if (new_name == NULL || strcmp(new_name, "default") == 0) - return; - - register_unlock_warehouse_handler(); - - /* lock new warehouse */ - if (strcmp(new_name, "default") != 0) - { - newwarehouse = GetGpWarehouseOid(new_name, false); - LockWarehouse(newwarehouse, ShareLock); - current_warehouse_oid = newwarehouse; - } - - /* unlock old warehouse */ - if (strcmp(old_name, "default") != 0) - { - oldwarehouse = GetGpWarehouseOid(old_name, false); - UnlockWarehouse(oldwarehouse, ShareLock); - } - - /* clear cache after warehouse name changed */ - cdbcomponent_destroyCdbComponents(); -} - -static void -unlock_warehouse_exit_callback(int code, Datum arg) -{ - if (Gp_role == GP_ROLE_EXECUTE) - return; - - /* unlock current warehouse */ - if (strcmp(current_warehouse, "default") != 0) - UnlockWarehouse(current_warehouse_oid, ShareLock); -} - -static void -register_unlock_warehouse_handler(void) -{ - static bool already_done = false; - - if (already_done) - return; - before_shmem_exit(unlock_warehouse_exit_callback, (Datum) 0); - already_done = true; -} - /* * Find CdbComponentDatabaseInfo in the array by segment index. */ @@ -1602,17 +1526,14 @@ dbid_get_dbinfo(int16 dbid) { Datum attr; bool isNull; - char *warehouse_name = NULL; + Oid warehouseid = InvalidOid; - attr = heap_getattr(tuple, Anum_gp_segment_configuration_warehouse_name, - RelationGetDescr(rel), &isNull); - if (!isNull) - warehouse_name = TextDatumGetCString(attr); - - attr = heap_getattr(tuple, Anum_gp_segment_configuration_content, + attr = heap_getattr(tuple, Anum_gp_segment_configuration_warehouseid, RelationGetDescr(rel), &isNull); Assert(!isNull); - if (DatumGetInt16(attr) == MASTER_CONTENT_ID || strcmp(warehouse_name, current_warehouse) == 0) + warehouseid = DatumGetObjectId(attr); + + if (!OidIsValid(warehouseid) || warehouseid == GetCurrentWarehouseId()) { i = palloc(sizeof(GpSegConfigEntry)); @@ -1748,9 +1669,9 @@ contentid_get_dbid(int16 contentid, char role, bool getPreferredRoleNotCurrentRo { nkeys++; ScanKeyInit(&scankey[2], - Anum_gp_segment_configuration_warehouse_name, - BTEqualStrategyNumber, F_TEXTEQ, - CStringGetTextDatum(current_warehouse)); + Anum_gp_segment_configuration_warehouseid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(GetCurrentWarehouseId())); } scan = systable_beginscan(rel, GpSegmentConfigContentPreferred_roleWarehouseIndexId, true, @@ -1774,9 +1695,9 @@ contentid_get_dbid(int16 contentid, char role, bool getPreferredRoleNotCurrentRo { nkeys++; ScanKeyInit(&scankey[2], - Anum_gp_segment_configuration_warehouse_name, - BTEqualStrategyNumber, F_TEXTEQ, - CStringGetTextDatum(current_warehouse)); + Anum_gp_segment_configuration_warehouseid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(GetCurrentWarehouseId())); } /* no index */ scan = systable_beginscan(rel, InvalidOid, false, diff --git a/src/backend/fts/fts.c b/src/backend/fts/fts.c index 732c8969478..3da780092d4 100644 --- a/src/backend/fts/fts.c +++ b/src/backend/fts/fts.c @@ -239,9 +239,9 @@ probeWalRepUpdateConfig(int16 dbid, int16 segindex, char role, BTEqualStrategyNumber, F_INT2EQ, Int16GetDatum(dbid)); ScanKeyInit(&scankey[1], - Anum_gp_segment_configuration_warehouse_name, - BTEqualStrategyNumber, F_TEXTEQ, - CStringGetTextDatum(current_warehouse)); + Anum_gp_segment_configuration_warehouseid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(GetCurrentWarehouseId())); sscan = systable_beginscan(configrel, GpSegmentConfigDbidWarehouseIndexId, true, NULL, 2, scankey); diff --git a/src/backend/utils/gp/segadmin.c b/src/backend/utils/gp/segadmin.c index ff0481df0a5..cc2eacb31fb 100644 --- a/src/backend/utils/gp/segadmin.c +++ b/src/backend/utils/gp/segadmin.c @@ -200,17 +200,14 @@ remove_segment_config(int16 dbid) { Datum attr; bool isNull; - char *warehouse_name = NULL; + Oid warehouseid = InvalidOid; - attr = heap_getattr(tuple, Anum_gp_segment_configuration_warehouse_name, - RelationGetDescr(rel), &isNull); - if (!isNull) - warehouse_name = TextDatumGetCString(attr); - - attr = heap_getattr(tuple, Anum_gp_segment_configuration_content, + attr = heap_getattr(tuple, Anum_gp_segment_configuration_warehouseid, RelationGetDescr(rel), &isNull); Assert(!isNull); - if (DatumGetInt16(attr) == MASTER_CONTENT_ID || strcmp(warehouse_name, current_warehouse) == 0) + warehouseid = DatumGetObjectId(attr); + + if (!OidIsValid(warehouseid) || warehouseid == GetCurrentWarehouseId()) { CatalogTupleDelete(rel, &tuple->t_self); numDel++; @@ -254,11 +251,8 @@ add_segment_config_entry(GpSegConfigEntry *i) CStringGetTextDatum(i->address); values[Anum_gp_segment_configuration_datadir - 1] = CStringGetTextDatum(i->datadir); - if (i->warehousename != NULL) - values[Anum_gp_segment_configuration_warehouse_name - 1] = - CStringGetTextDatum(i->warehousename); - else - nulls[Anum_gp_segment_configuration_warehouse_name - 1] = true; + values[Anum_gp_segment_configuration_warehouseid - 1] = + ObjectIdGetDatum(i->warehouseid); tuple = heap_form_tuple(RelationGetDescr(rel), values, nulls); @@ -413,9 +407,9 @@ gp_add_segment(PG_FUNCTION_ARGS) new.datadir = TextDatumGetCString(PG_GETARG_DATUM(9)); if (new.segindex == MASTER_CONTENT_ID) - new.warehousename = NULL; + new.warehouseid = InvalidOid; else - new.warehousename = current_warehouse; + new.warehouseid = GetCurrentWarehouseId(); mirroring_sanity_check(MASTER_ONLY | SUPERUSER, "gp_add_segment"); @@ -498,9 +492,9 @@ gp_add_segment_mirror(PG_FUNCTION_ARGS) new.datadir = TextDatumGetCString(PG_GETARG_DATUM(4)); if (new.segindex == MASTER_CONTENT_ID) - new.warehousename = NULL; + new.warehouseid = InvalidOid; else - new.warehousename = current_warehouse; + new.warehouseid = GetCurrentWarehouseId(); mirroring_sanity_check(MASTER_ONLY | SUPERUSER, "gp_add_segment_mirror"); @@ -639,7 +633,7 @@ gp_add_master_standby(PG_FUNCTION_ARGS) config->datadir = TextDatumGetCString(PG_GETARG_TEXT_P(2)); - config->warehousename = NULL; + config->warehouseid = InvalidOid; /* Use the new port number if specified */ if (PG_NARGS() > 3 && !PG_ARGISNULL(3)) @@ -870,9 +864,9 @@ gp_update_segment_configuration_mode_status(PG_FUNCTION_ARGS) BTEqualStrategyNumber, F_INT2EQ, Int16GetDatum(dbid)); ScanKeyInit(&scankey[1], - Anum_gp_segment_configuration_warehouse_name, - BTEqualStrategyNumber, F_TEXTEQ, - CStringGetTextDatum(current_warehouse)); + Anum_gp_segment_configuration_warehouseid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(GetCurrentWarehouseId())); sscan = systable_beginscan(rel, GpSegmentConfigDbidWarehouseIndexId, true, NULL, 2, scankey); diff --git a/src/backend/utils/init/miscinit.c b/src/backend/utils/init/miscinit.c index abdc2567914..a4214ad0662 100644 --- a/src/backend/utils/init/miscinit.c +++ b/src/backend/utils/init/miscinit.c @@ -478,6 +478,8 @@ static int SecurityRestrictionContext = 0; /* We also remember if a SET ROLE is currently active */ static bool SetRoleIsActive = false; +static Oid CurrentWarehouseId = InvalidOid; + /* * GetUserId - get the current effective user ID. * @@ -963,6 +965,18 @@ GetUserNameFromId(Oid roleid, bool noerr) return result; } +Oid +GetCurrentWarehouseId(void) +{ + return CurrentWarehouseId; +} + +void +SetCurrentWarehouseId(Oid warehouseid) +{ + AssertArg(OidIsValid(warehouseid)); + CurrentWarehouseId = warehouseid; +} /*------------------------------------------------------------------------- * Interlock-file support diff --git a/src/backend/utils/misc/guc_gp.c b/src/backend/utils/misc/guc_gp.c index ccce56dcd5f..2e39cea261c 100644 --- a/src/backend/utils/misc/guc_gp.c +++ b/src/backend/utils/misc/guc_gp.c @@ -101,8 +101,6 @@ static void assign_pljava_classpath_insecure(bool newval, void *extra); static bool check_gp_resource_group_bypass(bool *newval, void **extra, GucSource source); static int guc_array_compare(const void *a, const void *b); static bool check_max_running_tasks(int *newval, void **extra, GucSource source); -static bool check_current_warehouse(char **newval, void **extra, GucSource source); -static void assign_current_warehouse(const char *newval, void *extra); int listenerBacklog = 128; @@ -432,9 +430,6 @@ bool gp_enable_global_deadlock_detector = false; bool gp_enable_predicate_pushdown; int gp_predicate_pushdown_sample_rows; -/* Current warehouse */ -char *current_warehouse = "default"; - static const struct config_enum_entry gp_log_format_options[] = { {"text", 0}, {"csv", 1}, @@ -4544,16 +4539,6 @@ struct config_string ConfigureNamesString_gp[] = check_gp_default_storage_options, assign_gp_default_storage_options, NULL }, - { - {"warehouse", PGC_USERSET, CUSTOM_OPTIONS, - gettext_noop("Sets the current warehouse."), - NULL, - GUC_NO_RESET_ALL - }, - ¤t_warehouse, "default", - check_current_warehouse, assign_current_warehouse, NULL - }, - { {"task_timezone", PGC_POSTMASTER, TASK_SCHEDULE_OPTIONS, gettext_noop("Specify timezone used for cron task schedule."), @@ -5094,24 +5079,6 @@ check_max_running_tasks(int *newval, void **extra, GucSource source) return true; } -bool -check_current_warehouse(char **newval, void **extra, GucSource source) -{ -#ifdef USE_INTERNAL_FTS - return cdb_checkWarehouseName(*newval); -#else - return true; -#endif -} - -void assign_current_warehouse(const char *newval, void *extra) -{ -#ifdef USE_INTERNAL_FTS - char *new_name = pstrdup(newval); - cdb_assignCurrentWarehouse(current_warehouse, new_name); -#endif -} - /* * Malloc a new string representing current storage_opts. */ diff --git a/src/include/catalog/gp_segment_configuration.h b/src/include/catalog/gp_segment_configuration.h index 35119f18bac..1705924e23b 100644 --- a/src/include/catalog/gp_segment_configuration.h +++ b/src/include/catalog/gp_segment_configuration.h @@ -58,7 +58,7 @@ CATALOG(gp_segment_configuration,7026,GpSegmentConfigRelationId) BKI_SHARED_RELA text address; text datadir; - text warehouse_name BKI_DEFAULT(_null_); + Oid warehouseid BKI_DEFAULT(0); #endif } FormData_gp_segment_configuration; diff --git a/src/include/catalog/gp_segment_configuration_indexing.h b/src/include/catalog/gp_segment_configuration_indexing.h index 42c85fe001f..7d72782e6d4 100644 --- a/src/include/catalog/gp_segment_configuration_indexing.h +++ b/src/include/catalog/gp_segment_configuration_indexing.h @@ -14,9 +14,9 @@ #include "catalog/genbki.h" -DECLARE_UNIQUE_INDEX(gp_segment_config_content_preferred_role_index, 7139, on gp_segment_configuration using btree(content int2_ops, preferred_role char_ops, warehouse_name text_ops)); +DECLARE_UNIQUE_INDEX(gp_segment_config_content_preferred_role_warehouse_index, 7139, on gp_segment_configuration using btree(content int2_ops, preferred_role char_ops, warehouseid oid_ops)); #define GpSegmentConfigContentPreferred_roleWarehouseIndexId 7139 -DECLARE_UNIQUE_INDEX(gp_segment_config_dbid_index, 7140, on gp_segment_configuration using btree(dbid int2_ops, warehouse_name text_ops)); +DECLARE_UNIQUE_INDEX(gp_segment_config_dbid_warehouse_index, 7140, on gp_segment_configuration using btree(dbid int2_ops, warehouseid oid_ops)); #define GpSegmentConfigDbidWarehouseIndexId 7140 #endif // GP_SEGMENT_CONFIGURATION_INDEXING_H \ No newline at end of file diff --git a/src/include/catalog/gp_warehouse.h b/src/include/catalog/gp_warehouse.h index f637032ef41..87646361569 100644 --- a/src/include/catalog/gp_warehouse.h +++ b/src/include/catalog/gp_warehouse.h @@ -20,6 +20,4 @@ DECLARE_UNIQUE_INDEX(gp_warehouse_oid_index, 8086, on gp_warehouse using btree(o DECLARE_UNIQUE_INDEX(gp_warehouse_name_index, 8059, on gp_warehouse using btree(warehouse_name text_ops)); #define GpWarehouseNameIndexId 8059 -extern Oid GetGpWarehouseOid(char *warehouse_name, bool missing_ok); - #endif /* GP_WAREHOUSE_H */ diff --git a/src/include/cdb/cdbutil.h b/src/include/cdb/cdbutil.h index 27ba446cd2a..b2424b0efd6 100644 --- a/src/include/cdb/cdbutil.h +++ b/src/include/cdb/cdbutil.h @@ -172,8 +172,6 @@ extern int16 contentid_get_dbid(int16 contentid, char role, bool getPreferredRol extern int16 cdbcomponent_get_maxdbid(void); extern int16 cdbcomponent_get_availableDbId(void); extern int16 cdbcomponent_get_maxcontentid(void); -extern bool cdb_checkWarehouseName(char *new_name); -extern void cdb_assignCurrentWarehouse(char *old_name, char *new_name); extern int numsegmentsFromQD; /* diff --git a/src/include/miscadmin.h b/src/include/miscadmin.h index d67953e5126..07e375dfea0 100644 --- a/src/include/miscadmin.h +++ b/src/include/miscadmin.h @@ -460,6 +460,8 @@ extern void InitializeSessionUserIdStandalone(void); extern void SetSessionAuthorization(Oid userid, bool is_superuser); extern Oid GetCurrentRoleId(void); extern void SetCurrentRoleId(Oid roleid, bool is_superuser); +extern Oid GetCurrentWarehouseId(void); +extern void SetCurrentWarehouseId(Oid warehouseid); /* in utils/misc/superuser.c */ extern bool superuser(void); /* current user is superuser */ diff --git a/src/include/postmaster/fts_comm.h b/src/include/postmaster/fts_comm.h index feb3e0b7232..16fc53247c2 100644 --- a/src/include/postmaster/fts_comm.h +++ b/src/include/postmaster/fts_comm.h @@ -148,7 +148,7 @@ typedef struct GpSegConfigEntry char *hostname; /* name or ip address of host machine */ char *address; /* ip address of host machine */ char *datadir; /* absolute path to data directory on the host. */ - char *warehousename; + Oid warehouseid; /* additional cached info */ char *hostip; /* cached lookup of name */ diff --git a/src/include/utils/guc.h b/src/include/utils/guc.h index fd35f21cbe0..0e1c15bcf64 100644 --- a/src/include/utils/guc.h +++ b/src/include/utils/guc.h @@ -625,8 +625,6 @@ extern bool gp_enable_global_deadlock_detector; extern bool gp_enable_predicate_pushdown; extern int gp_predicate_pushdown_sample_rows; -extern char *current_warehouse; - typedef enum { INDEX_CHECK_NONE, diff --git a/src/include/utils/unsync_guc_name.h b/src/include/utils/unsync_guc_name.h index c439773a3e9..d35e7f72320 100644 --- a/src/include/utils/unsync_guc_name.h +++ b/src/include/utils/unsync_guc_name.h @@ -609,7 +609,6 @@ "vacuum_freeze_table_age", "vacuum_multixact_freeze_min_age", "vacuum_multixact_freeze_table_age", - "warehouse", "wait_for_replication_threshold", "wal_block_size", "wal_buffers", From 467844d2b04411c7cbe39938b37e468799ac9da6 Mon Sep 17 00:00:00 2001 From: roseduan Date: Thu, 10 Aug 2023 12:19:18 +0800 Subject: [PATCH 15/15] fix tests --- src/test/regress/expected/misc_sanity.out | 4 ++-- src/test/regress/output/dispatch.source | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/test/regress/expected/misc_sanity.out b/src/test/regress/expected/misc_sanity.out index 1ab8ae44ea2..4f7ef1d12b7 100644 --- a/src/test/regress/expected/misc_sanity.out +++ b/src/test/regress/expected/misc_sanity.out @@ -134,7 +134,7 @@ ORDER BY 1, 2; pg_task_run_history | return_message | text pg_task_run_history | status | text pg_task_run_history | username | text -(30 rows) +(31 rows) -- system catalogs without primary keys -- @@ -171,7 +171,7 @@ ORDER BY 1; pg_stat_last_operation pg_stat_last_shoperation pg_type_encoding -(22 rows) +(23 rows) -- system catalog unique indexes not wrapped in a constraint -- (There should be none.) diff --git a/src/test/regress/output/dispatch.source b/src/test/regress/output/dispatch.source index 3b0f54967f4..d4c6ed0f383 100644 --- a/src/test/regress/output/dispatch.source +++ b/src/test/regress/output/dispatch.source @@ -894,8 +894,8 @@ SELECT gp_inject_fault('cdbcomponent_recycle_idle_qe_error', 'interrupt', dbid, (1 row) select * from gp_segment_configuration a, t13393 ,gp_segment_configuration b where a.dbid = t13393.tc1 limit 0; - dbid | content | role | preferred_role | mode | status | port | hostname | address | datadir | warehouse_name | tc1 | dbid | content | role | preferred_role | mode | status | port | hostname | address | datadir | warehouse_name -------+---------+------+----------------+------+--------+------+----------+---------+---------+----------------+-----+------+---------+------+----------------+------+--------+------+----------+---------+---------+---------------- + dbid | content | role | preferred_role | mode | status | port | hostname | address | datadir | warehouseid | tc1 | dbid | content | role | preferred_role | mode | status | port | hostname | address | datadir | warehouseid +------+---------+------+----------------+------+--------+------+----------+---------+---------+-------------+-----+------+---------+------+----------------+------+--------+------+----------+---------+---------+------------- (0 rows) SELECT gp_inject_fault('cdbcomponent_recycle_idle_qe_error', 'reset', dbid, current_setting('gp_session_id')::int)