diff --git a/concourse/pipelines/pipeline_coverity.yml b/concourse/pipelines/pipeline_coverity.yml deleted file mode 100644 index c055129a906..00000000000 --- a/concourse/pipelines/pipeline_coverity.yml +++ /dev/null @@ -1,61 +0,0 @@ -resource_types: -- name: gcs - type: registry-image - source: - repository: frodenas/gcs-resource - -resources: -- name: gpdb_src - type: git - source: - branch: ((gpdb-git-branch)) - uri: ((gpdb-git-remote)) - -- name: centos-coverity - type: registry-image - source: - repository: pivotaldata/centos-coverity - username: ((docker_username)) - password: ((docker_password)) - -- name: coverity_daily - type: time - source: - location: America/Los_Angeles - days: [Monday, Tuesday, Wednesday, Thursday, Friday, Saturday, Sunday] - start: 4:00 AM - stop: 5:00 AM - -- name: libquicklz-centos7 - type: gcs - source: - bucket: ((gcs-bucket)) - json_key: ((concourse-gcs-resources-service-account-key))) - regexp: gp-internal-artifacts/centos7/libquicklz-(1\.5\.0-.*)-1.el7.x86_64.rpm - -- name: libquicklz-devel-centos7 - type: gcs - source: - bucket: ((gcs-bucket)) - json_key: ((concourse-gcs-resources-service-account-key)) - regexp: gp-internal-artifacts/centos7/libquicklz-devel-(1\.5\.0-.*)-1.el7.x86_64.rpm - -jobs: - -- name: coverity_scan - plan: - - get: coverity_daily - trigger: true - - aggregate: - - get: gpdb_src - - get: centos-coverity - - get: libquicklz-installer - resource: libquicklz-centos7 - - get: libquicklz-devel-installer - resource: libquicklz-devel-centos7 - - task: scan_with_coverity - file: gpdb_src/concourse/tasks/scan_with_coverity.yml - image: centos-coverity - params: - COVERITY_TOKEN: ((coverity_token)) - COVERITY_EMAIL: ((coverity_email)) diff --git a/concourse/scripts/scan_with_coverity.bash b/concourse/scripts/scan_with_coverity.bash deleted file mode 100755 index 5fab1bb37f2..00000000000 --- a/concourse/scripts/scan_with_coverity.bash +++ /dev/null @@ -1,101 +0,0 @@ -#!/bin/bash -l -set -exo pipefail - -BASE_DIR=$(pwd) -export GPDB_ARTIFACTS_DIR -GPDB_ARTIFACTS_DIR=$BASE_DIR/$OUTPUT_ARTIFACT_DIR - -function prep_env_for_centos() { - BLD_ARCH=rhel7_x86_64 - echo "Detecting java7 path ..." - java7_packages=($(rpm -qa | grep -F java-1.7)) - java7_bin="$(rpm -ql "${java7_packages[@]}" | grep /jre/bin/java$)" - alternatives --set java "$java7_bin" - export JAVA_HOME="${java7_bin/jre\/bin\/java/}" - ln -sf /usr/bin/xsubpp /usr/share/perl5/ExtUtils/xsubpp - - export PATH=${JAVA_HOME}/bin:${PATH} -} - -function generate_build_number() { - pushd gpdb_src - #Only if its git repro, add commit SHA as build number - # BUILD_NUMBER file is used by getversion file in GPDB to append to version - if [ -d .git ] ; then - echo "commit:$(git rev-parse HEAD)" > BUILD_NUMBER - fi - popd -} - -function make_sync_tools() { - pushd gpdb_src/gpAux - make sync_tools - # We have compiled LLVM with native zlib on CentOS6 and not from - # the zlib downloaded from artifacts. Therefore, remove the zlib - # downloaded from artifacts in order to use the native zlib. - find ext -name 'libz.*' -exec rm -f {} \; - popd -} - -function build_gpdb_and_scan_with_coverity() { - local cov_int_dir="$1" - - pushd gpdb_src/gpAux - cov-build --dir "$cov_int_dir" make BLD_TARGETS="gpdb" GPROOT=/usr/local - popd -} - -function upload_to_coverity() { - ( - set +x - local cov_int_base="$1" - local sha="$2" - local cov_int_tar="$cov_int_base"/cov-int.tgz - - tar czfp "$cov_int_tar" -C "$cov_int_base" cov-int - - response=$(curl --verbose \ - --progress-bar \ - --form token="$COVERITY_TOKEN" \ - --form email="$COVERITY_EMAIL" \ - --form file=@"$cov_int_tar" \ - --form version="$sha" \ - --form description="Generated by Concourse on https://gpdb.data.pivotal.ci/" \ - https://scan.coverity.com/builds?project=greenplum-db%2Fgpdb) - - ERROR_STRINGS=( - "quota for this project has been reached" - ) - - for ERR in "${ERROR_STRINGS[@]}"; do - if echo "$response" | grep -q "$ERR"; then - echo "Coverty returned: \"$response\"" - echo "Response matches following know error: \"$ERR\"" - exit 1 - fi - done - ) -} - -function install_deps_for_centos() { - # quicklz is proprietary code that we cannot put in our public Docker images. - rpm -i libquicklz-installer/libquicklz-*.rpm - rpm -i libquicklz-devel-installer/libquicklz-*.rpm -} - -function _main() { - install_deps_for_centos - prep_env_for_centos - generate_build_number - make_sync_tools - - /opt/prepare-coverity.bash - - mkdir -p "$GPDB_ARTIFACTS_DIR"/cov-int - build_gpdb_and_scan_with_coverity "$GPDB_ARTIFACTS_DIR"/cov-int - - sha=$(cd gpdb_src && git rev-parse HEAD) - upload_to_coverity "$GPDB_ARTIFACTS_DIR" "$sha" -} - -_main "$@" diff --git a/concourse/tasks/scan_with_coverity.yml b/concourse/tasks/scan_with_coverity.yml deleted file mode 100644 index b550e825d37..00000000000 --- a/concourse/tasks/scan_with_coverity.yml +++ /dev/null @@ -1,17 +0,0 @@ -platform: linux -image_resource: - type: registry-image - source: - repository: pivotaldata/centos-coverity -inputs: - - name: gpdb_src - - name: libquicklz-installer - - name: libquicklz-devel-installer -outputs: - - name: gpdb_coverity_artifacts -run: - path: gpdb_src/concourse/scripts/scan_with_coverity.bash -params: - OUTPUT_ARTIFACT_DIR: gpdb_coverity_artifacts - COVERITY_TOKEN: - COVERITY_EMAIL: diff --git a/gpMgmt/bin/gpload.py b/gpMgmt/bin/gpload.py index a6ce8f471dc..2d7c0a1eb7f 100755 --- a/gpMgmt/bin/gpload.py +++ b/gpMgmt/bin/gpload.py @@ -2254,9 +2254,9 @@ def get_reuse_staging_table_query(self, encoding_conditions): return: sql(string) ''' - sql = """SELECT oid::regclass - FROM pg_class - WHERE relname = 'staging_gpload_reusable_%s';""" % (encoding_conditions) + sql = """SELECT oid::regclass \ +FROM pg_class \ +WHERE relname = 'staging_gpload_reusable_%s';""" % (encoding_conditions) self.log(self.DEBUG, "query used to identify reusable temporary relations: %s" % sql) return sql @@ -2273,7 +2273,8 @@ def get_table_oid(self, tableName): pass return None - def get_ext_schematable(self, schemaName, tableName): + + def get_schematable(self, schemaName, tableName): ''' return formated table name ''' @@ -2448,7 +2449,7 @@ def create_external_table(self): AND n.nspname !~ '^pg_toast'""" result = self.db.query(sql).getresult() if len(result) > 0: - self.extSchemaTable = self.get_ext_schematable(quote_unident(self.extSchemaName), self.extTableName) + self.extSchemaTable = self.get_schematable(quote_unident(self.extSchemaName), self.extTableName) self.log(self.INFO, "reusing external staging table %s" % self.extSchemaTable) return # staging table is not specified, we need to find it manually @@ -2468,7 +2469,7 @@ def create_external_table(self): self.extTableName = (resultList[0])[0] # fast match result is only table name, so we need add schema info if self.fast_match: - self.extSchemaTable = self.get_ext_schematable(quote_unident(self.extSchemaName), self.extTableName) + self.extSchemaTable = self.get_schematable(quote_unident(self.extSchemaName), self.extTableName) else: self.extSchemaTable = self.extTableName self.log(self.INFO, "reusing external table %s" % self.extSchemaTable) @@ -2479,13 +2480,13 @@ def create_external_table(self): # around self.extTableName = "ext_gpload_reusable_%s" % self.unique_suffix - self.log(self.INFO, "did not find an external table to reuse. creating %s" % self.get_ext_schematable(self.extSchemaName, self.extTableName)) + self.log(self.INFO, "did not find an external table to reuse. creating %s" % self.get_schematable(self.extSchemaName, self.extTableName)) # process the single quotes in order to successfully create an external table. self.formatOpts = self.formatOpts.replace("'\''","E'\\''") # construct a CREATE EXTERNAL TABLE statement and execute it - self.extSchemaTable = self.get_ext_schematable(self.extSchemaName, self.extTableName) + self.extSchemaTable = self.get_schematable(self.extSchemaName, self.extTableName) sql = "create external table %s" % self.extSchemaTable sql += "(%s)" % ','.join(['%s %s' % (a[0], a[1]) for a in from_cols]) @@ -2566,10 +2567,12 @@ def create_staging_table(self): # we no longer need the timestamp, since we will never want to create few # tables with same encoding_conditions self.staging_table_name = "staging_gpload_reusable_%s" % (encoding_conditions) + self.staging_table_name = self.get_schematable(self.extSchemaName, self.staging_table_name) self.log(self.INFO, "did not find a staging table to reuse. creating %s" % self.staging_table_name) # MPP-14667 - self.reuse_tables should change one, and only one, aspect of how we build the following table, # and that is, whether it's a temp table or not. In other words, is_temp_table = '' iff self.reuse_tables == True. + sql = 'CREATE %sTABLE %s ' % (is_temp_table, self.staging_table_name) cols = ['"%s" %s' % (a[0], a[1]) for a in target_columns] sql += "(%s)" % ','.join(cols) @@ -2882,6 +2885,7 @@ def do_method(self): # Is the table to be truncated before the load? preload = self.getconfig('gpload:preload', list, default=None) method = self.getconfig('gpload:output:mode', str, 'insert').lower() + external = self.getconfig('gpload:external', list, default=None) self.log_errors = self.getconfig('gpload:input:log_errors', bool, False) truncate = False self.reuse_tables = False @@ -2889,6 +2893,10 @@ def do_method(self): if not self.options.no_auto_trans and not method=='insert': self.db.query("BEGIN") + self.extSchemaName = self.getconfig('gpload:external:schema', str, None) + if self.extSchemaName == '%': + self.extSchemaName = self.schema + if preload: truncate = self.getconfig('gpload:preload:truncate',bool,False) self.reuse_tables = self.getconfig('gpload:preload:reuse_tables',bool,False) diff --git a/gpMgmt/bin/gpload_test/gpload2/query33.ans b/gpMgmt/bin/gpload_test/gpload2/query33.ans index b69631bff50..93756fe0acf 100644 --- a/gpMgmt/bin/gpload_test/gpload2/query33.ans +++ b/gpMgmt/bin/gpload_test/gpload2/query33.ans @@ -1,7 +1,7 @@ 2018-07-24 06:14:29|INFO|gpload session started 2018-07-24 06:14:29 2018-07-24 06:14:29|INFO|setting schema 'public' for table 'texttable' 2018-07-24 06:14:29|INFO|started gpfdist -p 8081 -P 8082 -f "/home/gpadmin/workspace/gpdb/gpMgmt/bin/gpload_test/gpload2/data_file.txt" -t 30 -2018-07-24 06:14:29|INFO|did not find a staging table to reuse. creating staging_gpload_reusable_afbaac0da7ced19791c9ab9c537f41d3 +2018-07-24 06:14:29|INFO|did not find a staging table to reuse. creating test.staging_gpload_reusable_afbaac0da7ced19791c9ab9c537f41d3 2018-07-24 06:14:29|INFO|did not find an external table to reuse. creating test.ext_gpload_reusable_d2e95f76_8f08_11e8_8c76_0242ac110002 2018-07-24 06:14:29|INFO|running time: 0.40 seconds 2018-07-24 06:14:29|INFO|rows Inserted = 16 @@ -11,7 +11,7 @@ 2018-07-24 06:14:30|INFO|gpload session started 2018-07-24 06:14:30 2018-07-24 06:14:30|INFO|setting schema 'public' for table 'texttable' 2018-07-24 06:14:30|INFO|started gpfdist -p 8081 -P 8082 -f "/home/gpadmin/workspace/gpdb/gpMgmt/bin/gpload_test/gpload2/data_file.txt" -t 30 -2018-07-24 06:14:30|INFO|reusing staging table staging_gpload_reusable_afbaac0da7ced19791c9ab9c537f41d3 +2018-07-24 06:14:30|INFO|reusing staging table test.staging_gpload_reusable_afbaac0da7ced19791c9ab9c537f41d3 2018-07-24 06:14:30|INFO|reusing external table test.ext_gpload_reusable_d2e95f76_8f08_11e8_8c76_0242ac110002 2018-07-24 06:14:30|INFO|running time: 0.31 seconds 2018-07-24 06:14:30|INFO|rows Inserted = 0 diff --git a/gpMgmt/bin/gpstop b/gpMgmt/bin/gpstop index fb852d94f2e..6862e42ba90 100755 --- a/gpMgmt/bin/gpstop +++ b/gpMgmt/bin/gpstop @@ -34,6 +34,7 @@ try: from gppylib.gp_era import GpEraFile from gppylib.operations.utils import ParallelOperation, RemoteOperation from gppylib.operations.rebalanceSegments import ReconfigDetectionSQLQueryCommand + from gppylib.operations.detect_unreachable_hosts import get_unreachable_segment_hosts except ImportError as e: sys.exit('ERROR: Cannot import modules. Please check that you have sourced greenplum_path.sh. Detail: ' + str(e)) @@ -514,6 +515,10 @@ class GpStop: if self.gparray.standbyCoordinator: standby = self.gparray.standbyCoordinator + if get_unreachable_segment_hosts([standby.hostname], 1): + logger.warning("Standby is unreachable, skipping shutdown on standby") + return True + logger.info("Stopping coordinator standby host %s mode=%s" % (standby.hostname, self.mode)) try: cmd = SegmentStop("stopping coordinator standby", diff --git a/gpMgmt/test/behave/mgmt_utils/gpstop.feature b/gpMgmt/test/behave/mgmt_utils/gpstop.feature index e5ea0c41b3e..7ff5b86ae7c 100644 --- a/gpMgmt/test/behave/mgmt_utils/gpstop.feature +++ b/gpMgmt/test/behave/mgmt_utils/gpstop.feature @@ -27,3 +27,13 @@ Feature: gpstop behave tests And gpstop should print "There were 1 user connections at the start of the shutdown" to stdout And gpstop should print "'\(s\)mart_mode', '\(f\)ast_mode', '\(i\)mmediate_mode'" to stdout Then gpstop should return a return code of 0 + + @demo_cluster + Scenario: gpstop succeeds even if the standby host is unreachable + Given the database is running + And the catalog has a standby coordinator entry + When the standby host is made unreachable + And the user runs "gpstop -a" + Then gpstop should print "Standby is unreachable, skipping shutdown on standby" to stdout + And gpstop should return a return code of 0 + And the standby host is made reachable diff --git a/gpMgmt/test/behave/mgmt_utils/steps/gpstart.py b/gpMgmt/test/behave/mgmt_utils/steps/gpstart.py index cfb23d95fa8..2046dbcae4f 100644 --- a/gpMgmt/test/behave/mgmt_utils/steps/gpstart.py +++ b/gpMgmt/test/behave/mgmt_utils/steps/gpstart.py @@ -37,30 +37,36 @@ def change_hostname(content, preferred_role, hostname): def impl(context): change_hostname(-1, 'm', 'invalid_host') - def cleanup(context): - """ - Reverses the above SQL by starting up in coordinator-only utility mode. Since - the standby host is incorrect, a regular gpstart call won't work. - """ - utils.stop_database_if_started(context) - - subprocess.check_call(['gpstart', '-am']) - _run_sql(""" - SET allow_system_table_mods='true'; - UPDATE gp_segment_configuration - SET hostname = coordinator.hostname, - address = coordinator.address - FROM ( - SELECT hostname, address - FROM gp_segment_configuration - WHERE content = -1 and role = 'p' - ) coordinator - WHERE content = -1 AND role = 'm' - """, {'gp_role': 'utility'}) - subprocess.check_call(['gpstop', '-am']) - context.add_cleanup(cleanup, context) +@when('the standby host is made reachable') +@then('the standby host is made reachable') +def impl(context): + cleanup(context) + +""" +Reverses the changes done by change_hostname() function by starting up cluster in master-only utility mode. +Since the standby host is incorrect, a regular gpstart call won't work. +""" +def cleanup(context): + + utils.stop_database_if_started(context) + + subprocess.check_call(['gpstart', '-am']) + _run_sql(""" + SET allow_system_table_mods='true'; + UPDATE gp_segment_configuration + SET hostname = coordinator.hostname, + address = coordinator.address + FROM ( + SELECT hostname, address + FROM gp_segment_configuration + WHERE content = -1 and role = 'p' + ) coordinator + WHERE content = -1 AND role = 'm' + """, {'gp_role': 'utility'}) + subprocess.check_call(['gpstop', '-am']) + def _handle_sigpipe(): """ Work around https://bugs.python.org/issue1615376, which is not fixed until diff --git a/src/backend/access/aocs/aocsam.c b/src/backend/access/aocs/aocsam.c index 0ae994659b1..bbc42dc81c3 100644 --- a/src/backend/access/aocs/aocsam.c +++ b/src/backend/access/aocs/aocsam.c @@ -507,13 +507,15 @@ aocs_beginscan(Relation relation, RelationIncrementReferenceCount(relation); /* - * the append-only meta data should never be fetched with + * The append-only meta data should never be fetched with * SnapshotAny as bogus results are returned. + * We use SnapshotSelf for metadata, as regular MVCC snapshot can hide newly + * globally inserted tuples from global index build process. */ if (snapshot != SnapshotAny) aocsMetaDataSnapshot = snapshot; else - aocsMetaDataSnapshot = GetTransactionSnapshot(); + aocsMetaDataSnapshot = SnapshotSelf; seginfo = GetAllAOCSFileSegInfo(relation, aocsMetaDataSnapshot, &total_seg, NULL); return aocs_beginscan_internal(relation, diff --git a/src/backend/access/appendonly/appendonlyam.c b/src/backend/access/appendonly/appendonlyam.c index d4681bd4aa3..32a306f3ae0 100755 --- a/src/backend/access/appendonly/appendonlyam.c +++ b/src/backend/access/appendonly/appendonlyam.c @@ -1638,10 +1638,12 @@ appendonly_beginscan(Relation relation, if (appendOnlyMetaDataSnapshot == SnapshotAny) { /* - * the append-only meta data should never be fetched with + * The append-only meta data should never be fetched with * SnapshotAny as bogus results are returned. + * We use SnapshotSelf for metadata, as regular MVCC snapshot can hide + * newly globally inserted tuples from global index build process. */ - appendOnlyMetaDataSnapshot = GetTransactionSnapshot(); + appendOnlyMetaDataSnapshot = SnapshotSelf; } /* diff --git a/src/backend/access/transam/varsup.c b/src/backend/access/transam/varsup.c index fc866ba9d97..5ca18d1b77d 100644 --- a/src/backend/access/transam/varsup.c +++ b/src/backend/access/transam/varsup.c @@ -26,6 +26,7 @@ #include "storage/pmsignal.h" #include "storage/proc.h" #include "cdb/cdbutil.h" +#include "utils/faultinjector.h" #include "utils/guc.h" #include "utils/syscache.h" @@ -624,6 +625,26 @@ GetNewObjectIdUnderLock(void) (ShmemVariableCache->nextOid)++; (ShmemVariableCache->oidCount)--; +#ifdef FAULT_INJECTOR + if (SIMPLE_FAULT_INJECTOR("bump_oid") == FaultInjectorTypeSkip) + { + /* + * CDB: we encounter high oid issues several times, we should + * have some test-utils to verify logic under larger oid. + * + * NOTE: we do not have undo-bump, so take care when you decide to + * use this fault inject. Currently, only resgroup test job uses it, + * that is safe, becase resgroup job is an independent pipeline job. + */ + Oid large_oid = (1U<<31)+5; /* this value will overflow if taken as int32 */ + if (ShmemVariableCache->nextOid < large_oid) + { + ShmemVariableCache->nextOid = large_oid + 1; + result = large_oid; + } + } +#endif + return result; } diff --git a/src/backend/commands/foreigncmds.c b/src/backend/commands/foreigncmds.c index b39b2c69b43..a61b0b71d35 100644 --- a/src/backend/commands/foreigncmds.c +++ b/src/backend/commands/foreigncmds.c @@ -191,7 +191,6 @@ transformGenericOptions(Oid catalogId, if (catalogId != UserMappingRelationId) { SeparateOutMppExecute(&resultOptions); - SeparateOutNumSegments(&resultOptions); } if (OidIsValid(fdwvalidator)) diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index 3cc182ee5d3..982d4603aed 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -17581,8 +17581,11 @@ build_ctas_with_dist(Relation rel, DistributedBy *dist_clause, return queryDesc; } +/* + * GPDB: Convenience function to get reloptions for a given relation. + */ static Datum -new_rel_opts(Relation rel) +get_rel_opts(Relation rel) { Datum newOptions = PointerGetDatum(NULL); @@ -18127,7 +18130,6 @@ static void ATExecExpandTableCTAS(AlterTableCmd *rootCmd, Relation rel, AlterTableCmd *cmd, int numsegments) { RangeVar *tmprv; - Datum newOptions; Oid tmprelid; Oid relid = RelationGetRelid(rel); ReindexParams params = {0}; @@ -18167,10 +18169,8 @@ ATExecExpandTableCTAS(AlterTableCmd *rootCmd, Relation rel, AlterTableCmd *cmd, distby = make_distributedby_for_rel(rel); distby->numsegments = numsegments; - newOptions = new_rel_opts(rel); - queryDesc = build_ctas_with_dist(rel, distby, - untransformRelOptions(newOptions), + untransformRelOptions(get_rel_opts(rel)), &tmprv, true); @@ -18300,7 +18300,6 @@ ATExecSetDistributedBy(Relation rel, Node *node, AlterTableCmd *cmd) bool rand_pol = false; bool rep_pol = false; bool force_reorg = false; - Datum newOptions = PointerGetDatum(NULL); bool need_reorg; bool change_policy = false; int numsegments; @@ -18401,8 +18400,6 @@ ATExecSetDistributedBy(Relation rel, Node *node, AlterTableCmd *cmd) lwith = nlist; } - newOptions = new_rel_opts(rel); - if (ldistro) change_policy = true; @@ -18445,8 +18442,8 @@ ATExecSetDistributedBy(Relation rel, Node *node, AlterTableCmd *cmd) cmd->policy = policy; - /* only need to rebuild if have new storage options */ - if (!(DatumGetPointer(newOptions) || force_reorg)) + /* no need to rebuild if REORGANIZE=false*/ + if (!force_reorg) goto l_distro_fini; } } @@ -18465,12 +18462,9 @@ ATExecSetDistributedBy(Relation rel, Node *node, AlterTableCmd *cmd) policy = createReplicatedGpPolicy(ldistro->numsegments); - /* rebuild if have new storage options or policy changed */ - if (!DatumGetPointer(newOptions) && - GpPolicyIsReplicated(rel->rd_cdbpolicy)) - { + /* rebuild only if policy changed */ + if (GpPolicyIsReplicated(rel->rd_cdbpolicy)) goto l_distro_fini; - } /* * system columns is not visiable to users for replicated table, @@ -18573,11 +18567,9 @@ ATExecSetDistributedBy(Relation rel, Node *node, AlterTableCmd *cmd) ldistro->numsegments); /* - * See if the old policy is the same as the new one but - * remember, we still might have to rebuild if there are new - * storage options. + * See if the old policy is the same as the new one. */ - if (!DatumGetPointer(newOptions) && !force_reorg && + if (!force_reorg && (policy->nattrs == rel->rd_cdbpolicy->nattrs)) { int i; @@ -18704,7 +18696,7 @@ ATExecSetDistributedBy(Relation rel, Node *node, AlterTableCmd *cmd) /* Step (b) - build CTAS */ queryDesc = build_ctas_with_dist(rel, ldistro, - untransformRelOptions(newOptions), + untransformRelOptions(get_rel_opts(rel)), &tmprv, true); @@ -18796,7 +18788,6 @@ ATExecSetDistributedBy(Relation rel, Node *node, AlterTableCmd *cmd) backend_id = cmd->backendId - 1; tmprv = make_temp_table_name(rel, backend_id); - newOptions = new_rel_opts(rel); need_reorg = true; } @@ -18837,59 +18828,9 @@ ATExecSetDistributedBy(Relation rel, Node *node, AlterTableCmd *cmd) ReadNextMultiXactId(), NULL); - /* - * Make changes from swapping relation files visible before updating - * options below or else we get an already updated tuple error. - */ + /* Make changes from swapping relation files visible. */ CommandCounterIncrement(); - if (DatumGetPointer(newOptions)) - { - Datum repl_val[Natts_pg_class]; - bool repl_null[Natts_pg_class]; - bool repl_repl[Natts_pg_class]; - HeapTuple newOptsTuple; - HeapTuple tuple; - Relation relationRelation; - - /* - * All we need do here is update the pg_class row; the new - * options will be propagated into relcaches during - * post-commit cache inval. - */ - MemSet(repl_val, 0, sizeof(repl_val)); - MemSet(repl_null, false, sizeof(repl_null)); - MemSet(repl_repl, false, sizeof(repl_repl)); - - if (newOptions != (Datum) 0) - repl_val[Anum_pg_class_reloptions - 1] = newOptions; - else - repl_null[Anum_pg_class_reloptions - 1] = true; - - repl_repl[Anum_pg_class_reloptions - 1] = true; - - relationRelation = table_open(RelationRelationId, RowExclusiveLock); - tuple = SearchSysCache1(RELOID, ObjectIdGetDatum(tarrelid)); - - Assert(HeapTupleIsValid(tuple)); - newOptsTuple = heap_modify_tuple(tuple, RelationGetDescr(relationRelation), - repl_val, repl_null, repl_repl); - - CatalogTupleUpdate(relationRelation, &tuple->t_self, newOptsTuple); - - heap_freetuple(newOptsTuple); - - ReleaseSysCache(tuple); - - table_close(relationRelation, RowExclusiveLock); - - /* - * Increment cmd counter to make updates visible; this is - * needed because the same tuple has to be updated again - */ - CommandCounterIncrement(); - } - /* now, reindex */ reindex_relation(tarrelid, 0, ¶ms); } diff --git a/src/backend/executor/nodeShareInputScan.c b/src/backend/executor/nodeShareInputScan.c index 1bb05edf674..4886e5d377c 100644 --- a/src/backend/executor/nodeShareInputScan.c +++ b/src/backend/executor/nodeShareInputScan.c @@ -366,6 +366,15 @@ ExecShareInputScan(PlanState *pstate) /* if first time call, need to initialize the tuplestore state. */ if (!node->isready) init_tuplestore_state(node); + + /* + * Return NULL when necessary. + * This could help improve performance, especially when tuplestore is huge, because ShareInputScan + * do not need to read tuple from tuplestore when discard_output is true, which means current + * ShareInputScan is one but not the last one of Sequence's subplans. + */ + if (sisc->discard_output) + return NULL; slot = node->ss.ps.ps_ResultTupleSlot; diff --git a/src/backend/executor/nodeWindowAgg.c b/src/backend/executor/nodeWindowAgg.c index 75933ca5488..6a626f1de77 100644 --- a/src/backend/executor/nodeWindowAgg.c +++ b/src/backend/executor/nodeWindowAgg.c @@ -40,6 +40,7 @@ #include "executor/executor.h" #include "executor/nodeWindowAgg.h" #include "miscadmin.h" +#include "nodes/execnodes.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" #include "parser/parse_agg.h" @@ -49,6 +50,7 @@ #include "utils/builtins.h" #include "utils/datum.h" #include "utils/expandeddatum.h" +#include "utils/faultinjector.h" #include "utils/lsyscache.h" #include "utils/memutils.h" #include "utils/regproc.h" @@ -269,7 +271,7 @@ initialize_windowaggregate(WindowAggState *winstate, peraggstate->distinctLtOper, peraggstate->distinctColl, false, /* nullsFirstFlag */ - work_mem, + PlanStateOperatorMemKB((PlanState *) winstate), NULL, /* coordinate */ false); } @@ -686,6 +688,24 @@ perform_distinct_windowaggregate(WindowAggState *winstate, tuplesort_performsort(peraggstate->distinctSortState); +#ifdef FAULT_INJECTOR + /* + * This routine is used for tracing whether the sort operation of DISTINCT-qualified + * WindowAgg spills to disk. + */ + if (SIMPLE_FAULT_INJECTOR("distinct_winagg_perform_sort") == FaultInjectorTypeSkip) + { + TuplesortInstrumentation sortstats; + tuplesort_get_stats(peraggstate->distinctSortState, &sortstats); + if (sortstats.spaceType == SORT_SPACE_TYPE_MEMORY) + ereport(NOTICE, + (errmsg("distinct winagg sortstats: sort operation fitted in memory"))); + else + ereport(NOTICE, + (errmsg("distinct winagg sortstats: sort operation spilled to disk"))); + } +#endif + /* load the first tuple from spool */ if (tuplesort_getdatum(peraggstate->distinctSortState, true, &fcinfo->args[1].value, &fcinfo->args[1].isnull, NULL)) @@ -1360,7 +1380,9 @@ begin_partition(WindowAggState *winstate) } /* Create new tuplestore for this partition */ - winstate->buffer = tuplestore_begin_heap(false, false, work_mem); + winstate->buffer = + tuplestore_begin_heap(false, false, + PlanStateOperatorMemKB((PlanState *) winstate)); /* * Set up read pointers for the tuplestore. The current pointer doesn't @@ -2426,6 +2448,34 @@ ExecWindowAgg(PlanState *pstate) */ spool_tuples(winstate, winstate->currentpos); +#ifdef FAULT_INJECTOR + /* + * This routine is used for testing if we have allocated enough memory + * for the tuplestore (winstate->buffer) in begin_partition(). If all + * tuples of the current partition can be fitted in the memory, we + * emit a notice saying 'fitted in memory'. If they cannot be fitted in + * the memory, we emit a notice saying 'spilled to disk'. If there're + * no input rows, we emit a notice saying 'no input rows'. + * + * NOTE: The fault-injector only triggers once, we emit the notice when + * we finishes spooling all the tuples of the first partition. + */ + if (winstate->partition_spooled && + winstate->currentpos >= winstate->spooled_rows && + SIMPLE_FAULT_INJECTOR("winagg_after_spool_tuples") == FaultInjectorTypeSkip) + { + if (winstate->buffer) + { + if (tuplestore_in_memory(winstate->buffer)) + ereport(NOTICE, (errmsg("winagg: tuplestore fitted in memory"))); + else + ereport(NOTICE, (errmsg("winagg: tuplestore spilled to disk"))); + } + else + ereport(NOTICE, (errmsg("winagg: no input rows"))); + } +#endif + /* Move to the next partition if we reached the end of this partition */ if (winstate->partition_spooled && winstate->currentpos >= winstate->spooled_rows) diff --git a/src/backend/foreign/foreign.c b/src/backend/foreign/foreign.c index ed619d47348..4e55bf5d606 100644 --- a/src/backend/foreign/foreign.c +++ b/src/backend/foreign/foreign.c @@ -366,20 +366,13 @@ GetForeignTable(Oid relid) else ft->options = untransformRelOptions(datum); - ForeignServer *server = GetForeignServer(ft->serverid); - ft->exec_location = SeparateOutMppExecute(&ft->options); if (ft->exec_location == FTEXECLOCATION_NOT_DEFINED) { + ForeignServer *server = GetForeignServer(ft->serverid); ft->exec_location = server->exec_location; } - ft->num_segments = SeparateOutNumSegments(&ft->options); - if (ft->num_segments <= 0) - { - ft->num_segments = server->num_segments; - } - ReleaseSysCache(tp); return ft; diff --git a/src/backend/gpopt/translate/CTranslatorDXLToPlStmt.cpp b/src/backend/gpopt/translate/CTranslatorDXLToPlStmt.cpp index 0a75541c1a6..65894b7d779 100644 --- a/src/backend/gpopt/translate/CTranslatorDXLToPlStmt.cpp +++ b/src/backend/gpopt/translate/CTranslatorDXLToPlStmt.cpp @@ -3738,6 +3738,7 @@ CTranslatorDXLToPlStmt::TranslateDXLCTEProducerToSharedScan( // create the shared input scan representing the CTE Producer ShareInputScan *shared_input_scan = MakeNode(ShareInputScan); shared_input_scan->share_id = cte_id; + shared_input_scan->discard_output = true; Plan *plan = &(shared_input_scan->scan.plan); plan->plan_node_id = m_dxl_to_plstmt_context->GetNextPlanId(); @@ -3795,6 +3796,7 @@ CTranslatorDXLToPlStmt::TranslateDXLCTEConsumerToSharedScan( ShareInputScan *share_input_scan_cte_consumer = MakeNode(ShareInputScan); share_input_scan_cte_consumer->share_id = cte_id; + share_input_scan_cte_consumer->discard_output = false; Plan *plan = &(share_input_scan_cte_consumer->scan.plan); plan->plan_node_id = m_dxl_to_plstmt_context->GetNextPlanId(); diff --git a/src/backend/gpopt/translate/CTranslatorDXLToScalar.cpp b/src/backend/gpopt/translate/CTranslatorDXLToScalar.cpp index 6d87816dca4..331650e65bc 100644 --- a/src/backend/gpopt/translate/CTranslatorDXLToScalar.cpp +++ b/src/backend/gpopt/translate/CTranslatorDXLToScalar.cpp @@ -806,7 +806,8 @@ CTranslatorDXLToScalar::TranslateDXLScalarSubplanToScalar( SubLinkType slink = CTranslatorUtils::MapDXLSubplanToSublinkType( dxlop->GetDxlSubplanType()); Expr *test_expr = TranslateDXLSubplanTestExprToScalar( - dxlop->GetDxlTestExpr(), slink, colid_var, ¶m_ids); + dxlop->GetDxlTestExpr(), slink, colid_var, dxlop->FOuterParam(), + ¶m_ids); const CDXLColRefArray *outer_refs = dxlop->GetDxlOuterColRefsArray(); @@ -890,7 +891,7 @@ CTranslatorDXLToScalar::TranslateDXLScalarSubplanToScalar( Expr * CTranslatorDXLToScalar::TranslateDXLSubplanTestExprToScalar( CDXLNode *test_expr_node, SubLinkType slink, CMappingColIdVar *colid_var, - List **param_ids) + BOOL has_outer_refs, List **param_ids) { if (EXPR_SUBLINK == slink || EXISTS_SUBLINK == slink || NOT_EXISTS_SUBLINK == slink) @@ -923,19 +924,45 @@ CTranslatorDXLToScalar::TranslateDXLSubplanTestExprToScalar( CDXLNode *outer_child_node = (*test_expr_node)[0]; CDXLNode *inner_child_node = (*test_expr_node)[1]; + CContextDXLToPlStmt *dxl_to_plstmt_ctxt = + (dynamic_cast(colid_var)) + ->GetDXLToPlStmtContext(); + // translate outer expression (can be a deep scalar tree) - Expr *outer_arg_expr = TranslateDXLToScalar(outer_child_node, colid_var); - args = gpdb::LAppend(args, outer_arg_expr); + Expr *outer_arg_expr = nullptr; + if (has_outer_refs) + { + Param *param1 = MakeNode(Param); + param1->paramkind = PARAM_EXEC; + + // Ident + CDXLScalarIdent *outer_ident = + CDXLScalarIdent::Cast(outer_child_node->GetOperator()); + Expr *outer_expr = (Expr *) param1; + + // finalize outer expression + param1->paramtype = CMDIdGPDB::CastMdid(outer_ident->MdidType())->Oid(); + param1->paramtypmod = outer_ident->TypeModifier(); + param1->paramid = dxl_to_plstmt_ctxt->GetNextParamId(param1->paramtype); + + // test expression is used for non-scalar subplan, + // first arg of test expression must be an EXEC param1 referring to subplan output + args = gpdb::LAppend(args, outer_expr); + + // also, add this param1 to subplan param1 ids before translating other params + *param_ids = gpdb::LAppendInt(*param_ids, param1->paramid); + } + else + { + outer_arg_expr = TranslateDXLToScalar(outer_child_node, colid_var); + args = gpdb::LAppend(args, outer_arg_expr); + } // translate inner expression (only certain forms supported) // second arg must be an EXEC param which is replaced during query execution with subplan output Param *param = MakeNode(Param); param->paramkind = PARAM_EXEC; - CContextDXLToPlStmt *dxl_to_plstmt_ctxt = - (dynamic_cast(colid_var)) - ->GetDXLToPlStmtContext(); - CDXLScalarIdent *inner_ident = nullptr; Expr *inner_expr = nullptr; if (EdxlopScalarIdent == inner_child_node->GetOperator()->GetDXLOperator()) diff --git a/src/backend/gporca/concourse/test_explain_pipeline.yml b/src/backend/gporca/concourse/test_explain_pipeline.yml index 6d8a31da844..81759a2b82c 100644 --- a/src/backend/gporca/concourse/test_explain_pipeline.yml +++ b/src/backend/gporca/concourse/test_explain_pipeline.yml @@ -171,6 +171,28 @@ resources: json_key: ((concourse-gcs-resources-orca-service-account-key)) versioned_file: ((pipeline-name))/explain_results/explain_((workload4))_results.tar.gz +## resources for workload 6 +- name: explain_output_((workload6))_branch + type: gcs + source: + bucket: ((gcs-bucket-orca)) + json_key: ((concourse-gcs-resources-orca-service-account-key)) + versioned_file: ((pipeline-name))/explain_intermediate/explain_((workload6))_branch.tar.gz + +- name: explain_output_((workload6))_baseline + type: gcs + source: + bucket: ((gcs-bucket-orca)) + json_key: ((concourse-gcs-resources-orca-service-account-key)) + versioned_file: ((pipeline-name))/explain_intermediate/explain_((workload6))_baseline.tar.gz + +- name: explain_((workload6))_results + type: gcs + source: + bucket: ((gcs-bucket-orca)) + json_key: ((concourse-gcs-resources-orca-service-account-key)) + versioned_file: ((pipeline-name))/explain_results/explain_((workload6))_results.tar.gz + ## ====================================================================== ## jobs ## ====================================================================== @@ -631,3 +653,94 @@ jobs: put: explain_((workload4))_results params: file: diffs/explain_test_results.tar.gz + +## ====================================================================== +## workload6 +## ====================================================================== + +- name: run_explain_suite_((workload6)) + max_in_flight: 1 + plan: + - in_parallel: + - get: gporca-commits-to-test + passed: + - compile_branch + - get: bin_gpdb + passed: + - compile_branch + resource: bin_gpdb_centos7_branch + trigger: true + - get: gpdb_src + resource: gpdb_main_src + - get: gp_workloads + params: + disable_git_lfs: true + depth: 1 + - task: run_explain_suite + file: gpdb_src/concourse/tasks/run_explain_suite.yml + params: + MODE: orca + WORKLOAD: ((workload6)) + on_success: + try: + put: explain_output_((workload6))_branch + params: + file: output/explain_ouput.tar.gz + +- name: run_explain_suite_((workload6))_baseline + max_in_flight: 1 + plan: + - in_parallel: + - get: gporca-commits-to-test + passed: + - compile_baseline + - get: bin_gpdb + resource: bin_gpdb_centos7_baseline + passed: + - compile_baseline + trigger: true + - get: gpdb_src + resource: gpdb_main_src + params: + submodules: none + - get: gp_workloads + params: + disable_git_lfs: true + depth: 1 + - task: run_explain_suite_baseline + file: gpdb_src/concourse/tasks/run_explain_suite.yml + params: + MODE: orca + WORKLOAD: ((workload6)) + on_success: + try: + put: explain_output_((workload6))_baseline + params: + file: output/explain_ouput.tar.gz + +- name: diff_explain_results_((workload6)) + plan: + - in_parallel: + - get: gporca-commits-to-test + passed: + - run_explain_suite_((workload6)) + - run_explain_suite_((workload6))_baseline + trigger: true + - get: gpdb_src + resource: gpdb_main_src + params: + submodules: none + - get: explain_output + resource: explain_output_((workload6))_branch + passed: + - run_explain_suite_((workload6)) + - get: explain_output_baseline + resource: explain_output_((workload6))_baseline + passed: + - run_explain_suite_((workload6))_baseline + - task: diff_explain_results_((workload6)) + file: gpdb_src/concourse/tasks/diff_explain_results_with_baseline.yml + ensure: + put: explain_((workload6))_results + params: + file: diffs/explain_test_results.tar.gz diff --git a/src/backend/gporca/data/dxl/minidump/AnyPredicate-Over-UnionOfConsts.mdp b/src/backend/gporca/data/dxl/minidump/AnyPredicate-Over-UnionOfConsts.mdp index cb496402c58..66786c38994 100644 --- a/src/backend/gporca/data/dxl/minidump/AnyPredicate-Over-UnionOfConsts.mdp +++ b/src/backend/gporca/data/dxl/minidump/AnyPredicate-Over-UnionOfConsts.mdp @@ -1,5 +1,9 @@ + + diff --git a/src/backend/gporca/data/dxl/minidump/ScalarSubq-Eq-SubqAll-1.mdp b/src/backend/gporca/data/dxl/minidump/ScalarSubq-Eq-SubqAll-1.mdp new file mode 100644 index 00000000000..2fbe8cfa699 --- /dev/null +++ b/src/backend/gporca/data/dxl/minidump/ScalarSubq-Eq-SubqAll-1.mdp @@ -0,0 +1,332 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/backend/gporca/data/dxl/minidump/ScalarSubq-Eq-SubqAll-2.mdp b/src/backend/gporca/data/dxl/minidump/ScalarSubq-Eq-SubqAll-2.mdp new file mode 100644 index 00000000000..6a9b0fdb942 --- /dev/null +++ b/src/backend/gporca/data/dxl/minidump/ScalarSubq-Eq-SubqAll-2.mdp @@ -0,0 +1,360 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/backend/gporca/libgpopt/include/gpopt/base/CColRefSet.h b/src/backend/gporca/libgpopt/include/gpopt/base/CColRefSet.h index 3a38ec04f17..007d6671cff 100644 --- a/src/backend/gporca/libgpopt/include/gpopt/base/CColRefSet.h +++ b/src/backend/gporca/libgpopt/include/gpopt/base/CColRefSet.h @@ -120,6 +120,9 @@ class CColRefSet : public CBitSet, public DbgPrintMixin // convert to array CColRefArray *Pdrgpcr(CMemoryPool *mp) const; + // convert to id colref map + IntToColRefMap *Phmicr(CMemoryPool *mp) const; + // hash function ULONG HashValue(); diff --git a/src/backend/gporca/libgpopt/include/gpopt/operators/CLogicalLeftOuterCorrelatedApply.h b/src/backend/gporca/libgpopt/include/gpopt/operators/CLogicalLeftOuterCorrelatedApply.h index 7f3ed6b959f..63a3c0bc497 100644 --- a/src/backend/gporca/libgpopt/include/gpopt/operators/CLogicalLeftOuterCorrelatedApply.h +++ b/src/backend/gporca/libgpopt/include/gpopt/operators/CLogicalLeftOuterCorrelatedApply.h @@ -30,6 +30,8 @@ namespace gpopt class CLogicalLeftOuterCorrelatedApply : public CLogicalLeftOuterApply { private: + BOOL m_allow_predicate_pushdown{true}; + public: CLogicalLeftOuterCorrelatedApply(const CLogicalLeftOuterCorrelatedApply &) = delete; @@ -77,6 +79,12 @@ class CLogicalLeftOuterCorrelatedApply : public CLogicalLeftOuterApply return true; } + BOOL + IsPredicatePushDownAllowed() const + { + return m_allow_predicate_pushdown; + } + // conversion function static CLogicalLeftOuterCorrelatedApply * PopConvert(COperator *pop) diff --git a/src/backend/gporca/libgpopt/include/gpopt/translate/CTranslatorExprToDXL.h b/src/backend/gporca/libgpopt/include/gpopt/translate/CTranslatorExprToDXL.h index 269e4ec2fa6..aeea672aa96 100644 --- a/src/backend/gporca/libgpopt/include/gpopt/translate/CTranslatorExprToDXL.h +++ b/src/backend/gporca/libgpopt/include/gpopt/translate/CTranslatorExprToDXL.h @@ -625,7 +625,10 @@ class CTranslatorExprToDXL CDXLNode *PdxlnProjectBoolConst(CDXLNode *dxlnode, BOOL value); // helper to build a Result expression with project list restricted to required column - CDXLNode *PdxlnRestrictResult(CDXLNode *dxlnode, CColRef *colref); + CDXLNode *PdxlnRestrictResult(CDXLNode *dxlnode, const CColRef *colref); + + // helper to build a Result expression with project list restricted to required columns + CDXLNode *PdxlnRestrictResult(CDXLNode *dxlnode, const CColRefSet *colrefs); // helper to build subplans from correlated LOJ void BuildSubplansForCorrelatedLOJ( diff --git a/src/backend/gporca/libgpopt/include/gpopt/xforms/CSubqueryHandler.h b/src/backend/gporca/libgpopt/include/gpopt/xforms/CSubqueryHandler.h index bfe1d0987d7..b4dd303abf1 100644 --- a/src/backend/gporca/libgpopt/include/gpopt/xforms/CSubqueryHandler.h +++ b/src/backend/gporca/libgpopt/include/gpopt/xforms/CSubqueryHandler.h @@ -272,7 +272,8 @@ class CSubqueryHandler // build an expression for the quantified comparison of the subquery CExpression *PexprSubqueryPred(CExpression *pexprOuter, CExpression *pexprSubquery, - CExpression **ppexprResult); + CExpression **ppexprResult, + CSubqueryHandler::ESubqueryCtxt esqctxt); // main driver BOOL FProcess( diff --git a/src/backend/gporca/libgpopt/src/base/CColRefSet.cpp b/src/backend/gporca/libgpopt/src/base/CColRefSet.cpp index a3ba3ee37bc..716e071958b 100644 --- a/src/backend/gporca/libgpopt/src/base/CColRefSet.cpp +++ b/src/backend/gporca/libgpopt/src/base/CColRefSet.cpp @@ -88,7 +88,6 @@ CColRefSet::FMember(const CColRef *colref) const return CBitSet::Get(colref->Id()); } - //--------------------------------------------------------------------------- // @function: // CColRefSet::PcrAny @@ -294,6 +293,29 @@ CColRefSet::Pdrgpcr(CMemoryPool *mp) const } +//--------------------------------------------------------------------------- +// @function: +// CColRefSet::Phmicr +// +// @doc: +// Convert set into map +// +//--------------------------------------------------------------------------- +IntToColRefMap * +CColRefSet::Phmicr(CMemoryPool *mp) const +{ + IntToColRefMap *phmicr = GPOS_NEW(mp) IntToColRefMap(mp); + + CColRefSetIter crsi(*this); + while (crsi.Advance()) + { + phmicr->Insert(GPOS_NEW(mp) INT(crsi.Pcr()->Id()), crsi.Pcr()); + } + + return phmicr; +} + + //--------------------------------------------------------------------------- // @function: // CColRefSet::HashValue diff --git a/src/backend/gporca/libgpopt/src/operators/CLogicalLeftOuterCorrelatedApply.cpp b/src/backend/gporca/libgpopt/src/operators/CLogicalLeftOuterCorrelatedApply.cpp index 400442e8b01..7f9d1429c62 100644 --- a/src/backend/gporca/libgpopt/src/operators/CLogicalLeftOuterCorrelatedApply.cpp +++ b/src/backend/gporca/libgpopt/src/operators/CLogicalLeftOuterCorrelatedApply.cpp @@ -40,7 +40,49 @@ CLogicalLeftOuterCorrelatedApply::CLogicalLeftOuterCorrelatedApply( //--------------------------------------------------------------------------- CLogicalLeftOuterCorrelatedApply::CLogicalLeftOuterCorrelatedApply( CMemoryPool *mp, CColRefArray *pdrgpcrInner, EOperatorId eopidOriginSubq) - : CLogicalLeftOuterApply(mp, pdrgpcrInner, eopidOriginSubq) + : CLogicalLeftOuterApply(mp, pdrgpcrInner, eopidOriginSubq), + // In the case of subquery all, we cannot push down the predicate. + // Example query: + // + // SELECT (SELECT 1) = ALL (SELECT generate_series(1, 2)); + // + // Physical plan: + // ``` + // +--CPhysicalComputeScalar + // |--CPhysicalCorrelatedLeftOuterNLJoin + // | |--CPhysicalConstTableGet Columns: ["" (0)] + // | |--CPhysicalComputeScalar + // | | |--CPhysicalCorrelatedLeftOuterNLJoin + // | | | |--CPhysicalComputeScalar + // | | | | |--CPhysicalConstTableGet Columns: ["" (1)] Values: [(1)] + // | | | | +--CScalarProjectList + // | | | | +--CScalarProjectElement "generate_series" (2) + // | | | | +--CScalarFunc (generate_series) + // | | | | |--CScalarConst (1) + // | | | | +--CScalarConst (2) + // | | | |--CPhysicalComputeScalar + // | | | | |--CPhysicalConstTableGet Columns: ["" (3)] Values: [(1)] + // | | | | +--CScalarProjectList + // | | | | +--CScalarProjectElement "?column?" (4) + // | | | | +--CScalarConst (1) + // | | | +--CScalarConst (1) + // | | +--CScalarProjectList + // | | +--CScalarProjectElement "ColRef_0006" (6) + // | | +--CScalarConst (1) + // | +--CScalarCmp (=) + // | |--CScalarIdent "?column?" (4) + // | +--CScalarIdent "generate_series" (2) + // +--CScalarProjectList + // +--CScalarProjectElement "?column?" (5) + // +--CScalarIdent "ColRef_0006" (6) + // ``` + // + // If we push down CScalarCmp as a filter then we would incorrectly + // discard the tuple 2 output from generate_series. Instead we want to + // preserve it for a NULL match in the LOJ so that we correctly evaulate + // subplan ALL_SUBLINK. + m_allow_predicate_pushdown(COperator::EopScalarSubqueryAll != + eopidOriginSubq) { } diff --git a/src/backend/gporca/libgpopt/src/operators/CNormalizer.cpp b/src/backend/gporca/libgpopt/src/operators/CNormalizer.cpp index 9739ab89857..4b179a90ba3 100644 --- a/src/backend/gporca/libgpopt/src/operators/CNormalizer.cpp +++ b/src/backend/gporca/libgpopt/src/operators/CNormalizer.cpp @@ -18,6 +18,7 @@ #include "gpopt/base/CUtils.h" #include "gpopt/operators/CLogical.h" #include "gpopt/operators/CLogicalInnerJoin.h" +#include "gpopt/operators/CLogicalLeftOuterCorrelatedApply.h" #include "gpopt/operators/CLogicalLeftOuterJoin.h" #include "gpopt/operators/CLogicalNAryJoin.h" #include "gpopt/operators/CLogicalProject.h" @@ -1078,7 +1079,6 @@ CNormalizer::PushThru(CMemoryPool *mp, CExpression *pexprLogical, case COperator::EopLogicalInnerCorrelatedApply: case COperator::EopLogicalLeftOuterJoin: case COperator::EopLogicalLeftOuterApply: - case COperator::EopLogicalLeftOuterCorrelatedApply: case COperator::EopLogicalLeftSemiApply: case COperator::EopLogicalLeftSemiApplyIn: case COperator::EopLogicalLeftSemiCorrelatedApplyIn: @@ -1088,9 +1088,19 @@ CNormalizer::PushThru(CMemoryPool *mp, CExpression *pexprLogical, case COperator::EopLogicalLeftSemiJoin: PushThruJoin(mp, pexprLogical, pexprConj, ppexprResult); break; - + case COperator::EopLogicalLeftOuterCorrelatedApply: default: { + if (COperator::EopLogicalLeftOuterCorrelatedApply == + pexprLogical->Pop()->Eopid() && + CLogicalLeftOuterCorrelatedApply::PopConvert( + pexprLogical->Pop()) + ->IsPredicatePushDownAllowed()) + { + PushThruJoin(mp, pexprLogical, pexprConj, ppexprResult); + break; + } + // can't push predicates through, start a new normalization path CExpression *pexprNormalized = PexprRecursiveNormalize(mp, pexprLogical); diff --git a/src/backend/gporca/libgpopt/src/translate/CTranslatorExprToDXL.cpp b/src/backend/gporca/libgpopt/src/translate/CTranslatorExprToDXL.cpp index ae35120faaf..a3d421b8cd8 100644 --- a/src/backend/gporca/libgpopt/src/translate/CTranslatorExprToDXL.cpp +++ b/src/backend/gporca/libgpopt/src/translate/CTranslatorExprToDXL.cpp @@ -3317,10 +3317,35 @@ CTranslatorExprToDXL::BuildSubplans( // //--------------------------------------------------------------------------- CDXLNode * -CTranslatorExprToDXL::PdxlnRestrictResult(CDXLNode *dxlnode, CColRef *colref) +CTranslatorExprToDXL::PdxlnRestrictResult(CDXLNode *dxlnode, + const CColRef *colref) +{ + CDXLNode *dxlresult = nullptr; + CColRefSet *pcrInner = GPOS_NEW(m_mp) CColRefSet(m_mp); + + pcrInner->Include(colref); + dxlresult = PdxlnRestrictResult(dxlnode, pcrInner); + pcrInner->Release(); + + return dxlresult; +} + + +//--------------------------------------------------------------------------- +// @function: +// CTranslatorExprToDXL::PdxlnRestrictResult +// +// @doc: +// Helper to build a Result expression with project list +// restricted to required columns +// +//--------------------------------------------------------------------------- +CDXLNode * +CTranslatorExprToDXL::PdxlnRestrictResult(CDXLNode *dxlnode, + const CColRefSet *colrefs) { GPOS_ASSERT(nullptr != dxlnode); - GPOS_ASSERT(nullptr != colref); + GPOS_ASSERT(nullptr != colrefs); CDXLNode *pdxlnProjListOld = (*dxlnode)[0]; const ULONG ulPrjElems = pdxlnProjListOld->Arity(); @@ -3339,12 +3364,17 @@ CTranslatorExprToDXL::PdxlnRestrictResult(CDXLNode *dxlnode, CColRef *colref) CDXLScalarProjList *pdxlopPrL = GPOS_NEW(m_mp) CDXLScalarProjList(m_mp); CDXLNode *pdxlnProjListNew = GPOS_NEW(m_mp) CDXLNode(m_mp, pdxlopPrL); + IntToColRefMap *phmicr = colrefs->Phmicr(m_mp); + for (ULONG ul = 0; ul < ulPrjElems; ul++) { CDXLNode *child_dxlnode = (*pdxlnProjListOld)[ul]; CDXLScalarProjElem *pdxlPrjElem = CDXLScalarProjElem::Cast(child_dxlnode->GetOperator()); - if (pdxlPrjElem->Id() == colref->Id()) + + const INT colid = pdxlPrjElem->Id(); + CColRef *colref = phmicr->Find(&colid); + if (colref) { // create a new project element that simply points to required column, // we cannot re-use child_dxlnode here since it may have a deep expression with columns inaccessible @@ -3354,7 +3384,10 @@ CTranslatorExprToDXL::PdxlnRestrictResult(CDXLNode *dxlnode, CColRef *colref) pdxlnProjListNew->AddChild(pdxlnPrEl); } } - GPOS_ASSERT(1 == pdxlnProjListNew->Arity()); + + phmicr->Release(); + + GPOS_ASSERT(colrefs->Size() == pdxlnProjListNew->Arity()); pdxlnResult = GPOS_NEW(m_mp) CDXLNode(m_mp, GPOS_NEW(m_mp) CDXLPhysicalResult(m_mp)); @@ -3409,8 +3442,10 @@ CTranslatorExprToDXL::PdxlnQuantifiedSubplan( pulNonGatherMotions, pfDML, false /*fRemap*/, false /*fRoot*/); // find required column from inner child - CColRef *pcrInner = (*pdrgpcrInner)[0]; + CColRefSet *pcrInner = GPOS_NEW(m_mp) CColRefSet(m_mp); + pcrInner->Include((*pdrgpcrInner)[0]); + BOOL outerParam = false; if (fCorrelatedLOJ) { // overwrite required inner column based on scalar expression @@ -3421,14 +3456,24 @@ CTranslatorExprToDXL::PdxlnQuantifiedSubplan( pcrsUsed->Intersection(pcrsInner); if (0 < pcrsUsed->Size()) { - GPOS_ASSERT(1 == pcrsUsed->Size()); + GPOS_ASSERT(1 == pcrsUsed->Size() || 2 == pcrsUsed->Size()); + + // Both sides of the SubPlan test expression can come from the + // inner side. So we need to pass pcrsUsed instead of pcrInner into + // PdxlnRestrictResult() + outerParam = pcrsUsed->Size() > 1; - pcrInner = pcrsUsed->PcrFirst(); + pcrInner->Release(); + pcrInner = pcrsUsed; + } + else + { + pcrsUsed->Release(); } - pcrsUsed->Release(); } CDXLNode *inner_dxlnode = PdxlnRestrictResult(pdxlnInnerChild, pcrInner); + pcrInner->Release(); if (nullptr == inner_dxlnode) { GPOS_RAISE( @@ -3445,10 +3490,10 @@ CTranslatorExprToDXL::PdxlnQuantifiedSubplan( mdid->AddRef(); // construct a subplan node, with the inner child under it - CDXLNode *pdxlnSubPlan = GPOS_NEW(m_mp) CDXLNode( - m_mp, - GPOS_NEW(m_mp) CDXLScalarSubPlan(m_mp, mdid, dxl_colref_array, - dxl_subplan_type, dxlnode_test_expr)); + CDXLNode *pdxlnSubPlan = GPOS_NEW(m_mp) + CDXLNode(m_mp, GPOS_NEW(m_mp) CDXLScalarSubPlan( + m_mp, mdid, dxl_colref_array, dxl_subplan_type, + dxlnode_test_expr, outerParam)); pdxlnSubPlan->AddChild(inner_dxlnode); // add to hashmap diff --git a/src/backend/gporca/libgpopt/src/xforms/CSubqueryHandler.cpp b/src/backend/gporca/libgpopt/src/xforms/CSubqueryHandler.cpp index 5ab73c4ffd8..970de9e728d 100644 --- a/src/backend/gporca/libgpopt/src/xforms/CSubqueryHandler.cpp +++ b/src/backend/gporca/libgpopt/src/xforms/CSubqueryHandler.cpp @@ -179,7 +179,8 @@ CSubqueryHandler::PexprReplace(CMemoryPool *mp, CExpression *pexprInput, CExpression * CSubqueryHandler::PexprSubqueryPred(CExpression *pexprOuter, CExpression *pexprSubquery, - CExpression **ppexprResult) + CExpression **ppexprResult, + CSubqueryHandler::ESubqueryCtxt esqctxt) { GPOS_ASSERT(CUtils::FQuantifiedSubquery(pexprSubquery->Pop())); @@ -187,7 +188,6 @@ CSubqueryHandler::PexprSubqueryPred(CExpression *pexprOuter, CExpression *pexprNewLogical = nullptr; CExpression *pexprScalarChild = (*pexprSubquery)[1]; - CSubqueryHandler::ESubqueryCtxt esqctxt = CSubqueryHandler::EsqctxtFilter; // If pexprScalarChild is a non-scalar subquery such as follows, // EXPLAIN SELECT * FROM t3 WHERE (c = ANY(SELECT c FROM t2)) IN (SELECT b from t1); @@ -1227,7 +1227,7 @@ CSubqueryHandler::FCreateCorrelatedApplyForQuantifiedSubquery( CExpression *pexprResult = nullptr; CSubqueryHandler sh(mp, true /* fEnforceCorrelatedApply */); CExpression *pexprPredicate = - sh.PexprSubqueryPred(pexprInner, pexprSubquery, &pexprResult); + sh.PexprSubqueryPred(pexprInner, pexprSubquery, &pexprResult, esqctxt); pexprInner->AddRef(); if (EsqctxtFilter == esqctxt) @@ -1452,7 +1452,7 @@ CSubqueryHandler::FRemoveAnySubquery(CExpression *pexprOuter, // build subquery quantified comparison CExpression *pexprResult = nullptr; CExpression *pexprPredicate = - PexprSubqueryPred(pexprInner, pexprSubquery, &pexprResult); + PexprSubqueryPred(pexprInner, pexprSubquery, &pexprResult, esqctxt); // generate a select for the quantified predicate pexprInner->AddRef(); @@ -1623,8 +1623,8 @@ CSubqueryHandler::FRemoveAllSubquery(CExpression *pexprOuter, { // build subquery quantified comparison CExpression *pexprResult = nullptr; - CExpression *pexprPredicate = - PexprSubqueryPred(pexprInner, pexprSubquery, &pexprResult); + CExpression *pexprPredicate = PexprSubqueryPred( + pexprInner, pexprSubquery, &pexprResult, esqctxt); *ppexprResidualScalar = CUtils::PexprScalarConstBool(mp, true /*value*/); diff --git a/src/backend/gporca/libgpopt/src/xforms/CXformUtils.cpp b/src/backend/gporca/libgpopt/src/xforms/CXformUtils.cpp index 9472b5b9524..d206be48c80 100644 --- a/src/backend/gporca/libgpopt/src/xforms/CXformUtils.cpp +++ b/src/backend/gporca/libgpopt/src/xforms/CXformUtils.cpp @@ -847,7 +847,8 @@ CXformUtils::SubqueryAnyToAgg( CExpression *pexprResult = nullptr; CSubqueryHandler sh(mp, false /* fEnforceCorrelatedApply */); CExpression *pexprSubqPred = - sh.PexprSubqueryPred(pexprInner, pexprSubquery, &pexprResult); + sh.PexprSubqueryPred(pexprInner, pexprSubquery, &pexprResult, + CSubqueryHandler::EsqctxtFilter); CScalarCmp *scalarCmp = CScalarCmp::PopConvert(pexprSubqPred->Pop()); GPOS_ASSERT(nullptr != scalarCmp); diff --git a/src/backend/gporca/libnaucrates/include/naucrates/dxl/operators/CDXLScalarSubPlan.h b/src/backend/gporca/libnaucrates/include/naucrates/dxl/operators/CDXLScalarSubPlan.h index 54855430eec..00518f7b955 100644 --- a/src/backend/gporca/libnaucrates/include/naucrates/dxl/operators/CDXLScalarSubPlan.h +++ b/src/backend/gporca/libnaucrates/include/naucrates/dxl/operators/CDXLScalarSubPlan.h @@ -63,6 +63,9 @@ class CDXLScalarSubPlan : public CDXLScalar // test expression -- not null if quantified/existential subplan CDXLNode *m_dxlnode_test_expr; + // does test expression contain outer param + BOOL m_outer_param; + public: CDXLScalarSubPlan(CDXLScalarSubPlan &) = delete; @@ -70,7 +73,7 @@ class CDXLScalarSubPlan : public CDXLScalar CDXLScalarSubPlan(CMemoryPool *mp, IMDId *first_col_type_mdid, CDXLColRefArray *dxl_colref_array, EdxlSubPlanType dxl_subplan_type, - CDXLNode *dxlnode_test_expr); + CDXLNode *dxlnode_test_expr, BOOL outer_param = false); ~CDXLScalarSubPlan() override; @@ -108,6 +111,12 @@ class CDXLScalarSubPlan : public CDXLScalar return m_dxlnode_test_expr; } + BOOL + FOuterParam() const + { + return m_outer_param; + } + // serialize operator in DXL format void SerializeToDXL(CXMLSerializer *xml_serializer, const CDXLNode *dxlnode) const override; diff --git a/src/backend/gporca/libnaucrates/src/operators/CDXLScalarSubPlan.cpp b/src/backend/gporca/libnaucrates/src/operators/CDXLScalarSubPlan.cpp index 42c4bb4fe37..6274d9236a4 100644 --- a/src/backend/gporca/libnaucrates/src/operators/CDXLScalarSubPlan.cpp +++ b/src/backend/gporca/libnaucrates/src/operators/CDXLScalarSubPlan.cpp @@ -33,12 +33,14 @@ CDXLScalarSubPlan::CDXLScalarSubPlan(CMemoryPool *mp, IMDId *first_col_type_mdid, CDXLColRefArray *dxl_colref_array, EdxlSubPlanType dxl_subplan_type, - CDXLNode *dxlnode_test_expr) + CDXLNode *dxlnode_test_expr, + BOOL outer_param) : CDXLScalar(mp), m_first_col_type_mdid(first_col_type_mdid), m_dxl_colref_array(dxl_colref_array), m_dxl_subplan_type(dxl_subplan_type), - m_dxlnode_test_expr(dxlnode_test_expr) + m_dxlnode_test_expr(dxlnode_test_expr), + m_outer_param(outer_param) { GPOS_ASSERT(EdxlSubPlanTypeSentinel > dxl_subplan_type); GPOS_ASSERT_IMP(EdxlSubPlanTypeAny == dxl_subplan_type || diff --git a/src/backend/gporca/server/src/unittest/gpopt/xforms/CSubqueryHandlerTest.cpp b/src/backend/gporca/server/src/unittest/gpopt/xforms/CSubqueryHandlerTest.cpp index 0c730984807..8b87f813205 100644 --- a/src/backend/gporca/server/src/unittest/gpopt/xforms/CSubqueryHandlerTest.cpp +++ b/src/backend/gporca/server/src/unittest/gpopt/xforms/CSubqueryHandlerTest.cpp @@ -34,7 +34,9 @@ const CHAR *rgszSubqueryHandlerMinidumpFileNames[] = { "../data/dxl/minidump/CorrelatedSubqueryWithAggWindowFunc.mdp", "../data/dxl/minidump/AllSubqueryWithSubqueryInScalar.mdp", "../data/dxl/minidump/AnySubqueryWithAllSubqueryInScalar.mdp", - "../data/dxl/minidump/AnySubqueryWithSubqueryInScalar.mdp"}; + "../data/dxl/minidump/AnySubqueryWithSubqueryInScalar.mdp", + "../data/dxl/minidump/ScalarSubq-Eq-SubqAll-1.mdp", + "../data/dxl/minidump/ScalarSubq-Eq-SubqAll-2.mdp"}; //--------------------------------------------------------------------------- // @function: diff --git a/src/backend/optimizer/plan/planshare.c b/src/backend/optimizer/plan/planshare.c index 1b5c53f549e..75983b8ffe3 100644 --- a/src/backend/optimizer/plan/planshare.c +++ b/src/backend/optimizer/plan/planshare.c @@ -39,6 +39,7 @@ make_shareinputscan(PlannerInfo *root, Plan *inputplan) sisc->producer_slice_id = -1; sisc->this_slice_id = -1; sisc->nconsumers = 0; + sisc->discard_output = false; sisc->scan.plan.qual = NIL; sisc->scan.plan.righttree = NULL; diff --git a/src/backend/optimizer/plan/subselect.c b/src/backend/optimizer/plan/subselect.c index ebb5bbb10a0..aaed8b99c1f 100644 --- a/src/backend/optimizer/plan/subselect.c +++ b/src/backend/optimizer/plan/subselect.c @@ -156,7 +156,7 @@ get_first_col_type(Plan *plan, Oid *coltype, int32 *coltypmod, /** * Returns true if query refers to a distributed table. */ -bool QueryHasDistributedRelation(Query *q) +bool QueryHasDistributedRelation(Query *q, bool recursive) { ListCell *rt = NULL; @@ -164,6 +164,11 @@ bool QueryHasDistributedRelation(Query *q) { RangeTblEntry *rte = (RangeTblEntry *) lfirst(rt); + if (rte->rtekind == RTE_SUBQUERY + && recursive + && QueryHasDistributedRelation(rte->subquery, true)) + return true; + if (rte->relid != InvalidOid && rte->rtekind == RTE_RELATION) { @@ -325,7 +330,7 @@ make_subplan(PlannerInfo *root, Query *orig_subquery, if ((Gp_role == GP_ROLE_DISPATCH) && IsSubqueryMultiLevelCorrelated(subquery) - && QueryHasDistributedRelation(subquery)) + && QueryHasDistributedRelation(subquery, root->is_correlated_subplan)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), diff --git a/src/backend/optimizer/util/pathnode.c b/src/backend/optimizer/util/pathnode.c index 71b8ba267e1..9cf83acc338 100644 --- a/src/backend/optimizer/util/pathnode.c +++ b/src/backend/optimizer/util/pathnode.c @@ -3639,13 +3639,18 @@ create_foreignscan_path(PlannerInfo *root, RelOptInfo *rel, pathnode->path.total_cost = total_cost; pathnode->path.pathkeys = pathkeys; + ForeignServer *server = NULL; switch (rel->exec_location) { case FTEXECLOCATION_ANY: CdbPathLocus_MakeGeneral(&(pathnode->path.locus)); break; case FTEXECLOCATION_ALL_SEGMENTS: - CdbPathLocus_MakeStrewn(&(pathnode->path.locus), rel->num_segments, 0); + server = GetForeignServer(rel->serverid); + if (server) + CdbPathLocus_MakeStrewn(&(pathnode->path.locus), server->num_segments, 0); + else + CdbPathLocus_MakeStrewn(&(pathnode->path.locus), getgpsegmentCount(), 0); break; case FTEXECLOCATION_COORDINATOR: CdbPathLocus_MakeEntry(&(pathnode->path.locus)); @@ -3704,13 +3709,18 @@ create_foreign_join_path(PlannerInfo *root, RelOptInfo *rel, pathnode->path.total_cost = total_cost; pathnode->path.pathkeys = pathkeys; + ForeignServer *server = NULL; switch (rel->exec_location) { case FTEXECLOCATION_ANY: CdbPathLocus_MakeGeneral(&(pathnode->path.locus)); break; case FTEXECLOCATION_ALL_SEGMENTS: - CdbPathLocus_MakeStrewn(&(pathnode->path.locus), rel->num_segments, 0); + server = GetForeignServer(rel->serverid); + if (server) + CdbPathLocus_MakeStrewn(&(pathnode->path.locus), server->num_segments, 0); + else + CdbPathLocus_MakeStrewn(&(pathnode->path.locus), getgpsegmentCount(), 0); break; case FTEXECLOCATION_COORDINATOR: CdbPathLocus_MakeEntry(&(pathnode->path.locus)); diff --git a/src/backend/optimizer/util/plancat.c b/src/backend/optimizer/util/plancat.c index 1e64cfb06ee..95662fd05f8 100644 --- a/src/backend/optimizer/util/plancat.c +++ b/src/backend/optimizer/util/plancat.c @@ -60,7 +60,6 @@ #include "cdb/cdbappendonlyam.h" #include "cdb/cdbrelsize.h" -#include "cdb/cdbutil.h" #include "catalog/pg_appendonly.h" #include "catalog/pg_foreign_server.h" #include "catalog/pg_inherits.h" @@ -472,14 +471,12 @@ get_relation_info(PlannerInfo *root, Oid relationObjectId, bool inhparent, rel->serverid = GetForeignServerIdByRelId(RelationGetRelid(relation)); rel->fdwroutine = GetFdwRoutineForRelation(relation, true); rel->exec_location = GetForeignTable(RelationGetRelid(relation))->exec_location; - rel->num_segments = GetForeignTable(RelationGetRelid(relation))->num_segments; } else { rel->serverid = InvalidOid; rel->fdwroutine = NULL; rel->exec_location = FTEXECLOCATION_NOT_DEFINED; - rel->num_segments = getgpsegmentCount(); } /* Collect info about relation's foreign keys, if relevant */ diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c index 4c6f8646cb8..d104ccf86e1 100644 --- a/src/backend/replication/logical/tablesync.c +++ b/src/backend/replication/logical/tablesync.c @@ -700,9 +700,10 @@ fetch_remote_table_info(char *nspname, char *relname, WalRcvExecResult *res; StringInfoData cmd; TupleTableSlot *slot; - Oid tableRow[] = {OIDOID, CHAROID, CHAROID}; - Oid attrRow[] = {TEXTOID, OIDOID, BOOLOID}; + Oid tableRow[3] = {OIDOID, CHAROID, CHAROID}; + Oid attrRow[4] = {TEXTOID, OIDOID, INT4OID, BOOLOID}; bool isnull; + char relkind; int natt; lrel->nspname = nspname; @@ -738,9 +739,20 @@ fetch_remote_table_info(char *nspname, char *relname, Assert(!isnull); lrel->replident = DatumGetChar(slot_getattr(slot, 2, &isnull)); Assert(!isnull); - lrel->relkind = DatumGetChar(slot_getattr(slot, 3, &isnull)); + relkind = DatumGetChar(slot_getattr(slot, 3, &isnull)); Assert(!isnull); + /* + * Newer PG versions allow things that aren't plain tables to appear in + * publications. We don't handle that in this version, but try to provide + * a useful error message. + */ + if (relkind != RELKIND_RELATION) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("logical replication source relation \"%s.%s\" is not a table", + nspname, relname))); + ExecDropSingleTupleTableSlot(slot); walrcv_clear_result(res); diff --git a/src/backend/utils/cache/inval.c b/src/backend/utils/cache/inval.c index 5c41466a4ff..5a986349e2a 100644 --- a/src/backend/utils/cache/inval.c +++ b/src/backend/utils/cache/inval.c @@ -785,7 +785,18 @@ AcceptInvalidationMessages(void) * recursive reloads it's unlikely you'll learn more. *---------- */ -#ifdef DISCARD_CACHES_ENABLED +#if defined(CLOBBER_CACHE_ALWAYS) + { + static bool in_recursion = false; + + if (!in_recursion) + { + in_recursion = true; + InvalidateSystemCachesExtended(true); + in_recursion = false; + } + } +#elif defined(CLOBBER_CACHE_RECURSIVELY) { static int recursion_depth = 0; diff --git a/src/backend/utils/hyperloglog/gp_hyperloglog.c b/src/backend/utils/hyperloglog/gp_hyperloglog.c index c093111fe85..f9ce8e33245 100644 --- a/src/backend/utils/hyperloglog/gp_hyperloglog.c +++ b/src/backend/utils/hyperloglog/gp_hyperloglog.c @@ -573,6 +573,7 @@ gp_hll_compress_dense(GpHLLCounter hloglog) errmsg("out of memory"), errdetail("Failed on request of size %zu.", data_rawsize))); } + memset(data, 0, data_rawsize); /* put all registers in a normal array i.e. remove dense packing so * lz compression can work optimally */ diff --git a/src/backend/utils/resgroup/resgroup_helper.c b/src/backend/utils/resgroup/resgroup_helper.c index 68941a639c8..2148ffe3e34 100644 --- a/src/backend/utils/resgroup/resgroup_helper.c +++ b/src/backend/utils/resgroup/resgroup_helper.c @@ -105,7 +105,7 @@ getResUsage(ResGroupStatCtx *ctx, Oid inGroupId) initStringInfo(&buffer); appendStringInfo(&buffer, "SELECT groupid, cpu_usage, memory_usage " - "FROM pg_resgroup_get_status(%d)", + "FROM pg_resgroup_get_status(%u)", inGroupId); CdbDispatchCommand(buffer.data, DF_WITH_SNAPSHOT, &cdb_pgresults); @@ -133,8 +133,7 @@ getResUsage(ResGroupStatCtx *ctx, Oid inGroupId) { const char *result; ResGroupStat *row = &ctx->groups[j]; - Oid groupId = pg_atoi(PQgetvalue(pg_result, j, 0), - sizeof(Oid), 0); + Oid groupId = atooid(PQgetvalue(pg_result, j, 0)); Assert(groupId == row->groupId); diff --git a/src/include/foreign/foreign.h b/src/include/foreign/foreign.h index c8226060067..04405241b0c 100644 --- a/src/include/foreign/foreign.h +++ b/src/include/foreign/foreign.h @@ -59,8 +59,7 @@ typedef struct ForeignTable Oid relid; /* relation Oid */ Oid serverid; /* server Oid */ List *options; /* ftoptions as DefElem list */ - char exec_location; /* execute on COORDINATOR, ANY or ALL SEGMENTS, Cloudberry MPP specific */ - int32 num_segments; /* the number of segments of the foreign table */ + char exec_location; /* execute on COORDINATOR, ANY or ALL SEGMENTS, Greenplum MPP specific */ } ForeignTable; /* Flags for GetForeignServerExtended */ diff --git a/src/include/gpopt/translate/CTranslatorDXLToScalar.h b/src/include/gpopt/translate/CTranslatorDXLToScalar.h index e4fa180b69f..bd5672ccaea 100644 --- a/src/include/gpopt/translate/CTranslatorDXLToScalar.h +++ b/src/include/gpopt/translate/CTranslatorDXLToScalar.h @@ -150,6 +150,7 @@ class CTranslatorDXLToScalar Expr *TranslateDXLSubplanTestExprToScalar(CDXLNode *test_expr_node, SubLinkType slink, CMappingColIdVar *colid_var, + BOOL has_outer_refs, List **param_ids_list); // translate subplan parameters diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h index 695b4dbc5e3..6cb36977a2b 100644 --- a/src/include/nodes/pathnodes.h +++ b/src/include/nodes/pathnodes.h @@ -924,8 +924,7 @@ typedef struct RelOptInfo Oid serverid; /* identifies server for the table or join */ Oid userid; /* identifies user to check access as */ bool useridiscurrent; /* join is only valid for current user */ - char exec_location; /* execute on MASTER, ANY or ALL SEGMENTS, Cloudberry MPP specific */ - int32 num_segments; /* number of segments, Cloudberry MPP specific */ + char exec_location; /* execute on MASTER, ANY or ALL SEGMENTS, Greenplum MPP specific */ /* use "struct FdwRoutine" to avoid including fdwapi.h here */ struct FdwRoutine *fdwroutine; void *fdw_private; diff --git a/src/include/nodes/plannodes.h b/src/include/nodes/plannodes.h index 64679cbe527..fbac2ca02b9 100644 --- a/src/include/nodes/plannodes.h +++ b/src/include/nodes/plannodes.h @@ -1099,6 +1099,9 @@ typedef struct ShareInputScan /* Number of consumer slices participating, not including the producer. */ int nconsumers; + + /* Discard the scan output? True for ORCA CTE producer, false otherwise. */ + bool discard_output; } ShareInputScan; /* ---------------- diff --git a/src/include/optimizer/subselect.h b/src/include/optimizer/subselect.h index 08dfb95364e..7f4c9afa950 100644 --- a/src/include/optimizer/subselect.h +++ b/src/include/optimizer/subselect.h @@ -53,6 +53,6 @@ extern bool IsSubqueryMultiLevelCorrelated(Query *sq); extern List *generate_subquery_vars(PlannerInfo *root, List *tlist, Index varno); -extern bool QueryHasDistributedRelation(Query *q); +extern bool QueryHasDistributedRelation(Query *q, bool recursive); #endif /* SUBSELECT_H */ diff --git a/src/include/utils/guc.h b/src/include/utils/guc.h index d49f00a5bab..9d7f5f8909e 100644 --- a/src/include/utils/guc.h +++ b/src/include/utils/guc.h @@ -29,14 +29,6 @@ */ #define RESERVED_FTS_CONNECTIONS (1) - -/* - * Automatic configuration file name for ALTER SYSTEM. - * This file will be used to store values of configuration parameters - * set by ALTER SYSTEM command. - */ -#define PG_AUTOCONF_FILENAME "postgresql.auto.conf" - /* upper limit for GUC variables measured in kilobytes of memory */ /* note that various places assume the byte size fits in a "long" variable */ #if SIZEOF_SIZE_T > 4 && SIZEOF_LONG > 4 diff --git a/src/test/isolation2/expected/resgroup/resgroup_large_group_id.out b/src/test/isolation2/expected/resgroup/resgroup_large_group_id.out new file mode 100644 index 00000000000..558cfad58a3 --- /dev/null +++ b/src/test/isolation2/expected/resgroup/resgroup_large_group_id.out @@ -0,0 +1,31 @@ +-- Test resgroup oid larger than int32. +select gp_inject_fault('bump_oid', 'skip', dbid) from gp_segment_configuration where role = 'p' and content = -1; + gp_inject_fault +----------------- + Success: +(1 row) + +create resource group rg_large_oid with (cpu_rate_limit=20, memory_limit=10); +CREATE + +select gp_inject_fault('bump_oid', 'reset', dbid) from gp_segment_configuration where role = 'p' and content = -1; + gp_inject_fault +----------------- + Success: +(1 row) + +select max(oid)::bigint > (power(2,31) + 1)::bigint from pg_resgroup; + ?column? +---------- + t +(1 row) + +-- count(*) > 0 to run the SQL but do not display the result +select count(*) > 0 from pg_resgroup_get_status(NULL); + ?column? +---------- + t +(1 row) + +drop resource group rg_large_oid; +DROP diff --git a/src/test/isolation2/input/parallel_retrieve_cursor/fault_inject.source b/src/test/isolation2/input/parallel_retrieve_cursor/fault_inject.source index 3e592b5968c..c04643d14f8 100644 --- a/src/test/isolation2/input/parallel_retrieve_cursor/fault_inject.source +++ b/src/test/isolation2/input/parallel_retrieve_cursor/fault_inject.source @@ -114,8 +114,8 @@ insert into t1 select generate_series(1,100); 1: SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'reset', 2); 1: SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'reset', 3); 1: SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'reset', 4); -1: SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'sleep', '', '', '', 5, 6, 3, 2::smallint); -1: SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'sleep', '', '', '', 5, 6, 3, 4::smallint); +1: SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'suspend', '', '', '', 5, 5, 0, 2::smallint); +1: SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'suspend', '', '', '', 5, 5, 0, 4::smallint); 1: SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'interrupt', '', '', '', 5, 5, 0, 3::smallint); 1: BEGIN; @@ -132,8 +132,14 @@ insert into t1 select generate_series(1,100); 1U: SELECT state FROM gp_segment_endpoints() WHERE cursorname='c1'; 1R: @pre_run 'set_endpoint_variable @ENDPOINT5': RETRIEVE ALL FROM ENDPOINT "@ENDPOINT5"; +SELECT gp_wait_until_triggered_fault('fetch_tuples_from_endpoint', 1, 2); +SELECT gp_wait_until_triggered_fault('fetch_tuples_from_endpoint', 1, 4); + 1<: +SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'resume', 2); +SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'resume', 4); + 0R<: 2R<: @@ -144,9 +150,9 @@ insert into t1 select generate_series(1,100); 1: SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'reset', 4); -- Test6: close PARALLEL RETRIEVE CURSOR during retrieve -1: SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'sleep', '', '', '', 5, 6, 3, 2::smallint); -1: SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'sleep', '', '', '', 5, 6, 3, 4::smallint); -1: SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'sleep', '', '', '', 5, 6, 3, 3::smallint); +SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'suspend', '', '', '', 5, 5, 0, 2::smallint); +SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'suspend', '', '', '', 5, 5, 0, 4::smallint); +SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'suspend', '', '', '', 5, 5, 0, 3::smallint); 1: BEGIN; 1: DECLARE c1 PARALLEL RETRIEVE CURSOR FOR SELECT * from t1; @@ -164,6 +170,10 @@ insert into t1 select generate_series(1,100); 1: SELECT * FROM gp_wait_parallel_retrieve_cursor('c1', 0); 1: CLOSE c1; +SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'resume', 2); +SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'resume', 3); +SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'resume', 4); + 0R<: 1R<: 2R<: @@ -181,22 +191,22 @@ DROP TABLE IF EXISTS t2; CREATE TABLE t2 (a INT) DISTRIBUTED by (a); insert into t2 select generate_series(1,10000); -1: SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'reset', dbid) +SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'reset', dbid) FROM gp_segment_configuration WHERE content=1 AND role='p'; -1: SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'interrupt', '', '', '', 1000, 1000, 0, dbid) +SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'interrupt', '', '', '', 1000, 1000, 0, dbid) FROM gp_segment_configuration WHERE content=1 AND role='p'; -1: SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'reset', dbid) +SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'reset', dbid) FROM gp_segment_configuration WHERE content=0 AND role='p'; -1: SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'sleep', '', '', '', 900, 900, 2, dbid) +SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'suspend', '', '', '', 900, 900, 0, dbid) FROM gp_segment_configuration WHERE content=0 AND role='p'; -1: SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'reset', dbid) +SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'reset', dbid) FROM gp_segment_configuration WHERE content=2 AND role='p'; -1: SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'sleep', '', '', '', 800, 800, 2, dbid) +SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'suspend', '', '', '', 800, 800, 0, dbid) FROM gp_segment_configuration WHERE content=2 AND role='p'; @@ -216,6 +226,13 @@ insert into t2 select generate_series(1,10000); 1<: +SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'resume', dbid) + FROM gp_segment_configuration + WHERE content=0 AND role='p'; +SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'resume', dbid) + FROM gp_segment_configuration + WHERE content=2 AND role='p'; + 0R<: 2R<: diff --git a/src/test/isolation2/input/uao/snapshot_index_corruption.source b/src/test/isolation2/input/uao/snapshot_index_corruption.source new file mode 100644 index 00000000000..48c989b1af7 --- /dev/null +++ b/src/test/isolation2/input/uao/snapshot_index_corruption.source @@ -0,0 +1,35 @@ +-- @Description Test index corruption when invalid snapshot used. +-- +-- Create AO table, insert few rows on it. +drop table if exists test_ao; +create table test_ao(i bigint) using @amname@ distributed by (i); +insert into test_ao select generate_series(1,100); +-- Test 1 +-- Begin single-insert transaction. +1: begin; +1: insert into test_ao values(101); +-- Try to create index, it should hold on lock before commit below. +2&: create index test_ao_idx on test_ao(i); +-- Commit single-insert transaction, so index continues creation. +1: commit; +-- Force index usage and check row is here (false before fix). +2<: +2: set optimizer=off; +2: set enable_seqscan=off; +2: explain (costs off) select i from test_ao where i = 101; +2: select i from test_ao where i = 101; + +-- Test 2 +-- Drop incomplete index +1: drop index test_ao_idx; +-- Check row is here and start repeatable read transaction. +2: select i from test_ao where i = 100; +2: begin; +2: set transaction isolation level repeatable read; +2: select 1; +-- Update row selected above and create new index +1: update test_ao set i = 200 where i = 100; +1: create index test_ao_idx on test_ao(i); +-- For the repeatable read isolation level row still there. +2: explain (costs off) select i from test_ao where i = 100; +2: select i from test_ao where i = 100; diff --git a/src/test/isolation2/isolation2_resgroup_schedule b/src/test/isolation2/isolation2_resgroup_schedule index e94ee880a4b..c6aad426821 100644 --- a/src/test/isolation2/isolation2_resgroup_schedule +++ b/src/test/isolation2/isolation2_resgroup_schedule @@ -51,4 +51,7 @@ test: resgroup/resgroup_functions # dump info test: resgroup/resgroup_dumpinfo +# test larget group id +test: resgroup/resgroup_large_group_id + test: resgroup/disable_resgroup diff --git a/src/test/isolation2/isolation2_schedule b/src/test/isolation2/isolation2_schedule index 806c62a4d7f..03f9f7c467d 100644 --- a/src/test/isolation2/isolation2_schedule +++ b/src/test/isolation2/isolation2_schedule @@ -144,6 +144,7 @@ test: uao/select_while_vacuum_serializable2_row test: uao/selectinsert_while_vacuum_row test: uao/selectinsertupdate_while_vacuum_row test: uao/selectupdate_while_vacuum_row +test: uao/snapshot_index_corruption_row test: uao/update_while_vacuum_row test: uao/vacuum_self_serializable_row test: uao/vacuum_self_serializable2_row @@ -200,6 +201,7 @@ test: uao/select_while_vacuum_serializable2_column test: uao/selectinsert_while_vacuum_column test: uao/selectinsertupdate_while_vacuum_column test: uao/selectupdate_while_vacuum_column +test: uao/snapshot_index_corruption_column test: uao/update_while_vacuum_column test: uao/vacuum_self_serializable_column test: uao/vacuum_self_serializable2_column diff --git a/src/test/isolation2/output/parallel_retrieve_cursor/fault_inject.source b/src/test/isolation2/output/parallel_retrieve_cursor/fault_inject.source index a047a0c8294..044e288796b 100644 --- a/src/test/isolation2/output/parallel_retrieve_cursor/fault_inject.source +++ b/src/test/isolation2/output/parallel_retrieve_cursor/fault_inject.source @@ -457,12 +457,12 @@ ROLLBACK ----------------- Success: (1 row) -1: SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'sleep', '', '', '', 5, 6, 3, 2::smallint); +1: SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'suspend', '', '', '', 5, 5, 0, 2::smallint); gp_inject_fault ----------------- Success: (1 row) -1: SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'sleep', '', '', '', 5, 6, 3, 4::smallint); +1: SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'suspend', '', '', '', 5, 5, 0, 4::smallint); gp_inject_fault ----------------- Success: @@ -506,9 +506,31 @@ DECLARE 1R: @pre_run 'set_endpoint_variable @ENDPOINT5': RETRIEVE ALL FROM ENDPOINT "@ENDPOINT5"; ERROR: canceling statement due to user request +SELECT gp_wait_until_triggered_fault('fetch_tuples_from_endpoint', 1, 2); + gp_wait_until_triggered_fault +------------------------------- + Success: +(1 row) +SELECT gp_wait_until_triggered_fault('fetch_tuples_from_endpoint', 1, 4); + gp_wait_until_triggered_fault +------------------------------- + Success: +(1 row) + 1<: <... completed> ERROR: canceling MPP operation: "Endpoint retrieve statement aborted" +SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'resume', 2); + gp_inject_fault +----------------- + Success: +(1 row) +SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'resume', 4); + gp_inject_fault +----------------- + Success: +(1 row) + 0R<: <... completed> ERROR: endpoint is not available because the parallel retrieve cursor was aborted (cdbendpointretrieve.c:245) 2R<: <... completed> @@ -535,17 +557,17 @@ ROLLBACK (1 row) -- Test6: close PARALLEL RETRIEVE CURSOR during retrieve -1: SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'sleep', '', '', '', 5, 6, 3, 2::smallint); +SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'suspend', '', '', '', 5, 5, 0, 2::smallint); gp_inject_fault ----------------- Success: (1 row) -1: SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'sleep', '', '', '', 5, 6, 3, 4::smallint); +SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'suspend', '', '', '', 5, 5, 0, 4::smallint); gp_inject_fault ----------------- Success: (1 row) -1: SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'sleep', '', '', '', 5, 6, 3, 3::smallint); +SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'suspend', '', '', '', 5, 5, 0, 3::smallint); gp_inject_fault ----------------- Success: @@ -590,6 +612,22 @@ DECLARE 1: CLOSE c1; CLOSE +SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'resume', 2); + gp_inject_fault +----------------- + Success: +(1 row) +SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'resume', 3); + gp_inject_fault +----------------- + Success: +(1 row) +SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'resume', 4); + gp_inject_fault +----------------- + Success: +(1 row) + 0R<: <... completed> ERROR: endpoint is not available because the parallel retrieve cursor was aborted (cdbendpointretrieve.c:245) 1R<: <... completed> @@ -628,32 +666,32 @@ CREATE insert into t2 select generate_series(1,10000); INSERT 10000 -1: SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'reset', dbid) FROM gp_segment_configuration WHERE content=1 AND role='p'; +SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'reset', dbid) FROM gp_segment_configuration WHERE content=1 AND role='p'; gp_inject_fault ----------------- Success: (1 row) -1: SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'interrupt', '', '', '', 1000, 1000, 0, dbid) FROM gp_segment_configuration WHERE content=1 AND role='p'; +SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'interrupt', '', '', '', 1000, 1000, 0, dbid) FROM gp_segment_configuration WHERE content=1 AND role='p'; gp_inject_fault ----------------- Success: (1 row) -1: SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'reset', dbid) FROM gp_segment_configuration WHERE content=0 AND role='p'; +SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'reset', dbid) FROM gp_segment_configuration WHERE content=0 AND role='p'; gp_inject_fault ----------------- Success: (1 row) -1: SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'sleep', '', '', '', 900, 900, 2, dbid) FROM gp_segment_configuration WHERE content=0 AND role='p'; +SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'suspend', '', '', '', 900, 900, 0, dbid) FROM gp_segment_configuration WHERE content=0 AND role='p'; gp_inject_fault ----------------- Success: (1 row) -1: SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'reset', dbid) FROM gp_segment_configuration WHERE content=2 AND role='p'; +SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'reset', dbid) FROM gp_segment_configuration WHERE content=2 AND role='p'; gp_inject_fault ----------------- Success: (1 row) -1: SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'sleep', '', '', '', 800, 800, 2, dbid) FROM gp_segment_configuration WHERE content=2 AND role='p'; +SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'suspend', '', '', '', 800, 800, 0, dbid) FROM gp_segment_configuration WHERE content=2 AND role='p'; gp_inject_fault ----------------- Success: @@ -695,6 +733,17 @@ ERROR: canceling statement due to user request 1<: <... completed> ERROR: canceling MPP operation: "Endpoint retrieve statement aborted" +SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'resume', dbid) FROM gp_segment_configuration WHERE content=0 AND role='p'; + gp_inject_fault +----------------- + Success: +(1 row) +SELECT gp_inject_fault('fetch_tuples_from_endpoint', 'resume', dbid) FROM gp_segment_configuration WHERE content=2 AND role='p'; + gp_inject_fault +----------------- + Success: +(1 row) + 0R<: <... completed> ERROR: endpoint is not available because the parallel retrieve cursor was aborted (cdbendpointretrieve.c:245) 2R<: <... completed> diff --git a/src/test/isolation2/output/uao/snapshot_index_corruption.source b/src/test/isolation2/output/uao/snapshot_index_corruption.source new file mode 100644 index 00000000000..59b00e9f10c --- /dev/null +++ b/src/test/isolation2/output/uao/snapshot_index_corruption.source @@ -0,0 +1,83 @@ +-- @Description Test index corruption when invalid snapshot used. +-- +-- Create AO table, insert few rows on it. +drop table if exists test_ao; +DROP +create table test_ao(i bigint) using @amname@ distributed by (i); +CREATE +insert into test_ao select generate_series(1,100); +INSERT 100 +-- Test 1 +-- Begin single-insert transaction. +1: begin; +BEGIN +1: insert into test_ao values(101); +INSERT 1 +-- Try to create index, it should hold on lock before commit below. +2&: create index test_ao_idx on test_ao(i); +-- Commit single-insert transaction, so index continues creation. +1: commit; +COMMIT +-- Force index usage and check row is here (false before fix). +2<: <... completed> +CREATE +2: set optimizer=off; +SET +2: set enable_seqscan=off; +SET +2: explain (costs off) select i from test_ao where i = 101; + QUERY PLAN +---------------------------------------------- + Gather Motion 1:1 (slice1; segments: 1) + -> Bitmap Heap Scan on test_ao + Recheck Cond: (i = 101) + -> Bitmap Index Scan on test_ao_idx + Index Cond: (i = 101) + Optimizer: Postgres query optimizer +(6 rows) +2: select i from test_ao where i = 101; + i +----- + 101 +(1 row) + +-- Test 2 +-- Drop incomplete index +1: drop index test_ao_idx; +DROP +-- Check row is here and start repeatable read transaction. +2: select i from test_ao where i = 100; + i +----- + 100 +(1 row) +2: begin; +BEGIN +2: set transaction isolation level repeatable read; +SET +2: select 1; + ?column? +---------- + 1 +(1 row) +-- Update row selected above and create new index +1: update test_ao set i = 200 where i = 100; +UPDATE 1 +1: create index test_ao_idx on test_ao(i); +CREATE +-- For the repeatable read isolation level row still there. +2: explain (costs off) select i from test_ao where i = 100; + QUERY PLAN +---------------------------------------------- + Gather Motion 1:1 (slice1; segments: 1) + -> Bitmap Heap Scan on test_ao + Recheck Cond: (i = 100) + -> Bitmap Index Scan on test_ao_idx + Index Cond: (i = 100) + Optimizer: Postgres query optimizer +(6 rows) +2: select i from test_ao where i = 100; + i +----- + 100 +(1 row) diff --git a/src/test/isolation2/sql/resgroup/resgroup_large_group_id.sql b/src/test/isolation2/sql/resgroup/resgroup_large_group_id.sql new file mode 100644 index 00000000000..10ec72fa8b6 --- /dev/null +++ b/src/test/isolation2/sql/resgroup/resgroup_large_group_id.sql @@ -0,0 +1,13 @@ +-- Test resgroup oid larger than int32. +select gp_inject_fault('bump_oid', 'skip', dbid) from gp_segment_configuration where role = 'p' and content = -1; + +create resource group rg_large_oid with (cpu_rate_limit=20, memory_limit=10); + +select gp_inject_fault('bump_oid', 'reset', dbid) from gp_segment_configuration where role = 'p' and content = -1; + +select max(oid)::bigint > (power(2,31) + 1)::bigint from pg_resgroup; + +-- count(*) > 0 to run the SQL but do not display the result +select count(*) > 0 from pg_resgroup_get_status(NULL); + +drop resource group rg_large_oid; diff --git a/src/test/regress/expected/aggregates.out b/src/test/regress/expected/aggregates.out index c47e03dfd88..62548a4c69c 100644 --- a/src/test/regress/expected/aggregates.out +++ b/src/test/regress/expected/aggregates.out @@ -2773,6 +2773,8 @@ create aggregate my_avg_init2(int4) finalfunc = avg_finalfn, initcond = '(4,0)' ); +-- reset the plan cache, sometimes it would re-plan these prepared statements and log ORCA fallbacks +discard plans; -- state should be shared if INITCONDs are matching select my_sum_init(one),my_avg_init(one) from (values(1),(3)) t(one); NOTICE: avg_transfn called with 1 diff --git a/src/test/regress/expected/aggregates_optimizer.out b/src/test/regress/expected/aggregates_optimizer.out index 33b127d375e..87d3dd7ee53 100644 --- a/src/test/regress/expected/aggregates_optimizer.out +++ b/src/test/regress/expected/aggregates_optimizer.out @@ -2929,12 +2929,32 @@ create aggregate my_avg_init2(int4) finalfunc = avg_finalfn, initcond = '(4,0)' ); +-- reset the plan cache, sometimes it would re-plan these prepared statements and log ORCA fallbacks +discard plans; -- state should be shared if INITCONDs are matching select my_sum_init(one),my_avg_init(one) from (values(1),(3)) t(one); +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: GPDB Expression type: Query Parameter not supported in DXL NOTICE: avg_transfn called with 1 +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: GPDB Expression type: Query Parameter not supported in DXL +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: GPDB Expression type: Query Parameter not supported in DXL +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: GPDB Expression type: Query Parameter not supported in DXL +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: GPDB Expression type: Query Parameter not supported in DXL NOTICE: avg_transfn called with 1 NOTICE: avg_transfn called with 3 NOTICE: avg_transfn called with 3 +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: GPDB Expression type: Query Parameter not supported in DXL +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: GPDB Expression type: Query Parameter not supported in DXL +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: GPDB Expression type: Query Parameter not supported in DXL +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: GPDB Expression type: Query Parameter not supported in DXL my_sum_init | my_avg_init -------------+------------- 14 | 7 diff --git a/src/test/regress/expected/alter_distribution_policy.out b/src/test/regress/expected/alter_distribution_policy.out index a5e3875ddad..1d7e50df447 100644 --- a/src/test/regress/expected/alter_distribution_policy.out +++ b/src/test/regress/expected/alter_distribution_policy.out @@ -1550,3 +1550,53 @@ select check_redistributed('insert into t_distbya select * from t_reorganize', ' reset optimizer; reset gp_force_random_redistribution; +-- When reorganize=false, we won't reorganize and this shouldn't be affected by the existing reloptions. +CREATE TABLE public.t_reorganize_false ( +a integer, +b integer +) with (appendonly=false, autovacuum_enabled=false) DISTRIBUTED BY (a); +-- Insert values which will all be on one segment +INSERT INTO t_reorganize_false VALUES (0, generate_series(1,100)); +SELECT gp_segment_id,count(*) from t_reorganize_false GROUP BY 1; + gp_segment_id |  +---------------+----- + 1 | 100 +(1 row) + +-- Change the distribution policy but because REORGANIZE=false, it should NOT be re-distributed +ALTER TABLE t_reorganize_false SET WITH (REORGANIZE=false) DISTRIBUTED RANDOMLY; +SELECT gp_segment_id,count(*) from t_reorganize_false GROUP BY 1; + gp_segment_id |  +---------------+----- + 1 | 100 +(1 row) + +DROP TABLE t_reorganize_false; +-- Same rule should apply to partitioned table too +CREATE TABLE public.t_reorganize_false ( +a integer, +b integer +) +DISTRIBUTED BY (a) PARTITION BY RANGE(b) +( +PARTITION "00" START (0) END (1000) WITH (tablename='t_reorganize_false_0', appendonly='false', autovacuum_enabled=false), +PARTITION "01" START (1000) END (2000) WITH (tablename='t_reorganize_false_1', appendonly='false', autovacuum_enabled=false), +DEFAULT PARTITION def WITH (tablename='t_reorganize_false_def', appendonly='false', autovacuum_enabled=false) +); +-- Insert values which will all be on one segment +INSERT INTO t_reorganize_false VALUES (0, generate_series(1,100)); +SELECT gp_segment_id,count(*) from t_reorganize_false GROUP BY 1; + gp_segment_id | count +---------------+------- + 1 | 100 +(1 row) + +-- Should NOT be re-distributed +ALTER TABLE t_reorganize_false SET WITH (REORGANIZE=false) DISTRIBUTED RANDOMLY; +SELECT gp_segment_id,count(*) from t_reorganize_false GROUP BY 1; + gp_segment_id | count +---------------+------- + 1 | 100 +(1 row) + +DROP TABLE t_reorganize_false; diff --git a/src/test/regress/expected/correlated_subquery.out b/src/test/regress/expected/correlated_subquery.out new file mode 100644 index 00000000000..99625097746 --- /dev/null +++ b/src/test/regress/expected/correlated_subquery.out @@ -0,0 +1,128 @@ +SET optimizer_enforce_subplans = 1; +SET optimizer_trace_fallback=on; +SET client_min_messages=log; +SELECT a = ALL (SELECT generate_series(1, 2)), a FROM (values (1),(2)) v(a); +LOG: statement: SELECT a = ALL (SELECT generate_series(1, 2)), a FROM (values (1),(2)) v(a); + ?column? | a +----------+--- + f | 1 + f | 2 +(2 rows) + +SELECT a = ALL (SELECT generate_series(2, 2)), a FROM (values (1),(2)) v(a); +LOG: statement: SELECT a = ALL (SELECT generate_series(2, 2)), a FROM (values (1),(2)) v(a); + ?column? | a +----------+--- + f | 1 + t | 2 +(2 rows) + +SELECT 1 = ALL (SELECT generate_series(1, 2)) FROM (values (1),(2)) v(a); +LOG: statement: SELECT 1 = ALL (SELECT generate_series(1, 2)) FROM (values (1),(2)) v(a); + ?column? +---------- + f + f +(2 rows) + +SELECT 2 = ALL (SELECT generate_series(2, 2)) FROM (values (1),(2)) v(a); +LOG: statement: SELECT 2 = ALL (SELECT generate_series(2, 2)) FROM (values (1),(2)) v(a); + ?column? +---------- + t + t +(2 rows) + +SELECT 2 = ALL (SELECT generate_series(2, 3)) FROM (values (1),(2)) v(a); +LOG: statement: SELECT 2 = ALL (SELECT generate_series(2, 3)) FROM (values (1),(2)) v(a); + ?column? +---------- + f + f +(2 rows) + +SELECT 2+1 = ALL (SELECT generate_series(2, 3)) FROM (values (1),(2)) v(a); +LOG: statement: SELECT 2+1 = ALL (SELECT generate_series(2, 3)) FROM (values (1),(2)) v(a); + ?column? +---------- + f + f +(2 rows) + +SELECT 2+1 = ALL (SELECT generate_series(3, 3)) FROM (values (1),(2)) v(a); +LOG: statement: SELECT 2+1 = ALL (SELECT generate_series(3, 3)) FROM (values (1),(2)) v(a); + ?column? +---------- + t + t +(2 rows) + +SELECT (SELECT a) = ALL (SELECT generate_series(1, 2)), a FROM (values (1),(2)) v(a); +LOG: statement: SELECT (SELECT a) = ALL (SELECT generate_series(1, 2)), a FROM (values (1),(2)) v(a); + ?column? | a +----------+--- + f | 1 + f | 2 +(2 rows) + +SELECT (SELECT a) = ALL (SELECT generate_series(2, 2)), a FROM (values (1),(2)) v(a); +LOG: statement: SELECT (SELECT a) = ALL (SELECT generate_series(2, 2)), a FROM (values (1),(2)) v(a); + ?column? | a +----------+--- + f | 1 + t | 2 +(2 rows) + +SELECT (SELECT a+1) = ALL (SELECT generate_series(2, 2)), a FROM (values (1),(2)) v(a); +LOG: statement: SELECT (SELECT a+1) = ALL (SELECT generate_series(2, 2)), a FROM (values (1),(2)) v(a); + ?column? | a +----------+--- + t | 1 + f | 2 +(2 rows) + +SELECT (SELECT 1) = ALL (SELECT generate_series(1, 1)) FROM (values (1),(2)) v(a); +LOG: statement: SELECT (SELECT 1) = ALL (SELECT generate_series(1, 1)) FROM (values (1),(2)) v(a); + ?column? +---------- + t + t +(2 rows) + +SELECT (SELECT 1) = ALL (SELECT generate_series(1, 2)) FROM (values (1),(2)) v(a); +LOG: statement: SELECT (SELECT 1) = ALL (SELECT generate_series(1, 2)) FROM (values (1),(2)) v(a); + ?column? +---------- + f + f +(2 rows) + +SELECT (SELECT 3) = ALL (SELECT generate_series(3, 3)) FROM (values (1),(2)) v(a); +LOG: statement: SELECT (SELECT 3) = ALL (SELECT generate_series(3, 3)) FROM (values (1),(2)) v(a); + ?column? +---------- + t + t +(2 rows) + +SELECT (SELECT 1) = ALL (SELECT generate_series(1, 1)); +LOG: statement: SELECT (SELECT 1) = ALL (SELECT generate_series(1, 1)); + ?column? +---------- + t +(1 row) + +SELECT (SELECT 1) = ALL (SELECT generate_series(1, 2)); +LOG: statement: SELECT (SELECT 1) = ALL (SELECT generate_series(1, 2)); + ?column? +---------- + f +(1 row) + +SELECT (SELECT 3) = ALL (SELECT generate_series(3, 3)); +LOG: statement: SELECT (SELECT 3) = ALL (SELECT generate_series(3, 3)); + ?column? +---------- + t +(1 row) + diff --git a/src/test/regress/expected/gp_dqa.out b/src/test/regress/expected/gp_dqa.out index 36643b10471..110611f01c5 100644 --- a/src/test/regress/expected/gp_dqa.out +++ b/src/test/regress/expected/gp_dqa.out @@ -4,6 +4,7 @@ -- differences by setting 'extra_float_digits'. This isn't enough for all of -- the queries, so a few also use TO_CHAR() to truncate the results further. set extra_float_digits=0; +SET optimizer_trace_fallback to on; drop table if exists dqa_t1; NOTICE: table "dqa_t1" does not exist, skipping drop table if exists dqa_t2; diff --git a/src/test/regress/expected/gp_dqa_optimizer.out b/src/test/regress/expected/gp_dqa_optimizer.out index 3897ae46f1b..7c28cc1d7c2 100644 --- a/src/test/regress/expected/gp_dqa_optimizer.out +++ b/src/test/regress/expected/gp_dqa_optimizer.out @@ -4,6 +4,7 @@ -- differences by setting 'extra_float_digits'. This isn't enough for all of -- the queries, so a few also use TO_CHAR() to truncate the results further. set extra_float_digits=0; +SET optimizer_trace_fallback to on; drop table if exists dqa_t1; NOTICE: table "dqa_t1" does not exist, skipping drop table if exists dqa_t2; @@ -114,12 +115,16 @@ explain (costs off) select count(distinct d), sum(distinct d) from dqa_t1 group (11 rows) select count(distinct d), count(distinct dt) from dqa_t1; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer count | count -------+------- 23 | 34 (1 row) explain (costs off) select count(distinct d), count(distinct dt) from dqa_t1; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer QUERY PLAN ------------------------------------------------------------------ Finalize Aggregate @@ -136,12 +141,16 @@ explain (costs off) select count(distinct d), count(distinct dt) from dqa_t1; (11 rows) select count(distinct d), count(distinct c), count(distinct dt) from dqa_t1; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer count | count | count -------+-------+------- 23 | 10 | 34 (1 row) explain (costs off) select count(distinct d), count(distinct c), count(distinct dt) from dqa_t1; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer QUERY PLAN ------------------------------------------------------------------ Finalize Aggregate @@ -158,6 +167,8 @@ explain (costs off) select count(distinct d), count(distinct c), count(distinct (11 rows) select count(distinct d), count(distinct dt) from dqa_t1 group by c; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer count | count -------+------- 10 | 10 @@ -173,6 +184,8 @@ select count(distinct d), count(distinct dt) from dqa_t1 group by c; (10 rows) explain (costs off) select count(distinct d), count(distinct dt) from dqa_t1 group by c; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer QUERY PLAN ------------------------------------------------------------------------ Finalize HashAggregate @@ -194,6 +207,8 @@ explain (costs off) select count(distinct d), count(distinct dt) from dqa_t1 gro (16 rows) select count(distinct d), count(distinct dt) from dqa_t1 group by d; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer count | count -------+------- 1 | 5 @@ -222,6 +237,8 @@ select count(distinct d), count(distinct dt) from dqa_t1 group by d; (23 rows) explain (costs off) select count(distinct d), count(distinct dt) from dqa_t1 group by d; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer QUERY PLAN ------------------------------------------------------------------------ Finalize HashAggregate @@ -483,12 +500,16 @@ explain (costs off) select count(distinct i), sum(distinct i) from dqa_t1 group (9 rows) select count(distinct c), count(distinct dt) from dqa_t1; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer count | count -------+------- 10 | 34 (1 row) explain (costs off) select count(distinct c), count(distinct dt) from dqa_t1; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer QUERY PLAN ------------------------------------------------------------------ Finalize Aggregate @@ -505,6 +526,8 @@ explain (costs off) select count(distinct c), count(distinct dt) from dqa_t1; (11 rows) select count(distinct c), count(distinct dt), i from dqa_t1 group by i; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer count | count | i -------+-------+---- 5 | 9 | 3 @@ -522,6 +545,8 @@ select count(distinct c), count(distinct dt), i from dqa_t1 group by i; (12 rows) explain (costs off) select count(distinct c), count(distinct dt), i from dqa_t1 group by i; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer QUERY PLAN ------------------------------------------------------------------------ Finalize HashAggregate @@ -543,6 +568,8 @@ explain (costs off) select count(distinct c), count(distinct dt), i from dqa_t1 (16 rows) select count(distinct i), count(distinct c), d from dqa_t1 group by d; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer count | count | d -------+-------+---- 5 | 5 | 3 @@ -571,6 +598,8 @@ select count(distinct i), count(distinct c), d from dqa_t1 group by d; (23 rows) explain (costs off) select count(distinct i), count(distinct c), d from dqa_t1 group by d; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer QUERY PLAN ------------------------------------------------------------------------ Finalize HashAggregate @@ -702,6 +731,8 @@ explain (costs off) select count(distinct dqa_t1.dt) from dqa_t1, dqa_t2 where d -- multidqa with groupby and order by select sum(distinct d), count(distinct i), count(distinct c),i,c from dqa_t1 group by i,c order by i,c; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer sum | count | count | i | c -----+-------+-------+----+--- 14 | 1 | 1 | 0 | 0 @@ -767,6 +798,8 @@ select sum(distinct d), count(distinct i), count(distinct c),i,c from dqa_t1 gro (60 rows) explain (costs off) select sum(distinct d), count(distinct i), count(distinct c),i,c from dqa_t1 group by i,c order by i,c; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer QUERY PLAN ------------------------------------------------------------------------------------ Sort @@ -877,12 +910,16 @@ explain (costs off) select to_char(corr(distinct d, i), '9.99999999999999') from -- multi args multidqa select count(distinct c), corr(distinct d, i) from dqa_t1; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer count | corr -------+-------------------- 10 | 0.0824013341460019 (1 row) explain (costs off) select count(distinct c), corr(distinct d, i) from dqa_t1; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer QUERY PLAN ----------------------------------------------------------------------------------------------------- Finalize Aggregate @@ -899,12 +936,16 @@ explain (costs off) select count(distinct c), corr(distinct d, i) from dqa_t1; (11 rows) select count(distinct d), corr(distinct d, i) from dqa_t1; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer count | corr -------+-------------------- 23 | 0.0824013341460019 (1 row) explain (costs off) select count(distinct d), corr(distinct d, i) from dqa_t1; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer QUERY PLAN ----------------------------------------------------------------------------------------------------- Finalize Aggregate @@ -921,12 +962,16 @@ explain (costs off) select count(distinct d), corr(distinct d, i) from dqa_t1; (11 rows) select count(distinct d), count(distinct i), corr(distinct d, i) from dqa_t1; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer count | count | corr -------+-------+-------------------- 23 | 12 | 0.0824013341460019 (1 row) explain (costs off) select count(distinct d), count(distinct i), corr(distinct d, i) from dqa_t1; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer QUERY PLAN -------------------------------------------------------------------------------------------------------- Finalize Aggregate @@ -943,12 +988,16 @@ explain (costs off) select count(distinct d), count(distinct i), corr(distinct d (11 rows) select count(distinct c), count(distinct d), count(distinct i), corr(distinct d, i) from dqa_t1; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer count | count | count | corr -------+-------+-------+-------------------- 10 | 23 | 12 | 0.0824013341460019 (1 row) explain (costs off) select count(distinct c), count(distinct d), count(distinct i), corr(distinct d, i) from dqa_t1; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer QUERY PLAN ------------------------------------------------------------------------------------------------------------ Finalize Aggregate @@ -966,6 +1015,8 @@ explain (costs off) select count(distinct c), count(distinct d), count(distinct -- multi args multidqa with group by select count(distinct c), corr(distinct d, i), d from dqa_t1 group by d; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer count | corr | d -------+------+---- 5 | | 0 @@ -994,6 +1045,8 @@ select count(distinct c), corr(distinct d, i), d from dqa_t1 group by d; (23 rows) explain (costs off) select count(distinct c), corr(distinct d, i), d from dqa_t1 group by d; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer QUERY PLAN -------------------------------------------------------------------------------------------------------------- Finalize HashAggregate @@ -1015,6 +1068,8 @@ explain (costs off) select count(distinct c), corr(distinct d, i), d from dqa_t1 (16 rows) select count(distinct c), corr(distinct d, i), d, i from dqa_t1 group by d,i; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer count | corr | d | i -------+------+----+---- 1 | | 0 | 0 @@ -1120,6 +1175,8 @@ select count(distinct c), corr(distinct d, i), d, i from dqa_t1 group by d,i; (100 rows) explain (costs off) select count(distinct c), corr(distinct d, i), d, i from dqa_t1 group by d,i; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer QUERY PLAN ----------------------------------------------------------------------------------------------------------------- Finalize HashAggregate @@ -1141,6 +1198,8 @@ explain (costs off) select count(distinct c), corr(distinct d, i), d, i from dqa (16 rows) select count(distinct c), corr(distinct d, i), dt from dqa_t1 group by dt; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer count | corr | dt -------+--------------------+------------ 3 | 0.59603956067927 | 06-25-2009 @@ -1180,6 +1239,8 @@ select count(distinct c), corr(distinct d, i), dt from dqa_t1 group by dt; (34 rows) explain (costs off) select count(distinct c), corr(distinct d, i), dt from dqa_t1 group by dt; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer QUERY PLAN --------------------------------------------------------------------------------------------------------------- Finalize HashAggregate @@ -1201,6 +1262,8 @@ explain (costs off) select count(distinct c), corr(distinct d, i), dt from dqa_t (16 rows) select count(distinct d), corr(distinct d, i), i from dqa_t1 group by i; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer count | corr | i -------+------+---- 9 | | 0 @@ -1218,6 +1281,8 @@ select count(distinct d), corr(distinct d, i), i from dqa_t1 group by i; (12 rows) explain (costs off) select count(distinct d), corr(distinct d, i), i from dqa_t1 group by i; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer QUERY PLAN -------------------------------------------------------------------------------------------------------------- Finalize HashAggregate @@ -1239,6 +1304,8 @@ explain (costs off) select count(distinct d), corr(distinct d, i), i from dqa_t1 (16 rows) select count(distinct d), corr(distinct d, i), d from dqa_t1 group by d; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer count | corr | d -------+------+---- 1 | | 0 @@ -1267,6 +1334,8 @@ select count(distinct d), corr(distinct d, i), d from dqa_t1 group by d; (23 rows) explain (costs off) select count(distinct d), corr(distinct d, i), d from dqa_t1 group by d; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer QUERY PLAN ----------------------------------------------------------------------------------------------------------- Finalize HashAggregate @@ -1288,6 +1357,8 @@ explain (costs off) select count(distinct d), corr(distinct d, i), d from dqa_t1 (16 rows) select count(distinct d), to_char(corr(distinct d, i), '9.99999999999999'), c from dqa_t1 group by c; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer count | to_char | c -------+-------------------+--- 10 | .13670602618479 | 0 @@ -1303,6 +1374,8 @@ select count(distinct d), to_char(corr(distinct d, i), '9.99999999999999'), c f (10 rows) explain (costs off) select count(distinct d), to_char(corr(distinct d, i), '9.99999999999999'), c from dqa_t1 group by c; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer QUERY PLAN -------------------------------------------------------------------------------------------------------------- Finalize HashAggregate @@ -1359,6 +1432,8 @@ from fact_route_aggregation T218094 where ( T43883.device_id = T218094.device_id ) group by T43883.platform; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 | c9 ----+----+----+----+----+----+----+----+---- (0 rows) @@ -1384,6 +1459,8 @@ insert into t2_mdqa select i % 10 , i % 5, i || 'value' from generate_series(1, insert into t2_mdqa select i % 10 , i % 5, i || 'value' from generate_series(1, 20) i; -- simple mdqa select count(distinct t1.a), count(distinct t2.b), t1.c, t2.c from t1_mdqa t1, t2_mdqa t2 where t1.c = t2.c group by t1.c, t2.c order by t1.c; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer count | count | c | c -------+-------+---------+--------- 1 | 1 | 10value | 10value @@ -1410,6 +1487,8 @@ select count(distinct t1.a), count(distinct t2.b), t1.c, t2.c from t1_mdqa t1, t -- distinct on top of some mdqas select distinct sum(distinct t1.a), avg(t2.a), sum(distinct t2.b), t1.a, t2.b from t1_mdqa t1, t2_mdqa t2 where t1.a = t2.a group by t1.a, t2.b order by t1.a; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer sum | avg | sum | a | b -----+------------------------+-----+---+--- 0 | 0.00000000000000000000 | 0 | 0 | 0 @@ -1420,6 +1499,8 @@ select distinct sum(distinct t1.a), avg(t2.a), sum(distinct t2.b), t1.a, t2.b fr (5 rows) select distinct sum (distinct t1.a), avg(distinct t2.a), sum(distinct t2.b), t1.c from t1_mdqa t1, t2_mdqa t2 where t1.a = t2.a group by t1.c order by t1.c; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer sum | avg | sum | c -----+------------------------+-----+--------- 0 | 0.00000000000000000000 | 0 | 10value @@ -1446,6 +1527,8 @@ select distinct sum (distinct t1.a), avg(distinct t2.a), sum(distinct t2.b), t1. -- distinct on group by fields select distinct t1.c , sum(distinct t1.a), count(t2.b), sum(distinct t2.b) from t1_mdqa t1, t2_mdqa t2 where t1.a = t2.a group by t1.c order by t1.c; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer c | sum | count | sum ---------+-----+-------+----- 10value | 0 | 8 | 0 @@ -1472,6 +1555,8 @@ select distinct t1.c , sum(distinct t1.a), count(t2.b), sum(distinct t2.b) from -- distinct on normal aggregates select distinct sum(t1.a), avg(distinct t2.a), sum(distinct (t1.a + t2.a)), t1.a, t2.b from t1_mdqa t1, t2_mdqa t2 where t1.a = t2.a group by t1.a, t2.b order by t1.a; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer sum | avg | sum | a | b -----+------------------------+-----+---+--- 0 | 0.00000000000000000000 | 0 | 0 | 0 @@ -1482,6 +1567,8 @@ select distinct sum(t1.a), avg(distinct t2.a), sum(distinct (t1.a + t2.a)), t1.a (5 rows) select distinct avg(t1.a + t2.b), count(distinct t1.c), count(distinct char_length(t1.c)), t1.a, t2.b from t1_mdqa t1, t2_mdqa t2 where t1.a = t2.a group by t1.a, t2.b order by t1.a; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer avg | count | count | a | b ------------------------+-------+-------+---+--- 0.00000000000000000000 | 4 | 2 | 0 | 0 @@ -1508,6 +1595,8 @@ HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sur insert into gp_dqa_r select i , i %10, i%5 from generate_series(1,20) i; insert into gp_dqa_s select i, i %15, i%10 from generate_series(1,30) i; select a, d, count(distinct b) as c1, count(distinct c) as c2 from gp_dqa_r, gp_dqa_s where ( e = a ) group by d, a order by a,d; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer a | d | c1 | c2 ----+----+----+---- 1 | 1 | 1 | 1 @@ -1549,6 +1638,8 @@ d as c9 from gp_dqa_r, gp_dqa_s where ( e = a ) group by d order by c9; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer c1 | c2 | c3 | c2 | c9 ----+----+----+----+---- 1 | 2 | 1 | 1 | 1 @@ -1588,6 +1679,8 @@ d as c9 from gp_dqa_r, gp_dqa_s where ( e = a ) group by d order by c9; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer c1 | c2 | c9 ----+----+---- 1 | 1 | 1 @@ -1624,6 +1717,8 @@ select distinct count(distinct b) as c1, count(distinct c) as c2, d as c9 from gp_dqa_r, gp_dqa_s where ( e = a ) group by d order by c9; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer c1 | c2 | c9 ----+----+---- 1 | 1 | 1 @@ -1657,6 +1752,8 @@ group by d order by c9; (28 rows) select distinct d, count(distinct b) as c1, count(distinct c) as c2, d as c9 from gp_dqa_r, gp_dqa_s group by d order by c9; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer d | c1 | c2 | c9 ----+----+----+---- 1 | 10 | 5 | 1 @@ -1692,6 +1789,8 @@ select distinct d, count(distinct b) as c1, count(distinct c) as c2, d as c9 fro (30 rows) select distinct d, count(distinct b) as c1, count(distinct c) as c2, d as c9 from gp_dqa_r, gp_dqa_s group by d, a order by c9; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer d | c1 | c2 | c9 ----+----+----+---- 1 | 1 | 1 | 1 @@ -1727,18 +1826,24 @@ select distinct d, count(distinct b) as c1, count(distinct c) as c2, d as c9 fro (30 rows) select distinct count(distinct b) as c1, count(distinct c) as c2 from gp_dqa_r, gp_dqa_s; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer c1 | c2 ----+---- 10 | 5 (1 row) select distinct count(distinct b) as c1, count(distinct c) as c2 from gp_dqa_r; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer c1 | c2 ----+---- 10 | 5 (1 row) select distinct count(distinct b) as c1, count(distinct c) as c2, d, a from gp_dqa_r, gp_dqa_s where ( e = a)group by d, a order by a,d; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer c1 | c2 | d | a ----+----+----+---- 1 | 1 | 1 | 1 @@ -1776,6 +1881,8 @@ ERROR: for SELECT DISTINCT, ORDER BY expressions must appear in select list LINE 1: ...as c2, d from gp_dqa_r, gp_dqa_s group by d, a order by d,a; ^ select distinct count(distinct b) as c1, count(distinct c) as c2, d from gp_dqa_r, gp_dqa_s group by d, a order by d; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer c1 | c2 | d ----+----+---- 1 | 1 | 1 @@ -1811,6 +1918,8 @@ select distinct count(distinct b) as c1, count(distinct c) as c2, d from gp_dqa_ (30 rows) select distinct count(distinct b) as c1, count(distinct c) as c2, d from gp_dqa_r, gp_dqa_s group by d order by d; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer c1 | c2 | d ----+----+---- 10 | 5 | 1 @@ -1858,6 +1967,8 @@ create table gp_dqa_t2 (a int, c int) distributed by (a); insert into gp_dqa_t1 select i , i %5 from generate_series(1,10) i; insert into gp_dqa_t2 select i , i %4 from generate_series(1,10) i; select distinct A.a, sum(distinct A.b), count(distinct B.c) from gp_dqa_t1 A left join gp_dqa_t2 B on (A.a = B.a) group by A.a order by A.a; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer a | sum | count ----+-----+------- 1 | 1 | 1 @@ -1873,6 +1984,8 @@ select distinct A.a, sum(distinct A.b), count(distinct B.c) from gp_dqa_t1 A lef (10 rows) select distinct A.a, sum(distinct A.b), count(distinct B.c) from gp_dqa_t1 A right join gp_dqa_t2 B on (A.a = B.a) group by A.a order by A.a; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer a | sum | count ----+-----+------- 1 | 1 | 1 @@ -1925,12 +2038,16 @@ explain (costs off) select count(distinct d) from dqa_t1 group by i; (11 rows) select count(distinct d), count(distinct c), count(distinct dt) from dqa_t1; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer count | count | count -------+-------+------- 23 | 10 | 34 (1 row) select count(distinct c), count(distinct dt), i from dqa_t1 group by i; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer count | count | i -------+-------+---- 5 | 9 | 3 @@ -1954,11 +2071,15 @@ create table foo_mdqa(x int, y int); NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'x' as the Cloudberry Database data distribution key for this table. HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. SELECT distinct C.z, count(distinct FS.x), count(distinct FS.y) FROM (SELECT 1 AS z FROM generate_series(1,10)) C, foo_mdqa FS GROUP BY z; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer z | count | count ---+-------+------- (0 rows) SELECT distinct C.z, count(distinct FS.x), count(distinct FS.y) FROM (SELECT i AS z FROM generate_series(1,10) i) C, foo_mdqa FS GROUP BY z; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer z | count | count ---+-------+------- (0 rows) @@ -1977,6 +2098,10 @@ HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sur insert into nonullstab select 1, 1 from generate_series(1, 100); -- This returns wrong result. countall(distinct a) should return 1. select countall(distinct a), count(distinct b) from nonullstab; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: GPDB Expression type: Query Parameter not supported in DXL countall | count ----------+------- 1 | 1 @@ -1990,12 +2115,16 @@ create table dqa_f2(x int, y int, z int) distributed by (x); insert into dqa_f1 select i%17, i%5 , i%3 from generate_series(1,1000) i; insert into dqa_f2 select i % 13, i % 5 , i % 11 from generate_series(1,1000) i; select sum(distinct a) filter (where a > 0), sum(distinct b) filter (where a > 0) from dqa_f1; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Aggregate functions with FILTER sum | sum -----+----- 136 | 10 (1 row) select sum(distinct a) filter (where a > 0), sum(distinct b) filter (where a > 0) from dqa_f1 group by b; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Aggregate functions with FILTER sum | sum -----+----- 136 | 0 @@ -2006,6 +2135,8 @@ select sum(distinct a) filter (where a > 0), sum(distinct b) filter (where a > 0 (5 rows) select sum(distinct a) filter (where a > 0), sum(distinct b) filter (where a > 0) from dqa_f1 group by c; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Aggregate functions with FILTER sum | sum -----+----- 136 | 10 @@ -2014,12 +2145,16 @@ select sum(distinct a) filter (where a > 0), sum(distinct b) filter (where a > 0 (3 rows) select sum(distinct a) filter (where a in (select x from dqa_f2 where x = a)), sum(distinct b) filter (where a > 0) from dqa_f1; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Aggregate functions with FILTER sum | sum -----+----- 78 | 10 (1 row) select sum(distinct a) filter (where a in (select x from dqa_f2 where x = a)), sum(distinct b) filter (where a > 0) from dqa_f1 group by c; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Aggregate functions with FILTER sum | sum -----+----- 78 | 10 @@ -2028,12 +2163,16 @@ select sum(distinct a) filter (where a in (select x from dqa_f2 where x = a)), s (3 rows) select count(distinct a) filter (where a > 3),count( distinct b) filter (where a > 4), sum(distinct b) filter( where a > 4) from dqa_f1; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Aggregate functions with FILTER count | count | sum -------+-------+----- 13 | 5 | 10 (1 row) explain select sum(distinct a) filter (where a > 0), sum(distinct b) filter (where a > 0) from dqa_f1; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Aggregate functions with FILTER QUERY PLAN ------------------------------------------------------------------------------------------------------ Finalize Aggregate (cost=20.66..20.67 rows=1 width=16) @@ -2050,6 +2189,8 @@ explain select sum(distinct a) filter (where a > 0), sum(distinct b) filter (whe (11 rows) explain select sum(distinct a) filter (where a > 0), sum(distinct b) filter (where a > 0) from dqa_f1 group by b; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Aggregate functions with FILTER QUERY PLAN ------------------------------------------------------------------------------------------------------------ Finalize HashAggregate (cost=21.62..21.67 rows=5 width=20) @@ -2071,6 +2212,8 @@ explain select sum(distinct a) filter (where a > 0), sum(distinct b) filter (whe (16 rows) explain select sum(distinct a) filter (where a > 0), sum(distinct b) filter (where a > 0) from dqa_f1 group by c; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Aggregate functions with FILTER QUERY PLAN ------------------------------------------------------------------------------------------------------------- Finalize HashAggregate (cost=21.20..21.23 rows=3 width=20) @@ -2092,6 +2235,8 @@ explain select sum(distinct a) filter (where a > 0), sum(distinct b) filter (whe (16 rows) explain select sum(distinct a) filter (where a in (select x from dqa_f2 where x = a)), sum(distinct b) filter (where a > 0) from dqa_f1; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Aggregate functions with FILTER QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------ Finalize Aggregate (cost=96.41..96.42 rows=1 width=16) @@ -2114,6 +2259,8 @@ explain select sum(distinct a) filter (where a in (select x from dqa_f2 where x (17 rows) explain select sum(distinct a) filter (where a in (select x from dqa_f2 where x = a)), sum(distinct b) filter (where a > 0) from dqa_f1 group by c; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Aggregate functions with FILTER QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------ Finalize HashAggregate (cost=181.11..181.14 rows=3 width=20) @@ -2141,6 +2288,8 @@ explain select sum(distinct a) filter (where a in (select x from dqa_f2 where x (22 rows) explain select count(distinct a) filter (where a > 3),count( distinct b) filter (where a > 4), sum(distinct b) filter( where a > 4) from dqa_f1; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Aggregate functions with FILTER QUERY PLAN ------------------------------------------------------------------------------------------------------ Finalize Aggregate (cost=20.67..20.68 rows=1 width=24) @@ -2189,12 +2338,16 @@ insert into dqa_f3 values ('123', 2), ('213', 0), ('231', 2), ('312', 0), ('321' -- -> Seq Scan on public.dqa_f3 -- Output: b, a, (b)::text select count(distinct (b)::text) as b, count(distinct (a)::text) as a from dqa_f3; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer b | a ---+--- 3 | 7 (1 row) explain (verbose, costs off) select count(distinct (b)::text) as b, count(distinct (a)::text) as a from dqa_f3; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer QUERY PLAN ------------------------------------------------------------------------------------------------ Finalize Aggregate @@ -2220,12 +2373,16 @@ explain (verbose, costs off) select count(distinct (b)::text) as b, count(distin -- Case 2: Same as the above one, but convert the type of column 'a' to 'varchar' via binary-compatible types. select count(distinct (b)::text) as b, count(distinct (a)::text::varchar) as a from dqa_f3; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer b | a ---+--- 3 | 7 (1 row) explain (verbose, costs off) select count(distinct (b)::text) as b, count(distinct (a)::text::varchar) as a from dqa_f3; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer QUERY PLAN ------------------------------------------------------------------------------------------------------------- Finalize Aggregate @@ -2261,12 +2418,16 @@ explain (verbose, costs off) select count(distinct (b)::text) as b, count(distin -- -> Seq Scan on public.dqa_f3 -- Output: b, a, (b)::text, (a)::integer select count(distinct (b)::text) as b, count(distinct (a)::int) as a from dqa_f3; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer b | a ---+--- 3 | 7 (1 row) explain (verbose, costs off) select count(distinct (b)::text) as b, count(distinct (a)::int) as a from dqa_f3; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer QUERY PLAN --------------------------------------------------------------------------------------------------- Finalize Aggregate @@ -2293,12 +2454,16 @@ explain (verbose, costs off) select count(distinct (b)::text) as b, count(distin -- Case 4: When converting the type of column 'a' from 'varchar' to 'int' to 'varchar', TupleSplit should generate an additional -- column '(a)::integer::varchar' as part of hash-key in Redistribute-Motion. select count(distinct (b)::text) as b, count(distinct (a)::int::varchar) as a from dqa_f3; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer b | a ---+--- 3 | 7 (1 row) explain (verbose, costs off) select count(distinct (b)::text) as b, count(distinct (a)::int::varchar) as a from dqa_f3; +INFO: GPORCA failed to produce a plan, falling back to planner +DETAIL: Feature not supported: Multiple Distinct Qualified Aggregates are disabled in the optimizer QUERY PLAN ------------------------------------------------------------------------------------------------------------------------ Finalize Aggregate diff --git a/src/test/regress/expected/gp_foreign_data.out b/src/test/regress/expected/gp_foreign_data.out index c4dbd5bea93..500d395404d 100644 --- a/src/test/regress/expected/gp_foreign_data.out +++ b/src/test/regress/expected/gp_foreign_data.out @@ -26,18 +26,14 @@ CREATE FOREIGN TABLE ft3 ( CREATE FOREIGN TABLE ft4 ( c1 int ) SERVER s0 OPTIONS (delimiter ',', mpp_execute 'all segments'); --- Test num_segments option -CREATE SERVER s1 FOREIGN DATA WRAPPER dummy OPTIONS (num_segments '3'); -CREATE FOREIGN TABLE ft5 ( - c1 int -) SERVER s1 OPTIONS (delimiter ',', mpp_execute 'all segments', num_segments '5'); -\d+ ft5 - Foreign table "public.ft5" - Column | Type | Collation | Nullable | Default | FDW options | Storage | Stats target | Description ---------+---------+-----------+----------+---------+-------------+---------+--------------+------------- - c1 | integer | | | | | plain | | -Server: s1 -FDW options: (delimiter ',', mpp_execute 'all segments', num_segments '5') +-- CREATE FOREIGN SERVER WITH num_segments +CREATE SERVER s1 FOREIGN DATA WRAPPER dummy OPTIONS (num_segments '5'); +-- CHECK FOREIGN SERVER's OPTIONS +SELECT srvoptions FROM pg_foreign_server WHERE srvname = 's1'; + srvoptions +------------------ + {num_segments=5} +(1 row) --start_ignore DROP FOREIGN DATA WRAPPER dummy CASCADE; diff --git a/src/test/regress/expected/misc_jiras.out b/src/test/regress/expected/misc_jiras.out index 2c9e78fab5a..ab418abb849 100644 --- a/src/test/regress/expected/misc_jiras.out +++ b/src/test/regress/expected/misc_jiras.out @@ -1,4 +1,5 @@ drop schema if exists misc_jiras; +NOTICE: schema "misc_jiras" does not exist, skipping create schema misc_jiras; -- -- Test backward scanning of tuplestore spill files. @@ -11,12 +12,22 @@ create schema misc_jiras; -- create table misc_jiras.t1 (c1 int, c2 text, c3 smallint) distributed by (c1); insert into misc_jiras.t1 select i % 13, md5(i::text), i % 3 - from generate_series(1, 20000) i; --- tuplestore uses work_mem to control the in-memory data size, set a small --- value to trigger the spilling. -set work_mem to '64kB'; -WARNING: "work_mem": setting is deprecated, and may be removed in a future release. + from generate_series(1, 40000) i; +-- tuplestore in windowagg uses statement_mem to control the in-memory data size, +-- set a small value to trigger the spilling. +set statement_mem to '1024kB'; set extra_float_digits=0; -- the last decimal digits are somewhat random +-- Inject fault at 'winagg_after_spool_tuples' to show that the tuplestore spills +-- to disk. +SELECT gp_inject_fault('winagg_after_spool_tuples', 'skip', dbid) + FROM gp_segment_configuration WHERE role='p' AND content>=0; + gp_inject_fault +----------------- + Success: + Success: + Success: +(3 rows) + select sum(cc) from ( select c1 , c2 @@ -28,13 +39,24 @@ select sum(cc) from ( from misc_jiras.t1 group by 1, 2 ) tt; +NOTICE: winagg: tuplestore spilled to disk (seg0 slice1 127.0.0.1:7002 pid=54719) +NOTICE: winagg: tuplestore spilled to disk (seg1 slice1 127.0.0.1:7003 pid=54720) +NOTICE: winagg: tuplestore spilled to disk (seg2 slice1 127.0.0.1:7004 pid=54721) sum --------- - 10006.5 + 20006.5 (1 row) -reset work_mem; -WARNING: "work_mem": setting is deprecated, and may be removed in a future release. +SELECT gp_inject_fault('winagg_after_spool_tuples', 'reset', dbid) + FROM gp_segment_configuration WHERE role='p' AND content>=0; + gp_inject_fault +----------------- + Success: + Success: + Success: +(3 rows) + +reset statement_mem; -- non-ASCII multibyte character should show up correctly in error messages. select '溋' || (B'1'); ERROR: "溋" is not a valid binary digit diff --git a/src/test/regress/expected/qp_correlated_query.out b/src/test/regress/expected/qp_correlated_query.out index 07a3ab10a1c..d35e89ea6e0 100644 --- a/src/test/regress/expected/qp_correlated_query.out +++ b/src/test/regress/expected/qp_correlated_query.out @@ -1662,6 +1662,12 @@ select (select avg(x) from qp_csq_t1, qp_csq_t2 where qp_csq_t1.a = any (select 4.0000000000000000 (4 rows) +-- Planner should fail due to skip-level correlation not supported. Query should not cause segfault like in issue #12054. +select A.j, (select array_agg(a_B) from (select B.j, (select array_agg(a_C) from (select C.j from C where C.i = A.i) a_C) from B where B.i = A.i order by A.j) a_B) from A; +ERROR: correlated subquery with skip-level correlations is not supported +-- Planner should fail due to skip-level correlation not supported. Query should not return wrong results like in issue #12054. +select A.j, (select array_agg(a_B) from (select B.j, (select sum(a_C.j) from (select C.j from C where C.i = A.i) a_C) from B where B.i = A.i order by A.j) a_B) from A; +ERROR: correlated subquery with skip-level correlations is not supported -- ---------------------------------------------------------------------- -- Test: Correlated Subquery: CSQ with multiple columns (Heap) -- ---------------------------------------------------------------------- diff --git a/src/test/regress/expected/qp_correlated_query_optimizer.out b/src/test/regress/expected/qp_correlated_query_optimizer.out index c238b0633a4..a604629ae1e 100644 --- a/src/test/regress/expected/qp_correlated_query_optimizer.out +++ b/src/test/regress/expected/qp_correlated_query_optimizer.out @@ -1775,6 +1775,12 @@ select (select avg(x) from qp_csq_t1, qp_csq_t2 where qp_csq_t1.a = any (select 4.0000000000000000 (4 rows) +-- Planner should fail due to skip-level correlation not supported. Query should not cause segfault like in issue #12054. +select A.j, (select array_agg(a_B) from (select B.j, (select array_agg(a_C) from (select C.j from C where C.i = A.i) a_C) from B where B.i = A.i order by A.j) a_B) from A; +ERROR: correlated subquery with skip-level correlations is not supported +-- Planner should fail due to skip-level correlation not supported. Query should not return wrong results like in issue #12054. +select A.j, (select array_agg(a_B) from (select B.j, (select sum(a_C.j) from (select C.j from C where C.i = A.i) a_C) from B where B.i = A.i order by A.j) a_B) from A; +ERROR: correlated subquery with skip-level correlations is not supported -- ---------------------------------------------------------------------- -- Test: Correlated Subquery: CSQ with multiple columns (Heap) -- ---------------------------------------------------------------------- diff --git a/src/test/regress/expected/statement_mem_for_windowagg.out b/src/test/regress/expected/statement_mem_for_windowagg.out new file mode 100644 index 00000000000..d41610f6a30 --- /dev/null +++ b/src/test/regress/expected/statement_mem_for_windowagg.out @@ -0,0 +1,176 @@ +CREATE TABLE dummy_table(x int, y int) DISTRIBUTED BY (y); +INSERT INTO dummy_table SELECT generate_series(0, 20000), 0; +INSERT INTO dummy_table SELECT generate_series(0, 20000), 3; +INSERT INTO dummy_table SELECT generate_series(0, 20000), 10; +-- 1. Test that if we set statement_mem to a larger value, the tuplestore +-- for caching the tuples in partition used in WindowAgg is able to be fitted +-- in memory. +SET statement_mem TO '4096kB'; +SELECT gp_inject_fault('winagg_after_spool_tuples', 'skip', dbid) + FROM gp_segment_configuration WHERE role='p' AND content>=0; + gp_inject_fault +----------------- + Success: + Success: + Success: +(3 rows) + +EXPLAIN ANALYZE SELECT AVG(x) OVER (PARTITION BY y) FROM dummy_table; +NOTICE: winagg: tuplestore fitted in memory (seg1 slice1 127.0.0.1:7003 pid=43473) +NOTICE: winagg: tuplestore fitted in memory (seg0 slice1 127.0.0.1:7002 pid=43472) +NOTICE: winagg: tuplestore fitted in memory (seg2 slice1 127.0.0.1:7004 pid=43474) + QUERY PLAN +----------------------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..431.00 rows=1 width=8) (actual time=6.520..15.872 rows=30003 loops=1) + -> WindowAgg (cost=0.00..431.00 rows=1 width=8) (actual time=10.607..13.115 rows=10001 loops=1) + Partition By: y + -> Sort (cost=0.00..431.00 rows=1 width=8) (actual time=5.298..6.324 rows=10001 loops=1) + Sort Key: y + Sort Method: quicksort Memory: 5040kB + -> Seq Scan on dummy_table (cost=0.00..431.00 rows=1 width=8) (actual time=0.036..3.299 rows=10001 loops=1) + Planning Time: 5.241 ms + (slice0) Executor memory: 48K bytes. + (slice1) Executor memory: 606K bytes avg x 3 workers, 606K bytes max (seg0). Work_mem: 606K bytes max. + Memory used: 2048kB + Optimizer: Pivotal Optimizer (GPORCA) + Execution Time: 17.225 ms +(13 rows) + +SELECT gp_inject_fault('winagg_after_spool_tuples', 'reset', dbid) + FROM gp_segment_configuration WHERE role='p' AND content>=0; + gp_inject_fault +----------------- + Success: + Success: + Success: +(3 rows) + +-- 2. Test that if we set statement_mem to a smaller value, the tuplestore +-- for caching the tuples in partition used in WindowAgg will be spilled to disk. +SET statement_mem TO '1024kB'; +SELECT gp_inject_fault('winagg_after_spool_tuples', 'skip', dbid) + FROM gp_segment_configuration WHERE role='p' AND content>=0; + gp_inject_fault +----------------- + Success: + Success: + Success: +(3 rows) + +EXPLAIN ANALYZE SELECT AVG(x) OVER (PARTITION BY y) FROM dummy_table; +NOTICE: winagg: tuplestore spilled to disk (seg0 slice1 127.0.0.1:7002 pid=43472) +NOTICE: winagg: tuplestore spilled to disk (seg1 slice1 127.0.0.1:7003 pid=43473) +NOTICE: winagg: tuplestore spilled to disk (seg2 slice1 127.0.0.1:7004 pid=43474) + QUERY PLAN +----------------------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..431.00 rows=1 width=8) (actual time=8.784..13.923 rows=30003 loops=1) + -> WindowAgg (cost=0.00..431.00 rows=1 width=8) (actual time=8.390..9.720 rows=10001 loops=1) + Partition By: y + -> Sort (cost=0.00..431.00 rows=1 width=8) (actual time=3.125..4.135 rows=10001 loops=1) + Sort Key: y + Sort Method: external merge Disk: 6144kB + -> Seq Scan on dummy_table (cost=0.00..431.00 rows=1 width=8) (actual time=0.032..1.589 rows=10001 loops=1) + Planning Time: 3.174 ms + (slice0) Executor memory: 42K bytes. + (slice1) Executor memory: 391K bytes avg x 3 workers, 391K bytes max (seg0). Work_mem: 391K bytes max. + Memory used: 1024kB + Optimizer: Pivotal Optimizer (GPORCA) + Execution Time: 15.235 ms +(13 rows) + +SELECT gp_inject_fault('winagg_after_spool_tuples', 'reset', dbid) + FROM gp_segment_configuration WHERE role='p' AND content>=0; + gp_inject_fault +----------------- + Success: + Success: + Success: +(3 rows) + +-- 3. Test that if we set statement_mem to a larger value, the tuplesort +-- operation in DISTINCT-qualified WindowAgg is able to be fitted in memory. +SET statement_mem TO '4096kB'; +SELECT gp_inject_fault_infinite('distinct_winagg_perform_sort', 'skip', dbid) + FROM gp_segment_configuration WHERE role='p' AND content>=0; + gp_inject_fault_infinite +-------------------------- + Success: + Success: + Success: +(3 rows) + +EXPLAIN ANALYZE SELECT AVG(DISTINCT x) OVER (PARTITION BY y) FROM dummy_table; +NOTICE: distinct winagg sortstats: sort operation fitted in memory (seg0 slice1 127.0.0.1:7002 pid=43472) +NOTICE: distinct winagg sortstats: sort operation fitted in memory (seg1 slice1 127.0.0.1:7003 pid=43473) +NOTICE: distinct winagg sortstats: sort operation fitted in memory (seg2 slice1 127.0.0.1:7004 pid=43474) + QUERY PLAN +--------------------------------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=2446.06..4096.31 rows=86100 width=36) (actual time=10.348..16.027 rows=30003 loops=1) + -> WindowAgg (cost=2446.06..2948.31 rows=28700 width=36) (actual time=10.186..11.795 rows=10001 loops=1) + Partition By: y + -> Sort (cost=2446.06..2517.81 rows=28700 width=8) (actual time=3.678..4.683 rows=10001 loops=1) + Sort Key: y + Sort Method: external merge Disk: 6144kB + -> Seq Scan on dummy_table (cost=0.00..321.00 rows=28700 width=8) (actual time=0.030..1.919 rows=10001 loops=1) + Planning Time: 1.695 ms + (slice0) Executor memory: 50K bytes. + (slice1) Executor memory: 391K bytes avg x 3 workers, 391K bytes max (seg0). Work_mem: 391K bytes max. + Memory used: 1024kB + Optimizer: Postgres query optimizer + Execution Time: 18.747 ms +(13 rows) + +SELECT gp_inject_fault_infinite('distinct_winagg_perform_sort', 'reset', dbid) + FROM gp_segment_configuration WHERE role='p' AND content>=0; + gp_inject_fault_infinite +-------------------------- + Success: + Success: + Success: +(3 rows) + +-- 4. Test that if we set statement_mem to a smaller value, the tuplesort +-- operation in DISTINCT-qualified WindowAgg will be spilled to disk. +SET statement_mem TO '1024kB'; +SELECT gp_inject_fault_infinite('distinct_winagg_perform_sort', 'skip', dbid) + FROM gp_segment_configuration WHERE role='p' AND content>=0; + gp_inject_fault_infinite +-------------------------- + Success: + Success: + Success: +(3 rows) + +EXPLAIN ANALYZE SELECT AVG(DISTINCT x) OVER (PARTITION BY y) FROM dummy_table; +NOTICE: distinct winagg sortstats: sort operation spilled to disk (seg0 slice1 127.0.0.1:7002 pid=43472) +NOTICE: distinct winagg sortstats: sort operation spilled to disk (seg1 slice1 127.0.0.1:7003 pid=43473) +NOTICE: distinct winagg sortstats: sort operation spilled to disk (seg2 slice1 127.0.0.1:7004 pid=43474) + QUERY PLAN +--------------------------------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=2446.06..4096.31 rows=86100 width=36) (actual time=13.040..19.232 rows=30003 loops=1) + -> WindowAgg (cost=2446.06..2948.31 rows=28700 width=36) (actual time=12.768..14.527 rows=10001 loops=1) + Partition By: y + -> Sort (cost=2446.06..2517.81 rows=28700 width=8) (actual time=4.278..5.449 rows=10001 loops=1) + Sort Key: y + Sort Method: external merge Disk: 9216kB + -> Seq Scan on dummy_table (cost=0.00..321.00 rows=28700 width=8) (actual time=0.029..1.746 rows=10001 loops=1) + Planning Time: 1.509 ms + (slice0) Executor memory: 39K bytes. + (slice1) Executor memory: 275K bytes avg x 3 workers, 275K bytes max (seg0). Work_mem: 275K bytes max. + Memory used: 128kB + Optimizer: Postgres query optimizer + Execution Time: 22.056 ms +(13 rows) + +SELECT gp_inject_fault_infinite('distinct_winagg_perform_sort', 'reset', dbid) + FROM gp_segment_configuration WHERE role='p' AND content>=0; + gp_inject_fault_infinite +-------------------------- + Success: + Success: + Success: +(3 rows) + +-- Do some clean-ups. +DROP TABLE dummy_table; +RESET statement_mem; diff --git a/src/test/regress/greenplum_schedule b/src/test/regress/greenplum_schedule index 71d0e97eaa1..66c5129c356 100755 --- a/src/test/regress/greenplum_schedule +++ b/src/test/regress/greenplum_schedule @@ -54,10 +54,12 @@ test: leastsquares opr_sanity_gp decode_expr bitmapscan bitmapscan_ao case_gp li test: gpcopy test: orca_static_pruning orca_groupingsets_fallbacks -test: filter gpctas gpdist gpdist_opclasses gpdist_legacy_opclasses matrix sublink table_functions olap_setup complex opclass_ddl information_schema guc_env_var gp_explain distributed_transactions explain_format olap_plans misc_jiras gp_copy_dtx +test: filter gpctas gpdist gpdist_opclasses gpdist_legacy_opclasses matrix sublink table_functions olap_setup complex opclass_ddl information_schema guc_env_var gp_explain distributed_transactions explain_format olap_plans gp_copy_dtx # below test(s) inject faults so each of them need to be in a separate group test: guc_gp test: toast +test: misc_jiras +test: statement_mem_for_windowagg # namespace_gp test will show diff if concurrent tests use temporary tables. # So run it separately. @@ -191,7 +193,7 @@ test: qp_misc_jiras qp_with_clause qp_executor qp_olap_windowerr qp_olap_window test: qp_with_functional_inlining qp_with_functional_noinlining test: qp_functions_in_contexts_setup # test: qp_misc_rio_join_small qp_misc_rio qp_correlated_query qp_targeted_dispatch qp_gist_indexes2 qp_gist_indexes3 qp_gist_indexes4 qp_query_execution qp_functions_in_from qp_functions_in_select qp_functions_in_subquery qp_functions_in_subquery_column qp_functions_in_subquery_constant qp_functions_in_with -test: qp_misc_rio_join_small qp_correlated_query qp_targeted_dispatch qp_gist_indexes2 qp_gist_indexes3 qp_gist_indexes4 qp_query_execution qp_functions_in_from qp_functions_in_select qp_functions_in_subquery qp_functions_in_subquery_column qp_functions_in_with +test: qp_misc_rio_join_small qp_correlated_query qp_targeted_dispatch qp_gist_indexes2 qp_gist_indexes3 qp_gist_indexes4 qp_query_execution qp_functions_in_from qp_functions_in_select qp_functions_in_subquery qp_functions_in_subquery_column qp_functions_in_with correlated_subquery test: dpe qp_dpe qp_subquery qp_left_anti_semi_join qp_union_intersect qp_functions qp_functions_idf qp_regexp qp_resource_queue qp_orca_fallback diff --git a/src/test/regress/sql/aggregates.sql b/src/test/regress/sql/aggregates.sql index 9268a11b3df..16b811d9bbe 100644 --- a/src/test/regress/sql/aggregates.sql +++ b/src/test/regress/sql/aggregates.sql @@ -1103,6 +1103,8 @@ create aggregate my_avg_init2(int4) initcond = '(4,0)' ); +-- reset the plan cache, sometimes it would re-plan these prepared statements and log ORCA fallbacks +discard plans; -- state should be shared if INITCONDs are matching select my_sum_init(one),my_avg_init(one) from (values(1),(3)) t(one); diff --git a/src/test/regress/sql/alter_distribution_policy.sql b/src/test/regress/sql/alter_distribution_policy.sql index f6479f3a974..71d9490872e 100644 --- a/src/test/regress/sql/alter_distribution_policy.sql +++ b/src/test/regress/sql/alter_distribution_policy.sql @@ -531,3 +531,33 @@ select check_redistributed('insert into t_distbya select * from t_reorganize', ' reset optimizer; reset gp_force_random_redistribution; +-- When reorganize=false, we won't reorganize and this shouldn't be affected by the existing reloptions. +CREATE TABLE public.t_reorganize_false ( +a integer, +b integer +) with (appendonly=false, autovacuum_enabled=false) DISTRIBUTED BY (a); +-- Insert values which will all be on one segment +INSERT INTO t_reorganize_false VALUES (0, generate_series(1,100)); +SELECT gp_segment_id,count(*) from t_reorganize_false GROUP BY 1; +-- Change the distribution policy but because REORGANIZE=false, it should NOT be re-distributed +ALTER TABLE t_reorganize_false SET WITH (REORGANIZE=false) DISTRIBUTED RANDOMLY; +SELECT gp_segment_id,count(*) from t_reorganize_false GROUP BY 1; +DROP TABLE t_reorganize_false; +-- Same rule should apply to partitioned table too +CREATE TABLE public.t_reorganize_false ( +a integer, +b integer +) +DISTRIBUTED BY (a) PARTITION BY RANGE(b) +( +PARTITION "00" START (0) END (1000) WITH (tablename='t_reorganize_false_0', appendonly='false', autovacuum_enabled=false), +PARTITION "01" START (1000) END (2000) WITH (tablename='t_reorganize_false_1', appendonly='false', autovacuum_enabled=false), +DEFAULT PARTITION def WITH (tablename='t_reorganize_false_def', appendonly='false', autovacuum_enabled=false) +); +-- Insert values which will all be on one segment +INSERT INTO t_reorganize_false VALUES (0, generate_series(1,100)); +SELECT gp_segment_id,count(*) from t_reorganize_false GROUP BY 1; +-- Should NOT be re-distributed +ALTER TABLE t_reorganize_false SET WITH (REORGANIZE=false) DISTRIBUTED RANDOMLY; +SELECT gp_segment_id,count(*) from t_reorganize_false GROUP BY 1; +DROP TABLE t_reorganize_false; diff --git a/src/test/regress/sql/correlated_subquery.sql b/src/test/regress/sql/correlated_subquery.sql new file mode 100644 index 00000000000..b4a6d8283c3 --- /dev/null +++ b/src/test/regress/sql/correlated_subquery.sql @@ -0,0 +1,21 @@ +SET optimizer_enforce_subplans = 1; +SET optimizer_trace_fallback=on; +SET client_min_messages=log; + +SELECT a = ALL (SELECT generate_series(1, 2)), a FROM (values (1),(2)) v(a); +SELECT a = ALL (SELECT generate_series(2, 2)), a FROM (values (1),(2)) v(a); +SELECT 1 = ALL (SELECT generate_series(1, 2)) FROM (values (1),(2)) v(a); +SELECT 2 = ALL (SELECT generate_series(2, 2)) FROM (values (1),(2)) v(a); +SELECT 2 = ALL (SELECT generate_series(2, 3)) FROM (values (1),(2)) v(a); +SELECT 2+1 = ALL (SELECT generate_series(2, 3)) FROM (values (1),(2)) v(a); +SELECT 2+1 = ALL (SELECT generate_series(3, 3)) FROM (values (1),(2)) v(a); +SELECT (SELECT a) = ALL (SELECT generate_series(1, 2)), a FROM (values (1),(2)) v(a); +SELECT (SELECT a) = ALL (SELECT generate_series(2, 2)), a FROM (values (1),(2)) v(a); +SELECT (SELECT a+1) = ALL (SELECT generate_series(2, 2)), a FROM (values (1),(2)) v(a); +SELECT (SELECT 1) = ALL (SELECT generate_series(1, 1)) FROM (values (1),(2)) v(a); +SELECT (SELECT 1) = ALL (SELECT generate_series(1, 2)) FROM (values (1),(2)) v(a); +SELECT (SELECT 3) = ALL (SELECT generate_series(3, 3)) FROM (values (1),(2)) v(a); + +SELECT (SELECT 1) = ALL (SELECT generate_series(1, 1)); +SELECT (SELECT 1) = ALL (SELECT generate_series(1, 2)); +SELECT (SELECT 3) = ALL (SELECT generate_series(3, 3)); diff --git a/src/test/regress/sql/gp_dqa.sql b/src/test/regress/sql/gp_dqa.sql index ab8c0d91aac..808cee12c7e 100644 --- a/src/test/regress/sql/gp_dqa.sql +++ b/src/test/regress/sql/gp_dqa.sql @@ -5,6 +5,8 @@ -- the queries, so a few also use TO_CHAR() to truncate the results further. set extra_float_digits=0; +SET optimizer_trace_fallback to on; + drop table if exists dqa_t1; drop table if exists dqa_t2; diff --git a/src/test/regress/sql/gp_foreign_data.sql b/src/test/regress/sql/gp_foreign_data.sql index a26f96f9d50..23c630797da 100644 --- a/src/test/regress/sql/gp_foreign_data.sql +++ b/src/test/regress/sql/gp_foreign_data.sql @@ -2,6 +2,12 @@ -- Test foreign-data wrapper and server management. Cloudberry MPP specific -- +-- start_ignore +DROP SERVER s0 CASCADE; +DROP SERVER s1 CASCADE; +DROP FOREIGN DATA WRAPPER dummy CASCADE; +-- end_ignore + CREATE FOREIGN DATA WRAPPER dummy; COMMENT ON FOREIGN DATA WRAPPER dummy IS 'useless'; @@ -21,13 +27,14 @@ CREATE FOREIGN TABLE ft4 ( c1 int ) SERVER s0 OPTIONS (delimiter ',', mpp_execute 'all segments'); --- Test num_segments option -CREATE SERVER s1 FOREIGN DATA WRAPPER dummy OPTIONS (num_segments '3'); -CREATE FOREIGN TABLE ft5 ( - c1 int -) SERVER s1 OPTIONS (delimiter ',', mpp_execute 'all segments', num_segments '5'); -\d+ ft5 +-- CREATE FOREIGN SERVER WITH num_segments +CREATE SERVER s1 FOREIGN DATA WRAPPER dummy OPTIONS (num_segments '5'); + +-- CHECK FOREIGN SERVER's OPTIONS +SELECT srvoptions FROM pg_foreign_server WHERE srvname = 's1'; ---start_ignore +-- start_ignore +DROP SERVER s0 CASCADE; +DROP SERVER s1 CASCADE; DROP FOREIGN DATA WRAPPER dummy CASCADE; ---end_ignore +-- end_ignore diff --git a/src/test/regress/sql/misc_jiras.sql b/src/test/regress/sql/misc_jiras.sql index b90e908f37c..4e96bc11e52 100644 --- a/src/test/regress/sql/misc_jiras.sql +++ b/src/test/regress/sql/misc_jiras.sql @@ -13,14 +13,19 @@ create schema misc_jiras; create table misc_jiras.t1 (c1 int, c2 text, c3 smallint) distributed by (c1); insert into misc_jiras.t1 select i % 13, md5(i::text), i % 3 - from generate_series(1, 20000) i; + from generate_series(1, 40000) i; --- tuplestore uses work_mem to control the in-memory data size, set a small --- value to trigger the spilling. -set work_mem to '64kB'; +-- tuplestore in windowagg uses statement_mem to control the in-memory data size, +-- set a small value to trigger the spilling. +set statement_mem to '1024kB'; set extra_float_digits=0; -- the last decimal digits are somewhat random +-- Inject fault at 'winagg_after_spool_tuples' to show that the tuplestore spills +-- to disk. +SELECT gp_inject_fault('winagg_after_spool_tuples', 'skip', dbid) + FROM gp_segment_configuration WHERE role='p' AND content>=0; + select sum(cc) from ( select c1 , c2 @@ -33,7 +38,10 @@ select sum(cc) from ( group by 1, 2 ) tt; -reset work_mem; +SELECT gp_inject_fault('winagg_after_spool_tuples', 'reset', dbid) + FROM gp_segment_configuration WHERE role='p' AND content>=0; + +reset statement_mem; -- non-ASCII multibyte character should show up correctly in error messages. select '溋' || (B'1'); diff --git a/src/test/regress/sql/qp_correlated_query.sql b/src/test/regress/sql/qp_correlated_query.sql index 46048efc3b7..6c30e018391 100644 --- a/src/test/regress/sql/qp_correlated_query.sql +++ b/src/test/regress/sql/qp_correlated_query.sql @@ -342,6 +342,10 @@ SELECT a, (SELECT (SELECT d FROM qp_csq_t3 WHERE a=c)) FROM qp_csq_t1 GROUP BY a select A.i, (select C.j from C group by C.j having max(C.j) = any (select min(B.j) from B)) as C_j from A,B,C where A.i = 99 order by A.i, C_j limit 10; select (select avg(x) from qp_csq_t1, qp_csq_t2 where qp_csq_t1.a = any (select x)) as avg_x from qp_csq_t1 order by 1; +-- Planner should fail due to skip-level correlation not supported. Query should not cause segfault like in issue #12054. +select A.j, (select array_agg(a_B) from (select B.j, (select array_agg(a_C) from (select C.j from C where C.i = A.i) a_C) from B where B.i = A.i order by A.j) a_B) from A; +-- Planner should fail due to skip-level correlation not supported. Query should not return wrong results like in issue #12054. +select A.j, (select array_agg(a_B) from (select B.j, (select sum(a_C.j) from (select C.j from C where C.i = A.i) a_C) from B where B.i = A.i order by A.j) a_B) from A; -- ---------------------------------------------------------------------- -- Test: Correlated Subquery: CSQ with multiple columns (Heap) diff --git a/src/test/regress/sql/statement_mem_for_windowagg.sql b/src/test/regress/sql/statement_mem_for_windowagg.sql new file mode 100644 index 00000000000..5c4b96a1761 --- /dev/null +++ b/src/test/regress/sql/statement_mem_for_windowagg.sql @@ -0,0 +1,57 @@ +CREATE TABLE dummy_table(x int, y int) DISTRIBUTED BY (y); +INSERT INTO dummy_table SELECT generate_series(0, 20000), 0; +INSERT INTO dummy_table SELECT generate_series(0, 20000), 3; +INSERT INTO dummy_table SELECT generate_series(0, 20000), 10; + +-- 1. Test that if we set statement_mem to a larger value, the tuplestore +-- for caching the tuples in partition used in WindowAgg is able to be fitted +-- in memory. +SET statement_mem TO '4096kB'; + +SELECT gp_inject_fault('winagg_after_spool_tuples', 'skip', dbid) + FROM gp_segment_configuration WHERE role='p' AND content>=0; + +EXPLAIN ANALYZE SELECT AVG(x) OVER (PARTITION BY y) FROM dummy_table; + +SELECT gp_inject_fault('winagg_after_spool_tuples', 'reset', dbid) + FROM gp_segment_configuration WHERE role='p' AND content>=0; + +-- 2. Test that if we set statement_mem to a smaller value, the tuplestore +-- for caching the tuples in partition used in WindowAgg will be spilled to disk. +SET statement_mem TO '1024kB'; + +SELECT gp_inject_fault('winagg_after_spool_tuples', 'skip', dbid) + FROM gp_segment_configuration WHERE role='p' AND content>=0; + +EXPLAIN ANALYZE SELECT AVG(x) OVER (PARTITION BY y) FROM dummy_table; + +SELECT gp_inject_fault('winagg_after_spool_tuples', 'reset', dbid) + FROM gp_segment_configuration WHERE role='p' AND content>=0; + +-- 3. Test that if we set statement_mem to a larger value, the tuplesort +-- operation in DISTINCT-qualified WindowAgg is able to be fitted in memory. +SET statement_mem TO '4096kB'; + +SELECT gp_inject_fault_infinite('distinct_winagg_perform_sort', 'skip', dbid) + FROM gp_segment_configuration WHERE role='p' AND content>=0; + +EXPLAIN ANALYZE SELECT AVG(DISTINCT x) OVER (PARTITION BY y) FROM dummy_table; + +SELECT gp_inject_fault_infinite('distinct_winagg_perform_sort', 'reset', dbid) + FROM gp_segment_configuration WHERE role='p' AND content>=0; + +-- 4. Test that if we set statement_mem to a smaller value, the tuplesort +-- operation in DISTINCT-qualified WindowAgg will be spilled to disk. +SET statement_mem TO '1024kB'; + +SELECT gp_inject_fault_infinite('distinct_winagg_perform_sort', 'skip', dbid) + FROM gp_segment_configuration WHERE role='p' AND content>=0; + +EXPLAIN ANALYZE SELECT AVG(DISTINCT x) OVER (PARTITION BY y) FROM dummy_table; + +SELECT gp_inject_fault_infinite('distinct_winagg_perform_sort', 'reset', dbid) + FROM gp_segment_configuration WHERE role='p' AND content>=0; + +-- Do some clean-ups. +DROP TABLE dummy_table; +RESET statement_mem;