From 14d0ff064f9d583f3eb16a107d8501cf36124cf2 Mon Sep 17 00:00:00 2001 From: wangxiangyu Date: Tue, 19 Dec 2023 16:52:31 +0800 Subject: [PATCH] [Fix](multi-catalog) skip hms events if hms table is not supported. --- .../org/apache/doris/catalog/TableIf.java | 15 ++++++++++ .../catalog/external/HMSExternalTable.java | 20 +++++++------ .../datasource/ExternalMetaCacheMgr.java | 11 +++++++- .../exceptions/NotSupportedException.java | 28 +++++++++++++++++++ 4 files changed, 64 insertions(+), 10 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/exceptions/NotSupportedException.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java index 3f53fa1bfaef07..5fd8f563fd81d4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java @@ -18,6 +18,7 @@ package org.apache.doris.catalog; import org.apache.doris.alter.AlterCancelException; +import org.apache.doris.cluster.ClusterNamespace; import org.apache.doris.common.DdlException; import org.apache.doris.common.MetaNotFoundException; import org.apache.doris.statistics.AnalysisInfo; @@ -26,6 +27,7 @@ import org.apache.doris.statistics.TableStatsMeta; import org.apache.doris.thrift.TTableDescriptor; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import com.google.common.collect.Sets; import org.apache.logging.log4j.LogManager; @@ -249,6 +251,19 @@ default Partition getPartition(String name) { return null; } + + default List getFullQualifiers() { + return ImmutableList.of(getDatabase().getCatalog().getName(), + ClusterNamespace.getNameFromFullName(getDatabase().getFullName()), + getName()); + } + + default String getNameWithFullQualifiers() { + return String.format("%s.%s.%s", getDatabase().getCatalog().getName(), + ClusterNamespace.getNameFromFullName(getDatabase().getFullName()), + getName()); + } + default boolean isManagedTable() { return getType() == TableType.OLAP || getType() == TableType.MATERIALIZED_VIEW; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java index 0243ad12f75770..6937ba944f5c69 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java @@ -28,6 +28,7 @@ import org.apache.doris.datasource.HMSExternalCatalog; import org.apache.doris.datasource.hive.HiveMetaStoreCache; import org.apache.doris.datasource.hive.PooledHiveMetaStoreClient; +import org.apache.doris.nereids.exceptions.NotSupportedException; import org.apache.doris.statistics.AnalysisInfo; import org.apache.doris.statistics.BaseAnalysisTask; import org.apache.doris.statistics.ColumnStatistic; @@ -140,8 +141,13 @@ public HMSExternalTable(long id, String name, String dbName, HMSExternalCatalog } public boolean isSupportedHmsTable() { - makeSureInitialized(); - return dlaType != DLAType.UNKNOWN; + try { + makeSureInitialized(); + return true; + } catch (NotSupportedException e) { + LOG.warn("Not supported hms table, message: {}", e.getMessage()); + return false; + } } protected synchronized void makeSureInitialized() { @@ -149,7 +155,7 @@ protected synchronized void makeSureInitialized() { if (!objectCreated) { remoteTable = ((HMSExternalCatalog) catalog).getClient().getTable(dbName, name); if (remoteTable == null) { - dlaType = DLAType.UNKNOWN; + throw new IllegalArgumentException("Hms table not exists, table: " + getNameWithFullQualifiers()); } else { if (supportedIcebergTable()) { dlaType = DLAType.ICEBERG; @@ -158,7 +164,7 @@ protected synchronized void makeSureInitialized() { } else if (supportedHiveTable()) { dlaType = DLAType.HIVE; } else { - dlaType = DLAType.UNKNOWN; + throw new NotSupportedException("Unsupported dlaType for table: " + getNameWithFullQualifiers()); } } objectCreated = true; @@ -205,12 +211,8 @@ private boolean supportedHiveTable() { if (inputFileFormat == null) { return false; } - boolean supportedFileFormat = SUPPORTED_HIVE_FILE_FORMATS.contains(inputFileFormat); - if (!supportedFileFormat) { - throw new IllegalArgumentException("Unsupported hive input format: " + inputFileFormat); - } LOG.debug("hms table {} is {} with file format: {}", name, remoteTable.getTableType(), inputFileFormat); - return true; + return SUPPORTED_HIVE_FILE_FORMATS.contains(inputFileFormat); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java index ef62f498695a66..57aff50c621ecb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java @@ -17,12 +17,14 @@ package org.apache.doris.datasource; +import org.apache.doris.catalog.Type; import org.apache.doris.catalog.external.HMSExternalTable; import org.apache.doris.cluster.ClusterNamespace; import org.apache.doris.common.Config; import org.apache.doris.common.ThreadPoolManager; import org.apache.doris.datasource.hive.HiveMetaStoreCache; import org.apache.doris.fs.FileSystemCache; +import org.apache.doris.nereids.exceptions.NotSupportedException; import org.apache.doris.planner.external.hudi.HudiPartitionMgr; import org.apache.doris.planner.external.hudi.HudiPartitionProcessor; import org.apache.doris.planner.external.iceberg.IcebergMetadataCache; @@ -172,7 +174,14 @@ public void addPartitionsCache(long catalogId, HMSExternalTable table, List partitionColumnTypes; + try { + partitionColumnTypes = table.getPartitionColumnTypes(); + } catch (NotSupportedException e) { + LOG.warn("Ignore not supported hms table, message: {} ", e.getMessage()); + return; + } + metaCache.addPartitionsCache(dbName, table.getName(), partitionNames, partitionColumnTypes); } LOG.debug("add partition cache for {}.{} in catalog {}", dbName, table.getName(), catalogId); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/exceptions/NotSupportedException.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/exceptions/NotSupportedException.java new file mode 100644 index 00000000000000..bb707b6562088a --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/exceptions/NotSupportedException.java @@ -0,0 +1,28 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.exceptions; + +/** + * Exception for calling function only implement in bound expression or plan. + */ +public class NotSupportedException extends RuntimeException { + public NotSupportedException(String msg) { + super(String.format("Not Supported: %s", msg)); + } +} +