From 177ef032665d5db71523caf2ff02a5f3714b6f6c Mon Sep 17 00:00:00 2001 From: wuwenchi Date: Tue, 24 Sep 2024 13:48:09 +0800 Subject: [PATCH 1/8] fix --- .../paimon/PaimonDLFExternalCatalog.java | 67 +++++++++++++++++++ .../paimon/PaimonExternalCatalog.java | 1 + .../paimon/PaimonExternalCatalogFactory.java | 2 + .../property/constants/PaimonProperties.java | 1 + 4 files changed, 71 insertions(+) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonDLFExternalCatalog.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonDLFExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonDLFExternalCatalog.java new file mode 100644 index 00000000000000..29ee4cc50e7257 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonDLFExternalCatalog.java @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.paimon; + +import org.apache.doris.common.DdlException; +import org.apache.doris.datasource.property.PropertyConverter; +import org.apache.doris.datasource.property.constants.HMSProperties; +import org.apache.doris.datasource.property.constants.PaimonProperties; + +import com.aliyun.datalake.metastore.hive2.ProxyMetaStoreClient; +import com.google.common.collect.ImmutableList; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.List; +import java.util.Map; + +public class PaimonDLFExternalCatalog extends PaimonExternalCatalog { + private static final Logger LOG = LogManager.getLogger(PaimonDLFExternalCatalog.class); + private static final List REQUIRED_PROPERTIES = ImmutableList.of( + HMSProperties.HIVE_METASTORE_URIS + ); + + public PaimonDLFExternalCatalog(long catalogId, String name, String resource, + Map props, String comment) { + super(catalogId, name, resource, props, comment); + } + + @Override + protected void initLocalObjectsImpl() { + super.initLocalObjectsImpl(); + catalogType = PAIMON_DLF; + catalog = createCatalog(); + } + + @Override + protected void setPaimonCatalogOptions(Map properties, Map options) { + options.put(PaimonProperties.PAIMON_CATALOG_TYPE, getPaimonCatalogType(catalogType)); + options.put(PaimonProperties.PAIMON_METASTORE_CLIENT, ProxyMetaStoreClient.class.getName()); + options.putAll(PropertyConverter.convertToMetaProperties(properties)); + } + + @Override + public void checkProperties() throws DdlException { + super.checkProperties(); + for (String requiredProperty : REQUIRED_PROPERTIES) { + if (!catalogProperty.getProperties().containsKey(requiredProperty)) { + throw new DdlException("Required property '" + requiredProperty + "' is missing"); + } + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java index a4a2c092c61339..5a9e6feb5ad5b8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java @@ -49,6 +49,7 @@ public abstract class PaimonExternalCatalog extends ExternalCatalog { public static final String PAIMON_CATALOG_TYPE = "paimon.catalog.type"; public static final String PAIMON_FILESYSTEM = "filesystem"; public static final String PAIMON_HMS = "hms"; + public static final String PAIMON_DLF = "dlf"; protected String catalogType; protected Catalog catalog; protected AuthenticationConfig authConf; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalogFactory.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalogFactory.java index 5f7d991c2a679e..6c0d1c65505d73 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalogFactory.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalogFactory.java @@ -38,6 +38,8 @@ public static ExternalCatalog createCatalog(long catalogId, String name, String return new PaimonHMSExternalCatalog(catalogId, name, resource, props, comment); case PaimonExternalCatalog.PAIMON_FILESYSTEM: return new PaimonFileExternalCatalog(catalogId, name, resource, props, comment); + case PaimonExternalCatalog.PAIMON_DLF: + return new PaimonDLFExternalCatalog(catalogId, name, resource, props, comment); default: throw new DdlException("Unknown " + PaimonExternalCatalog.PAIMON_CATALOG_TYPE + " value: " + metastoreType); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/PaimonProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/PaimonProperties.java index 98739847758a5e..1a430fc997caf7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/PaimonProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/PaimonProperties.java @@ -38,6 +38,7 @@ public class PaimonProperties { public static final String PAIMON_OSS_SECRET_KEY = org.apache.hadoop.fs.aliyun.oss.Constants.ACCESS_KEY_SECRET; public static final String PAIMON_HMS_CATALOG = "hive"; public static final String PAIMON_FILESYSTEM_CATALOG = "filesystem"; + public static final String PAIMON_METASTORE_CLIENT = "metastore.client.class"; public static Map convertToS3Properties(Map properties, From 9d0eee887626de92e5a68e55a3a82f21b6c60b0e Mon Sep 17 00:00:00 2001 From: wuwenchi Date: Tue, 24 Sep 2024 22:05:29 +0800 Subject: [PATCH 2/8] fix --- .../paimon/PaimonDLFExternalCatalog.java | 13 +----------- .../paimon/PaimonExternalCatalogFactory.java | 2 ++ .../paimon/PaimonExternalTable.java | 5 +++-- .../property/PropertyConverter.java | 20 +++++++++++++++---- 4 files changed, 22 insertions(+), 18 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonDLFExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonDLFExternalCatalog.java index 29ee4cc50e7257..e514779b800299 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonDLFExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonDLFExternalCatalog.java @@ -17,7 +17,6 @@ package org.apache.doris.datasource.paimon; -import org.apache.doris.common.DdlException; import org.apache.doris.datasource.property.PropertyConverter; import org.apache.doris.datasource.property.constants.HMSProperties; import org.apache.doris.datasource.property.constants.PaimonProperties; @@ -50,18 +49,8 @@ protected void initLocalObjectsImpl() { @Override protected void setPaimonCatalogOptions(Map properties, Map options) { - options.put(PaimonProperties.PAIMON_CATALOG_TYPE, getPaimonCatalogType(catalogType)); + options.put(PaimonProperties.PAIMON_CATALOG_TYPE, PaimonProperties.PAIMON_HMS_CATALOG); options.put(PaimonProperties.PAIMON_METASTORE_CLIENT, ProxyMetaStoreClient.class.getName()); options.putAll(PropertyConverter.convertToMetaProperties(properties)); } - - @Override - public void checkProperties() throws DdlException { - super.checkProperties(); - for (String requiredProperty : REQUIRED_PROPERTIES) { - if (!catalogProperty.getProperties().containsKey(requiredProperty)) { - throw new DdlException("Required property '" + requiredProperty + "' is missing"); - } - } - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalogFactory.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalogFactory.java index 6c0d1c65505d73..2f1f744eb9c35a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalogFactory.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalogFactory.java @@ -19,6 +19,7 @@ import org.apache.doris.common.DdlException; import org.apache.doris.datasource.ExternalCatalog; +import org.apache.doris.datasource.property.constants.HMSProperties; import org.apache.commons.lang3.StringUtils; @@ -39,6 +40,7 @@ public static ExternalCatalog createCatalog(long catalogId, String name, String case PaimonExternalCatalog.PAIMON_FILESYSTEM: return new PaimonFileExternalCatalog(catalogId, name, resource, props, comment); case PaimonExternalCatalog.PAIMON_DLF: + props.put(HMSProperties.HIVE_METASTORE_TYPE, "dlf"); return new PaimonDLFExternalCatalog(catalogId, name, resource, props, comment); default: throw new DdlException("Unknown " + PaimonExternalCatalog.PAIMON_CATALOG_TYPE diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java index a3406dcbb57a99..2977bdf245fa50 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java @@ -165,8 +165,9 @@ protected Type paimonTypeToDorisType(org.apache.paimon.types.DataType type) { @Override public TTableDescriptor toThrift() { List schema = getFullSchema(); - if (PaimonExternalCatalog.PAIMON_HMS.equals(getPaimonCatalogType()) || PaimonExternalCatalog.PAIMON_FILESYSTEM - .equals(getPaimonCatalogType())) { + if (PaimonExternalCatalog.PAIMON_HMS.equals(getPaimonCatalogType()) || + PaimonExternalCatalog.PAIMON_FILESYSTEM.equals(getPaimonCatalogType()) || + PaimonExternalCatalog.PAIMON_DLF.equals(getPaimonCatalogType())) { THiveTable tHiveTable = new THiveTable(dbName, name, new HashMap<>()); TTableDescriptor tTableDescriptor = new TTableDescriptor(getId(), TTableType.HIVE_TABLE, schema.size(), 0, getName(), dbName); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java index 69c786f023f93d..9ca1feb8f8b52e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java @@ -444,10 +444,18 @@ private static void getPropertiesFromDLFProps(Map props, if (Strings.isNullOrEmpty(uid)) { throw new IllegalArgumentException("Required dlf property: " + DLFProperties.UID); } - String endpoint = props.get(DLFProperties.ENDPOINT); - props.put(DataLakeConfig.CATALOG_ENDPOINT, endpoint); - props.put(DataLakeConfig.CATALOG_REGION_ID, props.getOrDefault(DLFProperties.REGION, - S3Properties.getRegionOfEndpoint(endpoint))); + + // region + String region = props.get(DLFProperties.REGION); + if (Strings.isNullOrEmpty(region)) { + throw new IllegalArgumentException("Required dlf property: " + DLFProperties.REGION); + } + props.put(DataLakeConfig.CATALOG_REGION_ID, region); + + // endpoint + props.put(DataLakeConfig.CATALOG_ENDPOINT, + props.getOrDefault(DLFProperties.ENDPOINT, getDlfEndpointByRegion(region)); + props.put(DataLakeConfig.CATALOG_PROXY_MODE, props.getOrDefault(DLFProperties.PROXY_MODE, "DLF_ONLY")); props.put(DataLakeConfig.CATALOG_ACCESS_KEY_ID, credential.getAccessKey()); props.put(DataLakeConfig.CATALOG_ACCESS_KEY_SECRET, credential.getSecretKey()); @@ -508,6 +516,10 @@ private static String getOssEndpoint(String region, boolean publicAccess) { return prefix + region + suffix; } + private static String getDlfEndpointByRegion(String region) { + return "dlf." + region + ".aliyuncs.com"; + } + private static Map convertToGlueProperties(Map props, CloudCredential credential) { // convert doris glue property to glue properties, s3 client property and BE property String metastoreType = props.get(HMSProperties.HIVE_METASTORE_TYPE); From def4c3873274d0ac7a512605fa162fcd8beaab6e Mon Sep 17 00:00:00 2001 From: wuwenchi Date: Tue, 24 Sep 2024 22:39:38 +0800 Subject: [PATCH 3/8] fix --- .../apache/doris/datasource/property/PropertyConverter.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java index 9ca1feb8f8b52e..5d80b2af241d39 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java @@ -88,6 +88,7 @@ public static Map convertToMetaProperties(Map pr } metaProperties = convertToGlueProperties(props, credential); } else if (props.containsKey(DLFProperties.ENDPOINT) + || props.containsKey(DLFProperties.REGION) || props.containsKey(DataLakeConfig.CATALOG_ENDPOINT)) { metaProperties = convertToDLFProperties(props, DLFProperties.getCredential(props)); } else if (props.containsKey(S3Properties.Env.ENDPOINT)) { @@ -454,7 +455,7 @@ private static void getPropertiesFromDLFProps(Map props, // endpoint props.put(DataLakeConfig.CATALOG_ENDPOINT, - props.getOrDefault(DLFProperties.ENDPOINT, getDlfEndpointByRegion(region)); + props.getOrDefault(DLFProperties.ENDPOINT, getDlfEndpointByRegion(region))); props.put(DataLakeConfig.CATALOG_PROXY_MODE, props.getOrDefault(DLFProperties.PROXY_MODE, "DLF_ONLY")); props.put(DataLakeConfig.CATALOG_ACCESS_KEY_ID, credential.getAccessKey()); From 266566cfad11d472e8ed00657544a5f2bf92e9a3 Mon Sep 17 00:00:00 2001 From: wuwenchi Date: Wed, 25 Sep 2024 11:20:26 +0800 Subject: [PATCH 4/8] fix --- .../doris/datasource/paimon/PaimonDLFExternalCatalog.java | 7 ++++++- .../datasource/paimon/PaimonExternalCatalogFactory.java | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonDLFExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonDLFExternalCatalog.java index e514779b800299..bc1fd2779bd678 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonDLFExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonDLFExternalCatalog.java @@ -51,6 +51,11 @@ protected void initLocalObjectsImpl() { protected void setPaimonCatalogOptions(Map properties, Map options) { options.put(PaimonProperties.PAIMON_CATALOG_TYPE, PaimonProperties.PAIMON_HMS_CATALOG); options.put(PaimonProperties.PAIMON_METASTORE_CLIENT, ProxyMetaStoreClient.class.getName()); - options.putAll(PropertyConverter.convertToMetaProperties(properties)); + options.put(PaimonProperties.PAIMON_OSS_ENDPOINT, + properties.get(PaimonProperties.PAIMON_OSS_ENDPOINT)); + options.put(PaimonProperties.PAIMON_OSS_ACCESS_KEY, + properties.get(PaimonProperties.PAIMON_OSS_ACCESS_KEY)); + options.put(PaimonProperties.PAIMON_OSS_SECRET_KEY, + properties.get(PaimonProperties.PAIMON_OSS_SECRET_KEY)); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalogFactory.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalogFactory.java index 2f1f744eb9c35a..53e790d8c9e5ff 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalogFactory.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalogFactory.java @@ -40,7 +40,7 @@ public static ExternalCatalog createCatalog(long catalogId, String name, String case PaimonExternalCatalog.PAIMON_FILESYSTEM: return new PaimonFileExternalCatalog(catalogId, name, resource, props, comment); case PaimonExternalCatalog.PAIMON_DLF: - props.put(HMSProperties.HIVE_METASTORE_TYPE, "dlf"); + props.put(HMSProperties.HIVE_METASTORE_TYPE, HMSProperties.DLF_TYPE); return new PaimonDLFExternalCatalog(catalogId, name, resource, props, comment); default: throw new DdlException("Unknown " + PaimonExternalCatalog.PAIMON_CATALOG_TYPE From aefebb453ea14343bfa640b327ba9eb508d59ba4 Mon Sep 17 00:00:00 2001 From: wuwenchi Date: Wed, 25 Sep 2024 13:44:18 +0800 Subject: [PATCH 5/8] test --- .../paimon/test_paimon_dlf_catalog.out | 9 +++ .../paimon/test_paimon_dlf_catalog.groovy | 59 +++++++++++++++++++ 2 files changed, 68 insertions(+) create mode 100644 regression-test/data/external_table_p2/paimon/test_paimon_dlf_catalog.out create mode 100644 regression-test/suites/external_table_p2/paimon/test_paimon_dlf_catalog.groovy diff --git a/regression-test/data/external_table_p2/paimon/test_paimon_dlf_catalog.out b/regression-test/data/external_table_p2/paimon/test_paimon_dlf_catalog.out new file mode 100644 index 00000000000000..6e935c875440e9 --- /dev/null +++ b/regression-test/data/external_table_p2/paimon/test_paimon_dlf_catalog.out @@ -0,0 +1,9 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !c1 -- +1 a +2 b + +-- !c2 -- +1 a +2 b + diff --git a/regression-test/suites/external_table_p2/paimon/test_paimon_dlf_catalog.groovy b/regression-test/suites/external_table_p2/paimon/test_paimon_dlf_catalog.groovy new file mode 100644 index 00000000000000..9a5bcf0b748306 --- /dev/null +++ b/regression-test/suites/external_table_p2/paimon/test_paimon_dlf_catalog.groovy @@ -0,0 +1,59 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_paimon_dlf_catalog", "p2,external,paimon,external_remote,external_remote_paimon") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + return + } + + try { + String catalog = "test_paimon_dlf_catalog" + String uid = context.config.otherConfigs.get("dlf_uid") + String region = context.config.otherConfigs.get("dlf_region") + String catalog_id = context.config.otherConfigs.get("dlf_catalog_id") + String access_key = context.config.otherConfigs.get("dlf_access_key") + String secret_key = context.config.otherConfigs.get("dlf_secret_key") + + + sql """drop catalog if exists ${catalog};""" + sql """ + create catalog if not exists ${catalog} properties ( + "type" = "paimon", + "paimon.catalog.type" = "dlf", + "warehouse" = "oss://selectdb-qa-datalake-test/p2_regression_case", + "dlf.proxy.mode" = "DLF_ONLY", + "dlf.uid" = "${uid}", + "dlf.region" = "${region}", + "dlf.catalog.id" = "${catalog_id}", + "dlf.access_key" = "${access_key}", + "dlf.secret_key" = "${secret_key}" + ); + """ + + sql """ use ${catalog}.regression_paimon """ + + sql """set force_jni_scanner=false""" + qt_c1 """ select * from tb_simple order by id """ + sql """set force_jni_scanner=true""" + qt_c2 """ select * from tb_simple order by id """ + + } finally { + sql """set force_jni_scanner=false""" + } +} + From 3aa5764103ae1900cd401b91f44f67bf8275dab1 Mon Sep 17 00:00:00 2001 From: wuwenchi Date: Thu, 26 Sep 2024 10:58:26 +0800 Subject: [PATCH 6/8] fix --- .../datasource/paimon/PaimonDLFExternalCatalog.java | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonDLFExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonDLFExternalCatalog.java index bc1fd2779bd678..00363c1f799ef3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonDLFExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonDLFExternalCatalog.java @@ -17,23 +17,16 @@ package org.apache.doris.datasource.paimon; -import org.apache.doris.datasource.property.PropertyConverter; -import org.apache.doris.datasource.property.constants.HMSProperties; import org.apache.doris.datasource.property.constants.PaimonProperties; import com.aliyun.datalake.metastore.hive2.ProxyMetaStoreClient; -import com.google.common.collect.ImmutableList; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import java.util.List; import java.util.Map; public class PaimonDLFExternalCatalog extends PaimonExternalCatalog { private static final Logger LOG = LogManager.getLogger(PaimonDLFExternalCatalog.class); - private static final List REQUIRED_PROPERTIES = ImmutableList.of( - HMSProperties.HIVE_METASTORE_URIS - ); public PaimonDLFExternalCatalog(long catalogId, String name, String resource, Map props, String comment) { @@ -52,10 +45,10 @@ protected void setPaimonCatalogOptions(Map properties, Map Date: Fri, 27 Sep 2024 09:19:13 +0800 Subject: [PATCH 7/8] fix --- .../apache/doris/datasource/paimon/PaimonExternalTable.java | 6 +++--- .../main/java/org/apache/doris/persist/gson/GsonUtils.java | 4 +++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java index 2977bdf245fa50..4b364ef45ca321 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java @@ -165,9 +165,9 @@ protected Type paimonTypeToDorisType(org.apache.paimon.types.DataType type) { @Override public TTableDescriptor toThrift() { List schema = getFullSchema(); - if (PaimonExternalCatalog.PAIMON_HMS.equals(getPaimonCatalogType()) || - PaimonExternalCatalog.PAIMON_FILESYSTEM.equals(getPaimonCatalogType()) || - PaimonExternalCatalog.PAIMON_DLF.equals(getPaimonCatalogType())) { + if (PaimonExternalCatalog.PAIMON_HMS.equals(getPaimonCatalogType()) + || PaimonExternalCatalog.PAIMON_FILESYSTEM.equals(getPaimonCatalogType()) + || PaimonExternalCatalog.PAIMON_DLF.equals(getPaimonCatalogType())) { THiveTable tHiveTable = new THiveTable(dbName, name, new HashMap<>()); TTableDescriptor tTableDescriptor = new TTableDescriptor(getId(), TTableType.HIVE_TABLE, schema.size(), 0, getName(), dbName); diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java b/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java index f29dbaceab2313..dd8ad13ae6f361 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java @@ -157,6 +157,7 @@ import org.apache.doris.datasource.maxcompute.MaxComputeExternalCatalog; import org.apache.doris.datasource.maxcompute.MaxComputeExternalDatabase; import org.apache.doris.datasource.maxcompute.MaxComputeExternalTable; +import org.apache.doris.datasource.paimon.PaimonDLFExternalCatalog; import org.apache.doris.datasource.paimon.PaimonExternalCatalog; import org.apache.doris.datasource.paimon.PaimonExternalDatabase; import org.apache.doris.datasource.paimon.PaimonExternalTable; @@ -408,7 +409,8 @@ public class GsonUtils { .registerSubtype( TrinoConnectorExternalCatalog.class, TrinoConnectorExternalCatalog.class.getSimpleName()) .registerSubtype(LakeSoulExternalCatalog.class, LakeSoulExternalCatalog.class.getSimpleName()) - .registerSubtype(TestExternalCatalog.class, TestExternalCatalog.class.getSimpleName()); + .registerSubtype(TestExternalCatalog.class, TestExternalCatalog.class.getSimpleName()) + .registerSubtype(PaimonDLFExternalCatalog.class, PaimonDLFExternalCatalog.class.getSimpleName()); if (Config.isNotCloudMode()) { dsTypeAdapterFactory .registerSubtype(InternalCatalog.class, InternalCatalog.class.getSimpleName()); From 0c8f6df505b862029ca7d340d9c3c2b55a9f2dda Mon Sep 17 00:00:00 2001 From: wuwenchi Date: Fri, 27 Sep 2024 16:59:02 +0800 Subject: [PATCH 8/8] fix --- .../org/apache/doris/datasource/property/PropertyConverter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java index 5d80b2af241d39..7303f4e08c9fb9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java @@ -518,7 +518,7 @@ private static String getOssEndpoint(String region, boolean publicAccess) { } private static String getDlfEndpointByRegion(String region) { - return "dlf." + region + ".aliyuncs.com"; + return "dlf-vpc." + region + ".aliyuncs.com"; } private static Map convertToGlueProperties(Map props, CloudCredential credential) {