From a65e0fbc131bff3976f3d6837b114aff7619d5e1 Mon Sep 17 00:00:00 2001
From: Tiewei Fang <43782773+BePPPower@users.noreply.github.com>
Date: Sat, 2 Nov 2024 08:55:53 +0800
Subject: [PATCH] [Fix](PaimonCatalog) fix the problem that paimon catalog can
 not access to OSS-HDFS (#42585)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Fix the problem that paimon catalog can not access to OSS-HDFS.

There are 2 problems in paimon catalog:
1. Doris FE can not list paimon tables.
This is because we pass these three properties -- `fs.oss.endpoint /
fs.oss.accessKeyId / fs.oss.accessKeySecret` -- to the PaimonCatalog.
When PaimonCatalog get these three properties, it will use `OSSLoader`
rather than `HadoopFileIOLoader`.

2. Doris BE does not use libhdfs to access OSS-HDFS
This is because the `tmpLocation` in `LocationPath` does not contain
`oss-dls.aliyuncs`. We should use `endpoint` to judge if user wants to
access OSS-HDFS

What's more, if you want to access OSS-HDFS with PaimonCatalog, you
should:
1. Download Jindo SDK:
https://github.com/aliyun/alibabacloud-jindodata/blob/latest/docs/user/zh/jindosdk/jindosdk_download.md
2. copy `jindo-core.jar、jindo-sdk.jar` to `${DORIS_HOME}/fe/lib` and
`${DORIS_HOME}/be/lib/java_extensions/preload-extensions` directory.
---
 .../doris/common/util/LocationPath.java       | 16 ++++++++++++++--
 .../paimon/PaimonFileExternalCatalog.java     | 19 +++++++++++++------
 .../doris/common/util/LocationPathTest.java   |  4 +++-
 3 files changed, 30 insertions(+), 9 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/LocationPath.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/LocationPath.java
index 267e20a1f959bc..4604e4deabb2b7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/LocationPath.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/LocationPath.java
@@ -131,7 +131,19 @@ private LocationPath(String originLocation, Map<String, String> props, boolean c
                 tmpLocation = convertPath ? convertToS3(tmpLocation) : tmpLocation;
                 break;
             case FeConstants.FS_PREFIX_OSS:
-                if (isHdfsOnOssEndpoint(tmpLocation)) {
+                String endpoint = "";
+                if (props.containsKey(OssProperties.ENDPOINT)) {
+                    endpoint = props.get(OssProperties.ENDPOINT);
+                    if (endpoint.startsWith(OssProperties.OSS_PREFIX)) {
+                        // may use oss.oss-cn-beijing.aliyuncs.com
+                        endpoint = endpoint.replace(OssProperties.OSS_PREFIX, "");
+                    }
+                } else if (props.containsKey(S3Properties.ENDPOINT)) {
+                    endpoint = props.get(S3Properties.ENDPOINT);
+                } else if (props.containsKey(S3Properties.Env.ENDPOINT)) {
+                    endpoint = props.get(S3Properties.Env.ENDPOINT);
+                }
+                if (isHdfsOnOssEndpoint(endpoint)) {
                     this.scheme = Scheme.OSS_HDFS;
                 } else {
                     if (useS3EndPoint(props)) {
@@ -398,7 +410,7 @@ private static String normalizedLakefsPath(String location) {
         }
     }
 
-    private FileSystemType getFileSystemType() {
+    public FileSystemType getFileSystemType() {
         FileSystemType fsType;
         switch (scheme) {
             case S3:
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonFileExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonFileExternalCatalog.java
index 9b956a551d5b93..e74f3deeaf501e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonFileExternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonFileExternalCatalog.java
@@ -17,9 +17,11 @@
 
 package org.apache.doris.datasource.paimon;
 
+import org.apache.doris.common.util.LocationPath;
 import org.apache.doris.datasource.property.PropertyConverter;
 import org.apache.doris.datasource.property.constants.CosProperties;
 import org.apache.doris.datasource.property.constants.ObsProperties;
+import org.apache.doris.datasource.property.constants.OssProperties;
 import org.apache.doris.datasource.property.constants.PaimonProperties;
 
 import org.apache.logging.log4j.LogManager;
@@ -53,12 +55,17 @@ protected void setPaimonCatalogOptions(Map<String, String> properties, Map<Strin
             options.put(PaimonProperties.PAIMON_S3_SECRET_KEY,
                     properties.get(PaimonProperties.PAIMON_S3_SECRET_KEY));
         } else if (properties.containsKey(PaimonProperties.PAIMON_OSS_ENDPOINT)) {
-            options.put(PaimonProperties.PAIMON_OSS_ENDPOINT,
-                    properties.get(PaimonProperties.PAIMON_OSS_ENDPOINT));
-            options.put(PaimonProperties.PAIMON_OSS_ACCESS_KEY,
-                    properties.get(PaimonProperties.PAIMON_OSS_ACCESS_KEY));
-            options.put(PaimonProperties.PAIMON_OSS_SECRET_KEY,
-                    properties.get(PaimonProperties.PAIMON_OSS_SECRET_KEY));
+            boolean hdfsEnabled = Boolean.parseBoolean(properties.getOrDefault(
+                    OssProperties.OSS_HDFS_ENABLED, "false"));
+            if (!LocationPath.isHdfsOnOssEndpoint(properties.get(PaimonProperties.PAIMON_OSS_ENDPOINT))
+                    && !hdfsEnabled) {
+                options.put(PaimonProperties.PAIMON_OSS_ENDPOINT,
+                        properties.get(PaimonProperties.PAIMON_OSS_ENDPOINT));
+                options.put(PaimonProperties.PAIMON_OSS_ACCESS_KEY,
+                        properties.get(PaimonProperties.PAIMON_OSS_ACCESS_KEY));
+                options.put(PaimonProperties.PAIMON_OSS_SECRET_KEY,
+                        properties.get(PaimonProperties.PAIMON_OSS_SECRET_KEY));
+            }
         } else if (properties.containsKey(CosProperties.ENDPOINT)) {
             options.put(PaimonProperties.PAIMON_S3_ENDPOINT,
                     properties.get(CosProperties.ENDPOINT));
diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/util/LocationPathTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/util/LocationPathTest.java
index 23f052d61312b5..9d1edadd919b59 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/common/util/LocationPathTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/common/util/LocationPathTest.java
@@ -19,6 +19,7 @@
 
 import org.apache.doris.catalog.HdfsResource;
 import org.apache.doris.common.util.LocationPath.Scheme;
+import org.apache.doris.datasource.property.constants.OssProperties;
 import org.apache.doris.fs.FileSystemType;
 
 import org.junit.jupiter.api.Assertions;
@@ -119,13 +120,14 @@ public void testOSSLocationConvert() {
         Assertions.assertTrue(beLocation.startsWith("s3://"));
         Assertions.assertEquals(LocationPath.getFSIdentity(beLocation, null).first, FileSystemType.S3);
 
+        rangeProps.put(OssProperties.ENDPOINT, "oss-dls.aliyuncs.com");
         locationPath = new LocationPath("oss://test.oss-dls.aliyuncs.com/path", rangeProps);
         // FE
         Assertions.assertTrue(locationPath.get().startsWith("oss://test.oss-dls.aliyuncs"));
         // BE
         beLocation = locationPath.toStorageLocation().toString();
         Assertions.assertTrue(beLocation.startsWith("oss://test.oss-dls.aliyuncs"));
-        Assertions.assertEquals(LocationPath.getFSIdentity(beLocation, null).first, FileSystemType.DFS);
+        Assertions.assertEquals(locationPath.getFileSystemType(), FileSystemType.DFS);
 
     }