From 8095494402c8d57cab5656c3c613459bb2db17d8 Mon Sep 17 00:00:00 2001 From: pengxianyu Date: Fri, 13 May 2022 20:09:02 +0800 Subject: [PATCH 1/3] add StoragePolicyResource --- .../org/apache/doris/catalog/Resource.java | 6 +- .../org/apache/doris/catalog/ResourceMgr.java | 3 +- .../doris/catalog/StoragePolicyResource.java | 128 ++++++++++++++++++ 3 files changed, 135 insertions(+), 2 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/catalog/StoragePolicyResource.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Resource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Resource.java index d2c2e7f48f5049..726139d4c4efa2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Resource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Resource.java @@ -44,7 +44,8 @@ public enum ResourceType { UNKNOWN, SPARK, ODBC_CATALOG, - S3; + S3, + STORAGE_POLICY; public static ResourceType fromString(String resourceType) { for (ResourceType type : ResourceType.values()) { @@ -95,6 +96,9 @@ private static Resource getResourceInstance(ResourceType type, String name) thro case S3: resource = new S3Resource(name); break; + case STORAGE_POLICY: + resource = new StoragePolicyResource(name); + break; default: throw new DdlException("Unknown resource type: " + type); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/ResourceMgr.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/ResourceMgr.java index d7d57272ab21b7..19d39e4dc50ec6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/ResourceMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/ResourceMgr.java @@ -72,7 +72,8 @@ public ResourceMgr() { public void createResource(CreateResourceStmt stmt) throws DdlException { if (stmt.getResourceType() != ResourceType.SPARK && stmt.getResourceType() != ResourceType.ODBC_CATALOG - && stmt.getResourceType() != ResourceType.S3) { + && stmt.getResourceType() != ResourceType.S3 + && stmt.getResourceType() != ResourceType.STORAGE_POLICY) { throw new DdlException("Only support SPARK, ODBC_CATALOG and REMOTE_STORAGE resource."); } Resource resource = Resource.fromStmt(stmt); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/StoragePolicyResource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/StoragePolicyResource.java new file mode 100644 index 00000000000000..272a61728439fe --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/StoragePolicyResource.java @@ -0,0 +1,128 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.catalog; + +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.DdlException; +import org.apache.doris.common.proc.BaseProcResult; + +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.gson.annotations.SerializedName; + +import java.util.Map; + +/** + * Policy resource for olap table + * + * Syntax: + * CREATE RESOURCE "storage_policy_name" + * PROPERTIES( + * "type"="storage_policy", + * "cooldown_datetime" = "2022-06-01", // time when data is transfter to medium + * "cooldown_ttl" = "1h"， // data is transfter to medium after 1 hour + * "s3_resource" = "my_s3" // point to a s3 resource + * ); + */ +public class StoragePolicyResource extends Resource { + // required + private static final String STORAGE_RESOURCE = "storage_resource"; + // optional + private static final String COOLDOWN_DATETIME = "cooldown_datetime"; + private static final String COOLDOWN_TTL = "cooldown_ttl"; + + private static final String DEFAULT_COOLDOWN_DATETIME = "9999-01-01 00:00:00"; + private static final String DEFAULT_COOLDOWN_TTL = "1h"; + + @SerializedName(value = "properties") + private Map properties; + + public StoragePolicyResource(String name) { + this(name, Maps.newHashMap()); + } + + public StoragePolicyResource(String name, Map properties) { + super(name, ResourceType.STORAGE_POLICY); + this.properties = properties; + } + + public String getProperty(String propertyKey) { + return properties.get(propertyKey); + } + + @Override + protected void setProperties(Map properties) throws DdlException { + Preconditions.checkState(properties != null); + this.properties = properties; + // check properties + // required + checkRequiredProperty(STORAGE_RESOURCE); + // optional + checkOptionalProperty(COOLDOWN_DATETIME, DEFAULT_COOLDOWN_DATETIME); + checkOptionalProperty(COOLDOWN_TTL, DEFAULT_COOLDOWN_TTL); + } + + private void checkRequiredProperty(String propertyKey) throws DdlException { + String value = properties.get(propertyKey); + + if (Strings.isNullOrEmpty(value)) { + throw new DdlException("Missing [" + propertyKey + "] in properties."); + } + } + + private void checkOptionalProperty(String propertyKey, String defaultValue) { + this.properties.putIfAbsent(propertyKey, defaultValue); + } + + @Override + public void modifyProperties(Map properties) throws DdlException { + // modify properties + replaceIfEffectiveValue(this.properties, STORAGE_RESOURCE, properties.get(STORAGE_RESOURCE)); + replaceIfEffectiveValue(this.properties, COOLDOWN_DATETIME, properties.get(COOLDOWN_DATETIME)); + replaceIfEffectiveValue(this.properties, COOLDOWN_TTL, properties.get(COOLDOWN_TTL)); + } + + @Override + public void checkProperties(Map properties) throws AnalysisException { + // check properties + Map copiedProperties = Maps.newHashMap(properties); + copiedProperties.remove(STORAGE_RESOURCE); + copiedProperties.remove(COOLDOWN_DATETIME); + copiedProperties.remove(COOLDOWN_TTL); + + if (!copiedProperties.isEmpty()) { + throw new AnalysisException("Unknown S3 resource properties: " + copiedProperties); + } + } + + @Override + public Map getCopiedProperties() { + Map copiedProperties = Maps.newHashMap(properties); + return copiedProperties; + } + + @Override + protected void getProcNodeData(BaseProcResult result) { + String lowerCaseType = type.name().toLowerCase(); + for (Map.Entry entry : properties.entrySet()) { + result.addRow(Lists.newArrayList(name, lowerCaseType, entry.getKey(), entry.getValue())); + } + } +} From 09ad194a1527414718695ecf01edbbd6fca2c5fe Mon Sep 17 00:00:00 2001 From: pengxianyu Date: Mon, 16 May 2022 11:59:07 +0800 Subject: [PATCH 2/3] add StoragePolicyResource --- .../doris/catalog/StoragePolicyResource.java | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/StoragePolicyResource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/StoragePolicyResource.java index 272a61728439fe..cd0a40d9c44f37 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/StoragePolicyResource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/StoragePolicyResource.java @@ -77,6 +77,11 @@ protected void setProperties(Map properties) throws DdlException // optional checkOptionalProperty(COOLDOWN_DATETIME, DEFAULT_COOLDOWN_DATETIME); checkOptionalProperty(COOLDOWN_TTL, DEFAULT_COOLDOWN_TTL); + if (properties.containsKey(COOLDOWN_DATETIME) && properties.containsKey(COOLDOWN_TTL) && + !properties.get(COOLDOWN_DATETIME).isEmpty() && !properties.get(COOLDOWN_TTL).isEmpty()) { + throw new DdlException("Only one of [" + COOLDOWN_DATETIME + "] and [" + COOLDOWN_TTL + + "] can be specified in properties."); + } } private void checkRequiredProperty(String propertyKey) throws DdlException { @@ -93,6 +98,11 @@ private void checkOptionalProperty(String propertyKey, String defaultValue) { @Override public void modifyProperties(Map properties) throws DdlException { + if (properties.containsKey(COOLDOWN_DATETIME) && properties.containsKey(COOLDOWN_TTL) && + !properties.get(COOLDOWN_DATETIME).isEmpty() && !properties.get(COOLDOWN_TTL).isEmpty()) { + throw new DdlException("Only one of [" + COOLDOWN_DATETIME + "] and [" + COOLDOWN_TTL + + "] can be specified in properties."); + } // modify properties replaceIfEffectiveValue(this.properties, STORAGE_RESOURCE, properties.get(STORAGE_RESOURCE)); replaceIfEffectiveValue(this.properties, COOLDOWN_DATETIME, properties.get(COOLDOWN_DATETIME)); @@ -108,14 +118,13 @@ public void checkProperties(Map properties) throws AnalysisExcep copiedProperties.remove(COOLDOWN_TTL); if (!copiedProperties.isEmpty()) { - throw new AnalysisException("Unknown S3 resource properties: " + copiedProperties); + throw new AnalysisException("Unknown policy resource properties: " + copiedProperties); } } @Override public Map getCopiedProperties() { - Map copiedProperties = Maps.newHashMap(properties); - return copiedProperties; + return Maps.newHashMap(properties); } @Override From 403a6640c7cec907f57e0d5b791fbfd0e852ba4d Mon Sep 17 00:00:00 2001 From: pengxianyu Date: Mon, 16 May 2022 13:02:34 +0800 Subject: [PATCH 3/3] add StoragePolicyResource --- .../doris/catalog/StoragePolicyResource.java | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/StoragePolicyResource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/StoragePolicyResource.java index cd0a40d9c44f37..285be637f727c1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/StoragePolicyResource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/StoragePolicyResource.java @@ -30,8 +30,7 @@ import java.util.Map; /** - * Policy resource for olap table - * + * Policy resource for olap table. * Syntax: * CREATE RESOURCE "storage_policy_name" * PROPERTIES( @@ -77,10 +76,10 @@ protected void setProperties(Map properties) throws DdlException // optional checkOptionalProperty(COOLDOWN_DATETIME, DEFAULT_COOLDOWN_DATETIME); checkOptionalProperty(COOLDOWN_TTL, DEFAULT_COOLDOWN_TTL); - if (properties.containsKey(COOLDOWN_DATETIME) && properties.containsKey(COOLDOWN_TTL) && - !properties.get(COOLDOWN_DATETIME).isEmpty() && !properties.get(COOLDOWN_TTL).isEmpty()) { - throw new DdlException("Only one of [" + COOLDOWN_DATETIME + "] and [" + COOLDOWN_TTL + - "] can be specified in properties."); + if (properties.containsKey(COOLDOWN_DATETIME) && properties.containsKey(COOLDOWN_TTL) + && !properties.get(COOLDOWN_DATETIME).isEmpty() && !properties.get(COOLDOWN_TTL).isEmpty()) { + throw new DdlException("Only one of [" + COOLDOWN_DATETIME + "] and [" + COOLDOWN_TTL + + "] can be specified in properties."); } } @@ -98,10 +97,10 @@ private void checkOptionalProperty(String propertyKey, String defaultValue) { @Override public void modifyProperties(Map properties) throws DdlException { - if (properties.containsKey(COOLDOWN_DATETIME) && properties.containsKey(COOLDOWN_TTL) && - !properties.get(COOLDOWN_DATETIME).isEmpty() && !properties.get(COOLDOWN_TTL).isEmpty()) { - throw new DdlException("Only one of [" + COOLDOWN_DATETIME + "] and [" + COOLDOWN_TTL + - "] can be specified in properties."); + if (properties.containsKey(COOLDOWN_DATETIME) && properties.containsKey(COOLDOWN_TTL) + && !properties.get(COOLDOWN_DATETIME).isEmpty() && !properties.get(COOLDOWN_TTL).isEmpty()) { + throw new DdlException("Only one of [" + COOLDOWN_DATETIME + "] and [" + COOLDOWN_TTL + + "] can be specified in properties."); } // modify properties replaceIfEffectiveValue(this.properties, STORAGE_RESOURCE, properties.get(STORAGE_RESOURCE));