From 95124e220f4aedc8cffa7c16f375302bc643b4df Mon Sep 17 00:00:00 2001 From: morningman Date: Wed, 6 Dec 2023 20:23:35 +0800 Subject: [PATCH 1/2] [opt](resource-tag) root and admin user can use any resource tag by default --- docs/en/docs/admin-manual/multi-tenant.md | 4 ++-- docs/zh-CN/docs/admin-manual/multi-tenant.md | 4 ++-- .../doris/mysql/privilege/CommonUserProperties.java | 4 ---- .../apache/doris/mysql/privilege/UserPropertyMgr.java | 11 ++++++++++- .../org/apache/doris/catalog/UserPropertyTest.java | 3 +-- 5 files changed, 15 insertions(+), 11 deletions(-) diff --git a/docs/en/docs/admin-manual/multi-tenant.md b/docs/en/docs/admin-manual/multi-tenant.md index b5659a7897ebdc..d4f207ceae7908 100644 --- a/docs/en/docs/admin-manual/multi-tenant.md +++ b/docs/en/docs/admin-manual/multi-tenant.md @@ -138,7 +138,7 @@ Node resource division refers to setting tags for BE nodes in a Doris cluster, a After the setting is complete, when user1 initiates a query on the UserTable table, it will only access the data copy on the nodes in the `group_a` resource group, and the query will only use the node computing resources in the `group_a` resource group. The query of user3 can use copies and computing resources in any resource group. - > Note: By default, the user's `resource_tags.location` attribute is empty. In versions prior to 2.0.2 (inclusive), by default, users are not restricted by tags and can use any resource group. After version 2.0.3, users can only use the `default` resource group by default. + > Note: By default, the user's `resource_tags.location` attribute is empty. In versions prior to 2.0.2 (inclusive), by default, users are not restricted by tags and can use any resource group. After version 2.0.3, normal users can only use the `default` resource group by default. Root and Admin user can use any resource group. In this way, we have achieved physical resource isolation for different user queries by dividing nodes and restricting user resource usage. Furthermore, we can create different users for different business departments and restrict each user from using different resource groups. In order to avoid the use of resource interference between different business parts. For example, there is a business table in the cluster that needs to be shared by all 9 business departments, but it is hoped that resource preemption between different departments can be avoided as much as possible. Then we can create 3 copies of this table and store them in 3 resource groups. Next, we create 9 users for 9 business departments, and limit the use of one resource group for every 3 users. In this way, the degree of competition for resources is reduced from 9 to 3. @@ -268,4 +268,4 @@ Through the above 4 steps, we can smoothly use the resource division function af The table creation statements for table2, table3, and table4 do not need to specify `replication_allocation` again. - Note: Changing the replica distribution policy of the database will not affect existing tables. \ No newline at end of file + Note: Changing the replica distribution policy of the database will not affect existing tables. diff --git a/docs/zh-CN/docs/admin-manual/multi-tenant.md b/docs/zh-CN/docs/admin-manual/multi-tenant.md index ec24a513825aa0..7e346dbd8e536f 100644 --- a/docs/zh-CN/docs/admin-manual/multi-tenant.md +++ b/docs/zh-CN/docs/admin-manual/multi-tenant.md @@ -138,7 +138,7 @@ FE 不参与用户数据的处理计算等工作，因此是一个资源消耗 设置完成后，user1 在发起对 UserTable 表的查询时，只会访问 `group_a` 资源组内节点上的数据副本，并且查询仅会使用 `group_a` 资源组内的节点计算资源。而 user3 的查询可以使用任意资源组内的副本和计算资源。 - > 注：默认情况下，用户的 `resource_tags.location` 属性为空，在2.0.2（含）之前的版本中，默认情况下，用户不受 tag 的限制，可以使用任意资源组。在 2.0.3 版本之后，默认情况下，用户只能使用 `default` 资源组。 + > 注：默认情况下，用户的 `resource_tags.location` 属性为空，在2.0.2（含）之前的版本中，默认情况下，用户不受 tag 的限制，可以使用任意资源组。在 2.0.3 版本之后，默认情况下，普通用户只能使用 `default` 资源组。root 和 admin 用户可以使用任意资源组。 这样，我们通过对节点的划分，以及对用户的资源使用限制，实现了不同用户查询上的物理资源隔离。更进一步，我们可以给不同的业务部门创建不同的用户，并限制每个用户使用不同的资源组。以避免不同业务部分之间使用资源干扰。比如集群内有一张业务表需要共享给所有9个业务部门使用，但是希望能够尽量避免不同部门之间的资源抢占。则我们可以为这张表创建3个副本，分别存储在3个资源组中。接下来，我们为9个业务部门创建9个用户，每3个用户限制使用一个资源组。这样，资源的竞争程度就由9降低到了3。 @@ -266,4 +266,4 @@ FE 不参与用户数据的处理计算等工作，因此是一个资源消耗 table2，table3,table4的建表语句无需再指定`replication_allocation`。 - 注意事项：更改database的副本分布策略不会对已有的table产生影响。 \ No newline at end of file + 注意事项：更改database的副本分布策略不会对已有的table产生影响。 diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/CommonUserProperties.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/CommonUserProperties.java index 8fa28485ae5b7f..cf36639c0a931f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/CommonUserProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/CommonUserProperties.java @@ -110,10 +110,6 @@ public void setResourceTags(Set resourceTags) { } public Set getResourceTags() { - // If resource tags in user properties is empty, use default backend tag. - if (resourceTags.isEmpty()) { - return Sets.newHashSet(Tag.DEFAULT_BACKEND_TAG); - } return resourceTags; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserPropertyMgr.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserPropertyMgr.java index 7232417a2e8ac4..46f900f655ae95 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserPropertyMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/UserPropertyMgr.java @@ -29,6 +29,7 @@ import org.apache.doris.resource.Tag; import com.google.common.collect.Maps; +import com.google.common.collect.Sets; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -123,7 +124,15 @@ public Set getResourceTags(String qualifiedUser) { if (existProperty == null) { return UserProperty.INVALID_RESOURCE_TAGS; } - return existProperty.getCopiedResourceTags(); + Set tags = existProperty.getCopiedResourceTags(); + // only root and admin can return empty tag. + // empty tag means user can access all backends. + // for normal user, if tag is empty, use default tag. + if (tags.isEmpty() && !(qualifiedUser.equalsIgnoreCase(Auth.ROOT_USER) + || qualifiedUser.equalsIgnoreCase(Auth.ADMIN_USER))) { + tags = Sets.newHashSet(Tag.DEFAULT_BACKEND_TAG); + } + return tags; } public Pair getLoadClusterInfo(String qualifiedUser, String cluster) throws DdlException { diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/UserPropertyTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/UserPropertyTest.java index 3142d75691adde..2a3252caadc0ff 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/UserPropertyTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/UserPropertyTest.java @@ -24,7 +24,6 @@ import org.apache.doris.common.UserException; import org.apache.doris.load.DppConfig; import org.apache.doris.mysql.privilege.UserProperty; -import org.apache.doris.resource.Tag; import com.google.common.collect.Lists; import com.google.common.collect.Sets; @@ -118,7 +117,7 @@ public void testUpdate() throws UserException { Assert.assertEquals(new String[]{"rule1", "rule2"}, userProperty.getSqlBlockRules()); Assert.assertEquals(2, userProperty.getCpuResourceLimit()); Assert.assertEquals(500, userProperty.getQueryTimeout()); - Assert.assertEquals(Sets.newHashSet(Tag.DEFAULT_BACKEND_TAG), userProperty.getCopiedResourceTags()); + Assert.assertEquals(Sets.newHashSet(), userProperty.getCopiedResourceTags()); // fetch property List> rows = userProperty.fetchProperty(); From 726e4cb554a0046b49b9a062a2a7528c728d3657 Mon Sep 17 00:00:00 2001 From: morningman Date: Wed, 6 Dec 2023 21:38:44 +0800 Subject: [PATCH 2/2] 1 --- .../java/org/apache/doris/planner/ResourceTagQueryTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/ResourceTagQueryTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/ResourceTagQueryTest.java index 9bbc7ab55eb353..8bcd302a181688 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/ResourceTagQueryTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/ResourceTagQueryTest.java @@ -196,7 +196,7 @@ public void test() throws Exception { OlapTable tbl = (OlapTable) db.getTableNullable("tbl1"); Set userTags = Env.getCurrentEnv().getAuth().getResourceTags(Auth.ROOT_USER); - Assert.assertEquals(1, userTags.size()); + Assert.assertEquals(0, userTags.size()); // set default tag for root String setPropStr = "set property for 'root' 'resource_tags.location' = 'default';";