From fd03227176bb3ffd17e4986ad858aecb37e8ff2f Mon Sep 17 00:00:00 2001 From: lihangyu <15605149486@163.com> Date: Wed, 17 Apr 2024 23:03:11 +0800 Subject: [PATCH 1/3] [Fix](Variant Type) forbit distribution info contains variant columns (#33707) --- .../org/apache/doris/datasource/InternalCatalog.java | 6 ++++++ regression-test/suites/variant_p0/load.groovy | 12 ++++++++++++ 2 files changed, 18 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index b94c8b7d076588..c22cbf38c6fdf2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -2110,6 +2110,12 @@ private void createOlapTable(Database db, CreateTableStmt stmt) throws UserExcep Preconditions.checkNotNull(distributionDesc); DistributionInfo defaultDistributionInfo = distributionDesc.toDistributionInfo(baseSchema); + if (defaultDistributionInfo instanceof HashDistributionInfo + && ((HashDistributionInfo) defaultDistributionInfo).getDistributionColumns() + .stream().anyMatch(column -> column.getType().isVariantType())) { + throw new DdlException("Hash distribution info should not contain variant columns"); + } + // calc short key column count short shortKeyColumnCount = Env.calcShortKeyColumnCount(baseSchema, stmt.getProperties(), isKeysRequired); if (LOG.isDebugEnabled()) { diff --git a/regression-test/suites/variant_p0/load.groovy b/regression-test/suites/variant_p0/load.groovy index 3f3609ec877b81..b85dd686f84246 100644 --- a/regression-test/suites/variant_p0/load.groovy +++ b/regression-test/suites/variant_p0/load.groovy @@ -259,6 +259,18 @@ suite("regression_test_variant", "nonConcurrent"){ exception("errCode = 2, detailMessage = Doris hll, bitmap, array, map, struct, jsonb, variant column must use with specific function, and don't support filter, group by or order by") } + test { + sql """ + create table var( + `content` variant + )distributed by hash(`content`) buckets 8 + properties( + "replication_allocation" = "tag.location.default: 1" + ); + """ + exception("errCode = 2, detailMessage = Hash distribution info should not contain variant columns") + } + // 13. sparse columns table_name = "sparse_columns" create_table table_name From 9c2003f2189ed0ae38a18c1a0351de6e7281aa15 Mon Sep 17 00:00:00 2001 From: lihangyu <15605149486@163.com> Date: Thu, 18 Apr 2024 10:59:02 +0800 Subject: [PATCH 2/3] [Fix](Variant) VariantRootColumnIterator::read_by_rowids with wrong null map size (#33734) insert_range_from should start from `size` with `count` elements for null map --- .../olap/rowset/segment_v2/column_reader.cpp | 5 +- .../variant_p0/topn_opt_read_by_rowids.out | 14 +++ .../variant_p0/topn_opt_read_by_rowids.groovy | 104 ++++++++++++++++++ 3 files changed, 121 insertions(+), 2 deletions(-) create mode 100644 regression-test/data/variant_p0/topn_opt_read_by_rowids.out create mode 100644 regression-test/suites/variant_p0/topn_opt_read_by_rowids.groovy diff --git a/be/src/olap/rowset/segment_v2/column_reader.cpp b/be/src/olap/rowset/segment_v2/column_reader.cpp index 0506250f2470ee..9d19d1a64b4e30 100644 --- a/be/src/olap/rowset/segment_v2/column_reader.cpp +++ b/be/src/olap/rowset/segment_v2/column_reader.cpp @@ -1536,7 +1536,7 @@ Status VariantRootColumnIterator::read_by_rowids(const rowid_t* rowids, const si RETURN_IF_ERROR(_inner_iter->read_by_rowids(rowids, count, root_column)); obj.incr_num_rows(count); for (auto& entry : obj.get_subcolumns()) { - if (entry->data.size() != size + count) { + if (entry->data.size() != (size + count)) { entry->data.insertManyDefaults(count); } } @@ -1546,7 +1546,8 @@ Status VariantRootColumnIterator::read_by_rowids(const rowid_t* rowids, const si assert_cast(*dst).get_null_map_column(); vectorized::ColumnUInt8& src_null_map = assert_cast(*root_column).get_null_map_column(); - dst_null_map.insert_range_from(src_null_map, 0, src_null_map.size()); + DCHECK_EQ(src_null_map.size() - size, count); + dst_null_map.insert_range_from(src_null_map, size, count); } #ifndef NDEBUG obj.check_consistency(); diff --git a/regression-test/data/variant_p0/topn_opt_read_by_rowids.out b/regression-test/data/variant_p0/topn_opt_read_by_rowids.out new file mode 100644 index 00000000000000..6ee3844f7ef0a1 --- /dev/null +++ b/regression-test/data/variant_p0/topn_opt_read_by_rowids.out @@ -0,0 +1,14 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +2024-04-09T09:01:39 1712624474952 0004.00 740lp sit-iniwork-lcd-designer.qm.cn \N \N 630beed604d0513c 0000.0 \N https://sit-iniwork-lcd-designer.qm.cn/designer 1544512389907021826 https://sit-iniwork-lcd-designer.qm.cn/designer?caseName=44&caseCode=d4&appId=3fec598d32b10d26958d1d9119519c64&token=eyJhbGciOiJIUzUxMiJ9.eyJzdWIiOiI3NDBscCIsImNyZWF0ZWQiOjE3MTI1NDEzMTU0MzUsImlkbWlkIjpudWxsLCJleHAiOjE3MTMxNDYxMTUsInVwa2lkIjoiMTU0NDUxMjM4OTkwNzAyMTgyNiJ9.j8sJbrgrJzwPh5Ee1AoodqIJ7KsMT_qfUhZlyon6rRsMClRI5WpUYpSWkC7P-axolUUnWJi2Llw89hO_KyL_gg&formId=8f4f75e0c0aa93fcd0a4eae843114527&tenantId=YQJT&systemId=202203A-015&subType=0 202203A-015_APP_DES 20240409090114952 未知 {"browser":"Chrome 123.0.0.0","browserFlag":"a8ba48f1849c2423","browserName":"Chrome","browserVersion":"123.0.0.0","clientStr":"10.133.53.99","headerClient":"{X-Original-Forwarded-For=10.133.53.99, X-Forwarded-For=10.140.199.9, RemoteAddr=10.133.53.99, X-Real-IP=10.140.199.9}","os":"Windows 10","osName":"Windows","osVersion":"10","res":"1920x1080","type":"Computer"} 1712624474952 1712624475992 \N \N +2024-04-09T09:01:39 1712624474952 0004.00 740lp sit-iniwork-lcd-designer.qm.cn \N \N 630beed604d0513c 0000.0 \N https://sit-iniwork-lcd-designer.qm.cn/designer 1544512389907021826 https://sit-iniwork-lcd-designer.qm.cn/designer?caseName=44&caseCode=d4&appId=3fec598d32b10d26958d1d9119519c64&token=eyJhbGciOiJIUzUxMiJ9.eyJzdWIiOiI3NDBscCIsImNyZWF0ZWQiOjE3MTI1NDEzMTU0MzUsImlkbWlkIjpudWxsLCJleHAiOjE3MTMxNDYxMTUsInVwa2lkIjoiMTU0NDUxMjM4OTkwNzAyMTgyNiJ9.j8sJbrgrJzwPh5Ee1AoodqIJ7KsMT_qfUhZlyon6rRsMClRI5WpUYpSWkC7P-axolUUnWJi2Llw89hO_KyL_gg&formId=8f4f75e0c0aa93fcd0a4eae843114527&tenantId=YQJT&systemId=202203A-015&subType=0 202203A-015_APP_DES 20240409090114952 未知 {"browser":"Chrome 123.0.0.0","browserFlag":"a8ba48f1849c2423","browserName":"Chrome","browserVersion":"123.0.0.0","clientStr":"10.133.53.99","headerClient":"{X-Original-Forwarded-For=10.133.53.99, X-Forwarded-For=10.140.199.9, RemoteAddr=10.133.53.99, X-Real-IP=10.140.199.9}","os":"Windows 10","osName":"Windows","osVersion":"10","res":"1920x1080","type":"Computer"} 1712624474952 1712624475992 \N \N +2024-04-09T09:01:39 1712624474952 0004.00 740lp sit-iniwork-lcd-designer.qm.cn \N \N 630beed604d0513c 0000.0 \N https://sit-iniwork-lcd-designer.qm.cn/designer 1544512389907021826 https://sit-iniwork-lcd-designer.qm.cn/designer?caseName=44&caseCode=d4&appId=3fec598d32b10d26958d1d9119519c64&token=eyJhbGciOiJIUzUxMiJ9.eyJzdWIiOiI3NDBscCIsImNyZWF0ZWQiOjE3MTI1NDEzMTU0MzUsImlkbWlkIjpudWxsLCJleHAiOjE3MTMxNDYxMTUsInVwa2lkIjoiMTU0NDUxMjM4OTkwNzAyMTgyNiJ9.j8sJbrgrJzwPh5Ee1AoodqIJ7KsMT_qfUhZlyon6rRsMClRI5WpUYpSWkC7P-axolUUnWJi2Llw89hO_KyL_gg&formId=8f4f75e0c0aa93fcd0a4eae843114527&tenantId=YQJT&systemId=202203A-015&subType=0 202203A-015_APP_DES 20240409090114952 未知 {"browser":"Chrome 123.0.0.0","browserFlag":"a8ba48f1849c2423","browserName":"Chrome","browserVersion":"123.0.0.0","clientStr":"10.133.53.99","headerClient":"{X-Original-Forwarded-For=10.133.53.99, X-Forwarded-For=10.140.199.9, RemoteAddr=10.133.53.99, X-Real-IP=10.140.199.9}","os":"Windows 10","osName":"Windows","osVersion":"10","res":"1920x1080","type":"Computer"} 1712624474952 1712624475992 \N \N + +-- !sql -- +2024-04-09T09:02:31 1712624495211 004.00 740lp sit-iniwork-lcd-designer.qm.cn \N \N 630beed604d0513c 0000.0 \N https://sit-iniwork-lcd-designer.qm.cn/designer 1544512389907021826 https://sit-iniwork-lcd-designer.qm.cn/designer?caseName=44&caseCode=d4&appId=3fec598d32b10d26958d1d9119519c64&token=eyJhbGciOiJIUzUxMiJ9.eyJzdWIiOiI3NDBscCIsImNyZWF0ZWQiOjE3MTI1NDEzMTU0MzUsImlkbWlkIjpudWxsLCJleHAiOjE3MTMxNDYxMTUsInVwa2lkIjoiMTU0NDUxMjM4OTkwNzAyMTgyNiJ9.j8sJbrgrJzwPh5Ee1AoodqIJ7KsMT_qfUhZlyon6rRsMClRI5WpUYpSWkC7P-axolUUnWJi2Llw89hO_KyL_gg&formId=8f4f75e0c0aa93fcd0a4eae843114527&tenantId=YQJT&systemId=202203A-015&subType=0 202203A-015_APP_DES 20240409090114952 未知 {"browser1":"Chrome 123.0.0.0","browserFlag1":"a8ba48f1849c2423","browserName4":"Chrome","browserVersion7":"123.0.0.0","clientStr":"10.133.53.99","headerClient":"{X-Original-Forwarded-For=10.033.53.19, X-Forwarded-For=10.140.199.9, RemoteAddr=10.133.53.99, X-Real-IP=10.140.199.9}","os":"Windows 10","osName":"Windows","osVersion":"10","res":"1920x1080","type":"Computer"} 1712624474952 1712624475992 \N \N +2024-04-09T09:01:39 1712624474952 0004.00 740lp sit-iniwork-lcd-designer.qm.cn \N \N 630beed604d0513c 0000.0 \N https://sit-iniwork-lcd-designer.qm.cn/designer 1544512389907021826 https://sit-iniwork-lcd-designer.qm.cn/designer?caseName=44&caseCode=d4&appId=3fec598d32b10d26958d1d9119519c64&token=eyJhbGciOiJIUzUxMiJ9.eyJzdWIiOiI3NDBscCIsImNyZWF0ZWQiOjE3MTI1NDEzMTU0MzUsImlkbWlkIjpudWxsLCJleHAiOjE3MTMxNDYxMTUsInVwa2lkIjoiMTU0NDUxMjM4OTkwNzAyMTgyNiJ9.j8sJbrgrJzwPh5Ee1AoodqIJ7KsMT_qfUhZlyon6rRsMClRI5WpUYpSWkC7P-axolUUnWJi2Llw89hO_KyL_gg&formId=8f4f75e0c0aa93fcd0a4eae843114527&tenantId=YQJT&systemId=202203A-015&subType=0 202203A-015_APP_DES 20240409090114952 未知 {"browser":"Chrome 123.0.0.0","browserFlag":"a8ba48f1849c2423","browserName":"Chrome","browserVersion":"123.0.0.0","clientStr":"10.133.53.99","headerClient":"{X-Original-Forwarded-For=10.133.53.99, X-Forwarded-For=10.140.199.9, RemoteAddr=10.133.53.99, X-Real-IP=10.140.199.9}","os":"Windows 10","osName":"Windows","osVersion":"10","res":"1920x1080","type":"Computer"} 1712624474952 1712624475992 \N \N +2024-04-09T09:03:32 1712624474153 0014.00 740lp sit-iniwork-lcd-designer.qm.cn \N \N 630beed604d0513c 0000.0 \N https://sit-iniwork-lcd-designer.qm.cn/designer 1544512389907021826 https://sit-iniwork-lcd-designer.qm.cn/designer?caseName=44&caseCode=d4&appId=3fec598d32b10d26958d1d9119519c64&token=eyJhbGciOiJIUzUxMiJ9.eyJzdWIiOiI3NDBscCIsImNyZWF0ZWQiOjE3MTI1NDEzMTU0MzUsImlkbWlkIjpudWxsLCJleHAiOjE3MTMxNDYxMTUsInVwa2lkIjoiMTU0NDUxMjM4OTkwNzAyMTgyNiJ9.j8sJbrgrJzwPh5Ee1AoodqIJ7KsMT_qfUhZlyon6rRsMClRI5WpUYpSWkC7P-axolUUnWJi2Llw89hO_KyL_gg&formId=8f4f75e0c0aa93fcd0a4eae843114527&tenantId=YQJT&systemId=202203A-015&subType=0 202203A-015_APP_DES 20240409090114952 未知 {"browser2":"Chrome 123.0.0.0","browserFlag2":"a8ba48f1849c2423","browserName5":"Chrome","browserVersion8":"123.0.0.0","clientStr":"10.133.53.99","headerClient":"{X-Original-Forwarded-For=10.133.53.99, X-Forwarded-For=10.140.199.9, RemoteAddr=10.133.53.99, X-Real-IP=10.140.199.9}","os":"Windows 10","osName":"Windows","osVersion":"10","res":"1920x1080","type":"Computer"} 1712624474952 1712624475992 \N \N +2024-04-09T09:04:33 1712624474959 0024.00 740lp sit-iniwork-lcd-designer.qm.cn \N \N 630beed604d0513c 0000.0 \N https://sit-iniwork-lcd-designer.qm.cn/designer 1544512389907021826 https://sit-iniwork-lcd-designer.qm.cn/designer?caseName=44&caseCode=d4&appId=3fec598d32b10d26958d1d9119519c64&token=eyJhbGciOiJIUzUxMiJ9.eyJzdWIiOiI3NDBscCIsImNyZWF0ZWQiOjE3MTI1NDEzMTU0MzUsImlkbWlkIjpudWxsLCJleHAiOjE3MTMxNDYxMTUsInVwa2lkIjoiMTU0NDUxMjM4OTkwNzAyMTgyNiJ9.j8sJbrgrJzwPh5Ee1AoodqIJ7KsMT_qfUhZlyon6rRsMClRI5WpUYpSWkC7P-axolUUnWJi2Llw89hO_KyL_gg&formId=8f4f75e0c0aa93fcd0a4eae843114527&tenantId=YQJT&systemId=202203A-015&subType=0 202203A-015_APP_DES 20240409090114952 未知 {"browser3":"Chrome 123.0.0.0","browserFlag3":"a8ba48f1849c2423","browserName6":"Chrome","browserVersion9":"123.0.0.0","clientStr":"10.133.53.99","headerClient":"{X-Original-Forwarded-For=10.133.53.99, X-Forwarded-For=10.140.199.9, RemoteAddr=10.133.53.99, X-Real-IP=10.140.199.9}","os":"Windows 10","osName":"Windows","osVersion":"10","res":"1920x1080","type":"Computer"} 1712624474952 1712624475992 \N \N +2024-04-09T09:01:39 1712624474952 0004.00 740lp sit-iniwork-lcd-designer.qm.cn \N \N 630beed604d0513c 0000.0 \N https://sit-iniwork-lcd-designer.qm.cn/designer 1544512389907021826 https://sit-iniwork-lcd-designer.qm.cn/designer?caseName=44&caseCode=d4&appId=3fec598d32b10d26958d1d9119519c64&token=eyJhbGciOiJIUzUxMiJ9.eyJzdWIiOiI3NDBscCIsImNyZWF0ZWQiOjE3MTI1NDEzMTU0MzUsImlkbWlkIjpudWxsLCJleHAiOjE3MTMxNDYxMTUsInVwa2lkIjoiMTU0NDUxMjM4OTkwNzAyMTgyNiJ9.j8sJbrgrJzwPh5Ee1AoodqIJ7KsMT_qfUhZlyon6rRsMClRI5WpUYpSWkC7P-axolUUnWJi2Llw89hO_KyL_gg&formId=8f4f75e0c0aa93fcd0a4eae843114527&tenantId=YQJT&systemId=202203A-015&subType=0 202203A-015_APP_DES 20240409090114952 未知 {"browser":"Chrome 123.0.0.0","browserFlag":"a8ba48f1849c2423","browserName":"Chrome","browserVersion":"123.0.0.0","clientStr":"10.133.53.99","headerClient":"{X-Original-Forwarded-For=10.133.53.99, X-Forwarded-For=10.140.199.9, RemoteAddr=10.133.53.99, X-Real-IP=10.140.199.9}","os":"Windows 10","osName":"Windows","osVersion":"10","res":"1920x1080","type":"Computer"} 1712624474952 1712624475992 \N \N +2024-04-09T09:01:39 1712624474952 0004.00 740lp sit-iniwork-lcd-designer.qm.cn \N \N 630beed604d0513c 0000.0 \N https://sit-iniwork-lcd-designer.qm.cn/designer 1544512389907021826 https://sit-iniwork-lcd-designer.qm.cn/designer?caseName=44&caseCode=d4&appId=3fec598d32b10d26958d1d9119519c64&token=eyJhbGciOiJIUzUxMiJ9.eyJzdWIiOiI3NDBscCIsImNyZWF0ZWQiOjE3MTI1NDEzMTU0MzUsImlkbWlkIjpudWxsLCJleHAiOjE3MTMxNDYxMTUsInVwa2lkIjoiMTU0NDUxMjM4OTkwNzAyMTgyNiJ9.j8sJbrgrJzwPh5Ee1AoodqIJ7KsMT_qfUhZlyon6rRsMClRI5WpUYpSWkC7P-axolUUnWJi2Llw89hO_KyL_gg&formId=8f4f75e0c0aa93fcd0a4eae843114527&tenantId=YQJT&systemId=202203A-015&subType=0 202203A-015_APP_DES 20240409090114952 未知 {"browser":"Chrome 123.0.0.0","browserFlag":"a8ba48f1849c2423","browserName":"Chrome","browserVersion":"123.0.0.0","clientStr":"10.133.53.99","headerClient":"{X-Original-Forwarded-For=10.133.53.99, X-Forwarded-For=10.140.199.9, RemoteAddr=10.133.53.99, X-Real-IP=10.140.199.9}","os":"Windows 10","osName":"Windows","osVersion":"10","res":"1920x1080","type":"Computer"} 1712624474952 1712624475992 \N \N + diff --git a/regression-test/suites/variant_p0/topn_opt_read_by_rowids.groovy b/regression-test/suites/variant_p0/topn_opt_read_by_rowids.groovy new file mode 100644 index 00000000000000..dd597755a6dffb --- /dev/null +++ b/regression-test/suites/variant_p0/topn_opt_read_by_rowids.groovy @@ -0,0 +1,104 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("regression_test_variant_topn_opt_read_by_rowids", "p0"){ + sql "DROP TABLE IF EXISTS test_web_log" + sql """ + CREATE TABLE `test_web_log` ( + `timestampkey` DATETIME NULL DEFAULT CURRENT_TIMESTAMP COMMENT '', + `nodeName` VARCHAR(100) NULL COMMENT '', + `serviceCode` VARCHAR(100) NULL COMMENT '', + `pageTitle` VARCHAR(100) NULL COMMENT '', + `nodeIp` VARCHAR(100) NULL COMMENT '', + `clientTs` LARGEINT NULL COMMENT '', + `spanId` VARCHAR(1000) NULL COMMENT '', + `loginName` VARCHAR(100) NULL COMMENT '', + `host` VARCHAR(100) NULL COMMENT '', + `nameSpace` VARCHAR(100) NULL COMMENT '', + `event` VARIANT NULL COMMENT '', + `httpInfo` VARIANT NULL COMMENT '', + `iworkHost` VARCHAR(100) NULL COMMENT '', + `traceId` VARCHAR(1000) NULL COMMENT '', + `pSpanId` VARCHAR(1000) NULL COMMENT '', + `timing` VARIANT NULL COMMENT '', + `clusterFlag` VARCHAR(100) NULL COMMENT '', + `sessionId` VARCHAR(1000) NULL COMMENT '', + `serviceName` VARCHAR(100) NULL COMMENT '', + `userName` VARCHAR(100) NULL COMMENT '', + `uri` VARCHAR(1000) NULL COMMENT '', + `userId` VARCHAR(100) NULL COMMENT '', + `url` VARCHAR(1000) NULL COMMENT '', + `systemCode` VARCHAR(100) NULL COMMENT '', + `tenantId` VARCHAR(100) NULL COMMENT '', + `podName` VARCHAR(100) NULL COMMENT '', + `siteId` VARCHAR(100) NULL COMMENT '', + `time` VARCHAR(100) NULL COMMENT '', + `region` VARCHAR(100) NULL COMMENT '', + `device` VARIANT NULL COMMENT '', + `ts` VARCHAR(100) NULL COMMENT '', + `timestamp` LARGEINT NULL COMMENT '', + `resourceLoad` VARIANT NULL COMMENT '', + `scriptError` VARIANT NULL COMMENT '' +) ENGINE=OLAP +DUPLICATE KEY(`timestampkey`) +COMMENT 'OLAP' +PARTITION BY RANGE(`timestampkey`) +( +PARTITION p2024040909 VALUES [('2024-04-09 09:00:00'), ('2024-04-09 10:00:00')), +PARTITION p2024040910 VALUES [('2024-04-09 10:00:00'), ('2024-04-09 11:00:00')), +PARTITION p2024040911 VALUES [('2024-04-09 11:00:00'), ('2024-04-09 12:00:00')), +PARTITION p2024040923 VALUES [('2024-04-09 12:00:00'), ('2024-04-10 00:00:00'))) +DISTRIBUTED BY HASH(`traceId`) BUCKETS 25 +PROPERTIES ( +"replication_allocation" = "tag.location.default: 1" +); + """ + sql """ + INSERT INTO `test_web_log` VALUES ('2024-04-09 09:01:39', '', '', '', '', '1712624474952', '0004.00', '740lp', 'sit-iniwork-lcd-designer.qm.cn', '', NULL, NULL, '', '630beed604d0513c', '0000.0', NULL, '', '', '', '', 'https://sit-iniwork-lcd-designer.qm.cn/designer', '1544512389907021826', 'https://sit-iniwork-lcd-designer.qm.cn/designer?caseName=44&caseCode=d4&appId=3fec598d32b10d26958d1d9119519c64&token=eyJhbGciOiJIUzUxMiJ9.eyJzdWIiOiI3NDBscCIsImNyZWF0ZWQiOjE3MTI1NDEzMTU0MzUsImlkbWlkIjpudWxsLCJleHAiOjE3MTMxNDYxMTUsInVwa2lkIjoiMTU0NDUxMjM4OTkwNzAyMTgyNiJ9.j8sJbrgrJzwPh5Ee1AoodqIJ7KsMT_qfUhZlyon6rRsMClRI5WpUYpSWkC7P-axolUUnWJi2Llw89hO_KyL_gg&formId=8f4f75e0c0aa93fcd0a4eae843114527&tenantId=YQJT&systemId=202203A-015&subType=0', '', '', '', '202203A-015_APP_DES', '20240409090114952', '未知', '{\"browser\":\"Chrome 123.0.0.0\",\"browserFlag\":\"a8ba48f1849c2423\",\"browserName\":\"Chrome\",\"browserVersion\":\"123.0.0.0\",\"clientStr\":\"10.133.53.99\",\"headerClient\":\"{X-Original-Forwarded-For=10.133.53.99, X-Forwarded-For=10.140.199.9, RemoteAddr=10.133.53.99, X-Real-IP=10.140.199.9}\",\"os\":\"Windows 10\",\"osName\":\"Windows\",\"osVersion\":\"10\",\"res\":\"1920x1080\",\"type\":\"Computer\"}', '1712624474952', '1712624475992', NULL, NULL); + """ + sql """ + INSERT INTO `test_web_log` VALUES ('2024-04-09 09:01:39', '', '', '', '', '1712624474952', '0004.00', '740lp', 'sit-iniwork-lcd-designer.qm.cn', '', NULL, NULL, '', '630beed604d0513c', '0000.0', NULL, '', '', '', '', 'https://sit-iniwork-lcd-designer.qm.cn/designer', '1544512389907021826', 'https://sit-iniwork-lcd-designer.qm.cn/designer?caseName=44&caseCode=d4&appId=3fec598d32b10d26958d1d9119519c64&token=eyJhbGciOiJIUzUxMiJ9.eyJzdWIiOiI3NDBscCIsImNyZWF0ZWQiOjE3MTI1NDEzMTU0MzUsImlkbWlkIjpudWxsLCJleHAiOjE3MTMxNDYxMTUsInVwa2lkIjoiMTU0NDUxMjM4OTkwNzAyMTgyNiJ9.j8sJbrgrJzwPh5Ee1AoodqIJ7KsMT_qfUhZlyon6rRsMClRI5WpUYpSWkC7P-axolUUnWJi2Llw89hO_KyL_gg&formId=8f4f75e0c0aa93fcd0a4eae843114527&tenantId=YQJT&systemId=202203A-015&subType=0', '', '', '', '202203A-015_APP_DES', '20240409090114952', '未知', '{\"browser\":\"Chrome 123.0.0.0\",\"browserFlag\":\"a8ba48f1849c2423\",\"browserName\":\"Chrome\",\"browserVersion\":\"123.0.0.0\",\"clientStr\":\"10.133.53.99\",\"headerClient\":\"{X-Original-Forwarded-For=10.133.53.99, X-Forwarded-For=10.140.199.9, RemoteAddr=10.133.53.99, X-Real-IP=10.140.199.9}\",\"os\":\"Windows 10\",\"osName\":\"Windows\",\"osVersion\":\"10\",\"res\":\"1920x1080\",\"type\":\"Computer\"}', '1712624474952', '1712624475992', NULL, NULL); + """ + sql """ + INSERT INTO `test_web_log` VALUES ('2024-04-09 09:01:39', '', '', '', '', '1712624474952', '0004.00', '740lp', 'sit-iniwork-lcd-designer.qm.cn', '', NULL, NULL, '', '630beed604d0513c', '0000.0', NULL, '', '', '', '', 'https://sit-iniwork-lcd-designer.qm.cn/designer', '1544512389907021826', 'https://sit-iniwork-lcd-designer.qm.cn/designer?caseName=44&caseCode=d4&appId=3fec598d32b10d26958d1d9119519c64&token=eyJhbGciOiJIUzUxMiJ9.eyJzdWIiOiI3NDBscCIsImNyZWF0ZWQiOjE3MTI1NDEzMTU0MzUsImlkbWlkIjpudWxsLCJleHAiOjE3MTMxNDYxMTUsInVwa2lkIjoiMTU0NDUxMjM4OTkwNzAyMTgyNiJ9.j8sJbrgrJzwPh5Ee1AoodqIJ7KsMT_qfUhZlyon6rRsMClRI5WpUYpSWkC7P-axolUUnWJi2Llw89hO_KyL_gg&formId=8f4f75e0c0aa93fcd0a4eae843114527&tenantId=YQJT&systemId=202203A-015&subType=0', '', '', '', '202203A-015_APP_DES', '20240409090114952', '未知', '{\"browser\":\"Chrome 123.0.0.0\",\"browserFlag\":\"a8ba48f1849c2423\",\"browserName\":\"Chrome\",\"browserVersion\":\"123.0.0.0\",\"clientStr\":\"10.133.53.99\",\"headerClient\":\"{X-Original-Forwarded-For=10.133.53.99, X-Forwarded-For=10.140.199.9, RemoteAddr=10.133.53.99, X-Real-IP=10.140.199.9}\",\"os\":\"Windows 10\",\"osName\":\"Windows\",\"osVersion\":\"10\",\"res\":\"1920x1080\",\"type\":\"Computer\"}', '1712624474952', '1712624475992', NULL, NULL); + """ + sql "set topn_opt_limit_threshold = 1024" + qt_sql """SELECT + * FROM + test_web_log + WHERE + ts >= '1712480940849' + AND ts <= '1712805483291' + ORDER BY + ts DESC + LIMIT 10""" + sql """ + INSERT INTO `test_web_log` VALUES ('2024-04-09 09:02:31', '', '', '', '', '1712624495211', '004.00', '740lp', 'sit-iniwork-lcd-designer.qm.cn', '', NULL, NULL, '', '630beed604d0513c', '0000.0', NULL, '', '', '', '', 'https://sit-iniwork-lcd-designer.qm.cn/designer', '1544512389907021826', 'https://sit-iniwork-lcd-designer.qm.cn/designer?caseName=44&caseCode=d4&appId=3fec598d32b10d26958d1d9119519c64&token=eyJhbGciOiJIUzUxMiJ9.eyJzdWIiOiI3NDBscCIsImNyZWF0ZWQiOjE3MTI1NDEzMTU0MzUsImlkbWlkIjpudWxsLCJleHAiOjE3MTMxNDYxMTUsInVwa2lkIjoiMTU0NDUxMjM4OTkwNzAyMTgyNiJ9.j8sJbrgrJzwPh5Ee1AoodqIJ7KsMT_qfUhZlyon6rRsMClRI5WpUYpSWkC7P-axolUUnWJi2Llw89hO_KyL_gg&formId=8f4f75e0c0aa93fcd0a4eae843114527&tenantId=YQJT&systemId=202203A-015&subType=0', '', '', '', '202203A-015_APP_DES', '20240409090114952', '未知', '{\"browser1\":\"Chrome 123.0.0.0\",\"browserFlag1\":\"a8ba48f1849c2423\",\"browserName4\":\"Chrome\",\"browserVersion7\":\"123.0.0.0\",\"clientStr\":\"10.133.53.99\",\"headerClient\":\"{X-Original-Forwarded-For=10.033.53.19, X-Forwarded-For=10.140.199.9, RemoteAddr=10.133.53.99, X-Real-IP=10.140.199.9}\",\"os\":\"Windows 10\",\"osName\":\"Windows\",\"osVersion\":\"10\",\"res\":\"1920x1080\",\"type\":\"Computer\"}', '1712624474952', '1712624475992', NULL, NULL); + """ + sql """ + INSERT INTO `test_web_log` VALUES ('2024-04-09 09:03:32', '', '', '', '', '1712624474153', '0014.00', '740lp', 'sit-iniwork-lcd-designer.qm.cn', '', NULL, NULL, '', '630beed604d0513c', '0000.0', NULL, '', '', '', '', 'https://sit-iniwork-lcd-designer.qm.cn/designer', '1544512389907021826', 'https://sit-iniwork-lcd-designer.qm.cn/designer?caseName=44&caseCode=d4&appId=3fec598d32b10d26958d1d9119519c64&token=eyJhbGciOiJIUzUxMiJ9.eyJzdWIiOiI3NDBscCIsImNyZWF0ZWQiOjE3MTI1NDEzMTU0MzUsImlkbWlkIjpudWxsLCJleHAiOjE3MTMxNDYxMTUsInVwa2lkIjoiMTU0NDUxMjM4OTkwNzAyMTgyNiJ9.j8sJbrgrJzwPh5Ee1AoodqIJ7KsMT_qfUhZlyon6rRsMClRI5WpUYpSWkC7P-axolUUnWJi2Llw89hO_KyL_gg&formId=8f4f75e0c0aa93fcd0a4eae843114527&tenantId=YQJT&systemId=202203A-015&subType=0', '', '', '', '202203A-015_APP_DES', '20240409090114952', '未知', '{\"browser2\":\"Chrome 123.0.0.0\",\"browserFlag2\":\"a8ba48f1849c2423\",\"browserName5\":\"Chrome\",\"browserVersion8\":\"123.0.0.0\",\"clientStr\":\"10.133.53.99\",\"headerClient\":\"{X-Original-Forwarded-For=10.133.53.99, X-Forwarded-For=10.140.199.9, RemoteAddr=10.133.53.99, X-Real-IP=10.140.199.9}\",\"os\":\"Windows 10\",\"osName\":\"Windows\",\"osVersion\":\"10\",\"res\":\"1920x1080\",\"type\":\"Computer\"}', '1712624474952', '1712624475992', NULL, NULL); + """ + sql """ + INSERT INTO `test_web_log` VALUES ('2024-04-09 09:04:33', '', '', '', '', '1712624474959', '0024.00', '740lp', 'sit-iniwork-lcd-designer.qm.cn', '', NULL, NULL, '', '630beed604d0513c', '0000.0', NULL, '', '', '', '', 'https://sit-iniwork-lcd-designer.qm.cn/designer', '1544512389907021826', 'https://sit-iniwork-lcd-designer.qm.cn/designer?caseName=44&caseCode=d4&appId=3fec598d32b10d26958d1d9119519c64&token=eyJhbGciOiJIUzUxMiJ9.eyJzdWIiOiI3NDBscCIsImNyZWF0ZWQiOjE3MTI1NDEzMTU0MzUsImlkbWlkIjpudWxsLCJleHAiOjE3MTMxNDYxMTUsInVwa2lkIjoiMTU0NDUxMjM4OTkwNzAyMTgyNiJ9.j8sJbrgrJzwPh5Ee1AoodqIJ7KsMT_qfUhZlyon6rRsMClRI5WpUYpSWkC7P-axolUUnWJi2Llw89hO_KyL_gg&formId=8f4f75e0c0aa93fcd0a4eae843114527&tenantId=YQJT&systemId=202203A-015&subType=0', '', '', '', '202203A-015_APP_DES', '20240409090114952', '未知', '{\"browser3\":\"Chrome 123.0.0.0\",\"browserFlag3\":\"a8ba48f1849c2423\",\"browserName6\":\"Chrome\",\"browserVersion9\":\"123.0.0.0\",\"clientStr\":\"10.133.53.99\",\"headerClient\":\"{X-Original-Forwarded-For=10.133.53.99, X-Forwarded-For=10.140.199.9, RemoteAddr=10.133.53.99, X-Real-IP=10.140.199.9}\",\"os\":\"Windows 10\",\"osName\":\"Windows\",\"osVersion\":\"10\",\"res\":\"1920x1080\",\"type\":\"Computer\"}', '1712624474952', '1712624475992', NULL, NULL); + """ + qt_sql """SELECT + * FROM + test_web_log + ORDER BY + ts DESC + LIMIT 10""" +} \ No newline at end of file From 760eb6042d9c5a1e0799a811b19e630bbce87269 Mon Sep 17 00:00:00 2001 From: lihangyu <15605149486@163.com> Date: Thu, 18 Apr 2024 11:10:18 +0800 Subject: [PATCH 3/3] [Fix](Variant) check column index validation for extracted columns (#33766) --- .../rowset/segment_v2/inverted_index_writer.h | 22 +++ .../olap/rowset/segment_v2/segment_writer.cpp | 6 +- .../segment_v2/vertical_segment_writer.cpp | 5 +- be/src/olap/task/index_builder.cpp | 3 + .../data/variant_p0/with_index/var_index.out | 12 ++ .../variant_github_events_p0/load.groovy | 137 +++++++++++++++++- .../variant_p0/with_index/var_index.groovy | 8 +- 7 files changed, 184 insertions(+), 9 deletions(-) diff --git a/be/src/olap/rowset/segment_v2/inverted_index_writer.h b/be/src/olap/rowset/segment_v2/inverted_index_writer.h index 5ed34852c94032..77873905af107f 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_writer.h +++ b/be/src/olap/rowset/segment_v2/inverted_index_writer.h @@ -24,13 +24,16 @@ #include #include #include +#include #include "common/config.h" #include "common/status.h" #include "gutil/strings/split.h" #include "io/fs/file_system.h" #include "io/fs/local_file_system.h" +#include "olap/olap_common.h" #include "olap/options.h" +#include "olap/tablet_schema.h" namespace doris { class CollectionValue; @@ -70,6 +73,25 @@ class InvertedIndexColumnWriter { virtual void close_on_error() = 0; + // check if the column is valid for inverted index, some columns + // are generated from variant, but not all of them are supported + static bool check_column_valid(const TabletColumn& column) { + // bellow types are not supported in inverted index for extracted columns + static std::set invalid_types = { + FieldType::OLAP_FIELD_TYPE_DOUBLE, + FieldType::OLAP_FIELD_TYPE_JSONB, + FieldType::OLAP_FIELD_TYPE_ARRAY, + FieldType::OLAP_FIELD_TYPE_FLOAT, + }; + if (column.is_extracted_column() && (invalid_types.contains(column.type()))) { + return false; + } + if (column.is_variant_type()) { + return false; + } + return true; + } + private: DISALLOW_COPY_AND_ASSIGN(InvertedIndexColumnWriter); }; diff --git a/be/src/olap/rowset/segment_v2/segment_writer.cpp b/be/src/olap/rowset/segment_v2/segment_writer.cpp index dc85408accd827..284e0e0eaaa14d 100644 --- a/be/src/olap/rowset/segment_v2/segment_writer.cpp +++ b/be/src/olap/rowset/segment_v2/segment_writer.cpp @@ -42,6 +42,7 @@ #include "olap/rowset/rowset_writer_context.h" // RowsetWriterContext #include "olap/rowset/segment_v2/column_writer.h" // ColumnWriter #include "olap/rowset/segment_v2/inverted_index_file_writer.h" +#include "olap/rowset/segment_v2/inverted_index_writer.h" #include "olap/rowset/segment_v2/page_io.h" #include "olap/rowset/segment_v2/page_pointer.h" #include "olap/segment_loader.h" @@ -224,9 +225,8 @@ Status SegmentWriter::init(const std::vector& col_ids, bool has_key) { } // indexes for this column opts.indexes = std::move(_tablet_schema->get_indexes_for_column(column)); - if (column.is_variant_type() || (column.is_extracted_column() && column.is_jsonb_type()) || - (column.is_extracted_column() && column.is_array_type())) { - // variant and jsonb type skip write index + if (!InvertedIndexColumnWriter::check_column_valid(column)) { + // skip inverted index if invalid opts.indexes.clear(); opts.need_zone_map = false; opts.need_bloom_filter = false; diff --git a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp index d94bb0ce3fec6a..2a4f924b98a038 100644 --- a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp +++ b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp @@ -176,9 +176,8 @@ Status VerticalSegmentWriter::_create_column_writer(uint32_t cid, const TabletCo } // indexes for this column opts.indexes = _tablet_schema->get_indexes_for_column(column); - if (column.is_variant_type() || (column.is_extracted_column() && column.is_jsonb_type()) || - (column.is_extracted_column() && column.is_array_type())) { - // variant and jsonb type skip write index + if (!InvertedIndexColumnWriter::check_column_valid(column)) { + // skip inverted index if invalid opts.indexes.clear(); opts.need_zone_map = false; opts.need_bloom_filter = false; diff --git a/be/src/olap/task/index_builder.cpp b/be/src/olap/task/index_builder.cpp index 0e6abc2c1b14a2..09f745833d1c2f 100644 --- a/be/src/olap/task/index_builder.cpp +++ b/be/src/olap/task/index_builder.cpp @@ -346,6 +346,9 @@ Status IndexBuilder::handle_single_rowset(RowsetMetaSharedPtr output_rowset_meta continue; } auto column = output_rowset_schema->column(column_idx); + if (!InvertedIndexColumnWriter::check_column_valid(column)) { + continue; + } DCHECK(output_rowset_schema->has_inverted_index_with_index_id(index_id, "")); _olap_data_convertor->add_column_data_convertor(column); return_columns.emplace_back(column_idx); diff --git a/regression-test/data/variant_p0/with_index/var_index.out b/regression-test/data/variant_p0/with_index/var_index.out index d8b7417852b781..634b5125e843f1 100644 --- a/regression-test/data/variant_p0/with_index/var_index.out +++ b/regression-test/data/variant_p0/with_index/var_index.out @@ -8,3 +8,15 @@ 2 {"a":18811,"b":"hello world","c":1181111} 4 {"a":1234,"b":"hello xxx world","c":8181111} +-- !sql -- +1 {"a":123,"b":"xxxyyy","c":111999111} +2 {"a":18811,"b":"hello world","c":1181111} +3 {"a":18811,"b":"hello wworld","c":11111} +4 {"a":1234,"b":"hello xxx world","c":8181111} +5 {"a":123456789,"b":123456,"c":8181111} +6 {"timestamp":1713283200.060359} +7 {"timestamp":17.0} +8 {"timestamp":[123]} +9 {"timestamp":17.0} +10 {"timestamp":"17.0"} + diff --git a/regression-test/suites/variant_github_events_p0/load.groovy b/regression-test/suites/variant_github_events_p0/load.groovy index e05131d9153eed..befd1aa6103650 100644 --- a/regression-test/suites/variant_github_events_p0/load.groovy +++ b/regression-test/suites/variant_github_events_p0/load.groovy @@ -16,6 +16,97 @@ // under the License. suite("regression_test_variant_github_events_p0", "nonConcurrent"){ + // prepare test table + def timeout = 300000 + def delta_time = 1000 + def alter_res = "null" + def useTime = 0 + + def wait_for_latest_op_on_table_finish = { table_name, OpTimeout -> + for(int t = delta_time; t <= OpTimeout; t += delta_time){ + alter_res = sql """SHOW ALTER TABLE COLUMN WHERE TableName = "${table_name}" ORDER BY CreateTime DESC LIMIT 1;""" + alter_res = alter_res.toString() + if(alter_res.contains("FINISHED")) { + sleep(10000) // wait change table state to normal + logger.info(table_name + " latest alter job finished, detail: " + alter_res) + break + } + useTime = t + sleep(delta_time) + } + assertTrue(useTime <= OpTimeout, "wait_for_latest_op_on_table_finish timeout") + } + + def wait_for_build_index_on_partition_finish = { table_name, OpTimeout -> + for(int t = delta_time; t <= OpTimeout; t += delta_time){ + alter_res = sql """SHOW BUILD INDEX WHERE TableName = "${table_name}";""" + def expected_finished_num = alter_res.size(); + def finished_num = 0; + for (int i = 0; i < expected_finished_num; i++) { + logger.info(table_name + " build index job state: " + alter_res[i][7] + i) + if (alter_res[i][7] == "FINISHED") { + ++finished_num; + } + } + if (finished_num == expected_finished_num) { + sleep(10000) // wait change table state to normal + logger.info(table_name + " all build index jobs finished, detail: " + alter_res) + break + } + useTime = t + sleep(delta_time) + } + assertTrue(useTime <= OpTimeout, "wait_for_latest_build_index_on_partition_finish timeout") + } + + def wait_for_last_build_index_on_table_finish = { table_name, OpTimeout -> + for(int t = delta_time; t <= OpTimeout; t += delta_time){ + alter_res = sql """SHOW BUILD INDEX WHERE TableName = "${table_name}" ORDER BY JobId """ + + if (alter_res.size() == 0) { + logger.info(table_name + " last index job finished") + return "SKIPPED" + } + if (alter_res.size() > 0) { + def last_job_state = alter_res[alter_res.size()-1][7]; + if (last_job_state == "FINISHED" || last_job_state == "CANCELLED") { + sleep(10000) // wait change table state to normal + logger.info(table_name + " last index job finished, state: " + last_job_state + ", detail: " + alter_res) + return last_job_state; + } + } + useTime = t + sleep(delta_time) + } + logger.info("wait_for_last_build_index_on_table_finish debug: " + alter_res) + assertTrue(useTime <= OpTimeout, "wait_for_last_build_index_on_table_finish timeout") + return "wait_timeout" + } + + def wait_for_last_build_index_on_table_running = { table_name, OpTimeout -> + for(int t = delta_time; t <= OpTimeout; t += delta_time){ + alter_res = sql """SHOW BUILD INDEX WHERE TableName = "${table_name}" ORDER BY JobId """ + + if (alter_res.size() == 0) { + logger.info(table_name + " last index job finished") + return "SKIPPED" + } + if (alter_res.size() > 0) { + def last_job_state = alter_res[alter_res.size()-1][7]; + if (last_job_state == "RUNNING") { + logger.info(table_name + " last index job running, state: " + last_job_state + ", detail: " + alter_res) + return last_job_state; + } + } + useTime = t + sleep(delta_time) + } + logger.info("wait_for_last_build_index_on_table_running debug: " + alter_res) + assertTrue(useTime <= OpTimeout, "wait_for_last_build_index_on_table_running timeout") + return "wait_timeout" + } + + def backendId_to_backendIP = [:] def backendId_to_backendHttpPort = [:] getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); @@ -60,8 +151,8 @@ suite("regression_test_variant_github_events_p0", "nonConcurrent"){ sql """ CREATE TABLE IF NOT EXISTS ${table_name} ( k bigint, - v variant not null, - INDEX idx_var(v) USING INVERTED PROPERTIES("parser" = "english") COMMENT '' + v variant not null + -- INDEX idx_var(v) USING INVERTED PROPERTIES("parser" = "english") COMMENT '' ) DUPLICATE KEY(`k`) DISTRIBUTED BY HASH(k) BUCKETS 4 @@ -73,11 +164,53 @@ suite("regression_test_variant_github_events_p0", "nonConcurrent"){ load_json_data.call(table_name, """${getS3Url() + '/regression/gharchive.m/2015-01-01-1.json'}""") load_json_data.call(table_name, """${getS3Url() + '/regression/gharchive.m/2015-01-01-2.json'}""") load_json_data.call(table_name, """${getS3Url() + '/regression/gharchive.m/2015-01-01-3.json'}""") + + // build inverted index at middle of loading the data + // ADD INDEX + sql """ ALTER TABLE github_events ADD INDEX idx_var (`v`) USING INVERTED PROPERTIES("parser" = "chinese", "parser_mode" = "fine_grained", "support_phrase" = "true") """ + wait_for_latest_op_on_table_finish("github_events", timeout) + // 2022 load_json_data.call(table_name, """${getS3Url() + '/regression/gharchive.m/2022-11-07-16.json'}""") load_json_data.call(table_name, """${getS3Url() + '/regression/gharchive.m/2022-11-07-10.json'}""") load_json_data.call(table_name, """${getS3Url() + '/regression/gharchive.m/2022-11-07-22.json'}""") load_json_data.call(table_name, """${getS3Url() + '/regression/gharchive.m/2022-11-07-23.json'}""") + + // BUILD INDEX and expect state is FINISHED + sql """ BUILD INDEX idx_var ON github_events""" + state = wait_for_last_build_index_on_table_finish("github_events", timeout) + assertEquals("FINISHED", state) + + // add bloom filter at the end of loading data + + def tablets = sql_return_maparray """ show tablets from github_events; """ + // trigger compactions for all tablets in github_events + for (def tablet in tablets) { + String tablet_id = tablet.TabletId + backend_id = tablet.BackendId + (code, out, err) = be_run_cumulative_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) + logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def compactJson = parseJson(out.trim()) + } + + // wait for all compactions done + for (def tablet in tablets) { + boolean running = true + do { + Thread.sleep(1000) + String tablet_id = tablet.TabletId + backend_id = tablet.BackendId + (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) + logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def compactionStatus = parseJson(out.trim()) + assertEquals("success", compactionStatus.status.toLowerCase()) + running = compactionStatus.run_status + } while (running) + } + + // TODO fix compaction issue, this case could be stable qt_sql """select cast(v["payload"]["pull_request"]["additions"] as int) from github_events where cast(v["repo"]["name"] as string) = 'xpressengine/xe-core' order by 1;""" qt_sql """select * from github_events where cast(v["repo"]["name"] as string) = 'xpressengine/xe-core' order by 1 limit 10""" diff --git a/regression-test/suites/variant_p0/with_index/var_index.groovy b/regression-test/suites/variant_p0/with_index/var_index.groovy index bea90f7403b45b..8c7afaa4a26a8f 100644 --- a/regression-test/suites/variant_p0/with_index/var_index.groovy +++ b/regression-test/suites/variant_p0/with_index/var_index.groovy @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -suite("regression_test_variant_var_index", "variant_type"){ +suite("regression_test_variant_var_index", "p0"){ def table_name = "var_index" sql "DROP TABLE IF EXISTS var_index" sql """ @@ -36,4 +36,10 @@ suite("regression_test_variant_var_index", "variant_type"){ qt_sql """select * from var_index where cast(v["a"] as smallint) > 123 and cast(v["b"] as string) match 'hello' and cast(v["c"] as int) > 1024 order by k""" sql """insert into var_index values(5, '{"a" : 123456789, "b" : 123456, "c" : 8181111}')""" qt_sql """select * from var_index where cast(v["a"] as int) > 123 and cast(v["b"] as string) match 'hello' and cast(v["c"] as int) > 11111 order by k""" + // insert double/float/array/json + sql """insert into var_index values(6, '{"timestamp": 1713283200.060359}')""" + sql """insert into var_index values(7, '{"timestamp": 17.0}')""" + sql """insert into var_index values(8, '{"timestamp": [123]}')""" + sql """insert into var_index values(9, '{"timestamp": 17.0}'),(10, '{"timestamp": "17.0"}')""" + qt_sql "select * from var_index order by k limit 10" } \ No newline at end of file