[Pick](Variant) pick #33734 #33766 #33707 to branch-2.1

be/src/olap/rowset/segment_v2/column_reader.cpp

@@ -1536,7 +1536,7 @@ Status VariantRootColumnIterator::read_by_rowids(const rowid_t* rowids, const si
     RETURN_IF_ERROR(_inner_iter->read_by_rowids(rowids, count, root_column));
     obj.incr_num_rows(count);
     for (auto& entry : obj.get_subcolumns()) {
-        if (entry->data.size() != size + count) {
+        if (entry->data.size() != (size + count)) {
             entry->data.insertManyDefaults(count);
         }
     }
@@ -1546,7 +1546,8 @@ Status VariantRootColumnIterator::read_by_rowids(const rowid_t* rowids, const si
                 assert_cast<vectorized::ColumnNullable&>(*dst).get_null_map_column();
         vectorized::ColumnUInt8& src_null_map =
                 assert_cast<vectorized::ColumnNullable&>(*root_column).get_null_map_column();
-        dst_null_map.insert_range_from(src_null_map, 0, src_null_map.size());
+        DCHECK_EQ(src_null_map.size() - size, count);
+        dst_null_map.insert_range_from(src_null_map, size, count);
     }
 #ifndef NDEBUG
     obj.check_consistency();

be/src/olap/rowset/segment_v2/inverted_index_writer.h

@@ -24,13 +24,16 @@
 #include <atomic>
 #include <memory>
 #include <string>
+#include <vector>
 
 #include "common/config.h"
 #include "common/status.h"
 #include "gutil/strings/split.h"
 #include "io/fs/file_system.h"
 #include "io/fs/local_file_system.h"
+#include "olap/olap_common.h"
 #include "olap/options.h"
+#include "olap/tablet_schema.h"
 
 namespace doris {
 class CollectionValue;
@@ -70,6 +73,25 @@ class InvertedIndexColumnWriter {
 
     virtual void close_on_error() = 0;
 
+    // check if the column is valid for inverted index, some columns
+    // are generated from variant, but not all of them are supported
+    static bool check_column_valid(const TabletColumn& column) {
+        // bellow types are not supported in inverted index for extracted columns
+        static std::set<FieldType> invalid_types = {
+                FieldType::OLAP_FIELD_TYPE_DOUBLE,
+                FieldType::OLAP_FIELD_TYPE_JSONB,
+                FieldType::OLAP_FIELD_TYPE_ARRAY,
+                FieldType::OLAP_FIELD_TYPE_FLOAT,
+        };
+        if (column.is_extracted_column() && (invalid_types.contains(column.type()))) {
+            return false;
+        }
+        if (column.is_variant_type()) {
+            return false;
+        }
+        return true;
+    }
+
 private:
     DISALLOW_COPY_AND_ASSIGN(InvertedIndexColumnWriter);
 };

be/src/olap/rowset/segment_v2/segment_writer.cpp

@@ -42,6 +42,7 @@
 #include "olap/rowset/rowset_writer_context.h"    // RowsetWriterContext
 #include "olap/rowset/segment_v2/column_writer.h" // ColumnWriter
 #include "olap/rowset/segment_v2/inverted_index_file_writer.h"
+#include "olap/rowset/segment_v2/inverted_index_writer.h"
 #include "olap/rowset/segment_v2/page_io.h"
 #include "olap/rowset/segment_v2/page_pointer.h"
 #include "olap/segment_loader.h"
@@ -224,9 +225,8 @@ Status SegmentWriter::init(const std::vector<uint32_t>& col_ids, bool has_key) {
         }
         // indexes for this column
         opts.indexes = std::move(_tablet_schema->get_indexes_for_column(column));
-        if (column.is_variant_type() || (column.is_extracted_column() && column.is_jsonb_type()) ||
-            (column.is_extracted_column() && column.is_array_type())) {
-            // variant and jsonb type skip write index
+        if (!InvertedIndexColumnWriter::check_column_valid(column)) {
+            // skip inverted index if invalid
             opts.indexes.clear();
             opts.need_zone_map = false;
             opts.need_bloom_filter = false;

be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp

@@ -176,9 +176,8 @@ Status VerticalSegmentWriter::_create_column_writer(uint32_t cid, const TabletCo
     }
     // indexes for this column
     opts.indexes = _tablet_schema->get_indexes_for_column(column);
-    if (column.is_variant_type() || (column.is_extracted_column() && column.is_jsonb_type()) ||
-        (column.is_extracted_column() && column.is_array_type())) {
-        // variant and jsonb type skip write index
+    if (!InvertedIndexColumnWriter::check_column_valid(column)) {
+        // skip inverted index if invalid
         opts.indexes.clear();
         opts.need_zone_map = false;
         opts.need_bloom_filter = false;

be/src/olap/task/index_builder.cpp

@@ -346,6 +346,9 @@ Status IndexBuilder::handle_single_rowset(RowsetMetaSharedPtr output_rowset_meta
                     continue;
                 }
                 auto column = output_rowset_schema->column(column_idx);
+                if (!InvertedIndexColumnWriter::check_column_valid(column)) {
+                    continue;
+                }
                 DCHECK(output_rowset_schema->has_inverted_index_with_index_id(index_id, ""));
                 _olap_data_convertor->add_column_data_convertor(column);
                 return_columns.emplace_back(column_idx);

fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java

@@ -2110,6 +2110,12 @@ private void createOlapTable(Database db, CreateTableStmt stmt) throws UserExcep
         Preconditions.checkNotNull(distributionDesc);
         DistributionInfo defaultDistributionInfo = distributionDesc.toDistributionInfo(baseSchema);
 
+        if (defaultDistributionInfo instanceof HashDistributionInfo
+                    && ((HashDistributionInfo) defaultDistributionInfo).getDistributionColumns()
+                        .stream().anyMatch(column -> column.getType().isVariantType())) {
+            throw new DdlException("Hash distribution info should not contain variant columns");
+        }
+
         // calc short key column count
         short shortKeyColumnCount = Env.calcShortKeyColumnCount(baseSchema, stmt.getProperties(), isKeysRequired);
         if (LOG.isDebugEnabled()) {

regression-test/data/variant_p0/topn_opt_read_by_rowids.out

@@ -0,0 +1,14 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !sql --
+2024-04-09T09:01:39					1712624474952	0004.00	740lp	sit-iniwork-lcd-designer.qm.cn		\N	\N		630beed604d0513c	0000.0	\N					https://sit-iniwork-lcd-designer.qm.cn/designer	1544512389907021826	https://sit-iniwork-lcd-designer.qm.cn/designer?caseName=44&caseCode=d4&appId=3fec598d32b10d26958d1d9119519c64&token=eyJhbGciOiJIUzUxMiJ9.eyJzdWIiOiI3NDBscCIsImNyZWF0ZWQiOjE3MTI1NDEzMTU0MzUsImlkbWlkIjpudWxsLCJleHAiOjE3MTMxNDYxMTUsInVwa2lkIjoiMTU0NDUxMjM4OTkwNzAyMTgyNiJ9.j8sJbrgrJzwPh5Ee1AoodqIJ7KsMT_qfUhZlyon6rRsMClRI5WpUYpSWkC7P-axolUUnWJi2Llw89hO_KyL_gg&formId=8f4f75e0c0aa93fcd0a4eae843114527&tenantId=YQJT&systemId=202203A-015&subType=0				202203A-015_APP_DES	20240409090114952	未知	{"browser":"Chrome 123.0.0.0","browserFlag":"a8ba48f1849c2423","browserName":"Chrome","browserVersion":"123.0.0.0","clientStr":"10.133.53.99","headerClient":"{X-Original-Forwarded-For=10.133.53.99, X-Forwarded-For=10.140.199.9, RemoteAddr=10.133.53.99, X-Real-IP=10.140.199.9}","os":"Windows 10","osName":"Windows","osVersion":"10","res":"1920x1080","type":"Computer"}	1712624474952	1712624475992	\N	\N
+2024-04-09T09:01:39					1712624474952	0004.00	740lp	sit-iniwork-lcd-designer.qm.cn		\N	\N		630beed604d0513c	0000.0	\N					https://sit-iniwork-lcd-designer.qm.cn/designer	1544512389907021826	https://sit-iniwork-lcd-designer.qm.cn/designer?caseName=44&caseCode=d4&appId=3fec598d32b10d26958d1d9119519c64&token=eyJhbGciOiJIUzUxMiJ9.eyJzdWIiOiI3NDBscCIsImNyZWF0ZWQiOjE3MTI1NDEzMTU0MzUsImlkbWlkIjpudWxsLCJleHAiOjE3MTMxNDYxMTUsInVwa2lkIjoiMTU0NDUxMjM4OTkwNzAyMTgyNiJ9.j8sJbrgrJzwPh5Ee1AoodqIJ7KsMT_qfUhZlyon6rRsMClRI5WpUYpSWkC7P-axolUUnWJi2Llw89hO_KyL_gg&formId=8f4f75e0c0aa93fcd0a4eae843114527&tenantId=YQJT&systemId=202203A-015&subType=0				202203A-015_APP_DES	20240409090114952	未知	{"browser":"Chrome 123.0.0.0","browserFlag":"a8ba48f1849c2423","browserName":"Chrome","browserVersion":"123.0.0.0","clientStr":"10.133.53.99","headerClient":"{X-Original-Forwarded-For=10.133.53.99, X-Forwarded-For=10.140.199.9, RemoteAddr=10.133.53.99, X-Real-IP=10.140.199.9}","os":"Windows 10","osName":"Windows","osVersion":"10","res":"1920x1080","type":"Computer"}	1712624474952	1712624475992	\N	\N
+2024-04-09T09:01:39					1712624474952	0004.00	740lp	sit-iniwork-lcd-designer.qm.cn		\N	\N		630beed604d0513c	0000.0	\N					https://sit-iniwork-lcd-designer.qm.cn/designer	1544512389907021826	https://sit-iniwork-lcd-designer.qm.cn/designer?caseName=44&caseCode=d4&appId=3fec598d32b10d26958d1d9119519c64&token=eyJhbGciOiJIUzUxMiJ9.eyJzdWIiOiI3NDBscCIsImNyZWF0ZWQiOjE3MTI1NDEzMTU0MzUsImlkbWlkIjpudWxsLCJleHAiOjE3MTMxNDYxMTUsInVwa2lkIjoiMTU0NDUxMjM4OTkwNzAyMTgyNiJ9.j8sJbrgrJzwPh5Ee1AoodqIJ7KsMT_qfUhZlyon6rRsMClRI5WpUYpSWkC7P-axolUUnWJi2Llw89hO_KyL_gg&formId=8f4f75e0c0aa93fcd0a4eae843114527&tenantId=YQJT&systemId=202203A-015&subType=0				202203A-015_APP_DES	20240409090114952	未知	{"browser":"Chrome 123.0.0.0","browserFlag":"a8ba48f1849c2423","browserName":"Chrome","browserVersion":"123.0.0.0","clientStr":"10.133.53.99","headerClient":"{X-Original-Forwarded-For=10.133.53.99, X-Forwarded-For=10.140.199.9, RemoteAddr=10.133.53.99, X-Real-IP=10.140.199.9}","os":"Windows 10","osName":"Windows","osVersion":"10","res":"1920x1080","type":"Computer"}	1712624474952	1712624475992	\N	\N
+
+-- !sql --
+2024-04-09T09:02:31					1712624495211	004.00	740lp	sit-iniwork-lcd-designer.qm.cn		\N	\N		630beed604d0513c	0000.0	\N					https://sit-iniwork-lcd-designer.qm.cn/designer	1544512389907021826	https://sit-iniwork-lcd-designer.qm.cn/designer?caseName=44&caseCode=d4&appId=3fec598d32b10d26958d1d9119519c64&token=eyJhbGciOiJIUzUxMiJ9.eyJzdWIiOiI3NDBscCIsImNyZWF0ZWQiOjE3MTI1NDEzMTU0MzUsImlkbWlkIjpudWxsLCJleHAiOjE3MTMxNDYxMTUsInVwa2lkIjoiMTU0NDUxMjM4OTkwNzAyMTgyNiJ9.j8sJbrgrJzwPh5Ee1AoodqIJ7KsMT_qfUhZlyon6rRsMClRI5WpUYpSWkC7P-axolUUnWJi2Llw89hO_KyL_gg&formId=8f4f75e0c0aa93fcd0a4eae843114527&tenantId=YQJT&systemId=202203A-015&subType=0				202203A-015_APP_DES	20240409090114952	未知	{"browser1":"Chrome 123.0.0.0","browserFlag1":"a8ba48f1849c2423","browserName4":"Chrome","browserVersion7":"123.0.0.0","clientStr":"10.133.53.99","headerClient":"{X-Original-Forwarded-For=10.033.53.19, X-Forwarded-For=10.140.199.9, RemoteAddr=10.133.53.99, X-Real-IP=10.140.199.9}","os":"Windows 10","osName":"Windows","osVersion":"10","res":"1920x1080","type":"Computer"}	1712624474952	1712624475992	\N	\N
+2024-04-09T09:01:39					1712624474952	0004.00	740lp	sit-iniwork-lcd-designer.qm.cn		\N	\N		630beed604d0513c	0000.0	\N					https://sit-iniwork-lcd-designer.qm.cn/designer	1544512389907021826	https://sit-iniwork-lcd-designer.qm.cn/designer?caseName=44&caseCode=d4&appId=3fec598d32b10d26958d1d9119519c64&token=eyJhbGciOiJIUzUxMiJ9.eyJzdWIiOiI3NDBscCIsImNyZWF0ZWQiOjE3MTI1NDEzMTU0MzUsImlkbWlkIjpudWxsLCJleHAiOjE3MTMxNDYxMTUsInVwa2lkIjoiMTU0NDUxMjM4OTkwNzAyMTgyNiJ9.j8sJbrgrJzwPh5Ee1AoodqIJ7KsMT_qfUhZlyon6rRsMClRI5WpUYpSWkC7P-axolUUnWJi2Llw89hO_KyL_gg&formId=8f4f75e0c0aa93fcd0a4eae843114527&tenantId=YQJT&systemId=202203A-015&subType=0				202203A-015_APP_DES	20240409090114952	未知	{"browser":"Chrome 123.0.0.0","browserFlag":"a8ba48f1849c2423","browserName":"Chrome","browserVersion":"123.0.0.0","clientStr":"10.133.53.99","headerClient":"{X-Original-Forwarded-For=10.133.53.99, X-Forwarded-For=10.140.199.9, RemoteAddr=10.133.53.99, X-Real-IP=10.140.199.9}","os":"Windows 10","osName":"Windows","osVersion":"10","res":"1920x1080","type":"Computer"}	1712624474952	1712624475992	\N	\N
+2024-04-09T09:03:32					1712624474153	0014.00	740lp	sit-iniwork-lcd-designer.qm.cn		\N	\N		630beed604d0513c	0000.0	\N					https://sit-iniwork-lcd-designer.qm.cn/designer	1544512389907021826	https://sit-iniwork-lcd-designer.qm.cn/designer?caseName=44&caseCode=d4&appId=3fec598d32b10d26958d1d9119519c64&token=eyJhbGciOiJIUzUxMiJ9.eyJzdWIiOiI3NDBscCIsImNyZWF0ZWQiOjE3MTI1NDEzMTU0MzUsImlkbWlkIjpudWxsLCJleHAiOjE3MTMxNDYxMTUsInVwa2lkIjoiMTU0NDUxMjM4OTkwNzAyMTgyNiJ9.j8sJbrgrJzwPh5Ee1AoodqIJ7KsMT_qfUhZlyon6rRsMClRI5WpUYpSWkC7P-axolUUnWJi2Llw89hO_KyL_gg&formId=8f4f75e0c0aa93fcd0a4eae843114527&tenantId=YQJT&systemId=202203A-015&subType=0				202203A-015_APP_DES	20240409090114952	未知	{"browser2":"Chrome 123.0.0.0","browserFlag2":"a8ba48f1849c2423","browserName5":"Chrome","browserVersion8":"123.0.0.0","clientStr":"10.133.53.99","headerClient":"{X-Original-Forwarded-For=10.133.53.99, X-Forwarded-For=10.140.199.9, RemoteAddr=10.133.53.99, X-Real-IP=10.140.199.9}","os":"Windows 10","osName":"Windows","osVersion":"10","res":"1920x1080","type":"Computer"}	1712624474952	1712624475992	\N	\N
+2024-04-09T09:04:33					1712624474959	0024.00	740lp	sit-iniwork-lcd-designer.qm.cn		\N	\N		630beed604d0513c	0000.0	\N					https://sit-iniwork-lcd-designer.qm.cn/designer	1544512389907021826	https://sit-iniwork-lcd-designer.qm.cn/designer?caseName=44&caseCode=d4&appId=3fec598d32b10d26958d1d9119519c64&token=eyJhbGciOiJIUzUxMiJ9.eyJzdWIiOiI3NDBscCIsImNyZWF0ZWQiOjE3MTI1NDEzMTU0MzUsImlkbWlkIjpudWxsLCJleHAiOjE3MTMxNDYxMTUsInVwa2lkIjoiMTU0NDUxMjM4OTkwNzAyMTgyNiJ9.j8sJbrgrJzwPh5Ee1AoodqIJ7KsMT_qfUhZlyon6rRsMClRI5WpUYpSWkC7P-axolUUnWJi2Llw89hO_KyL_gg&formId=8f4f75e0c0aa93fcd0a4eae843114527&tenantId=YQJT&systemId=202203A-015&subType=0				202203A-015_APP_DES	20240409090114952	未知	{"browser3":"Chrome 123.0.0.0","browserFlag3":"a8ba48f1849c2423","browserName6":"Chrome","browserVersion9":"123.0.0.0","clientStr":"10.133.53.99","headerClient":"{X-Original-Forwarded-For=10.133.53.99, X-Forwarded-For=10.140.199.9, RemoteAddr=10.133.53.99, X-Real-IP=10.140.199.9}","os":"Windows 10","osName":"Windows","osVersion":"10","res":"1920x1080","type":"Computer"}	1712624474952	1712624475992	\N	\N
+2024-04-09T09:01:39					1712624474952	0004.00	740lp	sit-iniwork-lcd-designer.qm.cn		\N	\N		630beed604d0513c	0000.0	\N					https://sit-iniwork-lcd-designer.qm.cn/designer	1544512389907021826	https://sit-iniwork-lcd-designer.qm.cn/designer?caseName=44&caseCode=d4&appId=3fec598d32b10d26958d1d9119519c64&token=eyJhbGciOiJIUzUxMiJ9.eyJzdWIiOiI3NDBscCIsImNyZWF0ZWQiOjE3MTI1NDEzMTU0MzUsImlkbWlkIjpudWxsLCJleHAiOjE3MTMxNDYxMTUsInVwa2lkIjoiMTU0NDUxMjM4OTkwNzAyMTgyNiJ9.j8sJbrgrJzwPh5Ee1AoodqIJ7KsMT_qfUhZlyon6rRsMClRI5WpUYpSWkC7P-axolUUnWJi2Llw89hO_KyL_gg&formId=8f4f75e0c0aa93fcd0a4eae843114527&tenantId=YQJT&systemId=202203A-015&subType=0				202203A-015_APP_DES	20240409090114952	未知	{"browser":"Chrome 123.0.0.0","browserFlag":"a8ba48f1849c2423","browserName":"Chrome","browserVersion":"123.0.0.0","clientStr":"10.133.53.99","headerClient":"{X-Original-Forwarded-For=10.133.53.99, X-Forwarded-For=10.140.199.9, RemoteAddr=10.133.53.99, X-Real-IP=10.140.199.9}","os":"Windows 10","osName":"Windows","osVersion":"10","res":"1920x1080","type":"Computer"}	1712624474952	1712624475992	\N	\N
+2024-04-09T09:01:39					1712624474952	0004.00	740lp	sit-iniwork-lcd-designer.qm.cn		\N	\N		630beed604d0513c	0000.0	\N					https://sit-iniwork-lcd-designer.qm.cn/designer	1544512389907021826	https://sit-iniwork-lcd-designer.qm.cn/designer?caseName=44&caseCode=d4&appId=3fec598d32b10d26958d1d9119519c64&token=eyJhbGciOiJIUzUxMiJ9.eyJzdWIiOiI3NDBscCIsImNyZWF0ZWQiOjE3MTI1NDEzMTU0MzUsImlkbWlkIjpudWxsLCJleHAiOjE3MTMxNDYxMTUsInVwa2lkIjoiMTU0NDUxMjM4OTkwNzAyMTgyNiJ9.j8sJbrgrJzwPh5Ee1AoodqIJ7KsMT_qfUhZlyon6rRsMClRI5WpUYpSWkC7P-axolUUnWJi2Llw89hO_KyL_gg&formId=8f4f75e0c0aa93fcd0a4eae843114527&tenantId=YQJT&systemId=202203A-015&subType=0				202203A-015_APP_DES	20240409090114952	未知	{"browser":"Chrome 123.0.0.0","browserFlag":"a8ba48f1849c2423","browserName":"Chrome","browserVersion":"123.0.0.0","clientStr":"10.133.53.99","headerClient":"{X-Original-Forwarded-For=10.133.53.99, X-Forwarded-For=10.140.199.9, RemoteAddr=10.133.53.99, X-Real-IP=10.140.199.9}","os":"Windows 10","osName":"Windows","osVersion":"10","res":"1920x1080","type":"Computer"}	1712624474952	1712624475992	\N	\N
+

regression-test/data/variant_p0/with_index/var_index.out

@@ -8,3 +8,15 @@
 2	{"a":18811,"b":"hello world","c":1181111}
 4	{"a":1234,"b":"hello xxx world","c":8181111}
 
+-- !sql --
+1	{"a":123,"b":"xxxyyy","c":111999111}
+2	{"a":18811,"b":"hello world","c":1181111}
+3	{"a":18811,"b":"hello wworld","c":11111}
+4	{"a":1234,"b":"hello xxx world","c":8181111}
+5	{"a":123456789,"b":123456,"c":8181111}
+6	{"timestamp":1713283200.060359}
+7	{"timestamp":17.0}
+8	{"timestamp":[123]}
+9	{"timestamp":17.0}
+10	{"timestamp":"17.0"}
+