From aa17b10249b2740312962a2861a457ab04bad90a Mon Sep 17 00:00:00 2001 From: amorynan Date: Fri, 1 Nov 2024 12:35:37 +0800 Subject: [PATCH 1/9] fix ipv6 with rowstore --- .../data_types/serde/data_type_ipv6_serde.cpp | 5 +++- .../data_types/serde/data_type_serde_test.cpp | 29 +++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/be/src/vec/data_types/serde/data_type_ipv6_serde.cpp b/be/src/vec/data_types/serde/data_type_ipv6_serde.cpp index f09b6feb4a25ed..6d3e64e817b767 100644 --- a/be/src/vec/data_types/serde/data_type_ipv6_serde.cpp +++ b/be/src/vec/data_types/serde/data_type_ipv6_serde.cpp @@ -87,7 +87,10 @@ void DataTypeIPv6SerDe::write_one_cell_to_jsonb(const IColumn& column, result.writeKey(col_id); IPv6 data = assert_cast(column).get_element(row_num); IPv6Value ipv6_value(data); - result.writeString(ipv6_value.to_string()); + std::string ipv6_str = ipv6_value.to_string(); + result.writeStartString(); + result.writeString(ipv6_str.c_str(), ipv6_str.size()); + result.writeEndString(); } Status DataTypeIPv6SerDe::serialize_one_cell_to_json(const IColumn& column, int row_num, diff --git a/be/test/vec/data_types/serde/data_type_serde_test.cpp b/be/test/vec/data_types/serde/data_type_serde_test.cpp index ef235254db5136..e55409297e3c9e 100644 --- a/be/test/vec/data_types/serde/data_type_serde_test.cpp +++ b/be/test/vec/data_types/serde/data_type_serde_test.cpp @@ -220,4 +220,33 @@ TEST(DataTypeSerDeTest, DataTypeScalaSerDeTest) { serialize_and_deserialize_pb_test(); } +TEST(DataTypeSerDeTest, DataTypeRowStoreSerDeTest) { + // ipv6 + { + std::string ip = "5be8:dde9:7f0b:d5a7:bd01:b3be:9c69:573b"; + auto vec = vectorized::ColumnVector::create(); + IPv6Value ipv6; + EXPECT_TRUE(ipv6.from_string(ip)); + vec->insert(ipv6.value()); + vectorized::DataTypePtr data_type(std::make_shared()); + auto serde = data_type->get_serde(0); + JsonbWriterT jsonb_writer; + Arena pool; + serde->write_one_cell_to_jsonb(*vec, jsonb_writer, &pool, 0, 0); + jsonb_writer.writeStartObject(); + std::string jsonb_str = jsonb_writer.getOutput()->getBuffer(); + jsonb_writer.writeEndObject(); + auto jsonb_column = ColumnString::create(); + jsonb_column->insert_data(jsonb_writer.getOutput()->getBuffer(), + jsonb_writer.getOutput()->getSize()); + StringRef jsonb_data = jsonb_column->get_data_at(0); + auto* val = JsonbDocument::createValue(jsonb_data.data, jsonb_data.size); + serde->read_one_cell_from_jsonb(*vec, val); + EXPECT_TRUE(vec->size() == 2); + IPv6 data = vec->get_element(1); + IPv6Value ipv6_value(data); + EXPECT_EQ(ipv6_value.to_string(), ip); + } +} + } // namespace doris::vectorized From cf0ea2d3eec077f2ea02807e880b5110b393a9e6 Mon Sep 17 00:00:00 2001 From: amorynan Date: Fri, 1 Nov 2024 16:21:51 +0800 Subject: [PATCH 2/9] add regress tests --- .../data/datatype_p0/ip/test_ip_basic.out | 12 ++++++++++++ .../suites/datatype_p0/ip/test_ip_basic.groovy | 14 ++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/regression-test/data/datatype_p0/ip/test_ip_basic.out b/regression-test/data/datatype_p0/ip/test_ip_basic.out index 14ecbd47a46984..b69f9708a1b1fc 100644 --- a/regression-test/data/datatype_p0/ip/test_ip_basic.out +++ b/regression-test/data/datatype_p0/ip/test_ip_basic.out @@ -361,3 +361,15 @@ ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff 4 121.25.82.29 2620:44:a000::1 121.25.160.80 2001:418:0:5000::c2d +-- !sql -- +1 true 255.255.255.255 5be8:dde9:7f0b:d5a7:bd01:b3be:9c69:573b + +-- !sql -- +1 false 255.255.255.255 5be8:dde9:7f0b:d5a7:bd01:b3be:9c69:573b + +-- !sql -- +1 false 127.0.0.1 5be8:dde9:7f0b:d5a7:bd01:b3be:9c69:573b + +-- !sql -- +1 false 127.0.0.1 ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff + diff --git a/regression-test/suites/datatype_p0/ip/test_ip_basic.groovy b/regression-test/suites/datatype_p0/ip/test_ip_basic.groovy index 650dc86ad4e354..9e96496b6a0aa4 100644 --- a/regression-test/suites/datatype_p0/ip/test_ip_basic.groovy +++ b/regression-test/suites/datatype_p0/ip/test_ip_basic.groovy @@ -146,4 +146,18 @@ suite("test_ip_basic") { sql "DROP TABLE t0" sql "DROP TABLE t1" sql "DROP TABLE t2" + + // test ip with rowstore + sql """ SET enable_nereids_planner=true """ + sql """ SET enable_fallback_to_original_planner=false """ + sql """ DROP TABLE IF EXISTS table_ip """ + sql """ CREATE TABLE IF NOT EXISTS `table_ip` (`col0` bigint NOT NULL,`col1` boolean NOT NULL, `col24` ipv4 NOT NULL, `col25` ipv6 NOT NULL,INDEX col1 (`col1`) USING INVERTED, INDEX col25 (`col25`) USING INVERTED ) ENGINE=OLAP UNIQUE KEY(`col0`) DISTRIBUTED BY HASH(`col0`) BUCKETS 4 PROPERTIES ("replication_allocation" = "tag.location.default: 1") """ + sql """ insert into table_ip values (1, true, '255.255.255.255', "5be8:dde9:7f0b:d5a7:bd01:b3be:9c69:573b") """ + qt_sql """ select * from table_ip """ + sql """ Update table_ip set col1 = false where col0 = 1 """ + qt_sql """ select * from table_ip """ + sql """ Update table_ip set col24 = '127.0.0.1' where col0 = 1 """ + qt_sql """ select * from table_ip """ + sql """ Update table_ip set col25 = 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff' where col0 = 1 """ + qt_sql """ select * from table_ip """ } From b40e54ad23ee7c69900cdc5dbd749ff157860ddb Mon Sep 17 00:00:00 2001 From: amory Date: Mon, 4 Nov 2024 11:46:52 +0800 Subject: [PATCH 3/9] add "store_row_column" = "true" --- regression-test/suites/datatype_p0/ip/test_ip_basic.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regression-test/suites/datatype_p0/ip/test_ip_basic.groovy b/regression-test/suites/datatype_p0/ip/test_ip_basic.groovy index 9e96496b6a0aa4..d8aeda9a6f2b97 100644 --- a/regression-test/suites/datatype_p0/ip/test_ip_basic.groovy +++ b/regression-test/suites/datatype_p0/ip/test_ip_basic.groovy @@ -151,7 +151,7 @@ suite("test_ip_basic") { sql """ SET enable_nereids_planner=true """ sql """ SET enable_fallback_to_original_planner=false """ sql """ DROP TABLE IF EXISTS table_ip """ - sql """ CREATE TABLE IF NOT EXISTS `table_ip` (`col0` bigint NOT NULL,`col1` boolean NOT NULL, `col24` ipv4 NOT NULL, `col25` ipv6 NOT NULL,INDEX col1 (`col1`) USING INVERTED, INDEX col25 (`col25`) USING INVERTED ) ENGINE=OLAP UNIQUE KEY(`col0`) DISTRIBUTED BY HASH(`col0`) BUCKETS 4 PROPERTIES ("replication_allocation" = "tag.location.default: 1") """ + sql """ CREATE TABLE IF NOT EXISTS `table_ip` (`col0` bigint NOT NULL,`col1` boolean NOT NULL, `col24` ipv4 NOT NULL, `col25` ipv6 NOT NULL,INDEX col1 (`col1`) USING INVERTED, INDEX col25 (`col25`) USING INVERTED ) ENGINE=OLAP UNIQUE KEY(`col0`) DISTRIBUTED BY HASH(`col0`) BUCKETS 4 PROPERTIES ("replication_allocation" = "tag.location.default: 1", "store_row_column" = "true") """ sql """ insert into table_ip values (1, true, '255.255.255.255', "5be8:dde9:7f0b:d5a7:bd01:b3be:9c69:573b") """ qt_sql """ select * from table_ip """ sql """ Update table_ip set col1 = false where col0 = 1 """ From 5d1445194241f42118e9a4bce8a2032d4e2e66f2 Mon Sep 17 00:00:00 2001 From: amorynan Date: Mon, 4 Nov 2024 15:43:34 +0800 Subject: [PATCH 4/9] update ipvalue with int128 --- .../http/action/compaction_score_action.cpp | 4 ++-- .../data_types/serde/data_type_ipv6_serde.cpp | 19 ++++--------------- .../vec/exec/format/json/new_json_reader.cpp | 5 +++++ .../datatype_p0/ip/test_ip_basic.groovy | 4 ++-- 4 files changed, 13 insertions(+), 19 deletions(-) diff --git a/be/src/http/action/compaction_score_action.cpp b/be/src/http/action/compaction_score_action.cpp index 10b8cc6bdbab04..8c761f0951d369 100644 --- a/be/src/http/action/compaction_score_action.cpp +++ b/be/src/http/action/compaction_score_action.cpp @@ -62,8 +62,8 @@ constexpr std::string_view TABLET_ID = "tablet_id"; template concept CompactionScoreAccessble = requires(T t) { - { t.get_real_compaction_score() } -> std::same_as; -}; + { t.get_real_compaction_score() } -> std::same_as; + }; template std::vector calculate_compaction_scores( diff --git a/be/src/vec/data_types/serde/data_type_ipv6_serde.cpp b/be/src/vec/data_types/serde/data_type_ipv6_serde.cpp index 6d3e64e817b767..f9a692fc541f9b 100644 --- a/be/src/vec/data_types/serde/data_type_ipv6_serde.cpp +++ b/be/src/vec/data_types/serde/data_type_ipv6_serde.cpp @@ -69,28 +69,17 @@ Status DataTypeIPv6SerDe::write_column_to_mysql(const IColumn& column, } void DataTypeIPv6SerDe::read_one_cell_from_jsonb(IColumn& column, const JsonbValue* arg) const { - IPv6 val = 0; - const auto* str_value = static_cast(arg); - ReadBuffer rb(reinterpret_cast(str_value->getBlob()), - str_value->getBlobLen()); - if (!read_ipv6_text_impl(val, rb)) { - throw doris::Exception(ErrorCode::INVALID_ARGUMENT, "parse ipv6 fail, string: '{}'", - rb.to_string()); - } - assert_cast(column).insert_value(val); + const auto ip_value = static_cast(arg)->val(); + const IPv6* ip_val = reinterpret_cast(&ip_value); + assert_cast(column).insert_value(*ip_val); } void DataTypeIPv6SerDe::write_one_cell_to_jsonb(const IColumn& column, JsonbWriterT& result, Arena* mem_pool, int col_id, int row_num) const { - // we make ipv6 as string in jsonb result.writeKey(col_id); IPv6 data = assert_cast(column).get_element(row_num); - IPv6Value ipv6_value(data); - std::string ipv6_str = ipv6_value.to_string(); - result.writeStartString(); - result.writeString(ipv6_str.c_str(), ipv6_str.size()); - result.writeEndString(); + result.writeInt128(int128_t(data)); } Status DataTypeIPv6SerDe::serialize_one_cell_to_json(const IColumn& column, int row_num, diff --git a/be/src/vec/exec/format/json/new_json_reader.cpp b/be/src/vec/exec/format/json/new_json_reader.cpp index 4db2d62b9949e7..7dfc3c528cd88e 100644 --- a/be/src/vec/exec/format/json/new_json_reader.cpp +++ b/be/src/vec/exec/format/json/new_json_reader.cpp @@ -333,6 +333,11 @@ Status NewJsonReader::get_parsed_schema(std::vector* col_names, objectValue = _json_doc; } + if (!objectValue->IsObject()) { + return Status::DataQualityError("JSON data is not an object. but: {}", + objectValue->GetType()); + } + // use jsonpaths to col_names if (!_parsed_jsonpaths.empty()) { for (auto& _parsed_jsonpath : _parsed_jsonpaths) { diff --git a/regression-test/suites/datatype_p0/ip/test_ip_basic.groovy b/regression-test/suites/datatype_p0/ip/test_ip_basic.groovy index d8aeda9a6f2b97..468b6f6f146e14 100644 --- a/regression-test/suites/datatype_p0/ip/test_ip_basic.groovy +++ b/regression-test/suites/datatype_p0/ip/test_ip_basic.groovy @@ -157,7 +157,7 @@ suite("test_ip_basic") { sql """ Update table_ip set col1 = false where col0 = 1 """ qt_sql """ select * from table_ip """ sql """ Update table_ip set col24 = '127.0.0.1' where col0 = 1 """ - qt_sql """ select * from table_ip """ + qt_sql """ select * from table_ip where col0 = 1""" sql """ Update table_ip set col25 = 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff' where col0 = 1 """ - qt_sql """ select * from table_ip """ + qt_sql """ select * from table_ip where col0 = 1""" } From 4d3a8896d9864fa7ecdd583d8d9830013ddcd238 Mon Sep 17 00:00:00 2001 From: amorynan Date: Mon, 4 Nov 2024 15:45:16 +0800 Subject: [PATCH 5/9] delete file --- be/src/vec/exec/format/json/new_json_reader.cpp | 5 ----- 1 file changed, 5 deletions(-) diff --git a/be/src/vec/exec/format/json/new_json_reader.cpp b/be/src/vec/exec/format/json/new_json_reader.cpp index 7dfc3c528cd88e..4db2d62b9949e7 100644 --- a/be/src/vec/exec/format/json/new_json_reader.cpp +++ b/be/src/vec/exec/format/json/new_json_reader.cpp @@ -333,11 +333,6 @@ Status NewJsonReader::get_parsed_schema(std::vector* col_names, objectValue = _json_doc; } - if (!objectValue->IsObject()) { - return Status::DataQualityError("JSON data is not an object. but: {}", - objectValue->GetType()); - } - // use jsonpaths to col_names if (!_parsed_jsonpaths.empty()) { for (auto& _parsed_jsonpath : _parsed_jsonpaths) { From e64765c61f8da74d8540154c76eb866ee8b738b1 Mon Sep 17 00:00:00 2001 From: amorynan Date: Mon, 4 Nov 2024 15:48:29 +0800 Subject: [PATCH 6/9] format file --- be/src/http/action/compaction_score_action.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/be/src/http/action/compaction_score_action.cpp b/be/src/http/action/compaction_score_action.cpp index 8c761f0951d369..10b8cc6bdbab04 100644 --- a/be/src/http/action/compaction_score_action.cpp +++ b/be/src/http/action/compaction_score_action.cpp @@ -62,8 +62,8 @@ constexpr std::string_view TABLET_ID = "tablet_id"; template concept CompactionScoreAccessble = requires(T t) { - { t.get_real_compaction_score() } -> std::same_as; - }; + { t.get_real_compaction_score() } -> std::same_as; +}; template std::vector calculate_compaction_scores( From bc2ccedd85ae504b445198fa85546f2a0daa5166 Mon Sep 17 00:00:00 2001 From: amorynan Date: Mon, 4 Nov 2024 18:48:27 +0800 Subject: [PATCH 7/9] fix ut for ipv6 serde --- be/test/vec/data_types/serde/data_type_serde_test.cpp | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/be/test/vec/data_types/serde/data_type_serde_test.cpp b/be/test/vec/data_types/serde/data_type_serde_test.cpp index e55409297e3c9e..04798020f67122 100644 --- a/be/test/vec/data_types/serde/data_type_serde_test.cpp +++ b/be/test/vec/data_types/serde/data_type_serde_test.cpp @@ -228,20 +228,23 @@ TEST(DataTypeSerDeTest, DataTypeRowStoreSerDeTest) { IPv6Value ipv6; EXPECT_TRUE(ipv6.from_string(ip)); vec->insert(ipv6.value()); + vectorized::DataTypePtr data_type(std::make_shared()); auto serde = data_type->get_serde(0); JsonbWriterT jsonb_writer; Arena pool; - serde->write_one_cell_to_jsonb(*vec, jsonb_writer, &pool, 0, 0); jsonb_writer.writeStartObject(); - std::string jsonb_str = jsonb_writer.getOutput()->getBuffer(); + serde->write_one_cell_to_jsonb(*vec, jsonb_writer, &pool, 0, 0); jsonb_writer.writeEndObject(); auto jsonb_column = ColumnString::create(); jsonb_column->insert_data(jsonb_writer.getOutput()->getBuffer(), jsonb_writer.getOutput()->getSize()); StringRef jsonb_data = jsonb_column->get_data_at(0); - auto* val = JsonbDocument::createValue(jsonb_data.data, jsonb_data.size); - serde->read_one_cell_from_jsonb(*vec, val); + auto pdoc = JsonbDocument::createDocument(jsonb_data.data, jsonb_data.size); + JsonbDocument& doc = *pdoc; + for (auto it = doc->begin(); it != doc->end(); ++it) { + serde->read_one_cell_from_jsonb(*vec, it->value()); + } EXPECT_TRUE(vec->size() == 2); IPv6 data = vec->get_element(1); IPv6Value ipv6_value(data); From 77e21a0e0d1aed1bd0a9a2e97f8824ee98ff5abf Mon Sep 17 00:00:00 2001 From: amorynan Date: Mon, 4 Nov 2024 22:04:59 +0800 Subject: [PATCH 8/9] fix ipserde --- .../data_types/serde/data_type_ipv4_serde.cpp | 20 +++++++++++++ .../data_types/serde/data_type_ipv4_serde.h | 3 ++ .../data_types/serde/data_type_ipv6_serde.cpp | 14 +++++---- .../data_types/serde/data_type_serde_test.cpp | 30 +++++++++++++++++++ 4 files changed, 62 insertions(+), 5 deletions(-) diff --git a/be/src/vec/data_types/serde/data_type_ipv4_serde.cpp b/be/src/vec/data_types/serde/data_type_ipv4_serde.cpp index a2661fa203680a..e503b9328442b2 100644 --- a/be/src/vec/data_types/serde/data_type_ipv4_serde.cpp +++ b/be/src/vec/data_types/serde/data_type_ipv4_serde.cpp @@ -92,6 +92,26 @@ Status DataTypeIPv4SerDe::deserialize_one_cell_from_json(IColumn& column, Slice& return Status::OK(); } +void DataTypeIPv4SerDe::read_one_cell_from_jsonb(IColumn& column, const JsonbValue* arg) const { + if (arg->isInt32()) { + // old value format + assert_cast(column).insert_value( + static_cast(arg)->val()); + return; + } + const auto jsonb_ipv4 = static_cast(arg)->val(); + const IPv4* ip_val = reinterpret_cast(&jsonb_ipv4); + assert_cast(column).insert_value(*ip_val); +} + +void DataTypeIPv4SerDe::write_one_cell_to_jsonb(const IColumn& column, + JsonbWriterT& result, + Arena* mem_pool, int col_id, int row_num) const { + result.writeKey(col_id); + IPv4 data = assert_cast(column).get_element(row_num); + result.writeInt64(int64_t(data)); +} + Status DataTypeIPv4SerDe::write_column_to_pb(const IColumn& column, PValues& result, int start, int end) const { const auto& column_data = assert_cast(column).get_data(); diff --git a/be/src/vec/data_types/serde/data_type_ipv4_serde.h b/be/src/vec/data_types/serde/data_type_ipv4_serde.h index 65bd2469b5bbb2..4b8c139a4d793a 100644 --- a/be/src/vec/data_types/serde/data_type_ipv4_serde.h +++ b/be/src/vec/data_types/serde/data_type_ipv4_serde.h @@ -60,6 +60,9 @@ class DataTypeIPv4SerDe : public DataTypeNumberSerDe { const cctz::time_zone& ctz) const override; void read_column_from_arrow(IColumn& column, const arrow::Array* arrow_array, int start, int end, const cctz::time_zone& ctz) const override; + void read_one_cell_from_jsonb(IColumn& column, const JsonbValue* arg) const override; + void write_one_cell_to_jsonb(const IColumn& column, JsonbWriterT& result, + Arena* mem_pool, int unique_id, int row_num) const override; private: template diff --git a/be/src/vec/data_types/serde/data_type_ipv6_serde.cpp b/be/src/vec/data_types/serde/data_type_ipv6_serde.cpp index f9a692fc541f9b..612c9ce42227dd 100644 --- a/be/src/vec/data_types/serde/data_type_ipv6_serde.cpp +++ b/be/src/vec/data_types/serde/data_type_ipv6_serde.cpp @@ -69,17 +69,21 @@ Status DataTypeIPv6SerDe::write_column_to_mysql(const IColumn& column, } void DataTypeIPv6SerDe::read_one_cell_from_jsonb(IColumn& column, const JsonbValue* arg) const { - const auto ip_value = static_cast(arg)->val(); - const IPv6* ip_val = reinterpret_cast(&ip_value); - assert_cast(column).insert_value(*ip_val); + const auto* str_value = static_cast(arg); + column.deserialize_and_insert_from_arena(str_value->getBlob()); } void DataTypeIPv6SerDe::write_one_cell_to_jsonb(const IColumn& column, JsonbWriterT& result, Arena* mem_pool, int col_id, int row_num) const { + // we make ipv6 as BinaryValue in jsonb result.writeKey(col_id); - IPv6 data = assert_cast(column).get_element(row_num); - result.writeInt128(int128_t(data)); + const char* begin = nullptr; + // maybe serialize_value_into_arena should move to here later. + StringRef value = column.serialize_value_into_arena(row_num, *mem_pool, begin); + result.writeStartBinary(); + result.writeBinary(value.data, value.size); + result.writeEndBinary(); } Status DataTypeIPv6SerDe::serialize_one_cell_to_json(const IColumn& column, int row_num, diff --git a/be/test/vec/data_types/serde/data_type_serde_test.cpp b/be/test/vec/data_types/serde/data_type_serde_test.cpp index 04798020f67122..82674b0aa44762 100644 --- a/be/test/vec/data_types/serde/data_type_serde_test.cpp +++ b/be/test/vec/data_types/serde/data_type_serde_test.cpp @@ -250,6 +250,36 @@ TEST(DataTypeSerDeTest, DataTypeRowStoreSerDeTest) { IPv6Value ipv6_value(data); EXPECT_EQ(ipv6_value.to_string(), ip); } + + // ipv4 + { + std::string ip = "192.0.0.1"; + auto vec = vectorized::ColumnVector::create(); + IPv4Value ipv4; + EXPECT_TRUE(ipv4.from_string(ip)); + vec->insert(ipv4.value()); + + vectorized::DataTypePtr data_type(std::make_shared()); + auto serde = data_type->get_serde(0); + JsonbWriterT jsonb_writer; + Arena pool; + jsonb_writer.writeStartObject(); + serde->write_one_cell_to_jsonb(*vec, jsonb_writer, &pool, 0, 0); + jsonb_writer.writeEndObject(); + auto jsonb_column = ColumnString::create(); + jsonb_column->insert_data(jsonb_writer.getOutput()->getBuffer(), + jsonb_writer.getOutput()->getSize()); + StringRef jsonb_data = jsonb_column->get_data_at(0); + auto pdoc = JsonbDocument::createDocument(jsonb_data.data, jsonb_data.size); + JsonbDocument& doc = *pdoc; + for (auto it = doc->begin(); it != doc->end(); ++it) { + serde->read_one_cell_from_jsonb(*vec, it->value()); + } + EXPECT_TRUE(vec->size() == 2); + IPv4 data = vec->get_element(1); + IPv4Value ipv4_value(data); + EXPECT_EQ(ipv4_value.to_string(), ip); + } } } // namespace doris::vectorized From 48311fd4c772dbde480c3b921e8e198f7275a4fa Mon Sep 17 00:00:00 2001 From: amorynan Date: Tue, 5 Nov 2024 10:09:23 +0800 Subject: [PATCH 9/9] update ip --- .../data_types/serde/data_type_ipv4_serde.cpp | 20 ------------------- .../data_types/serde/data_type_ipv4_serde.h | 3 --- 2 files changed, 23 deletions(-) diff --git a/be/src/vec/data_types/serde/data_type_ipv4_serde.cpp b/be/src/vec/data_types/serde/data_type_ipv4_serde.cpp index e503b9328442b2..a2661fa203680a 100644 --- a/be/src/vec/data_types/serde/data_type_ipv4_serde.cpp +++ b/be/src/vec/data_types/serde/data_type_ipv4_serde.cpp @@ -92,26 +92,6 @@ Status DataTypeIPv4SerDe::deserialize_one_cell_from_json(IColumn& column, Slice& return Status::OK(); } -void DataTypeIPv4SerDe::read_one_cell_from_jsonb(IColumn& column, const JsonbValue* arg) const { - if (arg->isInt32()) { - // old value format - assert_cast(column).insert_value( - static_cast(arg)->val()); - return; - } - const auto jsonb_ipv4 = static_cast(arg)->val(); - const IPv4* ip_val = reinterpret_cast(&jsonb_ipv4); - assert_cast(column).insert_value(*ip_val); -} - -void DataTypeIPv4SerDe::write_one_cell_to_jsonb(const IColumn& column, - JsonbWriterT& result, - Arena* mem_pool, int col_id, int row_num) const { - result.writeKey(col_id); - IPv4 data = assert_cast(column).get_element(row_num); - result.writeInt64(int64_t(data)); -} - Status DataTypeIPv4SerDe::write_column_to_pb(const IColumn& column, PValues& result, int start, int end) const { const auto& column_data = assert_cast(column).get_data(); diff --git a/be/src/vec/data_types/serde/data_type_ipv4_serde.h b/be/src/vec/data_types/serde/data_type_ipv4_serde.h index 4b8c139a4d793a..65bd2469b5bbb2 100644 --- a/be/src/vec/data_types/serde/data_type_ipv4_serde.h +++ b/be/src/vec/data_types/serde/data_type_ipv4_serde.h @@ -60,9 +60,6 @@ class DataTypeIPv4SerDe : public DataTypeNumberSerDe { const cctz::time_zone& ctz) const override; void read_column_from_arrow(IColumn& column, const arrow::Array* arrow_array, int start, int end, const cctz::time_zone& ctz) const override; - void read_one_cell_from_jsonb(IColumn& column, const JsonbValue* arg) const override; - void write_one_cell_to_jsonb(const IColumn& column, JsonbWriterT& result, - Arena* mem_pool, int unique_id, int row_num) const override; private: template