Skip to content

[Fix](Json type) correct cast result for json type #34764

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
eldenmoon merged 3 commits into from
May 17, 2024
Merged

[Fix](Json type) correct cast result for json type #34764

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
3605ae6
[Fix](Json type) correct cast result for json type
eldenmoon May 7, 2024
c86291f
fix test case
eldenmoon May 13, 2024
f33dd13
Merge branch 'master' into json-cast
eldenmoon May 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion be/src/util/jsonb_document.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -498,7 +498,7 @@ class JsonbValue {
bool isNull() const { return (type_ == JsonbType::T_Null); }
bool isTrue() const { return (type_ == JsonbType::T_True); }
bool isFalse() const { return (type_ == JsonbType::T_False); }
bool isInt() const { return isInt8() || isInt16() || isInt32() || isInt64(); }
bool isInt() const { return isInt8() || isInt16() || isInt32() || isInt64() || isInt128(); }
bool isInt8() const { return (type_ == JsonbType::T_Int8); }
bool isInt16() const { return (type_ == JsonbType::T_Int16); }
bool isInt32() const { return (type_ == JsonbType::T_Int32); }
Expand Down
79 changes: 33 additions & 46 deletions be/src/vec/functions/function_cast.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,7 @@
#include "vec/data_types/data_type_struct.h"
#include "vec/data_types/data_type_time.h"
#include "vec/data_types/data_type_time_v2.h"
#include "vec/data_types/serde/data_type_serde.h"
#include "vec/functions/function.h"
#include "vec/functions/function_convert_tz.h"
#include "vec/functions/function_helpers.h"
Expand Down Expand Up @@ -762,29 +763,43 @@ struct ConvertImplGenericToJsonb {
auto column_string = ColumnString::create();
JsonbWriter writer;

ColumnUInt8::MutablePtr col_null_map_to = ColumnUInt8::create(col_from.size());
ColumnUInt8::Container* vec_null_map_to = &col_null_map_to->get_data();
DataTypeSerDe::FormatOptions format_options;
format_options.converted_from_string = true;
DataTypeSerDeSPtr from_serde = type.get_serde();
DataTypeSerDeSPtr to_serde = data_type_to->get_serde();
auto col_to = data_type_to->create_column();

auto tmp_col = ColumnString::create();
vectorized::DataTypeSerDe::FormatOptions options;
for (size_t i = 0; i < input_rows_count; i++) {
// convert to string
tmp_col->clear();
VectorBufferWriter write_buffer(*tmp_col.get());
type.to_string(col_from, i, write_buffer);
Status st =
from_serde->serialize_column_to_json(col_from, i, i + 1, write_buffer, options);
// if serialized failed, will return null
(*vec_null_map_to)[i] = !st.ok();
if (!st.ok()) {
col_to->insert_default();
continue;
}
write_buffer.commit();
writer.reset();
auto str_ref = tmp_col->get_data_at(0);
ReadBuffer read_buffer((char*)(str_ref.data), str_ref.size);
Slice data((char*)(str_ref.data), str_ref.size);
// first try to parse string
Status st = data_type_to->from_string(read_buffer, column_string.get());
st = to_serde->deserialize_one_cell_from_json(*col_to, data, format_options);
// if parsing failed, will return null
(*vec_null_map_to)[i] = !st.ok();
if (!st.ok()) {
// write raw string to jsonb
writer.writeStartString();
writer.writeString(str_ref.data, str_ref.size);
writer.writeEndString();
column_string->insert_data(writer.getOutput()->getBuffer(),
writer.getOutput()->getSize());
col_to->insert_default();
}
}

block.replace_by_position(result, std::move(column_string));
block.replace_by_position(
result, ColumnNullable::create(std::move(col_to), std::move(col_null_map_to)));
return Status::OK();
}
};
Expand Down Expand Up @@ -845,48 +860,21 @@ struct ConvertImplFromJsonb {
null_map[i] = 1;
res[i] = 0;
}
} else if constexpr (type_index == TypeIndex::Int8) {
if (value->isInt8()) {
res[i] = (int8_t)((const JsonbIntVal*)value)->val();
} else {
null_map[i] = 1;
res[i] = 0;
}
} else if constexpr (type_index == TypeIndex::Int16) {
if (value->isInt8() || value->isInt16()) {
res[i] = (int16_t)((const JsonbIntVal*)value)->val();
} else {
null_map[i] = 1;
res[i] = 0;
}
} else if constexpr (type_index == TypeIndex::Int32) {
if (value->isInt8() || value->isInt16() || value->isInt32()) {
res[i] = (int32_t)((const JsonbIntVal*)value)->val();
} else {
null_map[i] = 1;
res[i] = 0;
}
} else if constexpr (type_index == TypeIndex::Int64) {
if (value->isInt8() || value->isInt16() || value->isInt32() ||
value->isInt64()) {
res[i] = (int64_t)((const JsonbIntVal*)value)->val();
} else {
null_map[i] = 1;
res[i] = 0;
}
} else if constexpr (type_index == TypeIndex::Int128) {
if (value->isInt8() || value->isInt16() || value->isInt32() ||
value->isInt64() || value->isInt128()) {
res[i] = (int128_t)((const JsonbIntVal*)value)->val();
} else if constexpr (type_index == TypeIndex::Int8 ||
type_index == TypeIndex::Int16 ||
type_index == TypeIndex::Int32 ||
type_index == TypeIndex::Int64 ||
type_index == TypeIndex::Int128) {
if (value->isInt()) {
res[i] = ((const JsonbIntVal*)value)->val();
Copy link
Contributor

@xiaokang xiaokang May 16, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Will it cause wrong result, eg int64 -> int8 ?

Copy link
Member Author

@eldenmoon eldenmoon May 17, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

the same as cast bigint to tinyint

} else {
null_map[i] = 1;
res[i] = 0;
}
} else if constexpr (type_index == TypeIndex::Float64) {
if (value->isDouble()) {
res[i] = ((const JsonbDoubleVal*)value)->val();
} else if (value->isInt8() || value->isInt16() || value->isInt32() ||
value->isInt64()) {
} else if (value->isInt()) {
res[i] = ((const JsonbIntVal*)value)->val();
} else {
null_map[i] = 1;
Expand Down Expand Up @@ -2082,7 +2070,6 @@ class FunctionCast final : public IFunctionBase {
const auto& col_with_type_and_name = block.get_by_position(arguments[0]);
auto& from_type = col_with_type_and_name.type;
auto& col_from = col_with_type_and_name.column;

// set variant root column/type to from column/type
auto variant = ColumnObject::create(true /*always nullable*/);
variant->create_root(from_type, col_from->assume_mutable());
Expand Down
15 changes: 8 additions & 7 deletions be/test/vec/function/function_jsonb_test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1271,11 +1271,11 @@ TEST(FunctionJsonbTEST, JsonbCastToOtherTest) {
{{STRING("null"), static_cast<int8_t>(TypeIndex::Int8)}, Null()},
{{STRING("true"), static_cast<int8_t>(TypeIndex::Int8)}, Null()},
{{STRING("false"), static_cast<int8_t>(TypeIndex::Int8)}, Null()},
{{STRING("100"), static_cast<int8_t>(TypeIndex::Int8)}, TINYINT(100)}, //int8
{{STRING("10000"), static_cast<int8_t>(TypeIndex::Int8)}, Null()}, // int16
{{STRING("1000000000"), static_cast<int8_t>(TypeIndex::Int8)}, Null()}, // int32
{{STRING("100"), static_cast<int8_t>(TypeIndex::Int8)}, TINYINT(100)}, //int8
{{STRING("10000"), static_cast<int8_t>(TypeIndex::Int8)}, TINYINT(16)}, // int16
{{STRING("1000000000"), static_cast<int8_t>(TypeIndex::Int8)}, TINYINT(0)}, // int32
{{STRING("1152921504606846976"), static_cast<int8_t>(TypeIndex::Int8)},
Null()}, // int64
TINYINT(0)}, // int64
{{STRING("6.18"), static_cast<int8_t>(TypeIndex::Int8)}, Null()}, // double
{{STRING(R"("abcd")"), static_cast<int8_t>(TypeIndex::Int8)}, Null()}, // string
{{STRING("{}"), static_cast<int8_t>(TypeIndex::Int8)}, Null()}, // empty object
Expand Down Expand Up @@ -1306,9 +1306,10 @@ TEST(FunctionJsonbTEST, JsonbCastToOtherTest) {
{{STRING("false"), static_cast<int16_t>(TypeIndex::Int16)}, Null()},
{{STRING("100"), static_cast<int16_t>(TypeIndex::Int16)}, SMALLINT(100)}, //int8
{{STRING("10000"), static_cast<int16_t>(TypeIndex::Int16)}, SMALLINT(10000)}, // int16
{{STRING("1000000000"), static_cast<int16_t>(TypeIndex::Int16)}, Null()}, // int32
{{STRING("1000000000"), static_cast<int16_t>(TypeIndex::Int16)},
SMALLINT(-13824)}, // int32
{{STRING("1152921504606846976"), static_cast<int16_t>(TypeIndex::Int16)},
Null()}, // int64
SMALLINT(0)}, // int64
{{STRING("6.18"), static_cast<int16_t>(TypeIndex::Int16)}, Null()}, // double
{{STRING(R"("abcd")"), static_cast<int16_t>(TypeIndex::Int16)}, Null()}, // string
{{STRING("{}"), static_cast<int16_t>(TypeIndex::Int16)}, Null()}, // empty object
Expand Down Expand Up @@ -1342,7 +1343,7 @@ TEST(FunctionJsonbTEST, JsonbCastToOtherTest) {
{{STRING("1000000000"), static_cast<int32_t>(TypeIndex::Int32)},
INT(1000000000)}, // int32
{{STRING("1152921504606846976"), static_cast<int32_t>(TypeIndex::Int32)},
Null()}, // int64
INT(0)}, // int64
{{STRING("6.18"), static_cast<int32_t>(TypeIndex::Int32)}, Null()}, // double
{{STRING(R"("abcd")"), static_cast<int32_t>(TypeIndex::Int32)}, Null()}, // string
{{STRING("{}"), static_cast<int32_t>(TypeIndex::Int32)}, Null()}, // empty object
Expand Down
43 changes: 43 additions & 0 deletions regression-test/data/datatype_p0/json/json_cast.out
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !sql1 --
10

-- !sql2 --
23

-- !sql3 --
-28649

-- !sql4 --
-679213870

-- !sql5 --
1234

-- !sql6 --
-46

-- !sql7 --
true

-- !sql8 --
\N

-- !sql9 --
1000.1111

-- !sql10 --
\N

-- !sql11 --
["CXO0N: 1045901740","HMkTa: 1348450505","44 HHD: 915015173","j9WoJ: -1517316688"]

-- !sql12 --
111111

-- !sql13 --
111111

-- !sql14 --
1.1111

52 changes: 26 additions & 26 deletions regression-test/data/json_p0/test_json_load_and_function.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5151,8 +5151,8 @@
4 false \N
5 100 100
6 10000 10000
7 1000000000 \N
8 1152921504606846976 \N
7 1000000000 -13824
8 1152921504606846976 0
9 6.18 \N
10 "abcd" \N
11 {} \N
Expand All @@ -5166,9 +5166,9 @@
26 \N \N
27 {"k1":"v1","k2":200} \N
28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N
29 12524337771678448270 \N
30 -9223372036854775808 \N
31 18446744073709551615 \N
29 12524337771678448270 -17778
30 -9223372036854775808 0
31 18446744073709551615 -1

-- !select --
1 \N \N
Expand All @@ -5178,7 +5178,7 @@
5 100 100
6 10000 10000
7 1000000000 1000000000
8 1152921504606846976 \N
8 1152921504606846976 0
9 6.18 \N
10 "abcd" \N
11 {} \N
Expand All @@ -5192,9 +5192,9 @@
26 \N \N
27 {"k1":"v1","k2":200} \N
28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N
29 12524337771678448270 \N
30 -9223372036854775808 \N
31 18446744073709551615 \N
29 12524337771678448270 -1209615730
30 -9223372036854775808 0
31 18446744073709551615 -1

-- !select --
1 \N \N
Expand All @@ -5218,9 +5218,9 @@
26 \N \N
27 {"k1":"v1","k2":200} \N
28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N
29 12524337771678448270 \N
29 12524337771678448270 -5922406302031103346
30 -9223372036854775808 -9223372036854775808
31 18446744073709551615 \N
31 18446744073709551615 -1

-- !select --
1 \N \N
Expand All @@ -5244,9 +5244,9 @@
26 \N \N
27 {"k1":"v1","k2":200} \N
28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N
29 12524337771678448270 \N
29 12524337771678448270 1.2524337771678448E19
30 -9223372036854775808 -9.223372036854776E18
31 18446744073709551615 \N
31 18446744073709551615 1.8446744073709552E19

-- !select --
1 \N \N
Expand Down Expand Up @@ -5307,8 +5307,8 @@
4 false \N
5 100 100
6 10000 10000
7 1000000000 \N
8 1152921504606846976 \N
7 1000000000 -13824
8 1152921504606846976 0
9 6.18 \N
10 "abcd" \N
11 {} \N
Expand All @@ -5322,9 +5322,9 @@
26 \N \N
27 {"k1":"v1","k2":200} \N
28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N
29 12524337771678448270 \N
30 -9223372036854775808 \N
31 18446744073709551615 \N
29 12524337771678448270 -17778
30 -9223372036854775808 0
31 18446744073709551615 -1

-- !select --
1 \N \N
Expand All @@ -5334,7 +5334,7 @@
5 100 100
6 10000 10000
7 1000000000 1000000000
8 1152921504606846976 \N
8 1152921504606846976 0
9 6.18 \N
10 "abcd" \N
11 {} \N
Expand All @@ -5348,9 +5348,9 @@
26 \N \N
27 {"k1":"v1","k2":200} \N
28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N
29 12524337771678448270 \N
30 -9223372036854775808 \N
31 18446744073709551615 \N
29 12524337771678448270 -1209615730
30 -9223372036854775808 0
31 18446744073709551615 -1

-- !select --
1 \N \N
Expand All @@ -5374,9 +5374,9 @@
26 \N \N
27 {"k1":"v1","k2":200} \N
28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N
29 12524337771678448270 \N
29 12524337771678448270 -5922406302031103346
30 -9223372036854775808 -9223372036854775808
31 18446744073709551615 \N
31 18446744073709551615 -1

-- !select --
1 \N \N
Expand All @@ -5400,9 +5400,9 @@
26 \N \N
27 {"k1":"v1","k2":200} \N
28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N
29 12524337771678448270 \N
29 12524337771678448270 1.25243377716784e+19
30 -9223372036854775808 -9.22337203685478e+18
31 18446744073709551615 \N
31 18446744073709551615 1.84467440737096e+19

-- !select --
1 \N \N
Expand Down
12 changes: 6 additions & 6 deletions regression-test/data/json_p0/test_json_load_unique_key_and_function.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3959,8 +3959,8 @@
4 false \N
5 100 100
6 10000 10000
7 1000000000 \N
8 1152921504606846976 \N
7 1000000000 -13824
8 1152921504606846976 0
9 6.18 \N
10 "abcd" \N
11 {} \N
Expand All @@ -3983,7 +3983,7 @@
5 100 100
6 10000 10000
7 1000000000 1000000000
8 1152921504606846976 \N
8 1152921504606846976 0
9 6.18 \N
10 "abcd" \N
11 {} \N
Expand Down Expand Up @@ -4097,8 +4097,8 @@
4 false \N
5 100 100
6 10000 10000
7 1000000000 \N
8 1152921504606846976 \N
7 1000000000 -13824
8 1152921504606846976 0
9 6.18 \N
10 "abcd" \N
11 {} \N
Expand All @@ -4121,7 +4121,7 @@
5 100 100
6 10000 10000
7 1000000000 1000000000
8 1152921504606846976 \N
8 1152921504606846976 0
9 6.18 \N
10 "abcd" \N
11 {} \N
Expand Down
Loading