Skip to content

[Fix](mutli-catalog) Fix decimal v3 precision loss issues in the multi-catalog module. #18835

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
morningman merged 1 commit into from
Apr 20, 2023
Merged

[Fix](mutli-catalog) Fix decimal v3 precision loss issues in the multi-catalog module. #18835

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions be/src/runtime/types.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,24 @@ struct TypeDescriptor {
return ret;
}

static TypeDescriptor create_decimalv3_type(int precision, int scale) {
DCHECK_LE(precision, MAX_PRECISION);
DCHECK_LE(scale, MAX_SCALE);
DCHECK_GE(precision, 0);
DCHECK_LE(scale, precision);
TypeDescriptor ret;
if (precision <= MAX_DECIMAL4_PRECISION) {
ret.type = TYPE_DECIMAL32;
} else if (precision <= MAX_DECIMAL8_PRECISION) {
ret.type = TYPE_DECIMAL64;
} else {
ret.type = TYPE_DECIMAL128I;
}
ret.precision = precision;
ret.scale = scale;
return ret;
}

static TypeDescriptor from_thrift(const TTypeDesc& t) {
int idx = 0;
TypeDescriptor result(t.types, &idx);
Expand Down
5 changes: 4 additions & 1 deletion be/src/vec/exec/format/json/new_json_reader.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -878,8 +878,11 @@ Status NewJsonReader::_write_data_to_column(rapidjson::Value::ConstValueIterator
wbytes = snprintf(tmp_buf, sizeof(tmp_buf), "%" PRIu64, value->GetUint64());
} else if (value->IsInt64()) {
wbytes = snprintf(tmp_buf, sizeof(tmp_buf), "%" PRId64, value->GetInt64());
} else if (value->IsFloat() || value->IsDouble()) {
auto end = fmt::format_to(tmp_buf, "{}", value->GetDouble());
wbytes = end - tmp_buf;
} else {
wbytes = snprintf(tmp_buf, sizeof(tmp_buf), "%f", value->GetDouble());
return Status::InternalError("It should not here.");
}
str_value = tmp_buf;
break;
Expand Down
4 changes: 2 additions & 2 deletions be/src/vec/exec/format/orc/vorc_reader.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -580,8 +580,8 @@ TypeDescriptor OrcReader::_convert_to_doris_type(const orc::Type* orc_type) {
case orc::TypeKind::TIMESTAMP:
return TypeDescriptor(PrimitiveType::TYPE_DATETIMEV2);
case orc::TypeKind::DECIMAL:
// TODO: using decimal v3 instead
return TypeDescriptor(PrimitiveType::TYPE_DECIMALV2);
return TypeDescriptor::create_decimalv3_type(orc_type->getPrecision(),
orc_type->getScale());
case orc::TypeKind::DATE:
return TypeDescriptor(PrimitiveType::TYPE_DATEV2);
case orc::TypeKind::VARCHAR:
Expand Down
15 changes: 9 additions & 6 deletions be/src/vec/exec/format/parquet/schema_desc.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -170,7 +170,7 @@ TypeDescriptor FieldDescriptor::get_doris_type(const tparquet::SchemaElement& ph
if (physical_schema.__isset.logicalType) {
type = convert_to_doris_type(physical_schema.logicalType);
} else if (physical_schema.__isset.converted_type) {
type = convert_to_doris_type(physical_schema.converted_type);
type = convert_to_doris_type(physical_schema);
}
// use physical type instead
if (type.type == INVALID_TYPE) {
Expand Down Expand Up @@ -211,7 +211,8 @@ TypeDescriptor FieldDescriptor::convert_to_doris_type(tparquet::LogicalType logi
if (logicalType.__isset.STRING) {
type = TypeDescriptor(TYPE_STRING);
} else if (logicalType.__isset.DECIMAL) {
type = TypeDescriptor(TYPE_DECIMALV2);
type = TypeDescriptor::create_decimalv3_type(logicalType.DECIMAL.precision,
logicalType.DECIMAL.scale);
} else if (logicalType.__isset.DATE) {
type = TypeDescriptor(TYPE_DATEV2);
} else if (logicalType.__isset.INTEGER) {
Expand All @@ -238,14 +239,16 @@ TypeDescriptor FieldDescriptor::convert_to_doris_type(tparquet::LogicalType logi
return type;
}

TypeDescriptor FieldDescriptor::convert_to_doris_type(tparquet::ConvertedType::type convertedType) {
TypeDescriptor FieldDescriptor::convert_to_doris_type(
const tparquet::SchemaElement& physical_schema) {
TypeDescriptor type;
switch (convertedType) {
switch (physical_schema.converted_type) {
case tparquet::ConvertedType::type::UTF8:
type = TypeDescriptor(TYPE_STRING);
break;
case tparquet::ConvertedType::type::DECIMAL:
type = TypeDescriptor(TYPE_DECIMALV2);
type = TypeDescriptor::create_decimalv3_type(physical_schema.precision,
physical_schema.scale);
break;
case tparquet::ConvertedType::type::DATE:
type = TypeDescriptor(TYPE_DATEV2);
Expand Down Expand Up @@ -281,7 +284,7 @@ TypeDescriptor FieldDescriptor::convert_to_doris_type(tparquet::ConvertedType::t
type = TypeDescriptor(TYPE_BIGINT);
break;
default:
LOG(WARNING) << "Not supported parquet ConvertedType: " << convertedType;
LOG(WARNING) << "Not supported parquet ConvertedType: " << physical_schema.converted_type;
type = TypeDescriptor(INVALID_TYPE);
break;
}
Expand Down
2 changes: 1 addition & 1 deletion be/src/vec/exec/format/parquet/schema_desc.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ class FieldDescriptor {

TypeDescriptor convert_to_doris_type(tparquet::LogicalType logicalType);

TypeDescriptor convert_to_doris_type(tparquet::ConvertedType::type convertedType);
TypeDescriptor convert_to_doris_type(const tparquet::SchemaElement& physical_schema);

TypeDescriptor get_doris_type(const tparquet::SchemaElement& physical_schema);

Expand Down
4 changes: 4 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,10 @@ public Column(String name, PrimitiveType dataType, boolean isAllowNull) {
this(name, ScalarType.createType(dataType), isAllowNull);
}

public Column(String name, PrimitiveType dataType, int len, int precision, int scale, boolean isAllowNull) {
this(name, ScalarType.createType(dataType, len, precision, scale), isAllowNull);
}

public Column(String name, Type type, boolean isAllowNull) {
this(name, type, false, null, isAllowNull, null, "");
}
Expand Down
2 changes: 1 addition & 1 deletion fe/fe-core/src/main/java/org/apache/doris/catalog/HiveMetaStoreClientHelper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -797,7 +797,7 @@ public static Type hiveTypeToDorisType(String hiveType, int timeScale) {
if (match.find()) {
scale = Integer.parseInt(match.group(1));
}
return ScalarType.createDecimalType(precision, scale);
return ScalarType.createDecimalV3Type(precision, scale);
}
return Type.UNSUPPORTED;
}
Expand Down
5 changes: 4 additions & 1 deletion fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -380,7 +380,9 @@ private void fillColumns(InternalService.PFetchTableSchemaResult result)
// only support ScalarType.
PScalarType scalarType = typeNode.getScalarType();
TPrimitiveType tPrimitiveType = TPrimitiveType.findByValue(scalarType.getType());
columns.add(new Column(colName, PrimitiveType.fromThrift(tPrimitiveType), true));
columns.add(new Column(colName, PrimitiveType.fromThrift(tPrimitiveType),
scalarType.getLen() <= 0 ? -1 : scalarType.getLen(), scalarType.getPrecision(),
scalarType.getScale(), true));
}
}
}
Expand Down Expand Up @@ -426,3 +428,4 @@ private PFetchTableSchemaRequest getFetchTableStructureRequest() throws Analysis
.setFileScanRange(ByteString.copyFrom(new TSerializer().serialize(fileScanRange))).build();
}
}

82 changes: 41 additions & 41 deletions regression-test/data/correctness_p0/table_valued_function/test_hdfs_tvf.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -176,48 +176,48 @@
10 nami 18

-- !parquet --
1 Supplier#000000001 N kD4on9OM Ipw3,gf0JBoQDd7tgrzrddZ 17 27-918-335-1736 5755 each slyly above the careful
2 Supplier#000000002 89eJ5ksX3ImxJQBvxObC, 5 15-679-861-2259 4032 slyly bold instructions. idle dependen
3 Supplier#000000003 q1,G3Pj6OjIuUYfUoH18BFTKP5aU9bEV3 1 11-383-516-1199 4192 blithely silent requests after the express dependencies are sl
4 Supplier#000000004 Bk7ah4CK8SYQTepEmvMkkgMwg 15 25-843-787-7479 4641 riously even requests above the exp
5 Supplier#000000005 Gcdm2rJRzl5qlTVzc 11 21-151-690-3663 -283 . slyly regular pinto bea
6 Supplier#000000006 tQxuVm7s7CnK 14 24-696-997-4969 1365 final accounts. regular dolphins use against the furiously ironic decoys.
7 Supplier#000000007 s,4TicNGB4uO6PaSqNBUq 23 33-990-965-2201 6820 s unwind silently furiously regular courts. final requests are deposits. requests wake quietly blit
8 Supplier#000000008 9Sq4bBH2FQEmaFOocY45sRTxo6yuoG 17 27-498-742-3860 7627 al pinto beans. asymptotes haggl
9 Supplier#000000009 1KhUgZegwM3ua7dsYmekYBsK 10 20-403-398-8662 5302 s. unusual, even requests along the furiously regular pac
10 Supplier#000000010 Saygah3gYWMp72i PY 24 34-852-489-8585 3891 ing waters. regular requests ar
11 Supplier#000000011 JfwTs,LZrV, M,9C 18 28-613-996-1505 3393 y ironic packages. slyly ironic accounts affix furiously; ironically unusual excuses across the flu
12 Supplier#000000012 aLIW q0HYd 8 18-179-925-7181 1432 al packages nag alongside of the bold instructions. express, daring accounts
13 Supplier#000000013 HK71HQyWoqRWOX8GI FpgAifW,2PoH 3 13-727-620-7813 9107 requests engage regularly instructions. furiously special requests ar
14 Supplier#000000014 EXsnO5pTNj4iZRm 15 25-656-247-5058 9189 l accounts boost. fluffily bold warhorses wake
15 Supplier#000000015 olXVbNBfVzRqgokr1T,Ie 8 18-453-357-6394 308 across the furiously regular platelets wake even deposits. quickly express she
16 Supplier#000000016 YjP5C55zHDXL7LalK27zfQnwejdpin4AMpvh 22 32-822-502-4215 2972 ously express ideas haggle quickly dugouts? fu
17 Supplier#000000017 c2d,ESHRSkK3WYnxpgw6aOqN0q 19 29-601-884-9219 1687 eep against the furiously bold ideas. fluffily bold packa
18 Supplier#000000018 PGGVE5PWAMwKDZw 16 26-729-551-1115 7040 accounts snooze slyly furiously bold
19 Supplier#000000019 edZT3es,nBFD8lBXTGeTl 24 34-278-310-2731 6150 refully final foxes across the dogged theodolites sleep slyly abou
20 Supplier#000000020 iybAE,RmTymrZVYaFZva2SH,j 3 13-715-945-6730 530 n, ironic ideas would nag blithely about the slyly regular accounts. silent, expr
1 Supplier#000000001 N kD4on9OM Ipw3,gf0JBoQDd7tgrzrddZ 17 27-918-335-1736 5755.94 each slyly above the careful
2 Supplier#000000002 89eJ5ksX3ImxJQBvxObC, 5 15-679-861-2259 4032.68 slyly bold instructions. idle dependen
3 Supplier#000000003 q1,G3Pj6OjIuUYfUoH18BFTKP5aU9bEV3 1 11-383-516-1199 4192.40 blithely silent requests after the express dependencies are sl
4 Supplier#000000004 Bk7ah4CK8SYQTepEmvMkkgMwg 15 25-843-787-7479 4641.08 riously even requests above the exp
5 Supplier#000000005 Gcdm2rJRzl5qlTVzc 11 21-151-690-3663 -283.84 . slyly regular pinto bea
6 Supplier#000000006 tQxuVm7s7CnK 14 24-696-997-4969 1365.79 final accounts. regular dolphins use against the furiously ironic decoys.
7 Supplier#000000007 s,4TicNGB4uO6PaSqNBUq 23 33-990-965-2201 6820.35 s unwind silently furiously regular courts. final requests are deposits. requests wake quietly blit
8 Supplier#000000008 9Sq4bBH2FQEmaFOocY45sRTxo6yuoG 17 27-498-742-3860 7627.85 al pinto beans. asymptotes haggl
9 Supplier#000000009 1KhUgZegwM3ua7dsYmekYBsK 10 20-403-398-8662 5302.37 s. unusual, even requests along the furiously regular pac
10 Supplier#000000010 Saygah3gYWMp72i PY 24 34-852-489-8585 3891.91 ing waters. regular requests ar
11 Supplier#000000011 JfwTs,LZrV, M,9C 18 28-613-996-1505 3393.08 y ironic packages. slyly ironic accounts affix furiously; ironically unusual excuses across the flu
12 Supplier#000000012 aLIW q0HYd 8 18-179-925-7181 1432.69 al packages nag alongside of the bold instructions. express, daring accounts
13 Supplier#000000013 HK71HQyWoqRWOX8GI FpgAifW,2PoH 3 13-727-620-7813 9107.22 requests engage regularly instructions. furiously special requests ar
14 Supplier#000000014 EXsnO5pTNj4iZRm 15 25-656-247-5058 9189.82 l accounts boost. fluffily bold warhorses wake
15 Supplier#000000015 olXVbNBfVzRqgokr1T,Ie 8 18-453-357-6394 308.56 across the furiously regular platelets wake even deposits. quickly express she
16 Supplier#000000016 YjP5C55zHDXL7LalK27zfQnwejdpin4AMpvh 22 32-822-502-4215 2972.26 ously express ideas haggle quickly dugouts? fu
17 Supplier#000000017 c2d,ESHRSkK3WYnxpgw6aOqN0q 19 29-601-884-9219 1687.81 eep against the furiously bold ideas. fluffily bold packa
18 Supplier#000000018 PGGVE5PWAMwKDZw 16 26-729-551-1115 7040.82 accounts snooze slyly furiously bold
19 Supplier#000000019 edZT3es,nBFD8lBXTGeTl 24 34-278-310-2731 6150.38 refully final foxes across the dogged theodolites sleep slyly abou
20 Supplier#000000020 iybAE,RmTymrZVYaFZva2SH,j 3 13-715-945-6730 530.82 n, ironic ideas would nag blithely about the slyly regular accounts. silent, expr

-- !orc --
1 goldenrod lavender spring chocolate lace Manufacturer#1 Brand#13 PROMO BURNISHED COPPER 7 JUMBO PKG 901 ly. slyly ironi
2 blush thistle blue yellow saddle Manufacturer#1 Brand#13 LARGE BRUSHED BRASS 1 LG CASE 902 lar accounts amo
3 spring green yellow purple cornsilk Manufacturer#4 Brand#42 STANDARD POLISHED BRASS 21 WRAP CASE 903 egular deposits hag
4 cornflower chocolate smoke green pink Manufacturer#3 Brand#34 SMALL PLATED BRASS 14 MED DRUM 904 p furiously r
5 forest brown coral puff cream Manufacturer#3 Brand#32 STANDARD POLISHED TIN 15 SM PKG 905 wake carefully
6 bisque cornflower lawn forest magenta Manufacturer#2 Brand#24 PROMO PLATED STEEL 4 MED BAG 906 sual a
7 moccasin green thistle khaki floral Manufacturer#1 Brand#11 SMALL PLATED COPPER 45 SM BAG 907 lyly. ex
8 misty lace thistle snow royal Manufacturer#4 Brand#44 PROMO BURNISHED TIN 41 LG DRUM 908 eposi
9 thistle dim navajo dark gainsboro Manufacturer#4 Brand#43 SMALL BURNISHED STEEL 12 WRAP CASE 909 ironic foxe
10 linen pink saddle puff powder Manufacturer#5 Brand#54 LARGE BURNISHED STEEL 44 LG CAN 910 ithely final deposit
11 spring maroon seashell almond orchid Manufacturer#2 Brand#25 STANDARD BURNISHED NICKEL 43 WRAP BOX 911 ng gr
12 cornflower wheat orange maroon ghost Manufacturer#3 Brand#33 MEDIUM ANODIZED STEEL 25 JUMBO CASE 912 quickly
13 ghost olive orange rosy thistle Manufacturer#5 Brand#55 MEDIUM BURNISHED NICKEL 1 JUMBO PACK 913 osits.
14 khaki seashell rose cornsilk navajo Manufacturer#1 Brand#13 SMALL POLISHED STEEL 28 JUMBO BOX 914 kages c
15 blanched honeydew sky turquoise medium Manufacturer#1 Brand#15 LARGE ANODIZED BRASS 45 LG CASE 915 usual ac
16 deep sky turquoise drab peach Manufacturer#3 Brand#32 PROMO PLATED TIN 2 MED PACK 916 unts a
17 indian navy coral pink deep Manufacturer#4 Brand#43 ECONOMY BRUSHED STEEL 16 LG BOX 917 regular accounts
18 turquoise indian lemon lavender misty Manufacturer#1 Brand#11 SMALL BURNISHED STEEL 42 JUMBO PACK 918 s cajole slyly a
19 chocolate navy tan deep brown Manufacturer#2 Brand#23 SMALL ANODIZED NICKEL 33 WRAP BOX 919 pending acc
20 ivory navy honeydew sandy midnight Manufacturer#1 Brand#12 LARGE POLISHED NICKEL 48 MED BAG 920 are across the asympt
1 goldenrod lavender spring chocolate lace Manufacturer#1 Brand#13 PROMO BURNISHED COPPER 7 JUMBO PKG 901.00 ly. slyly ironi
2 blush thistle blue yellow saddle Manufacturer#1 Brand#13 LARGE BRUSHED BRASS 1 LG CASE 902.00 lar accounts amo
3 spring green yellow purple cornsilk Manufacturer#4 Brand#42 STANDARD POLISHED BRASS 21 WRAP CASE 903.00 egular deposits hag
4 cornflower chocolate smoke green pink Manufacturer#3 Brand#34 SMALL PLATED BRASS 14 MED DRUM 904.00 p furiously r
5 forest brown coral puff cream Manufacturer#3 Brand#32 STANDARD POLISHED TIN 15 SM PKG 905.00 wake carefully
6 bisque cornflower lawn forest magenta Manufacturer#2 Brand#24 PROMO PLATED STEEL 4 MED BAG 906.00 sual a
7 moccasin green thistle khaki floral Manufacturer#1 Brand#11 SMALL PLATED COPPER 45 SM BAG 907.00 lyly. ex
8 misty lace thistle snow royal Manufacturer#4 Brand#44 PROMO BURNISHED TIN 41 LG DRUM 908.00 eposi
9 thistle dim navajo dark gainsboro Manufacturer#4 Brand#43 SMALL BURNISHED STEEL 12 WRAP CASE 909.00 ironic foxe
10 linen pink saddle puff powder Manufacturer#5 Brand#54 LARGE BURNISHED STEEL 44 LG CAN 910.01 ithely final deposit
11 spring maroon seashell almond orchid Manufacturer#2 Brand#25 STANDARD BURNISHED NICKEL 43 WRAP BOX 911.01 ng gr
12 cornflower wheat orange maroon ghost Manufacturer#3 Brand#33 MEDIUM ANODIZED STEEL 25 JUMBO CASE 912.01 quickly
13 ghost olive orange rosy thistle Manufacturer#5 Brand#55 MEDIUM BURNISHED NICKEL 1 JUMBO PACK 913.01 osits.
14 khaki seashell rose cornsilk navajo Manufacturer#1 Brand#13 SMALL POLISHED STEEL 28 JUMBO BOX 914.01 kages c
15 blanched honeydew sky turquoise medium Manufacturer#1 Brand#15 LARGE ANODIZED BRASS 45 LG CASE 915.01 usual ac
16 deep sky turquoise drab peach Manufacturer#3 Brand#32 PROMO PLATED TIN 2 MED PACK 916.01 unts a
17 indian navy coral pink deep Manufacturer#4 Brand#43 ECONOMY BRUSHED STEEL 16 LG BOX 917.01 regular accounts
18 turquoise indian lemon lavender misty Manufacturer#1 Brand#11 SMALL BURNISHED STEEL 42 JUMBO PACK 918.01 s cajole slyly a
19 chocolate navy tan deep brown Manufacturer#2 Brand#23 SMALL ANODIZED NICKEL 33 WRAP BOX 919.01 pending acc
20 ivory navy honeydew sandy midnight Manufacturer#1 Brand#12 LARGE POLISHED NICKEL 48 MED BAG 920.02 are across the asympt

-- !json --
1 beijing 2345671
Expand Down Expand Up @@ -293,6 +293,6 @@ s_name TEXT Yes false \N NONE
s_address TEXT Yes false \N NONE
s_nationkey INT Yes false \N NONE
s_phone TEXT Yes false \N NONE
s_acctbal DECIMAL(9, 0) Yes false \N NONE
s_acctbal DECIMAL(12, 2) Yes false \N NONE
s_comment TEXT Yes false \N NONE

Loading