Skip to content

branch-3.0: [fix](es) Fix array not supporting json subtype #55738 #56042

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
github-actions wants to merge 1 commit into from
Closed

branch-3.0: [fix](es) Fix array not supporting json subtype #55738 #56042

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 21 additions & 0 deletions be/src/exec/es/es_scroll_parser.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -529,6 +529,24 @@ Status process_date_column(const rapidjson::Value& col, PrimitiveType sub_type,
return Status::OK();
}

Status process_jsonb_column(const rapidjson::Value& col, PrimitiveType sub_type,
bool pure_doc_value, vectorized::Array& array) {
if (!col.IsArray()) {
JsonBinaryValue jsonb_value;
RETURN_IF_ERROR(jsonb_value.from_json_string(json_value_to_string(col)));
vectorized::JsonbField json(jsonb_value.value(), jsonb_value.size());
array.push_back(vectorized::Field::create_field<TYPE_JSONB>(json));
} else {
for (const auto& sub_col : col.GetArray()) {
JsonBinaryValue jsonb_value;
RETURN_IF_ERROR(jsonb_value.from_json_string(json_value_to_string(sub_col)));
vectorized::JsonbField json(jsonb_value.value(), jsonb_value.size());
array.push_back(vectorized::Field::create_field<TYPE_JSONB>(json));
}
}
return Status::OK();
}

Status ScrollParser::parse_column(const rapidjson::Value& col, PrimitiveType sub_type,
bool pure_doc_value, vectorized::Array& array,
const cctz::time_zone& time_zone) {
Expand Down Expand Up @@ -564,6 +582,9 @@ Status ScrollParser::parse_column(const rapidjson::Value& col, PrimitiveType sub
return process_date_column<DateV2Value<DateTimeV2ValueType>, uint64_t>(
col, sub_type, pure_doc_value, array, time_zone);
}
case TYPE_JSONB: {
return process_jsonb_column(col, sub_type, pure_doc_value, array);
}
default:
LOG(ERROR) << "Do not support Array type: " << sub_type;
return Status::InternalError("Unsupported type");
Expand Down
20 changes: 10 additions & 10 deletions docker/thirdparties/docker-compose/elasticsearch/scripts/data/composite_type_array_bulk.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
{"name": "Andy", "sports": "soccer", "scores": 100}
{"name": "Betty", "sports": "pingpong ball", "scores": 90}
{"name": "Cindy", "sports": "武术", "scores": 89}
{"name": "David", "sports": ["volleyball"], "scores": [77]}
{"name": "Emily", "sports": ["baseball", "golf", "hockey"], "scores": [56, 78, 99]}
{"name": "Frank", "sports": ["rugby", "cricket", "boxing"], "scores": [45, 67, 88]}
{"name": "Grace", "sports": ["table tennis", "badminton", "athletics"], "scores": [34, 56, 78]}
{"name": "Henry", "sports": ["archery", "fencing", "weightlifting"], "scores": [23, 45, 67]}
{"name": "Ivy", "sports": ["judo", "karate", "taekwondo"], "scores": [12, 34, 56]}
{"name": "Jack", "sports": ["wrestling", "gymnastics", "surfing"], "scores": [1, 23, 45]}
{"name": "Andy", "sports": "soccer", "scores": 100, "z_details": [{"position": "forward", "club": "Manchester", "salary": 50000}]}
{"name": "Betty", "sports": "pingpong ball", "scores": 90, "z_details": [{"position": "attacker", "club": "Beijing", "salary": 30000}]}
{"name": "Cindy", "sports": "武术", "scores": 89, "z_details": [{"position": "fighter", "club": "Shaolin", "salary": 25000}]}
{"name": "David", "sports": ["volleyball"], "scores": [77], "z_details": [{"position": "spiker", "club": "Tokyo", "salary": 40000}]}
{"name": "Emily", "sports": ["baseball", "golf", "hockey"], "scores": [56, 78, 99], "z_details": [{"position": "pitcher", "club": "Yankees", "salary": 80000}, {"position": "driver", "club": "PGA", "salary": 120000}, {"position": "center", "club": "Rangers", "salary": 90000}]}
{"name": "Frank", "sports": ["rugby", "cricket", "boxing"], "scores": [45, 67, 88], "z_details": [{"position": "scrum-half", "club": "Leicester", "salary": 60000}, {"position": "batsman", "club": "Mumbai", "salary": 70000}, {"position": "heavyweight", "club": "Vegas", "salary": 150000}]}
{"name": "Grace", "sports": ["table tennis", "badminton", "athletics"], "scores": [34, 56, 78], "z_details": [{"position": "attacker", "club": "China", "salary": 35000}, {"position": "singles", "club": "Denmark", "salary": 45000}, {"position": "sprinter", "club": "Jamaica", "salary": 85000}]}
{"name": "Henry", "sports": ["archery", "fencing", "weightlifting"], "scores": [23, 45, 67], "z_details": [{"position": "recurve", "club": "Korea", "salary": 20000}, {"position": "epee", "club": "France", "salary": 30000}, {"position": "heavyweight", "club": "Bulgaria", "salary": 55000}]}
{"name": "Ivy", "sports": ["judo", "karate", "taekwondo"], "scores": [12, 34, 56], "z_details": [{"position": "lightweight", "club": "Japan", "salary": 25000}, {"position": "kata", "club": "Okinawa", "salary": 22000}, {"position": "middleweight", "club": "Korea", "salary": 28000}]}
{"name": "Jack", "sports": ["wrestling", "gymnastics", "surfing"], "scores": [1, 23, 45], "z_details": [{"position": "freestyle", "club": "Russia", "salary": 15000}, {"position": "rings", "club": "Romania", "salary": 35000}, {"position": "longboard", "club": "Hawaii", "salary": 50000}]}
3 changes: 2 additions & 1 deletion ...rdparties/docker-compose/elasticsearch/scripts/index/array_meta_composite_type_array.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@
"doris":{
"array_fields":[
"sports",
"scores"
"scores",
"z_details"
]
}
}
Expand Down
10 changes: 9 additions & 1 deletion docker/thirdparties/docker-compose/elasticsearch/scripts/index/es6_composite_type_array.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,15 @@
"properties": {
"name": { "type": "keyword" },
"sports": { "type": "keyword", "doc_values": false},
"scores": { "type": "integer", "doc_values": false}
"scores": { "type": "integer", "doc_values": false},
"z_details": {
"type": "object",
"properties": {
"position": { "type": "keyword" },
"club": { "type": "keyword" },
"salary": { "type": "integer" }
}
}
}
}
}
Expand Down
10 changes: 9 additions & 1 deletion docker/thirdparties/docker-compose/elasticsearch/scripts/index/es7_composite_type_array.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,15 @@
"properties": {
"name": { "type": "keyword" },
"sports": { "type": "keyword", "doc_values": false},
"scores": { "type": "integer", "doc_values": false}
"scores": { "type": "integer", "doc_values": false},
"z_details": {
"type": "object",
"properties": {
"position": { "type": "keyword" },
"club": { "type": "keyword" },
"salary": { "type": "integer" }
}
}
}
}
}
Loading
Loading