[Optimize](simd json reader) Cached search results for previous row (keyed as index in JSON object) - used as a hint.

be/src/vec/exec/format/json/new_json_reader.cpp

@@ -1042,9 +1042,10 @@ Status NewJsonReader::_simdjson_init_reader() {
     }
     _ondemand_json_parser = std::make_unique<simdjson::ondemand::parser>();
     for (int i = 0; i < _file_slot_descs.size(); ++i) {
-        _slot_desc_index.emplace(_file_slot_descs[i]->col_name(), i);
+        _slot_desc_index[_file_slot_descs[i]->col_name()] = i;
     }
     _simdjson_ondemand_padding_buffer.resize(_padded_size);
+    _prev_positions.resize(_file_slot_descs.size());
     return Status::OK();
 }
 
@@ -1303,27 +1304,50 @@ Status NewJsonReader::_simdjson_handle_nested_complex_json(
     return Status::OK();
 }
 
+size_t NewJsonReader::_column_index(const StringRef& name, size_t key_index) {
+    /// Optimization by caching the order of fields (which is almost always the same)
+    /// and a quick check to match the next expected field, instead of searching the hash table.
+    if (_prev_positions.size() > key_index && _prev_positions[key_index] &&
+        name == _prev_positions[key_index]->get_first()) {
+        return _prev_positions[key_index]->get_second();
+    } else {
+        auto* it = _slot_desc_index.find(name);
+        if (it) {
+            if (key_index < _prev_positions.size()) {
+                _prev_positions[key_index] = it;
+            }
+            return it->get_second();
+        } else {
+            return size_t(-1);
+        }
+    }
+}
+
 Status NewJsonReader::_simdjson_set_column_value(simdjson::ondemand::object* value,
                                                  std::vector<MutableColumnPtr>& columns,
                                                  const std::vector<SlotDescriptor*>& slot_descs,
                                                  bool* valid) {
     // set
+    _seen_columns.assign(columns.size(), false);
     size_t cur_row_count = columns[0]->size();
     bool has_valid_value = false;
     // iterate through object, simdjson::ondemond will parsing on the fly
+    size_t key_index = 0;
     for (auto field : *value) {
         std::string_view key = field.unescaped_key();
-        auto iter = _slot_desc_index.find(std::string(key));
-        if (iter == _slot_desc_index.end()) {
+        StringRef name_ref(key.data(), key.size());
+        const size_t column_index = _column_index(name_ref, key_index++);
+        if (UNLIKELY(ssize_t(column_index) < 0)) {
             // This key is not exist in slot desc, just ignore
             continue;
         }
         simdjson::ondemand::value val = field.value();
-        RETURN_IF_ERROR(_simdjson_write_data_to_column(val, slot_descs[iter->second],
-                                                       columns[iter->second].get(), valid));
+        RETURN_IF_ERROR(_simdjson_write_data_to_column(val, slot_descs[column_index],
+                                                       columns[column_index].get(), valid));
         if (!(*valid)) {
             return Status::OK();
         }
+        _seen_columns[column_index] = true;
         has_valid_value = true;
     }
     if (!has_valid_value) {
@@ -1334,22 +1358,31 @@ Status NewJsonReader::_simdjson_set_column_value(simdjson::ondemand::object* val
 
     // fill missing slot
     int nullcount = 0;
-    int ctx_idx = 0;
-    for (auto slot_desc : slot_descs) {
+    for (size_t i = 0; i < slot_descs.size(); ++i) {
+        if (_seen_columns[i]) {
+            continue;
+        }
+        auto slot_desc = slot_descs[i];
         if (!slot_desc->is_materialized()) {
             continue;
         }
-        int dest_index = ctx_idx++;
-        auto* column_ptr = columns[dest_index].get();
+        auto* column_ptr = columns[i].get();
         if (column_ptr->size() < cur_row_count + 1) {
             DCHECK(column_ptr->size() == cur_row_count);
             column_ptr->assume_mutable()->insert_default();
             ++nullcount;
         }
         DCHECK(column_ptr->size() == cur_row_count + 1);
     }
+
+#ifndef NDEBUG
+    // Check all columns rows matched
+    for (size_t i = 0; i < columns.size(); ++i) {
+        DCHECK_EQ(columns[i]->size(), cur_row_count + 1);
+    }
     // There is at least one valid value here
     DCHECK(nullcount < columns.size());
+#endif
     *valid = true;
     return Status::OK();
 }
@@ -1389,27 +1422,13 @@ Status NewJsonReader::_simdjson_write_data_to_column(simdjson::ondemand::value&
         }
         break;
     }
-    case simdjson::ondemand::json_type::object:
-    case simdjson::ondemand::json_type::array: {
-        auto str_view = simdjson::to_json_string(value).value();
-        std::string value_str(str_view.data(), str_view.size());
-        // compact json value
-        value_str.erase(std::remove_if(value_str.begin(), value_str.end(),
-                                       [](const char& c) {
-                                           // white space
-                                           return c == ' ' || c == '\t' || c == '\n' || c == '\r' ||
-                                                  c == '\f' || c == '\v';
-                                       }),
-                        value_str.end());
-        nullable_column->get_null_map_data().push_back(0);
-        column_string->insert_data(value_str.data(), value_str.length());
-        break;
-    }
     default: {
         auto str_view = simdjson::to_json_string(value).value();
         if (value.type() == simdjson::ondemand::json_type::string) {
+            nullable_column->get_null_map_data().push_back(0);
             // trim
-            str_view = str_view.substr(1, str_view.length() - 2);
+            column_string->insert_data(str_view.data() + 1, str_view.length() - 2);
+            break;
         }
         nullable_column->get_null_map_data().push_back(0);
         column_string->insert_data(str_view.data(), str_view.length());

be/src/vec/exec/format/json/new_json_reader.h

@@ -24,6 +24,7 @@
 
 #include "io/fs/file_reader.h"
 #include "olap/iterators.h"
+#include "vec/common/hash_table/hash_map.h"
 #include "vec/exec/format/generic_reader.h"
 
 namespace doris {
@@ -138,6 +139,8 @@ class NewJsonReader : public GenericReader {
     Status _append_error_msg(simdjson::ondemand::object* obj, std::string error_msg,
                              std::string col_name, bool* valid);
 
+    size_t _column_index(const StringRef& name, size_t key_index);
+
     Status (NewJsonReader::*_vhandle_json_callback)(
             std::vector<vectorized::MutableColumnPtr>& columns,
             const std::vector<SlotDescriptor*>& slot_descs, bool* is_empty_row, bool* eof);
@@ -194,8 +197,16 @@ class NewJsonReader : public GenericReader {
     RuntimeProfile::Counter* _file_read_timer;
 
     bool _is_dynamic_schema = false;
+
+    // ======SIMD JSON======
     // name mapping
-    phmap::flat_hash_map<String, size_t> _slot_desc_index;
+    /// Hash table match `field name -> position in the block`. NOTE You can use perfect hash map.
+    using NameMap = HashMap<StringRef, size_t, StringRefHash>;
+    NameMap _slot_desc_index;
+    /// Cached search results for previous row (keyed as index in JSON object) - used as a hint.
+    std::vector<NameMap::LookupResult> _prev_positions;
+    /// Set of columns which already met in row. Exception is thrown if there are more than one column with the same name.
+    std::vector<UInt8> _seen_columns;
     // simdjson
     static constexpr size_t _init_buffer_size = 1024 * 1024 * 8;
     size_t _padded_size = _init_buffer_size + simdjson::SIMDJSON_PADDING;

regression-test/data/load_p0/stream_load/invalid_nest_json2.json

@@ -1,16 +1,16 @@
 {"no": 1, "item": {"id": 1, "city": "beijing", "code": 2345671}}
-{"no": 2, "item": {"id": 2, "city": "shanghai", "code": 2345672}}
-{"no": 3, "item": ["id": 3, "city"], "hangzhou", "code": 2345673}}
-{"no": 3, "xxxxx": ["id": 3, "city"], "hangzhou", "code": 2345673}}
-{"no": 3, xxxxx": ["id": 3, "city"], "hangzhou", "code": 2345673}}
-{"no": 4, "item": {"id": 4, "city": "shenzhen", "code": 2345674}}
-{"no": 5, "item": {"id": 5, "city": "guangzhou", "code": 2345675}}
-{"no": 5, "item": {"id": 5, "city": ["guangzhou"], "code": 2345675}}
-{"no": 5, "item": {"id": 5, "city": {"guangzhou": 1}, "code": 2345675}}
-{"no": 5, "item": {"id": 5, "city": ["guangzhou", "code": 2345675]]}
-{"no": 5, "item": {"id": 5, "city": ["guangzhou", "code": 2345675}}
-{"no": 5, "item": {"id": 5, "city": {"guangzhou", "code": 2345675}}
-{"no": 5, "item": {"id": 5, "city": {"guangzhou":1, "code": 2345675}}}
+{"no": 2, "item": {"id": 2, "city": "shanghai","code": 2345672}}
+{"no": 3, "item": ["id": 3, "city"], "hangzhou","code": 2345673}}
+{"no": 3, "xxxxx": ["id": 3, "city"], "hangzhou","code": 2345673}}
+{"no": 3, xxxxx": ["id": 3, "city"], "hangzhou","code":2345673}}
+{"no": 4, "item": {"id": 4, "city": "shenzhen","code":2345674}}
+{"no": 5, "item": {"id": 5, "city": "guangzhou","code":2345675}}
+{"no": 5, "item": {"id": 5, "city": ["guangzhou"],"code":2345675}}
+{"no": 5, "item": {"id": 5, "city": {"guangzhou":1},"code":2345675}}
+{"no": 5, "item": {"id": 5, "city": ["guangzhou","code":2345675]]}
+{"no": 5, "item": {"id": 5, "city": ["guangzhou","code":2345675}}
+{"no": 5, "item": {"id": 5, "city": {"guangzhou","code":2345675}}
+{"no": 5, "item": {"id": 5, "city": {"guangzhou":1,"code":2345675}}}
 {"no": 5, "item": {"id": 5, "city": "1}}}
 {"no": 5, "item": {"id": 5, "city": "1]}}}
-{"no": 5, ["item": {"id": 5, "city": {"guangzhou": 1, "code": 2345675}]}
+{"no": 5, ["item": {"id": 5, "city": {"guangzhou":1,"code":2345675}]}

regression-test/data/load_p0/stream_load/invalid_nest_json3.json

@@ -1,5 +1,5 @@
 no": 1, "item": {"id": 1, "city": "beijing", "code": 2345671}}
 {"no": , "item": {"id": 2, "city": "shanghai", "code": 2345672}}
 {"no": 3, "item": ["id": 3, "city": "hangzhou", "code": 2345673}}
-{"no": 4, item": {"id": 4, "city": "shenzhen", "code": 2345674}}
-{"no": 5, "item": {"id": 5, "city": guangzhou", "code": 2345675}}
+{"no": 4, item": {"id": 4, "city":"shenzhen","code":2345674}}
+{"no": 5, "item": {"id": 5, "city":guangzhou","code":2345675}}