Set tablet as bad when loading index failed (#1146)

morningman · imay · commit 02f36c23ede1 · 2019-05-13T10:22:04.000+08:00
Bad tablet will be reported to FE and be handled

And add a config auto_recover_index_loading_failure to control the index loading failure processing
diff --git a/be/src/common/config.h b/be/src/common/config.h
@@ -399,6 +399,10 @@ namespace config {
 
     // max consumer num in one data consumer group, for routine load
     CONF_Int32(max_consumer_num_per_group, "3");
+
+    // Is set to true, index loading failure will not causing BE exit,
+    // and the tablet will be marked as bad, so that FE will try to repair it.
+    CONF_Bool(auto_recover_index_loading_failure, "false");
 } // namespace config
 
 } // namespace doris
diff --git a/be/src/exec/olap_meta_reader.cpp b/be/src/exec/olap_meta_reader.cpp
@@ -43,13 +43,14 @@ Status EngineMetaReader::get_hints(
         RuntimeProfile* profile) {
     auto tablet_id = scan_range->scan_range().tablet_id;
     int32_t schema_hash = strtoul(scan_range->scan_range().schema_hash.c_str(), NULL, 10);
+    std::string err;
     OLAPTablePtr table = OLAPEngine::get_instance()->get_table(
-        tablet_id, schema_hash);
+        tablet_id, schema_hash, true, &err);
     if (table.get() == NULL) {
-        LOG(WARNING) << "tablet does not exist. tablet_id=" << tablet_id << ", schema_hash="
-            << schema_hash;
         std::stringstream ss;
-        ss << "tablet does not exist: " << tablet_id;
+        ss << "failed to get tablet: " << tablet_id << "with schema hash: "
+            << schema_hash << ", reason: " << err;
+        LOG(WARNING) << ss.str();
         return Status(ss.str());
     }
 
diff --git a/be/src/exec/olap_scanner.cpp b/be/src/exec/olap_scanner.cpp
@@ -79,13 +79,13 @@ Status OlapScanner::_prepare(
     VersionHash version_hash =
         strtoul(scan_range->scan_range().version_hash.c_str(), nullptr, 10);
     {
-        _olap_table = OLAPEngine::get_instance()->get_table(tablet_id, schema_hash);
+        std::string err;
+        _olap_table = OLAPEngine::get_instance()->get_table(tablet_id, schema_hash, true, &err);
         if (_olap_table.get() == nullptr) {
-            OLAP_LOG_WARNING("tablet does not exist. [tablet_id=%ld schema_hash=%d]",
-                             tablet_id, schema_hash);
-
             std::stringstream ss;
-            ss << "tablet does not exist: " << tablet_id;
+            ss << "failed to get tablet: " << tablet_id << " with schema hash: " << schema_hash
+               << ", reason: " << err;
+            LOG(WARNING) << ss.str();
             return Status(ss.str());
         }
         {
diff --git a/be/src/olap/olap_engine.cpp b/be/src/olap/olap_engine.cpp
@@ -785,7 +785,7 @@ OLAPTablePtr OLAPEngine::_get_table_with_no_lock(TTabletId tablet_id, SchemaHash
     return olap_table;
 }
 
-OLAPTablePtr OLAPEngine::get_table(TTabletId tablet_id, SchemaHash schema_hash, bool load_table) {
+OLAPTablePtr OLAPEngine::get_table(TTabletId tablet_id, SchemaHash schema_hash, bool load_table, std::string* err) {
     _tablet_map_lock.rdlock();
     OLAPTablePtr olap_table;
     olap_table = _get_table_with_no_lock(tablet_id, schema_hash);
@@ -794,13 +794,18 @@ OLAPTablePtr OLAPEngine::get_table(TTabletId tablet_id, SchemaHash schema_hash,
     if (olap_table.get() != NULL) {
         if (!olap_table->is_used()) {
             OLAP_LOG_WARNING("olap table cannot be used. [table=%ld]", tablet_id);
+            if (err != nullptr) { *err = "tablet cannot be used"; }
             olap_table.reset();
         } else if (load_table && !olap_table->is_loaded()) {
-            if (olap_table->load() != OLAP_SUCCESS) {
+            OLAPStatus ost = olap_table->load();
+            if (ost != OLAP_SUCCESS) {
                 OLAP_LOG_WARNING("fail to load olap table. [table=%ld]", tablet_id);
+                if (err != nullptr) { *err = "load tablet failed"; }
                 olap_table.reset();
             }
         }
+    } else if (err != nullptr) {
+        *err = "tablet does not exist";
     }
 
     return olap_table;
@@ -835,6 +840,10 @@ OLAPStatus OLAPEngine::get_tables_by_id(
                 it = table_list->erase(it);
                 continue;
             }
+        } else if ((*it)->is_used()) {
+            LOG(WARNING) << "table is bad: " << (*it)->full_name().c_str();
+            it = table_list->erase(it); 
+            continue;
         }
         ++it;
     }
@@ -1884,7 +1893,7 @@ OLAPTablePtr OLAPEngine::_find_best_tablet_to_compaction(CompactionType compacti
     OLAPTablePtr best_table;
     for (tablet_map_t::value_type& table_ins : _tablet_map){
         for (OLAPTablePtr& table_ptr : table_ins.second.table_arr) {
-            if (!table_ptr->is_loaded() || !_can_do_compaction(table_ptr)) {
+            if (!table_ptr->is_used() || !table_ptr->is_loaded() || !_can_do_compaction(table_ptr)) {
                 continue;
             }
 
diff --git a/be/src/olap/olap_engine.h b/be/src/olap/olap_engine.h
@@ -86,7 +86,11 @@ class OLAPEngine {
     }
 
     // Get table pointer
-    OLAPTablePtr get_table(TTabletId tablet_id, SchemaHash schema_hash, bool load_table = true);
+    // TODO(cmy): I think it is better to return Status instead of OLAPTablePtr,
+    // so that the caller can decide what to do next based on Status.
+    // Currently, I just add a new parameter 'err' to save the error msg.
+    // This should be redesigned later.
+    OLAPTablePtr get_table(TTabletId tablet_id, SchemaHash schema_hash, bool load_table = true, std::string* err = nullptr);
 
     OLAPStatus get_tables_by_id(TTabletId tablet_id, std::list<OLAPTablePtr>* table_list);    
 
diff --git a/be/src/olap/olap_table.cpp b/be/src/olap/olap_table.cpp
@@ -148,7 +148,8 @@ OLAPTable::OLAPTable(OLAPHeader* header, OlapStore* store) :
         _num_key_fields(0),
         _id(0),
         _store(store),
-        _is_loaded(false) {
+        _is_loaded(false),
+        _is_bad(false) {
     if (header == NULL) {
         return;  // for convenience of mock test.
     }
@@ -310,13 +311,18 @@ OLAPStatus OLAPTable::load() {
                      << "res=" << res << ", root=" << one_schema_root;
         goto EXIT;
     } else if (res != OLAP_SUCCESS) {
-        OLAPEngine::get_instance()->drop_table(tablet_id(), schema_hash(), true);
-        return res;
+        // OLAPEngine::get_instance()->drop_table(tablet_id(), schema_hash(), true);
+        goto EXIT;
     }
     res = load_indices();
 
     if (res != OLAP_SUCCESS) {
-        LOG(FATAL) << "fail to load indices. [res=" << res << " table='" << _full_name << "']";
+        if (config::auto_recover_index_loading_failure) {
+            LOG(WARNING) << "fail to load indices. [res=" << res << " table='" << _full_name << "']";
+        } else {
+            // fatal log will let BE process exit
+            LOG(FATAL) << "fail to load indices. [res=" << res << " table='" << _full_name << "']";
+        }
         goto EXIT;
     }
 
@@ -333,11 +339,14 @@ OLAPStatus OLAPTable::load() {
     }
     release_header_lock();
 
+EXIT:
+    // always set _is_loaded to true, so that this tablet will be not loaded again
     _is_loaded = true;
 
-EXIT:
     if (res != OLAP_SUCCESS) {
-        OLAPEngine::get_instance()->drop_table(tablet_id(), schema_hash());
+        _is_bad = true;
+        // Do not drop table directly here, FE will get the report and handle it.
+        //  OLAPEngine::get_instance()->drop_table(tablet_id(), schema_hash());
     }
 
     return res;
@@ -2227,7 +2236,7 @@ void OLAPTable::set_io_error() {
 }
 
 bool OLAPTable::is_used() {
-    return _store->is_used();
+    return !_is_bad && _store->is_used();
 }
 
 VersionEntity OLAPTable::get_version_entity_by_version(const Version& version) {
diff --git a/be/src/olap/olap_table.h b/be/src/olap/olap_table.h
@@ -626,6 +626,8 @@ class OLAPTable : public std::enable_shared_from_this<OLAPTable> {
 
     bool is_used();
 
+    void set_bad(bool is_bad) { _is_bad = is_bad; }
+
     // 得到当前table的root path路径，路径末尾不带斜杠(/)
     std::string storage_root_path_name() {
         return _storage_root_path;
@@ -753,6 +755,7 @@ class OLAPTable : public std::enable_shared_from_this<OLAPTable> {
     std::string _tablet_path;
 
     bool _table_for_check;
+    std::atomic<bool> _is_bad;   // if this tablet is broken, set to true. default is false
 
     DISALLOW_COPY_AND_ASSIGN(OLAPTable);
 };

Original file line number	Diff line number	Diff line change
`@@ -79,13 +79,13 @@ Status OlapScanner::_prepare(`
`79`	`79`	`VersionHash version_hash =`
`80`	`80`	`strtoul(scan_range->scan_range().version_hash.c_str(), nullptr, 10);`
`81`	`81`	`{`
`82`		`- _olap_table = OLAPEngine::get_instance()->get_table(tablet_id, schema_hash);`
	`82`	`+ std::string err;`
	`83`	`+ _olap_table = OLAPEngine::get_instance()->get_table(tablet_id, schema_hash, true, &err);`
`83`	`84`	`if (_olap_table.get() == nullptr) {`
`84`		`- OLAP_LOG_WARNING("tablet does not exist. [tablet_id=%ld schema_hash=%d]",`
`85`		`- tablet_id, schema_hash);`
`86`		`-`
`87`	`85`	`std::stringstream ss;`
`88`		`- ss << "tablet does not exist: " << tablet_id;`
	`86`	`+ ss << "failed to get tablet: " << tablet_id << " with schema hash: " << schema_hash`
	`87`	`+ << ", reason: " << err;`
	`88`	`+ LOG(WARNING) << ss.str();`
`89`	`89`	`return Status(ss.str());`
`90`	`90`	`}`
`91`	`91`	`{`
Original file line number	Diff line number	Diff line change
`@@ -785,7 +785,7 @@ OLAPTablePtr OLAPEngine::_get_table_with_no_lock(TTabletId tablet_id, SchemaHash`
`785`	`785`	`return olap_table;`
`786`	`786`	`}`
`787`	`787`
`788`		`-OLAPTablePtr OLAPEngine::get_table(TTabletId tablet_id, SchemaHash schema_hash, bool load_table) {`
	`788`	`+OLAPTablePtr OLAPEngine::get_table(TTabletId tablet_id, SchemaHash schema_hash, bool load_table, std::string* err) {`
`789`	`789`	`_tablet_map_lock.rdlock();`
`790`	`790`	`OLAPTablePtr olap_table;`
`791`	`791`	`olap_table = _get_table_with_no_lock(tablet_id, schema_hash);`
`@@ -794,13 +794,18 @@ OLAPTablePtr OLAPEngine::get_table(TTabletId tablet_id, SchemaHash schema_hash,`
`794`	`794`	`if (olap_table.get() != NULL) {`
`795`	`795`	`if (!olap_table->is_used()) {`
`796`	`796`	`OLAP_LOG_WARNING("olap table cannot be used. [table=%ld]", tablet_id);`
	`797`	`+ if (err != nullptr) { *err = "tablet cannot be used"; }`
`797`	`798`	`olap_table.reset();`
`798`	`799`	`} else if (load_table && !olap_table->is_loaded()) {`
`799`		`- if (olap_table->load() != OLAP_SUCCESS) {`
	`800`	`+ OLAPStatus ost = olap_table->load();`
	`801`	`+ if (ost != OLAP_SUCCESS) {`
`800`	`802`	`OLAP_LOG_WARNING("fail to load olap table. [table=%ld]", tablet_id);`
	`803`	`+ if (err != nullptr) { *err = "load tablet failed"; }`
`801`	`804`	`olap_table.reset();`
`802`	`805`	`}`
`803`	`806`	`}`
	`807`	`+ } else if (err != nullptr) {`
	`808`	`+ *err = "tablet does not exist";`
`804`	`809`	`}`
`805`	`810`
`806`	`811`	`return olap_table;`
`@@ -835,6 +840,10 @@ OLAPStatus OLAPEngine::get_tables_by_id(`
`835`	`840`	`it = table_list->erase(it);`
`836`	`841`	`continue;`
`837`	`842`	`}`
	`843`	`+ } else if ((*it)->is_used()) {`
	`844`	`+ LOG(WARNING) << "table is bad: " << (*it)->full_name().c_str();`
	`845`	`+ it = table_list->erase(it);`
	`846`	`+ continue;`
`838`	`847`	`}`
`839`	`848`	`++it;`
`840`	`849`	`}`
`@@ -1884,7 +1893,7 @@ OLAPTablePtr OLAPEngine::_find_best_tablet_to_compaction(CompactionType compacti`
`1884`	`1893`	`OLAPTablePtr best_table;`
`1885`	`1894`	`for (tablet_map_t::value_type& table_ins : _tablet_map){`
`1886`	`1895`	`for (OLAPTablePtr& table_ptr : table_ins.second.table_arr) {`
`1887`		`- if (!table_ptr->is_loaded() \|\| !_can_do_compaction(table_ptr)) {`
	`1896`	`+ if (!table_ptr->is_used() \|\| !table_ptr->is_loaded() \|\| !_can_do_compaction(table_ptr)) {`
`1888`	`1897`	`continue;`
`1889`	`1898`	`}`
`1890`	`1899`