[Tablet][Recovery] Support using empty tablet to repair the damaged or missing tablet

be/src/agent/task_worker_pool.cpp

@@ -168,10 +168,6 @@ void TaskWorkerPool::start() {
         _worker_count = 1;
         _callback_function = _move_dir_thread_callback;
         break;
-    case TaskWorkerType::RECOVER_TABLET:
-        _worker_count = 1;
-        _callback_function = _recover_tablet_thread_callback;
-        break;
     case TaskWorkerType::UPDATE_TABLET_META_INFO:
         _worker_count = 1;
         _callback_function = _update_tablet_meta_worker_thread_callback;
@@ -1041,7 +1037,6 @@ void* TaskWorkerPool::_report_task_worker_thread_callback(void* arg_this) {
     TaskWorkerPool* worker_pool_this = (TaskWorkerPool*)arg_this;
 
     TReportRequest request;
-    request.__set_force_recovery(config::force_recovery);
     request.__set_backend(worker_pool_this->_backend);
 
 #ifndef BE_TEST
@@ -1075,7 +1070,6 @@ void* TaskWorkerPool::_report_disk_state_worker_thread_callback(void* arg_this)
     TaskWorkerPool* worker_pool_this = (TaskWorkerPool*)arg_this;
 
     TReportRequest request;
-    request.__set_force_recovery(config::force_recovery);
     request.__set_backend(worker_pool_this->_backend);
 
 #ifndef BE_TEST
@@ -1131,7 +1125,6 @@ void* TaskWorkerPool::_report_tablet_worker_thread_callback(void* arg_this) {
     TaskWorkerPool* worker_pool_this = (TaskWorkerPool*)arg_this;
 
     TReportRequest request;
-    request.__set_force_recovery(config::force_recovery);
     request.__set_backend(worker_pool_this->_backend);
     request.__isset.tablets = true;
     AgentStatus status = DORIS_SUCCESS;
@@ -1552,64 +1545,4 @@ AgentStatus TaskWorkerPool::_move_dir(const TTabletId tablet_id, const TSchemaHa
     return DORIS_SUCCESS;
 }
 
-void* TaskWorkerPool::_recover_tablet_thread_callback(void* arg_this) {
-    TaskWorkerPool* worker_pool_this = (TaskWorkerPool*)arg_this;
-
-    while (true) {
-        TAgentTaskRequest agent_task_req;
-        TRecoverTabletReq recover_tablet_req;
-        {
-            MutexLock worker_thread_lock(&(worker_pool_this->_worker_thread_lock));
-            while (worker_pool_this->_tasks.empty()) {
-                worker_pool_this->_worker_thread_condition_variable.wait();
-            }
-
-            agent_task_req = worker_pool_this->_tasks.front();
-            recover_tablet_req = agent_task_req.recover_tablet_req;
-            worker_pool_this->_tasks.pop_front();
-        }
-
-        TStatusCode::type status_code = TStatusCode::OK;
-        vector<string> error_msgs;
-        TStatus task_status;
-
-        LOG(INFO) << "begin to recover tablet."
-                  << ", tablet_id:" << recover_tablet_req.tablet_id << "."
-                  << recover_tablet_req.schema_hash << ", version:" << recover_tablet_req.version
-                  << "-" << recover_tablet_req.version_hash;
-        OLAPStatus status =
-                worker_pool_this->_env->storage_engine()->recover_tablet_until_specfic_version(
-                        recover_tablet_req);
-        if (status != OLAP_SUCCESS) {
-            status_code = TStatusCode::RUNTIME_ERROR;
-            LOG(WARNING) << "failed to recover tablet."
-                         << "signature:" << agent_task_req.signature
-                         << ", table:" << recover_tablet_req.tablet_id << "."
-                         << recover_tablet_req.schema_hash
-                         << ", version:" << recover_tablet_req.version << "-"
-                         << recover_tablet_req.version_hash;
-        } else {
-            LOG(WARNING) << "succeed to recover tablet."
-                         << "signature:" << agent_task_req.signature
-                         << ", table:" << recover_tablet_req.tablet_id << "."
-                         << recover_tablet_req.schema_hash
-                         << ", version:" << recover_tablet_req.version << "-"
-                         << recover_tablet_req.version_hash;
-        }
-
-        task_status.__set_status_code(status_code);
-        task_status.__set_error_msgs(error_msgs);
-
-        TFinishTaskRequest finish_task_request;
-        finish_task_request.__set_backend(worker_pool_this->_backend);
-        finish_task_request.__set_task_type(agent_task_req.task_type);
-        finish_task_request.__set_signature(agent_task_req.signature);
-        finish_task_request.__set_task_status(task_status);
-
-        worker_pool_this->_finish_task(finish_task_request);
-        worker_pool_this->_remove_task_info(agent_task_req.task_type, agent_task_req.signature);
-    }
-    return (void*)0;
-}
-
 } // namespace doris

be/src/agent/task_worker_pool.h

@@ -109,7 +109,6 @@ class TaskWorkerPool {
     static void* _make_snapshot_thread_callback(void* arg_this);
     static void* _release_snapshot_thread_callback(void* arg_this);
     static void* _move_dir_thread_callback(void* arg_this);
-    static void* _recover_tablet_thread_callback(void* arg_this);
     static void* _update_tablet_meta_worker_thread_callback(void* arg_this);
 
     void _alter_tablet(

be/src/common/config.h

@@ -446,9 +446,6 @@ namespace config {
     // result buffer cancelled time (unit: second)
     CONF_mInt32(result_buffer_cancelled_interval_time, "300");
 
-    // can perform recovering tablet
-    CONF_Bool(force_recovery, "false");
-
     // the increased frequency of priority for remaining tasks in BlockingPriorityQueue
     CONF_mInt32(priority_queue_remaining_tasks_increased_frequency, "512");
 

be/src/olap/storage_engine.cpp

@@ -832,16 +832,6 @@ OLAPStatus StorageEngine::create_tablet(const TCreateTabletReq& request) {
     return _tablet_manager->create_tablet(request, stores);
 }
 
-OLAPStatus StorageEngine::recover_tablet_until_specfic_version(
-        const TRecoverTabletReq& recover_tablet_req) {
-    TabletSharedPtr tablet = _tablet_manager->get_tablet(recover_tablet_req.tablet_id,
-                                   recover_tablet_req.schema_hash);
-    if (tablet == nullptr) { return OLAP_ERR_TABLE_NOT_FOUND; }
-    RETURN_NOT_OK(tablet->recover_tablet_until_specfic_version(recover_tablet_req.version,
-                                                        recover_tablet_req.version_hash));
-    return OLAP_SUCCESS;
-}
-
 OLAPStatus StorageEngine::obtain_shard_path(
         TStorageMedium::type storage_medium, std::string* shard_path, DataDir** store) {
     LOG(INFO) << "begin to process obtain root path. storage_medium=" << storage_medium;

be/src/olap/storage_engine.h

@@ -119,8 +119,6 @@ class StorageEngine {
     void start_delete_unused_rowset();
     void add_unused_rowset(RowsetSharedPtr rowset);
 
-    OLAPStatus recover_tablet_until_specfic_version(const TRecoverTabletReq& recover_tablet_req);
-
     // Obtain shard path for new tablet.
     //
     // @param [out] shard_path choose an available root_path to clone new tablet

be/src/olap/tablet.cpp

@@ -639,11 +639,6 @@ OLAPStatus Tablet::set_alter_state(AlterTabletState state) {
     return _tablet_meta->set_alter_state(state);
 }
 
-OLAPStatus Tablet::recover_tablet_until_specfic_version(const int64_t& spec_version,
-                                                        const int64_t& version_hash) {
-    return OLAP_SUCCESS;
-}
-
 bool Tablet::can_do_compaction() {
     // 如果table正在做schema change，则通过选路判断数据是否转换完成
     // 如果选路成功，则转换完成，可以进行compaction

be/src/olap/tablet.h

@@ -179,11 +179,6 @@ class Tablet : public BaseTablet {
             uint64_t request_block_row_count,
             vector<OlapTuple>* ranges);
 
-    // operation for recover tablet
-    // Deprected, remove it later
-    OLAPStatus recover_tablet_until_specfic_version(const int64_t& spec_version,
-                                                    const int64_t& version_hash);
-
     void set_bad(bool is_bad) { _is_bad = is_bad; }
 
     int64_t last_cumu_compaction_failure_time() { return _last_cumu_compaction_failure_millis; }

docs/.vuepress/sidebar/en.js

@@ -95,6 +95,7 @@ module.exports = [
           "multi-tenant",
           "tablet-meta-tool",
           "tablet-repair-and-balance",
+          "tablet-restore-tool",
           {
             title: "Metrics",
             directoryPath: "monitor-metrics/",

docs/en/administrator-guide/config/fe_config.md

@@ -674,3 +674,11 @@ The time interval of the latest partitioned version of the table refers to the t
 ### `cache_result_max_row_count`
 
 In order to avoid occupying too much memory, the maximum number of rows that can be cached is 2000 by default. If this threshold is exceeded, the cache cannot be set.
+
+### `recover_with_empty_tablet`
+
+In some very special circumstances, such as code bugs, or human misoperation, etc., all replicas of some tablets may be lost. In this case, the data has been substantially lost. However, in some scenarios, the business still hopes to ensure that the query will not report errors even if there is data loss, and reduce the perception of the user layer. At this point, we can use the blank Tablet to fill the missing replica to ensure that the query can be executed normally.
+
+Set to true so that Doris will automatically use blank replicas to fill tablets which all replicas have been damaged or missing.
+
+Default is false.

docs/en/administrator-guide/operation/tablet-restore-tool.md

@@ -0,0 +1,136 @@
+---
+{
+    "title": "Tablet Restore Tool",
+    "language": "en"
+}
+---
+
+<!-- 
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Tablet Restore Tool
+
+## Restore data from BE Recycle Bin
+
+During the user's use of Doris, some valid tablets (including metadata and data) may be deleted due to some misoperations or online bugs. In order to prevent data loss in these abnormal situations, Doris provides a recycle bin mechanism to protect user data. Tablet data deleted by users will not be deleted directly, but will be stored in the recycle bin for a period of time. After a period of time, there will be a regular cleaning mechanism to delete expired data. The data in the recycle bin includes: tablet data file (.dat), tablet index file (.idx) and tablet metadata file (.hdr). The data will be stored in a path in the following format:
+
+```
+/root_path/trash/time_label/tablet_id/schema_hash/
+```
+
+* `root_path`: a data root directory corresponding to the BE node.
+* `trash`: The directory of the recycle bin.
+* `time_label`: Time label, for the uniqueness of the data directory in the recycle bin, while recording the data time, use the time label as a subdirectory.
+
+When a user finds that online data has been deleted by mistake, he needs to recover the deleted tablet from the recycle bin. This tablet data recovery function is needed.
+
+BE provides http interface and `restore_tablet_tool.sh` script to achieve this function, and supports single tablet operation (single mode) and batch operation mode (batch mode).
+
+* In single mode, data recovery of a single tablet is supported.
+* In batch mode, support batch tablet data recovery.
+
+### Operation
+
+#### single mode
+
+1. http request method
+
+    BE provides an http interface for single tablet data recovery, the interface is as follows:
+
+    ```
+    curl -X POST "http://be_host:be_webserver_port/api/restore_tablet?tablet_id=11111\&schema_hash=12345"
+    ```
+
+    The successful results are as follows:
+
+    ```
+    {"status": "Success", "msg": "OK"}
+    ```
+
+    If it fails, the corresponding failure reason will be returned. One possible result is as follows:
+
+    ```
+    {"status": "Failed", "msg": "create link path failed"}
+    ```
+
+2. Script mode
+
+    `restore_tablet_tool.sh` can be used to realize the function of single tablet data recovery.
+
+    ```
+    sh tools/restore_tablet_tool.sh -b "http://127.0.0.1:8040" -t 12345 -s 11111
+    sh tools/restore_tablet_tool.sh --backend "http://127.0.0.1:8040" --tablet_id 12345 --schema_hash 11111
+    ```
+
+#### batch mode
+
+The batch recovery mode is used to realize the function of recovering multiple tablet data.
+
+When using, you need to put the restored tablet id and schema hash in a file in a comma-separated format in advance, one tablet per line.
+
+The format is as follows:
+
+```
+12345,11111
+12346,11111
+12347,11111
+```
+
+Then perform the recovery with the following command (assuming the file name is: `tablets.txt`):
+
+```
+sh restore_tablet_tool.sh -b "http://127.0.0.1:8040" -f tablets.txt
+sh restore_tablet_tool.sh --backend "http://127.0.0.1:8040" --file tablets.txt
+```
+
+## Repair missing or damaged Tablet
+
+In some very special circumstances, such as code bugs, or human misoperation, etc., all replicas of some tablets may be lost. In this case, the data has been substantially lost. However, in some scenarios, the business still hopes to ensure that the query will not report errors even if there is data loss, and reduce the perception of the user layer. At this point, we can use the blank Tablet to fill the missing replica to ensure that the query can be executed normally.
+
+**Note: This operation is only used to avoid the problem of error reporting due to the inability to find a queryable replica, and it is impossible to recover the data that has been substantially lost.**
+
+1. View Master FE log `fe.log`
+
+    If there is data loss, there will be a log similar to the following in the log:
+
+    ```
+    backend [10001] invalid situation. tablet[20000] has few replica[1], replica num setting is [3]
+    ```
+
+    This log indicates that all replicas of tablet 20000 have been damaged or lost.
+
+2. Use blank replicas to fill in missing copies
+
+    After confirming that the data cannot be recovered, you can execute the following command to generate blank replicas.
+
+    ```
+    ADMIN SET FRONTEND CONFIG ("recover_with_empty_tablet" = "true");
+    ```
+
+    * Note: You can first check whether the current version supports this parameter through the `AMDIN SHOW FRONTEND CONFIG;` command.
+
+3. A few minutes after the setup is complete, you should see the following log in the Master FE log `fe.log`:
+
+    ```
+    tablet 20000 has only one replica 20001 on backend 10001 and it is lost. create an empty replica to recover it.
+    ```
+
+    The log indicates that the system has created a blank tablet to fill in the missing replica.
+
+4. Judge whether it has been repaired successfully through query.

docs/zh-CN/administrator-guide/config/fe_config.md

@@ -670,3 +670,15 @@ thrift_client_timeout_ms 的值被设置为大于0来避免线程卡在java.net.
 ### `cache_result_max_row_count`
 
 为了避免过多占用内存，能够被缓存最大的行数，默认2000，超过这个阈值将不能缓存置。
+
+### `recover_with_empty_tablet`
+
+在某些极特殊情况下，如代码BUG、或人为误操作等，可能导致部分分片的全部副本都丢失。这种情况下，数据已经实质性的丢失。但是在某些场景下，业务依然希望能够在即使有数据丢失的情况下，保证查询正常不报错，降低用户层的感知程度。此时，我们可以通过使用空白Tablet填充丢失副本的功能，来保证查询能够正常执行。
+
+将此参数设置为 true，则 Doris 会自动使用空白副本填充所有副本都以损坏或丢失的 Tablet。
+
+默认为 false。
+
+
+
+