add garbase collect by rowsetid #1374
Conversation
be/src/olap/storage_engine.cpp
Outdated
| } | ||
| } | ||
|
|
||
| bool StorageEngine::check_path_in_unused_rowsets(const string& path) { |
There was a problem hiding this comment.
Add some comment to describe the path strategy
| // gc thread should get tablet include deleted tablet | ||
| // or it will delete rowset file before tablet is garbage collected | ||
| RowsetId rowset_id = -1; | ||
| bool is_rowset_file = _tablet_manager->get_rowset_id_from_path(path, &rowset_id); |
There was a problem hiding this comment.
get_rowset_id_from_path should in rowset class?
There was a problem hiding this comment.
I don't think so, because this is a path pattern. And more here you can not get a rowset to get the rowsetid
| std::atomic<bool> _is_bad; // if this tablet is broken, set to true. default is false | ||
| std::atomic<int64_t> _last_compaction_failure_time; // timestamp of last compaction failure | ||
| RowsetId _start_rowset_id; | ||
|
|
There was a problem hiding this comment.
start_rowset_id is useless?
There was a problem hiding this comment.
OK, I will remove it.
| } | ||
| for (auto& inc_version_rowset : _inc_rs_version_map) { | ||
| bool ret = inc_version_rowset.second->check_path(path_to_check); | ||
| if (ret) { |
There was a problem hiding this comment.
maybe only check rowset id not the full path?
There was a problem hiding this comment.
OK， I will add check_rowset_id to do this
be/src/olap/data_dir.cpp
Outdated
| if (is_rowset_file) { | ||
| TabletSharedPtr tablet = _tablet_manager->get_tablet(tablet_id, schema_hash, true); | ||
| if (tablet != nullptr) { | ||
| bool valid = tablet->check_path(path); |
There was a problem hiding this comment.
should sleep in path scan method to prevent io 100%
There was a problem hiding this comment.
yes, I have sleep in line 944
| _process_garbage_path(path); | ||
| } | ||
| } | ||
| } |
There was a problem hiding this comment.
every path_gc_check_step paths, I will sleep 10ms, it is a config.
be/src/olap/storage_engine.cpp
Outdated
| _gc_mutex.lock(); | ||
| for (auto& _unused_rowset_pair : _unused_rowsets) { | ||
| if (_unused_rowset_pair.second->check_path(path)) { | ||
| LOG(INFO) << "path is found in unused rowsets, path:" << path; |
There was a problem hiding this comment.
not suggest to print log in lock
be/src/olap/tablet.cpp
Outdated
| } | ||
|
|
||
| bool Tablet::check_rowset_id(RowsetId rowset_id) { | ||
| bool ret = false; |
There was a problem hiding this comment.
_rs_version_map will be modified when you call this function.
A readlock is necessary.
be/src/olap/tablet.cpp
Outdated
| bool Tablet::check_rowset_id(RowsetId rowset_id) { | ||
| bool ret = false; | ||
| for (auto& version_rowset : _rs_version_map) { | ||
| ret = version_rowset.second->rowset_id() == rowset_id; |
There was a problem hiding this comment.
if (version_rowset.second->rowset_id() == rowset_id) {
return true;
}
be/src/olap/tablet.cpp
Outdated
| } | ||
|
|
||
| for (auto& inc_version_rowset : _inc_rs_version_map) { | ||
| ret = inc_version_rowset.second->rowset_id() == rowset_id; |
There was a problem hiding this comment.
if (inc_version_rowset.second->rowset_id() == rowset_id) {
return true;
}
3 participants
No description provided.