Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions be/src/olap/rowset/segment_v2/inverted_index_desc.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -27,21 +27,21 @@ namespace doris::segment_v2 {
// {tmp_dir}/{rowset_id}_{seg_id}_{index_id}@{suffix}
std::string InvertedIndexDescriptor::get_temporary_index_path(std::string_view tmp_dir_path,
std::string_view rowset_id,
int64_t seg_id, uint64_t uuid,
int64_t seg_id, int64_t index_id,
std::string_view index_path_suffix) {
std::string suffix =
index_path_suffix.empty() ? "" : std::string {"@"} + index_path_suffix.data();
return fmt::format("{}/{}_{}_{}{}", tmp_dir_path, rowset_id, seg_id, uuid, suffix);
return fmt::format("{}/{}_{}_{}{}", tmp_dir_path, rowset_id, seg_id, index_id, suffix);
}

// InvertedIndexStorageFormat V1
// {prefix}_{index_id}@{suffix}.idx
std::string InvertedIndexDescriptor::get_index_path_v1(std::string_view index_path_prefix,
uint64_t uuid,
int64_t index_id,
std::string_view index_path_suffix) {
std::string suffix =
index_path_suffix.empty() ? "" : std::string {"@"} + index_path_suffix.data();
return fmt::format("{}_{}{}{}", index_path_prefix, uuid, suffix, index_suffix);
return fmt::format("{}_{}{}{}", index_path_prefix, index_id, suffix, index_suffix);
}

// InvertedIndexStorageFormat V2
Expand Down
5 changes: 3 additions & 2 deletions be/src/olap/rowset/segment_v2/inverted_index_desc.h
Original file line number Diff line number Diff line change
Expand Up @@ -32,9 +32,10 @@ class InvertedIndexDescriptor {
static constexpr std::string_view index_suffix = ".idx";
static std::string get_temporary_index_path(std::string_view tmp_dir_path,
std::string_view rowset_id, int64_t seg_id,
uint64_t uuid, std::string_view index_path_suffix);
int64_t index_id,
std::string_view index_path_suffix);
// InvertedIndexStorageFormat V1
static std::string get_index_path_v1(std::string_view index_path_prefix, uint64_t uuid,
static std::string get_index_path_v1(std::string_view index_path_prefix, int64_t index_id,
std::string_view index_path_suffix);
// InvertedIndexStorageFormat V2
static std::string get_index_path_v2(std::string_view index_path_prefix);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ Status InvertedIndexFileReader::_init_from_v2(int32_t read_buffer_size) {
ReaderFileEntry* entry = nullptr;

for (int32_t i = 0; i < numIndices; ++i) {
int64_t indexId = _stream->readInt(); // Read index ID
int64_t indexId = _stream->readLong(); // Read index ID
int32_t suffix_length = _stream->readInt(); // Read suffix length
std::vector<uint8_t> suffix_data(suffix_length);
_stream->readBytes(suffix_data.data(), suffix_length);
Expand Down
39 changes: 20 additions & 19 deletions be/src/olap/rowset/segment_v2/inverted_index_file_writer.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -99,22 +99,23 @@ Status InvertedIndexFileWriter::delete_index(const TabletIndex* index_meta) {
size_t InvertedIndexFileWriter::headerLength() {
size_t header_size = 0;
header_size +=
sizeof(int) * 2; // Account for the size of the version number and number of indices
sizeof(int32_t) * 2; // Account for the size of the version number and number of indices

for (const auto& entry : _indices_dirs) {
auto suffix = entry.first.second;
header_size += sizeof(int); // index id
header_size += 4; // index suffix name size
const auto& suffix = entry.first.second;
header_size += sizeof(int64_t); // index id
header_size += sizeof(int32_t); // index suffix name size
header_size += suffix.length(); // index suffix name
header_size += sizeof(int); // index file count
header_size += sizeof(int32_t); // index file count
const auto& dir = entry.second;
std::vector<std::string> files;
dir->list(&files);

for (auto file : files) {
header_size += 4; // file name size
header_size += file.length(); // file name
header_size += 8; // file offset
header_size += 8; // file size
for (const auto& file : files) {
header_size += sizeof(int32_t); // file name size
header_size += file.length(); // file name
header_size += sizeof(int64_t); // file offset
header_size += sizeof(int64_t); // file size
}
}
return header_size;
Expand Down Expand Up @@ -192,29 +193,29 @@ size_t InvertedIndexFileWriter::write() {
}
// sort file list by file length
std::vector<std::pair<std::string, int64_t>> sorted_files;
for (auto file : files) {
for (const auto& file : files) {
sorted_files.emplace_back(file, dir->fileLength(file.c_str()));
}
// TODO: need to optimize

std::sort(sorted_files.begin(), sorted_files.end(),
[](const std::pair<std::string, int64_t>& a,
const std::pair<std::string, int64_t>& b) { return (a.second < b.second); });

int32_t file_count = sorted_files.size();

// Write the index ID and the number of files
compound_file_output->writeInt(index_id);
const auto* index_suffix_str = reinterpret_cast<const uint8_t*>(index_suffix.c_str());
compound_file_output->writeInt(index_suffix.length());
compound_file_output->writeBytes(index_suffix_str, index_suffix.length());
compound_file_output->writeLong(index_id);
compound_file_output->writeInt(static_cast<int32_t>(index_suffix.length()));
compound_file_output->writeBytes(reinterpret_cast<const uint8_t*>(index_suffix.data()),
index_suffix.length());
compound_file_output->writeInt(file_count);

// Calculate the offset for each file and write the file metadata
for (const auto& file : sorted_files) {
int64_t file_length = dir->fileLength(file.first.c_str());
const auto* file_name = reinterpret_cast<const uint8_t*>(file.first.c_str());
compound_file_output->writeInt(file.first.length());
compound_file_output->writeBytes(file_name, file.first.length());
compound_file_output->writeInt(static_cast<int32_t>(file.first.length()));
compound_file_output->writeBytes(reinterpret_cast<const uint8_t*>(file.first.data()),
file.first.length());
compound_file_output->writeLong(current_offset);
compound_file_output->writeLong(file_length);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2620,7 +2620,7 @@ public class Config extends ConfigBase {
"倒排索引默认存储格式",
"Default storage format of inverted index, the default value is V1."
})
public static String inverted_index_storage_format = "V1";
public static String inverted_index_storage_format = "V2";

@ConfField(description = {
"是否开启 Proxy Protocol 支持",
Expand Down
Loading