From 1d38eef332a570606168240b47758f4e03f73ca2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Massot?= Date: Mon, 21 Nov 2022 10:38:35 +0100 Subject: [PATCH] Unify merge & janitor max write throughput paramters into one parameter. Fix #2228. --- quickwit/quickwit-config/src/index_config/mod.rs | 14 ++++---------- .../src/actors/delete_task_pipeline.rs | 2 +- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/quickwit/quickwit-config/src/index_config/mod.rs b/quickwit/quickwit-config/src/index_config/mod.rs index bc831ac9e4d..4576d48c0c3 100644 --- a/quickwit/quickwit-config/src/index_config/mod.rs +++ b/quickwit/quickwit-config/src/index_config/mod.rs @@ -74,18 +74,13 @@ pub struct IndexingResources { pub __num_threads_deprecated: IgnoredAny, // DEPRECATED #[serde(default = "IndexingResources::default_heap_size")] pub heap_size: Byte, - /// Sets the maximum write IO throughput for the merge pipeline, - /// in bytes per secs. On hardware where IO is limited, this parameter can help limiting - /// the impact of merges on indexing. + /// Sets the maximum write IO throughput in bytes/sec for the merge and delete pipelines. + /// The IO limit is applied both to the downloader and to the merge executor. + /// On hardware where IO is limited, this parameter can help limiting the impact of + /// merges/deletes on indexing. #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] pub max_merge_write_throughput: Option, - /// Sets the maximum write IO throughput for the janitor, in bytes per secs. - /// On hardware where IO is limited, this parameter can help limiting - /// the impact on indexing. - #[serde(default)] - #[serde(skip_serializing_if = "Option::is_none")] - pub max_janitor_write_throughput: Option, } impl PartialEq for IndexingResources { @@ -113,7 +108,6 @@ impl Default for IndexingResources { Self { heap_size: Self::default_heap_size(), max_merge_write_throughput: None, - max_janitor_write_throughput: None, __num_threads_deprecated: IgnoredAny, } } diff --git a/quickwit/quickwit-janitor/src/actors/delete_task_pipeline.rs b/quickwit/quickwit-janitor/src/actors/delete_task_pipeline.rs index 7f457196bb7..8f97051d46b 100644 --- a/quickwit/quickwit-janitor/src/actors/delete_task_pipeline.rs +++ b/quickwit/quickwit-janitor/src/actors/delete_task_pipeline.rs @@ -182,7 +182,7 @@ impl DeleteTaskPipeline { let throughput_limit: f64 = index_config .indexing_settings .resources - .max_janitor_write_throughput + .max_merge_write_throughput .as_ref() .map(|bytes_per_sec| bytes_per_sec.get_bytes() as f64) .unwrap_or(f64::INFINITY);