From 6dfcdcb4658ff843a2ec4f373c329e344fc80b50 Mon Sep 17 00:00:00 2001 From: fbrv Date: Mon, 12 Aug 2024 09:35:05 +0100 Subject: [PATCH 1/3] logs in separate containers --- crates/cli/src/docker_init.rs | 77 +++++++++++++---------------------- 1 file changed, 29 insertions(+), 48 deletions(-) diff --git a/crates/cli/src/docker_init.rs b/crates/cli/src/docker_init.rs index 8bcd128f..9a806733 100644 --- a/crates/cli/src/docker_init.rs +++ b/crates/cli/src/docker_init.rs @@ -1,4 +1,7 @@ -use std::{path::Path, vec}; +use std::{ + path::{Path, PathBuf}, + vec, +}; use cb_common::{ config::{ @@ -23,7 +26,6 @@ pub(super) const CB_COMPOSE_FILE: &str = "cb.docker-compose.yml"; pub(super) const CB_ENV_FILE: &str = ".cb.env"; pub(super) const CB_TARGETS_FILE: &str = "targets.json"; // needs to match prometheus.yml pub(super) const PROMETHEUS_DATA_VOLUME: &str = "prometheus-data"; -pub(super) const GRAFANA_DATA_VOLUME: &str = "grafana-data"; const METRICS_NETWORK: &str = "monitoring_network"; const SIGNER_NETWORK: &str = "signer_network"; @@ -42,19 +44,14 @@ pub fn handle_docker_init(config_path: String, output_dir: String) -> Result<()> // config volume to pass to all services let config_volume = Volumes::Simple(format!("./{}:{}:ro", config_path, CB_CONFIG_NAME)); - let log_volume = Volumes::Simple(format!( - "{}:{}", - cb_config.logs.log_dir_path.to_str().unwrap(), - CB_BASE_LOG_PATH - )); let mut jwts = IndexMap::new(); // envs to write in .env file let mut envs = IndexMap::from([(CB_CONFIG_ENV.into(), CB_CONFIG_NAME.into())]); + // targets to pass to prometheus let mut targets = Vec::new(); let metrics_port = 10000; - let cadvisor_port = 8080; // address for signer API communication let signer_port = 20000; @@ -118,6 +115,10 @@ pub fn handle_docker_init(config_path: String, output_dir: String) -> Result<()> envs.insert(jwt_name.clone(), jwt.clone()); jwts.insert(module.id.clone(), jwt); + let log_volume = get_log_volume( + cb_config.logs.log_dir_path.to_str().unwrap().into(), + &module.id, + ); Service { container_name: Some(module_cid.clone()), @@ -127,7 +128,7 @@ pub fn handle_docker_init(config_path: String, output_dir: String) -> Result<()> METRICS_NETWORK.to_owned(), SIGNER_NETWORK.to_owned(), ]), - volumes: vec![config_volume.clone(), log_volume.clone()], + volumes: vec![config_volume.clone(), log_volume], environment: Environment::KvPair(module_envs), depends_on: DependsOnOptions::Simple(vec!["cb_signer".to_owned()]), ..Service::default() @@ -150,12 +151,16 @@ pub fn handle_docker_init(config_path: String, output_dir: String) -> Result<()> } builder_events_modules.push(format!("{module_cid}:{builder_events_port}")); + let log_volume = get_log_volume( + cb_config.logs.log_dir_path.to_str().unwrap().into(), + &module.id, + ); Service { container_name: Some(module_cid.clone()), image: Some(module.docker_image), networks: Networks::Simple(vec![METRICS_NETWORK.to_owned()]), - volumes: vec![config_volume.clone(), log_volume.clone()], + volumes: vec![config_volume.clone(), log_volume], environment: Environment::KvPair(module_envs), depends_on: DependsOnOptions::Simple(vec!["cb_pbs".to_owned()]), ..Service::default() @@ -172,7 +177,7 @@ pub fn handle_docker_init(config_path: String, output_dir: String) -> Result<()> let (k, v) = get_env_val(BUILDER_SERVER_ENV, &env); pbs_envs.insert(k, v); } - + let log_volume = get_log_volume(cb_config.logs.log_dir_path.to_str().unwrap().into(), "pbs"); let pbs_service = Service { container_name: Some("cb_pbs".to_owned()), image: Some(cb_config.pbs.docker_image), @@ -181,7 +186,7 @@ pub fn handle_docker_init(config_path: String, output_dir: String) -> Result<()> cb_config.pbs.pbs_config.port, cb_config.pbs.pbs_config.port )]), networks: Networks::Simple(vec![METRICS_NETWORK.to_owned()]), - volumes: vec![config_volume.clone(), log_volume.clone()], + volumes: vec![config_volume.clone(), log_volume], environment: Environment::KvPair(pbs_envs), ..Service::default() }; @@ -194,7 +199,9 @@ pub fn handle_docker_init(config_path: String, output_dir: String) -> Result<()> if let Some(signer_config) = cb_config.signer { if needs_signer_module { - let mut volumes = vec![config_volume.clone(), log_volume.clone()]; + let log_volume = + get_log_volume(cb_config.logs.log_dir_path.to_str().unwrap().into(), "signer"); + let mut volumes = vec![config_volume.clone(), log_volume]; targets.push(PrometheusTargetConfig { targets: vec![format!("cb_signer:{metrics_port}")], @@ -287,8 +294,6 @@ pub fn handle_docker_init(config_path: String, output_dir: String) -> Result<()> let data_volume = Volumes::Simple(format!("{}:/prometheus", PROMETHEUS_DATA_VOLUME)); - let grafana_data_volume = Volumes::Simple(format!("{}:/var/lib/grafana", GRAFANA_DATA_VOLUME)); - volumes.insert( PROMETHEUS_DATA_VOLUME.to_owned(), MapOrEmpty::Map(ComposeVolume { @@ -300,17 +305,6 @@ pub fn handle_docker_init(config_path: String, output_dir: String) -> Result<()> }), ); - volumes.insert( - GRAFANA_DATA_VOLUME.to_owned(), - MapOrEmpty::Map(ComposeVolume { - driver: Some("local".to_owned()), - driver_opts: IndexMap::default(), - external: None, - labels: Labels::default(), - name: None, - }), - ); - let prometheus_service = Service { container_name: Some("cb_prometheus".to_owned()), image: Some("prom/prometheus:latest".to_owned()), @@ -338,7 +332,6 @@ pub fn handle_docker_init(config_path: String, output_dir: String) -> Result<()> Volumes::Simple( "./grafana/datasources:/etc/grafana/provisioning/datasources".to_owned(), ), - grafana_data_volume, ], // TODO: re-enable logging here once we move away from docker logs logging: Some(LoggingParameters { driver: Some("none".to_owned()), options: None }), @@ -348,27 +341,6 @@ pub fn handle_docker_init(config_path: String, output_dir: String) -> Result<()> services.insert("cb_grafana".to_owned(), Some(grafana_service)); } - services.insert( - "cb_cadvisor".to_owned(), - Some(Service { - container_name: Some("cb_cadvisor".to_owned()), - image: Some("gcr.io/cadvisor/cadvisor".to_owned()), - ports: Ports::Short(vec![format!("{cadvisor_port}:8080")]), - networks: Networks::Simple(vec![METRICS_NETWORK.to_owned()]), - volumes: vec![ - Volumes::Simple("/var/run/docker.sock:/var/run/docker.sock:ro".to_owned()), - Volumes::Simple("/sys:/sys:ro".to_owned()), - Volumes::Simple("/var/lib/docker/:/var/lib/docker:ro".to_owned()), - ], - ..Service::default() - }), - ); - - targets.push(PrometheusTargetConfig { - targets: vec![format!("cb_cadvisor:{cadvisor_port}")], - labels: PrometheusLabelsConfig { job: "cadvisor".to_owned() }, - }); - compose.services = Services(services); compose.volumes = TopLevelVolumes(volumes); @@ -434,3 +406,12 @@ struct PrometheusTargetConfig { struct PrometheusLabelsConfig { job: String, } + +fn get_log_volume(host_path: PathBuf, module_id: &str) -> Volumes { + let p = host_path.join(module_id); + Volumes::Simple(format!( + "{}:{}", + p.to_str().expect("could not convert pathbuf to str"), + CB_BASE_LOG_PATH + )) +} From 731107c216195d804da47aa2c22728130f20acd8 Mon Sep 17 00:00:00 2001 From: fbrv Date: Mon, 12 Aug 2024 10:00:00 +0100 Subject: [PATCH 2/3] merge --- crates/cli/src/docker_init.rs | 57 ++++++++++++++++++++++++++--------- 1 file changed, 43 insertions(+), 14 deletions(-) diff --git a/crates/cli/src/docker_init.rs b/crates/cli/src/docker_init.rs index 9a806733..e5b467eb 100644 --- a/crates/cli/src/docker_init.rs +++ b/crates/cli/src/docker_init.rs @@ -26,6 +26,7 @@ pub(super) const CB_COMPOSE_FILE: &str = "cb.docker-compose.yml"; pub(super) const CB_ENV_FILE: &str = ".cb.env"; pub(super) const CB_TARGETS_FILE: &str = "targets.json"; // needs to match prometheus.yml pub(super) const PROMETHEUS_DATA_VOLUME: &str = "prometheus-data"; +pub(super) const GRAFANA_DATA_VOLUME: &str = "grafana-data"; const METRICS_NETWORK: &str = "monitoring_network"; const SIGNER_NETWORK: &str = "signer_network"; @@ -48,10 +49,10 @@ pub fn handle_docker_init(config_path: String, output_dir: String) -> Result<()> let mut jwts = IndexMap::new(); // envs to write in .env file let mut envs = IndexMap::from([(CB_CONFIG_ENV.into(), CB_CONFIG_NAME.into())]); - // targets to pass to prometheus let mut targets = Vec::new(); let metrics_port = 10000; + let cadvisor_port = 8080; // address for signer API communication let signer_port = 20000; @@ -115,11 +116,8 @@ pub fn handle_docker_init(config_path: String, output_dir: String) -> Result<()> envs.insert(jwt_name.clone(), jwt.clone()); jwts.insert(module.id.clone(), jwt); - let log_volume = get_log_volume( - cb_config.logs.log_dir_path.to_str().unwrap().into(), - &module.id, - ); - + let log_volume = + get_log_volume(cb_config.logs.log_dir_path.clone(), &module.id); Service { container_name: Some(module_cid.clone()), image: Some(module.docker_image), @@ -151,11 +149,8 @@ pub fn handle_docker_init(config_path: String, output_dir: String) -> Result<()> } builder_events_modules.push(format!("{module_cid}:{builder_events_port}")); - let log_volume = get_log_volume( - cb_config.logs.log_dir_path.to_str().unwrap().into(), - &module.id, - ); - + let log_volume = + get_log_volume(cb_config.logs.log_dir_path.clone(), &module.id); Service { container_name: Some(module_cid.clone()), image: Some(module.docker_image), @@ -177,7 +172,7 @@ pub fn handle_docker_init(config_path: String, output_dir: String) -> Result<()> let (k, v) = get_env_val(BUILDER_SERVER_ENV, &env); pbs_envs.insert(k, v); } - let log_volume = get_log_volume(cb_config.logs.log_dir_path.to_str().unwrap().into(), "pbs"); + let log_volume = get_log_volume(cb_config.logs.log_dir_path.clone(), "pbs"); let pbs_service = Service { container_name: Some("cb_pbs".to_owned()), image: Some(cb_config.pbs.docker_image), @@ -199,8 +194,7 @@ pub fn handle_docker_init(config_path: String, output_dir: String) -> Result<()> if let Some(signer_config) = cb_config.signer { if needs_signer_module { - let log_volume = - get_log_volume(cb_config.logs.log_dir_path.to_str().unwrap().into(), "signer"); + let log_volume = get_log_volume(cb_config.logs.log_dir_path.clone(), "signer"); let mut volumes = vec![config_volume.clone(), log_volume]; targets.push(PrometheusTargetConfig { @@ -294,6 +288,8 @@ pub fn handle_docker_init(config_path: String, output_dir: String) -> Result<()> let data_volume = Volumes::Simple(format!("{}:/prometheus", PROMETHEUS_DATA_VOLUME)); + let grafana_data_volume = Volumes::Simple(format!("{}:/var/lib/grafana", GRAFANA_DATA_VOLUME)); + volumes.insert( PROMETHEUS_DATA_VOLUME.to_owned(), MapOrEmpty::Map(ComposeVolume { @@ -305,6 +301,17 @@ pub fn handle_docker_init(config_path: String, output_dir: String) -> Result<()> }), ); + volumes.insert( + GRAFANA_DATA_VOLUME.to_owned(), + MapOrEmpty::Map(ComposeVolume { + driver: Some("local".to_owned()), + driver_opts: IndexMap::default(), + external: None, + labels: Labels::default(), + name: None, + }), + ); + let prometheus_service = Service { container_name: Some("cb_prometheus".to_owned()), image: Some("prom/prometheus:latest".to_owned()), @@ -332,6 +339,7 @@ pub fn handle_docker_init(config_path: String, output_dir: String) -> Result<()> Volumes::Simple( "./grafana/datasources:/etc/grafana/provisioning/datasources".to_owned(), ), + grafana_data_volume, ], // TODO: re-enable logging here once we move away from docker logs logging: Some(LoggingParameters { driver: Some("none".to_owned()), options: None }), @@ -341,6 +349,27 @@ pub fn handle_docker_init(config_path: String, output_dir: String) -> Result<()> services.insert("cb_grafana".to_owned(), Some(grafana_service)); } + services.insert( + "cb_cadvisor".to_owned(), + Some(Service { + container_name: Some("cb_cadvisor".to_owned()), + image: Some("gcr.io/cadvisor/cadvisor".to_owned()), + ports: Ports::Short(vec![format!("{cadvisor_port}:8080")]), + networks: Networks::Simple(vec![METRICS_NETWORK.to_owned()]), + volumes: vec![ + Volumes::Simple("/var/run/docker.sock:/var/run/docker.sock:ro".to_owned()), + Volumes::Simple("/sys:/sys:ro".to_owned()), + Volumes::Simple("/var/lib/docker/:/var/lib/docker:ro".to_owned()), + ], + ..Service::default() + }), + ); + + targets.push(PrometheusTargetConfig { + targets: vec![format!("cb_cadvisor:{cadvisor_port}")], + labels: PrometheusLabelsConfig { job: "cadvisor".to_owned() }, + }); + compose.services = Services(services); compose.volumes = TopLevelVolumes(volumes); From c45c28cb038e17993bdd015dcc610981afd528fe Mon Sep 17 00:00:00 2001 From: fbrv Date: Mon, 12 Aug 2024 10:11:32 +0100 Subject: [PATCH 3/3] use const --- crates/cli/src/docker_init.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/crates/cli/src/docker_init.rs b/crates/cli/src/docker_init.rs index e5b467eb..06235d55 100644 --- a/crates/cli/src/docker_init.rs +++ b/crates/cli/src/docker_init.rs @@ -7,8 +7,9 @@ use cb_common::{ config::{ CommitBoostConfig, ModuleKind, BUILDER_SERVER_ENV, CB_BASE_LOG_PATH, CB_CONFIG_ENV, CB_CONFIG_NAME, JWTS_ENV, METRICS_SERVER_ENV, MODULE_ID_ENV, MODULE_JWT_ENV, - SIGNER_DIR_KEYS, SIGNER_DIR_KEYS_ENV, SIGNER_DIR_SECRETS, SIGNER_DIR_SECRETS_ENV, - SIGNER_KEYS, SIGNER_KEYS_ENV, SIGNER_SERVER_ENV, + PBS_MODULE_NAME, SIGNER_DIR_KEYS, SIGNER_DIR_KEYS_ENV, SIGNER_DIR_SECRETS, + SIGNER_DIR_SECRETS_ENV, SIGNER_KEYS, SIGNER_KEYS_ENV, SIGNER_MODULE_NAME, + SIGNER_SERVER_ENV, }, loader::SignerLoader, utils::{random_jwt, MAX_LOG_FILES_ENV, ROLLING_DURATION_ENV, RUST_LOG_ENV}, @@ -172,7 +173,7 @@ pub fn handle_docker_init(config_path: String, output_dir: String) -> Result<()> let (k, v) = get_env_val(BUILDER_SERVER_ENV, &env); pbs_envs.insert(k, v); } - let log_volume = get_log_volume(cb_config.logs.log_dir_path.clone(), "pbs"); + let log_volume = get_log_volume(cb_config.logs.log_dir_path.clone(), PBS_MODULE_NAME); let pbs_service = Service { container_name: Some("cb_pbs".to_owned()), image: Some(cb_config.pbs.docker_image), @@ -194,7 +195,8 @@ pub fn handle_docker_init(config_path: String, output_dir: String) -> Result<()> if let Some(signer_config) = cb_config.signer { if needs_signer_module { - let log_volume = get_log_volume(cb_config.logs.log_dir_path.clone(), "signer"); + let log_volume = + get_log_volume(cb_config.logs.log_dir_path.clone(), SIGNER_MODULE_NAME); let mut volumes = vec![config_volume.clone(), log_volume]; targets.push(PrometheusTargetConfig {