From 07ec029f06754190fb1b0059ad772aaa5f76a207 Mon Sep 17 00:00:00 2001 From: shreyamalpani Date: Mon, 28 Oct 2024 16:38:12 -0400 Subject: [PATCH 1/9] send cpu metrics --- bottlecap/Cargo.lock | 1 + bottlecap/Cargo.toml | 1 + bottlecap/src/bin/bottlecap/main.rs | 22 +- bottlecap/src/lifecycle/invocation/context.rs | 66 +++- .../src/lifecycle/invocation/processor.rs | 43 ++- bottlecap/src/metrics/enhanced/constants.rs | 8 + bottlecap/src/metrics/enhanced/lambda.rs | 297 +++++++++++++++++- bottlecap/src/proc/clock.rs | 20 ++ bottlecap/src/proc/constants.rs | 2 + bottlecap/src/proc/mod.rs | 195 +++++++++++- .../stat/invalid_stat_malformed_first_line | 2 + .../stat/invalid_stat_malformed_per_cpu_line | 10 + .../proc/stat/invalid_stat_missing_cpun_data | 8 + .../stat/invalid_stat_non_numerical_value_1 | 2 + .../stat/invalid_stat_non_numerical_value_2 | 2 + bottlecap/tests/proc/stat/valid_stat | 10 + .../tests/proc/uptime/invalid_data_uptime | 1 + bottlecap/tests/proc/uptime/malformed_uptime | 1 + bottlecap/tests/proc/uptime/valid_uptime | 1 + 19 files changed, 653 insertions(+), 39 deletions(-) create mode 100644 bottlecap/src/proc/clock.rs create mode 100644 bottlecap/tests/proc/stat/invalid_stat_malformed_first_line create mode 100644 bottlecap/tests/proc/stat/invalid_stat_malformed_per_cpu_line create mode 100644 bottlecap/tests/proc/stat/invalid_stat_missing_cpun_data create mode 100644 bottlecap/tests/proc/stat/invalid_stat_non_numerical_value_1 create mode 100644 bottlecap/tests/proc/stat/invalid_stat_non_numerical_value_2 create mode 100644 bottlecap/tests/proc/stat/valid_stat create mode 100644 bottlecap/tests/proc/uptime/invalid_data_uptime create mode 100644 bottlecap/tests/proc/uptime/malformed_uptime create mode 100644 bottlecap/tests/proc/uptime/valid_uptime diff --git a/bottlecap/Cargo.lock b/bottlecap/Cargo.lock index d61984248..fbae75bbc 100644 --- a/bottlecap/Cargo.lock +++ b/bottlecap/Cargo.lock @@ -426,6 +426,7 @@ dependencies = [ "httpmock", "hyper 0.14.30", "lazy_static", + "libc", "log", "proptest", "protobuf", diff --git a/bottlecap/Cargo.toml b/bottlecap/Cargo.toml index 44474479a..8fe260530 100644 --- a/bottlecap/Cargo.toml +++ b/bottlecap/Cargo.toml @@ -19,6 +19,7 @@ dogstatsd = { git = "https://github.com/DataDog/libdatadog", branch = "main" } figment = { version = "0.10", default-features = false, features = ["yaml", "env"] } hyper = { version = "0.14", default-features = false, features = ["server"] } lazy_static = { version = "1.5", default-features = false } +libc = "0.2" log = { version = "0.4", default-features = false } protobuf = { version = "3.5", default-features = false } regex = { version = "1.10", default-features = false } diff --git a/bottlecap/src/bin/bottlecap/main.rs b/bottlecap/src/bin/bottlecap/main.rs index 28909ad18..e2e936aca 100644 --- a/bottlecap/src/bin/bottlecap/main.rs +++ b/bottlecap/src/bin/bottlecap/main.rs @@ -376,7 +376,7 @@ async fn extension_loop_active( ); lambda_enhanced_metrics.increment_invocation_metric(); let mut p = invocation_processor.lock().await; - p.on_invoke_event(request_id); + p.on_invoke_event(request_id, lambda_enhanced_metrics.config.enhanced_metrics); drop(p); } Ok(NextEventResponse::Shutdown { @@ -425,8 +425,9 @@ async fn extension_loop_active( .. } => { let mut p = invocation_processor.lock().await; + let mut enhanced_metric_data = None; if let Some(metrics) = metrics { - p.on_platform_runtime_done( + enhanced_metric_data = p.on_platform_runtime_done( &request_id, metrics.duration_ms, config.clone(), @@ -462,6 +463,11 @@ async fn extension_loop_active( stats_flusher.manual_flush() ); } + + if let Some(offsets) = enhanced_metric_data { + lambda_enhanced_metrics.set_cpu_utilization_enhanced_metrics(offsets.cpu_offset, offsets.uptime_offset); + } + break; } TelemetryRecord::PlatformReport { @@ -476,11 +482,13 @@ async fn extension_loop_active( ); lambda_enhanced_metrics.set_report_log_metrics(&metrics); let mut p = invocation_processor.lock().await; - if let Some((post_runtime_duration_ms, network_offset)) = p.on_platform_report(&request_id, metrics.duration_ms) { - lambda_enhanced_metrics.set_post_runtime_duration_metric( - post_runtime_duration_ms, - ); - lambda_enhanced_metrics.set_network_enhanced_metrics(network_offset); + let (post_runtime_duration_ms, enhanced_metric_data) = p.on_platform_report(&request_id, metrics.duration_ms); + if let Some(duration) = post_runtime_duration_ms { + lambda_enhanced_metrics.set_post_runtime_duration_metric(duration); + } + if let Some(offsets) = enhanced_metric_data { + lambda_enhanced_metrics.set_network_enhanced_metrics(offsets.network_offset); + lambda_enhanced_metrics.set_cpu_time_enhanced_metrics(offsets.cpu_offset); } drop(p); diff --git a/bottlecap/src/lifecycle/invocation/context.rs b/bottlecap/src/lifecycle/invocation/context.rs index 9c00dea30..6621ccedf 100644 --- a/bottlecap/src/lifecycle/invocation/context.rs +++ b/bottlecap/src/lifecycle/invocation/context.rs @@ -1,4 +1,4 @@ -use crate::proc::NetworkData; +use crate::metrics::enhanced::lambda::EnhancedMetricData; use std::collections::VecDeque; use tracing::debug; @@ -9,7 +9,7 @@ pub struct Context { pub runtime_duration_ms: f64, pub init_duration_ms: f64, pub start_time: i64, - pub network_offset: Option, + pub enhanced_metric_data: Option, } impl Context { @@ -19,14 +19,14 @@ impl Context { runtime_duration_ms: f64, init_duration_ms: f64, start_time: i64, - network_offset: Option, + enhanced_metric_data: Option, ) -> Self { Context { request_id, runtime_duration_ms, init_duration_ms, start_time, - network_offset, + enhanced_metric_data, } } } @@ -150,18 +150,28 @@ impl ContextBuffer { } } - /// Adds the network offset to a `Context` in the buffer. If the `Context` is not found, a new + /// Adds the enhanced metric offsets to a `Context` in the buffer. If the `Context` is not found, a new /// `Context` is created and added to the buffer. /// - pub fn add_network_offset(&mut self, request_id: &String, network_data: Option) { + pub fn add_enhanced_metric_data( + &mut self, + request_id: &String, + enhanced_metric_data: Option, + ) { if let Some(context) = self .buffer .iter_mut() .find(|context| context.request_id == *request_id) { - context.network_offset = network_data; + context.enhanced_metric_data = enhanced_metric_data; } else { - self.insert(Context::new(request_id.clone(), 0.0, 0.0, 0, network_data)); + self.insert(Context::new( + request_id.clone(), + 0.0, + 0.0, + 0, + enhanced_metric_data, + )); } } @@ -176,6 +186,9 @@ impl ContextBuffer { #[cfg(test)] #[allow(clippy::unwrap_used)] mod tests { + use crate::proc::{CPUData, NetworkData}; + use std::collections::HashMap; + use super::*; #[test] @@ -323,7 +336,7 @@ mod tests { } #[test] - fn test_add_network_offset() { + fn test_add_enhanced_metric_data() { let mut buffer = ContextBuffer::with_capacity(2); let request_id = String::from("1"); @@ -337,19 +350,40 @@ mod tests { tx_bytes: 254.0, }); - buffer.add_network_offset(&request_id, network_offset); + let mut individual_cpu_idle_times = HashMap::new(); + individual_cpu_idle_times.insert("cpu0".to_string(), 10.0); + individual_cpu_idle_times.insert("cpu1".to_string(), 20.0); + let cpu_offset = Some(CPUData { + total_user_time_ms: 100.0, + total_system_time_ms: 53.0, + total_idle_time_ms: 20.0, + individual_cpu_idle_times: individual_cpu_idle_times, + }); + + let uptime_offset = Some(50.0); + + let enhanced_metric_data = Some(EnhancedMetricData { + network_offset: network_offset, + cpu_offset: cpu_offset, + uptime_offset: uptime_offset, + }); + + buffer.add_enhanced_metric_data(&request_id, enhanced_metric_data.clone()); assert_eq!( - buffer.get(&request_id).unwrap().network_offset, - network_offset, + buffer.get(&request_id).unwrap().enhanced_metric_data, + enhanced_metric_data, ); - // Add network offset to a context that doesn't exist + // Adds enhanced metric offsets to a context that doesn't exist let unexistent_request_id = String::from("unexistent"); - buffer.add_network_offset(&unexistent_request_id, network_offset); + buffer.add_enhanced_metric_data(&unexistent_request_id, enhanced_metric_data.clone()); assert_eq!(buffer.size(), 2); assert_eq!( - buffer.get(&unexistent_request_id).unwrap().network_offset, - network_offset + buffer + .get(&unexistent_request_id) + .unwrap() + .enhanced_metric_data, + enhanced_metric_data ); } } diff --git a/bottlecap/src/lifecycle/invocation/processor.rs b/bottlecap/src/lifecycle/invocation/processor.rs index 112791e9a..1476f2870 100644 --- a/bottlecap/src/lifecycle/invocation/processor.rs +++ b/bottlecap/src/lifecycle/invocation/processor.rs @@ -14,7 +14,8 @@ use tracing::debug; use crate::{ config::{self, AwsConfig}, lifecycle::invocation::{context::ContextBuffer, span_inferrer::SpanInferrer}, - proc::{self, NetworkData}, + metrics::enhanced::lambda::EnhancedMetricData, + proc::{self, CPUData, NetworkData}, tags::provider, traces::{ context::SpanContext, @@ -76,12 +77,21 @@ impl Processor { } } - /// Given a `request_id`, add the enhanced metric offsets to the context buffer. + /// Given a `request_id`, creates the context and adds the enhanced metric offsets to the context buffer. /// - pub fn on_invoke_event(&mut self, request_id: String) { - let network_offset: Option = proc::get_network_data().ok(); - self.context_buffer - .add_network_offset(&request_id, network_offset); + pub fn on_invoke_event(&mut self, request_id: String, collect_enhanced_data: bool) { + if collect_enhanced_data { + let network_offset: Option = proc::get_network_data().ok(); + let cpu_offset: Option = proc::get_cpu_data().ok(); + let uptime_offset: Option = proc::get_uptime().ok(); + let enhanced_metric_offsets = Some(EnhancedMetricData { + network_offset: network_offset, + cpu_offset: cpu_offset, + uptime_offset: uptime_offset, + }); + self.context_buffer + .add_enhanced_metric_data(&request_id, enhanced_metric_offsets); + } } /// Given a `request_id` and the time of the platform start, add the start time to the context buffer. @@ -108,10 +118,11 @@ impl Processor { tags_provider: Arc, trace_processor: Arc, trace_agent_tx: Sender, - ) { + ) -> Option { self.context_buffer .add_runtime_duration(request_id, duration_ms); + let mut enhanced_metric_data: Option = None; if let Some(context) = self.context_buffer.get(request_id) { let span = &mut self.span; // `round` is intentionally meant to be a whole integer @@ -128,6 +139,8 @@ impl Processor { // - error.stack // - trigger tags (from inferred spans) // - metrics tags (for asm) + + enhanced_metric_data = context.enhanced_metric_data.clone(); } self.inferrer.complete_inferred_span(&self.span); @@ -164,6 +177,8 @@ impl Processor { debug!("Failed to send invocation span to agent: {e}"); } } + + enhanced_metric_data } /// Given a `request_id` and the duration in milliseconds of the platform report, @@ -176,18 +191,18 @@ impl Processor { &mut self, request_id: &String, duration_ms: f64, - ) -> Option<(f64, Option)> { + ) -> (Option, Option) { if let Some(context) = self.context_buffer.remove(request_id) { - if context.runtime_duration_ms == 0.0 { - return None; - } + let mut post_runtime_duration_ms: Option = None; - let post_runtime_duration_ms = duration_ms - context.runtime_duration_ms; + if context.runtime_duration_ms != 0.0 { + post_runtime_duration_ms = Some(duration_ms - context.runtime_duration_ms); + } - return Some((post_runtime_duration_ms, context.network_offset)); + return (post_runtime_duration_ms, context.enhanced_metric_data); } - None + (None, None) } /// If this method is called, it means that we are operating in a Universally Instrumented diff --git a/bottlecap/src/metrics/enhanced/constants.rs b/bottlecap/src/metrics/enhanced/constants.rs index e82f48057..2d17e73ec 100644 --- a/bottlecap/src/metrics/enhanced/constants.rs +++ b/bottlecap/src/metrics/enhanced/constants.rs @@ -24,5 +24,13 @@ pub const INVOCATIONS_METRIC: &str = "aws.lambda.enhanced.invocations"; pub const RX_BYTES_METRIC: &str = "aws.lambda.enhanced.rx_bytes"; pub const TX_BYTES_METRIC: &str = "aws.lambda.enhanced.tx_bytes"; pub const TOTAL_NETWORK_METRIC: &str = "aws.lambda.enhanced.total_network"; +pub const CPU_SYSTEM_TIME_METRIC: &str = "aws.lambda.enhanced.cpu_system_time"; +pub const CPU_USER_TIME_METRIC: &str = "aws.lambda.enhanced.cpu_user_time"; +pub const CPU_TOTAL_TIME_METRIC: &str = "aws.lambda.enhanced.cpu_total_time"; +pub const CPU_TOTAL_UTILIZATION_PCT_METRIC: &str = "aws.lambda.enhanced.cpu_total_utilization_pct"; +pub const CPU_TOTAL_UTILIZATION_METRIC: &str = "aws.lambda.enhanced.cpu_total_utilization"; +pub const NUM_CORES_METRIC: &str = "aws.lambda.enhanced.num_cores"; +pub const CPU_MAX_UTILIZATION_METRIC: &str = "aws.lambda.enhanced.cpu_max_utilization"; +pub const CPU_MIN_UTILIZATION_METRIC: &str = "aws.lambda.enhanced.cpu_min_utilization"; //pub const ASM_INVOCATIONS_METRIC: &str = "aws.lambda.enhanced.asm.invocations"; pub const ENHANCED_METRICS_ENV_VAR: &str = "DD_ENHANCED_METRICS"; diff --git a/bottlecap/src/metrics/enhanced/lambda.rs b/bottlecap/src/metrics/enhanced/lambda.rs index 657e57442..97cdbbc35 100644 --- a/bottlecap/src/metrics/enhanced/lambda.rs +++ b/bottlecap/src/metrics/enhanced/lambda.rs @@ -1,5 +1,5 @@ use super::constants::{self, BASE_LAMBDA_INVOCATION_PRICE}; -use crate::proc::{self, NetworkData}; +use crate::proc::{self, CPUData, NetworkData}; use crate::telemetry::events::ReportMetrics; use dogstatsd::aggregator::Aggregator; use dogstatsd::metric; @@ -166,6 +166,180 @@ impl Lambda { } } + pub(crate) fn generate_cpu_time_enhanced_metrics( + cpu_data_offset: &CPUData, + cpu_data_end: &CPUData, + aggr: &mut std::sync::MutexGuard, + ) { + let cpu_user_time = cpu_data_end.total_user_time_ms - cpu_data_offset.total_user_time_ms; + let cpu_system_time = + cpu_data_end.total_system_time_ms - cpu_data_offset.total_system_time_ms; + let cpu_total_time = cpu_user_time + cpu_system_time; + + let metric = Metric::new( + constants::CPU_USER_TIME_METRIC.into(), + MetricValue::distribution(cpu_user_time), + None, + ); + if let Err(e) = aggr.insert(metric) { + error!("Failed to insert cpu_user_time metric: {}", e); + } + + let metric = Metric::new( + constants::CPU_SYSTEM_TIME_METRIC.into(), + MetricValue::distribution(cpu_system_time), + None, + ); + if let Err(e) = aggr.insert(metric) { + error!("Failed to insert cpu_system_time metric: {}", e); + } + + let metric = Metric::new( + constants::CPU_TOTAL_TIME_METRIC.into(), + MetricValue::distribution(cpu_total_time), + None, + ); + if let Err(e) = aggr.insert(metric) { + error!("Failed to insert cpu_total_time metric: {}", e); + } + } + + pub fn set_cpu_time_enhanced_metrics( + &self, + cpu_offset: Option, + ) { + if !self.config.enhanced_metrics { + return; + } + + let mut aggr: std::sync::MutexGuard = + self.aggregator.lock().expect("lock poisoned"); + + let cpu_data = proc::get_cpu_data(); + match (cpu_offset, cpu_data) { + (Some(cpu_offset), Ok(cpu_data)) => { + Self::generate_cpu_time_enhanced_metrics(&cpu_offset, &cpu_data, &mut aggr); + } + (_, _) => { + debug!("Could not find data to generate cpu time enhanced metrics"); + } + } + } + + pub(crate) fn generate_cpu_utilization_enhanced_metrics( + cpu_data_offset: &CPUData, + cpu_data_end: &CPUData, + uptime_data_offset: f64, + uptime_data_end: f64, + aggr: &mut std::sync::MutexGuard, + ) { + let num_cores = cpu_data_end.individual_cpu_idle_times.len() as f64; + let uptime = uptime_data_end - uptime_data_offset; + let total_idle_time = cpu_data_end.total_idle_time_ms - cpu_data_offset.total_idle_time_ms; + + let mut max_idle_time = 0.0; + let mut min_idle_time = f64::MAX; + + for (cpu_name, cpu_idle_time) in &cpu_data_end.individual_cpu_idle_times { + if let Some(cpu_idle_time_offset) = + cpu_data_offset.individual_cpu_idle_times.get(cpu_name) + { + let idle_time = cpu_idle_time - cpu_idle_time_offset; + if idle_time < min_idle_time { + min_idle_time = idle_time; + } + if idle_time > max_idle_time { + max_idle_time = idle_time; + } + } + } + + // Maximally utilized CPU is the one with the least time spent in the idle process + let cpu_max_utilization = ((uptime - min_idle_time) / uptime) * 100.0; + // Minimally utilized CPU is the one with the most time spent in the idle process + let cpu_min_utilization = ((uptime - max_idle_time) / uptime) * 100.0; + + let cpu_total_utilization_decimal = + ((uptime * num_cores) - total_idle_time) / (uptime * num_cores); + let cpu_total_utilization_pct = cpu_total_utilization_decimal * 100.0; + let cpu_total_utilization = cpu_total_utilization_decimal * num_cores; + + let metric = Metric::new( + constants::CPU_TOTAL_UTILIZATION_PCT_METRIC.into(), + MetricValue::distribution(cpu_total_utilization_pct), + None, + ); + if let Err(e) = aggr.insert(metric) { + error!("Failed to insert cpu_total_utilization_pct metric: {}", e); + } + + let metric = Metric::new( + constants::CPU_TOTAL_UTILIZATION_METRIC.into(), + MetricValue::distribution(cpu_total_utilization), + None, + ); + if let Err(e) = aggr.insert(metric) { + error!("Failed to insert cpu_total_utilization metric: {}", e); + } + + let metric = Metric::new( + constants::NUM_CORES_METRIC.into(), + MetricValue::distribution(num_cores), + None, + ); + if let Err(e) = aggr.insert(metric) { + error!("Failed to insert num_cores metric: {}", e); + } + + let metric = Metric::new( + constants::CPU_MAX_UTILIZATION_METRIC.into(), + MetricValue::distribution(cpu_max_utilization), + None, + ); + if let Err(e) = aggr.insert(metric) { + error!("Failed to insert cpu_max_utilization metric: {}", e); + } + + let metric = Metric::new( + constants::CPU_MIN_UTILIZATION_METRIC.into(), + MetricValue::distribution(cpu_min_utilization), + None, + ); + if let Err(e) = aggr.insert(metric) { + error!("Failed to insert cpu_min_utilization metric: {}", e); + } + } + + pub fn set_cpu_utilization_enhanced_metrics( + &self, + cpu_offset: Option, + uptime_offset: Option, + ) { + if !self.config.enhanced_metrics { + return; + } + + let mut aggr: std::sync::MutexGuard = + self.aggregator.lock().expect("lock poisoned"); + + let cpu_data = proc::get_cpu_data(); + let uptime_data = proc::get_uptime(); + match (cpu_offset, cpu_data, uptime_offset, uptime_data) { + (Some(cpu_offset), Ok(cpu_data), Some(uptime_offset), Ok(uptime_data)) => { + Self::generate_cpu_utilization_enhanced_metrics( + &cpu_offset, + &cpu_data, + uptime_offset, + uptime_data, + &mut aggr, + ); + } + (_, _, _, _) => { + debug!("Could not find data to generate cpu utilization enhanced metrics"); + } + } + } + fn calculate_estimated_cost_usd(billed_duration_ms: u64, memory_size_mb: u64) -> f64 { let gb_seconds = (billed_duration_ms as f64 * constants::MS_TO_SEC) * (memory_size_mb as f64 / constants::MB_TO_GB); @@ -234,9 +408,18 @@ impl Lambda { } } +#[derive(Clone, Debug, PartialEq)] +pub struct EnhancedMetricData { + pub network_offset: Option, + pub cpu_offset: Option, + pub uptime_offset: Option, +} + #[cfg(test)] #[allow(clippy::unwrap_used)] mod tests { + use std::collections::HashMap; + use super::*; use crate::config; use dogstatsd::metric::EMPTY_TAGS; @@ -346,6 +529,39 @@ mod tests { assert!(aggr .get_entry_by_id(constants::ESTIMATED_COST_METRIC.into(), &None) .is_none()); + assert!(aggr + .get_entry_by_id(constants::RX_BYTES_METRIC.into(), &None) + .is_none()); + assert!(aggr + .get_entry_by_id(constants::TX_BYTES_METRIC.into(), &None) + .is_none()); + assert!(aggr + .get_entry_by_id(constants::TOTAL_NETWORK_METRIC.into(), &None) + .is_none()); + assert!(aggr + .get_entry_by_id(constants::CPU_USER_TIME_METRIC.into(), &None) + .is_none()); + assert!(aggr + .get_entry_by_id(constants::CPU_SYSTEM_TIME_METRIC.into(), &None) + .is_none()); + assert!(aggr + .get_entry_by_id(constants::CPU_TOTAL_TIME_METRIC.into(), &None) + .is_none()); + assert!(aggr + .get_entry_by_id(constants::CPU_TOTAL_UTILIZATION_PCT_METRIC.into(), &None) + .is_none()); + assert!(aggr + .get_entry_by_id(constants::CPU_TOTAL_UTILIZATION_METRIC.into(), &None) + .is_none()); + assert!(aggr + .get_entry_by_id(constants::NUM_CORES_METRIC.into(), &None) + .is_none()); + assert!(aggr + .get_entry_by_id(constants::CPU_MIN_UTILIZATION_METRIC.into(), &None) + .is_none()); + assert!(aggr + .get_entry_by_id(constants::CPU_MAX_UTILIZATION_METRIC.into(), &None) + .is_none()); } #[test] @@ -393,4 +609,83 @@ mod tests { assert_sketch(&metrics_aggr, constants::TX_BYTES_METRIC, 74746.0); assert_sketch(&metrics_aggr, constants::TOTAL_NETWORK_METRIC, 94746.0); } + + #[test] + fn test_set_cpu_time_enhanced_metrics() { + let (metrics_aggr, my_config) = setup(); + let lambda = Lambda::new(metrics_aggr.clone(), my_config); + + let mut individual_cpu_idle_time_offsets = HashMap::new(); + individual_cpu_idle_time_offsets.insert("cpu0".to_string(), 10.0); + individual_cpu_idle_time_offsets.insert("cpu1".to_string(), 20.0); + let cpu_offset = CPUData { + total_user_time_ms: 100.0, + total_system_time_ms: 3.0, + total_idle_time_ms: 20.0, + individual_cpu_idle_times: individual_cpu_idle_time_offsets, + }; + + let mut individual_cpu_idle_times_end = HashMap::new(); + individual_cpu_idle_times_end.insert("cpu0".to_string(), 30.0); + individual_cpu_idle_times_end.insert("cpu1".to_string(), 80.0); + let cpu_data = CPUData { + total_user_time_ms: 200.0, + total_system_time_ms: 56.0, + total_idle_time_ms: 100.0, + individual_cpu_idle_times: individual_cpu_idle_times_end, + }; + + Lambda::generate_cpu_time_enhanced_metrics( + &cpu_offset, + &cpu_data, + &mut lambda.aggregator.lock().expect("lock poisoned"), + ); + + assert_sketch(&metrics_aggr, constants::CPU_USER_TIME_METRIC, 100.0); + assert_sketch(&metrics_aggr, constants::CPU_SYSTEM_TIME_METRIC, 53.0); + assert_sketch(&metrics_aggr, constants::CPU_TOTAL_TIME_METRIC, 153.0); + } + + #[test] + fn test_set_cpu_utilization_enhanced_metrics() { + let (metrics_aggr, my_config) = setup(); + let lambda = Lambda::new(metrics_aggr.clone(), my_config); + + let mut individual_cpu_idle_time_offsets = HashMap::new(); + individual_cpu_idle_time_offsets.insert("cpu0".to_string(), 10.0); + individual_cpu_idle_time_offsets.insert("cpu1".to_string(), 30.0); + let cpu_offset = CPUData { + total_user_time_ms: 50.0, + total_system_time_ms: 10.0, + total_idle_time_ms: 10.0, + individual_cpu_idle_times: individual_cpu_idle_time_offsets, + }; + let uptime_offset = 1891100.0; + + let mut individual_cpu_idle_times_end = HashMap::new(); + individual_cpu_idle_times_end.insert("cpu0".to_string(), 570.0); + individual_cpu_idle_times_end.insert("cpu1".to_string(), 600.0); + let cpu_data = CPUData { + total_user_time_ms: 200.0, + total_system_time_ms: 170.0, + total_idle_time_ms: 1130.0, + individual_cpu_idle_times: individual_cpu_idle_times_end, + }; + let uptime_data = 1891900.0; + + Lambda::generate_cpu_utilization_enhanced_metrics( + &cpu_offset, + &cpu_data, + uptime_offset, + uptime_data, + &mut lambda.aggregator.lock().expect("lock poisoned"), + ); + + // the differences above and metric values below are from an invocation using the go agent to verify the calculations + assert_sketch(&metrics_aggr, constants::CPU_TOTAL_UTILIZATION_PCT_METRIC, 30.0); + assert_sketch(&metrics_aggr, constants::CPU_TOTAL_UTILIZATION_METRIC, 0.6); + assert_sketch(&metrics_aggr, constants::NUM_CORES_METRIC, 2.0); + assert_sketch(&metrics_aggr, constants::CPU_MAX_UTILIZATION_METRIC, 30.0); + assert_sketch(&metrics_aggr, constants::CPU_MIN_UTILIZATION_METRIC, 28.75); + } } diff --git a/bottlecap/src/proc/clock.rs b/bottlecap/src/proc/clock.rs new file mode 100644 index 000000000..0455554e5 --- /dev/null +++ b/bottlecap/src/proc/clock.rs @@ -0,0 +1,20 @@ +use libc; +use std::io; + +#[cfg(not(target_os = "windows"))] +pub fn get_clk_tck() -> Result { + let clk_tck = unsafe { libc::sysconf(libc::_SC_CLK_TCK) }; + if clk_tck == -1 { + return Err(io::Error::new( + io::ErrorKind::Other, + "Failed to get clock ticks per second", + )); + } + Ok(clk_tck as u64) +} + +#[cfg(target_os = "windows")] +pub fn get_clk_tck() -> Result { + // Windows does not have this concept + Ok(1) +} diff --git a/bottlecap/src/proc/constants.rs b/bottlecap/src/proc/constants.rs index be7986e53..fe06b908d 100644 --- a/bottlecap/src/proc/constants.rs +++ b/bottlecap/src/proc/constants.rs @@ -1,3 +1,5 @@ pub const PROC_NET_DEV_PATH: &str = "/proc/net/dev"; +pub const PROC_STAT_PATH: &str = "/proc/stat"; +pub const PROC_UPTIME_PATH: &str = "/proc/uptime"; pub const LAMDBA_NETWORK_INTERFACE: &str = "vinternal_1"; diff --git a/bottlecap/src/proc/mod.rs b/bottlecap/src/proc/mod.rs index c33a2984e..2a2f894a1 100644 --- a/bottlecap/src/proc/mod.rs +++ b/bottlecap/src/proc/mod.rs @@ -1,11 +1,15 @@ +pub mod clock; pub mod constants; use std::{ + collections::HashMap, fs::File, io::{self, BufRead}, }; -use constants::{LAMDBA_NETWORK_INTERFACE, PROC_NET_DEV_PATH}; +use constants::{ + LAMDBA_NETWORK_INTERFACE, PROC_NET_DEV_PATH, PROC_STAT_PATH, PROC_UPTIME_PATH, +}; #[derive(Copy, Clone, Debug, PartialEq)] pub struct NetworkData { @@ -58,6 +62,122 @@ fn get_network_data_from_path(path: &str) -> Result { )) } +#[derive(Clone, Debug, PartialEq)] +pub struct CPUData { + pub total_user_time_ms: f64, + pub total_system_time_ms: f64, + pub total_idle_time_ms: f64, + pub individual_cpu_idle_times: HashMap, +} + +pub fn get_cpu_data() -> Result { + get_cpu_data_from_path(PROC_STAT_PATH) +} + +fn get_cpu_data_from_path(path: &str) -> Result { + let file = File::open(path)?; + let reader = io::BufReader::new(file); + + let mut cpu_data = CPUData { + total_user_time_ms: 0.0, + total_system_time_ms: 0.0, + total_idle_time_ms: 0.0, + individual_cpu_idle_times: HashMap::new(), + }; + + // SC_CLK_TCK is the system clock frequency in ticks per second + // We'll use this to convert CPU times from user HZ to milliseconds + let clktck = clock::get_clk_tck()? as f64; + + for line in reader.lines() { + let line = line?; + let mut values = line.split_whitespace(); + + if let Some(label) = values.next() { + if label == "cpu" { + // Parse CPU times for total user, system, and idle + let user: Option = values.next().and_then(|s| s.parse().ok()); + values.next(); // skip "nice" + let system: Option = values.next().and_then(|s| s.parse().ok()); + let idle: Option = values.next().and_then(|s| s.parse().ok()); + + match (user, system, idle) { + (Some(user_val), Some(system_val), Some(idle_val)) => { + cpu_data.total_user_time_ms = (1000.0 * user_val) / clktck; + cpu_data.total_system_time_ms = (1000.0 * system_val) / clktck; + cpu_data.total_idle_time_ms = (1000.0 * idle_val) / clktck; + } + (_, _, _) => { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Failed to parse CPU data", + )) + } + } + } else if label.starts_with("cpu") { // i.e. "cpu0", "cpu1" + // Parse per-core idle times + // Skip the first three values (user, nice, system) and get the 4th value (idle) + let idle: Option = values.nth(3).and_then(|s| s.parse().ok()); + + match idle { + Some(idle_val) => { + cpu_data + .individual_cpu_idle_times + .insert(label.to_string(), (1000.0 * idle_val) / clktck); + } + None => { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Failed to parse per-core CPU data", + )) + } + } + } + } + } + + if cpu_data.individual_cpu_idle_times.is_empty() { + return Err(io::Error::new( + io::ErrorKind::NotFound, + "Per-core CPU data not found", + )); + } + + Ok(cpu_data) +} + +pub fn get_uptime() -> Result { + get_uptime_from_path(PROC_UPTIME_PATH) +} + +fn get_uptime_from_path(path: &str) -> Result { + let file = File::open(path)?; + let reader = io::BufReader::new(file); + + if let Some(Ok(line)) = reader.lines().next() { + let mut values = line.split_whitespace(); + + let uptime: Option = values.next().and_then(|s| s.parse().ok()); + let idle: Option = values.next().and_then(|s| s.parse().ok()); + + match (uptime, idle) { + // Check that the file is correctly formatted (i.e. has both values) + (Some(uptime_val), Some(_idle_val)) => return Ok(uptime_val * 1000.0), + (_, _) => { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Failed to parse uptime data", + )); + } + } + } + + Err(io::Error::new( + io::ErrorKind::NotFound, + "Uptime data not found", + )) +} + #[cfg(test)] #[allow(clippy::unwrap_used)] mod tests { @@ -89,4 +209,77 @@ mod tests { let network_data_result = get_network_data_from_path(&path); assert!(network_data_result.is_err()); } + + #[test] + #[allow(clippy::float_cmp)] + fn test_get_cpu_data() { + let path = "./tests/proc/stat/valid_stat"; + let cpu_data_result = get_cpu_data_from_path(&path); + assert!(!cpu_data_result.is_err()); + let cpu_data = cpu_data_result.unwrap(); + assert_eq!(cpu_data.total_user_time_ms, 23370.0); + assert_eq!(cpu_data.total_system_time_ms, 1880.0); + assert_eq!(cpu_data.total_idle_time_ms, 178380.0); + assert_eq!(cpu_data.individual_cpu_idle_times.len(), 2); + assert_eq!( + *cpu_data + .individual_cpu_idle_times + .get("cpu0") + .expect("cpu0 not found"), + 91880.0 + ); + assert_eq!( + *cpu_data + .individual_cpu_idle_times + .get("cpu1") + .expect("cpu1 not found"), + 86490.0 + ); + + let path = "./tests/proc/stat/invalid_stat_non_numerical_value_1"; + let cpu_data_result = get_cpu_data_from_path(&path); + assert!(cpu_data_result.is_err()); + + let path = "./tests/proc/stat/invalid_stat_non_numerical_value_2"; + let cpu_data_result = get_cpu_data_from_path(&path); + assert!(cpu_data_result.is_err()); + + let path = "./tests/proc/stat/invalid_stat_malformed_first_line"; + let cpu_data_result = get_cpu_data_from_path(&path); + assert!(cpu_data_result.is_err()); + + let path = "./tests/proc/stat/invalid_stat_malformed_per_cpu_line"; + let cpu_data_result = get_cpu_data_from_path(&path); + assert!(cpu_data_result.is_err()); + + let path = "./tests/proc/stat/invalid_stat_missing_cpun_data"; + let cpu_data_result = get_cpu_data_from_path(&path); + assert!(cpu_data_result.is_err()); + + let path = "./tests/proc/stat/nonexistent_stat"; + let cpu_data_result = get_cpu_data_from_path(&path); + assert!(cpu_data_result.is_err()); + } + + #[test] + #[allow(clippy::float_cmp)] + fn test_get_uptime_data() { + let path = "./tests/proc/uptime/valid_uptime"; + let uptime_data_result = get_uptime_from_path(&path); + assert!(!uptime_data_result.is_err()); + let uptime_data = uptime_data_result.unwrap(); + assert_eq!(uptime_data, 3213103123000.0); + + let path = "./tests/proc/uptime/invalid_data_uptime"; + let uptime_data_result = get_uptime_from_path(&path); + assert!(uptime_data_result.is_err()); + + let path = "./tests/proc/uptime/malformed_uptime"; + let uptime_data_result = get_uptime_from_path(&path); + assert!(uptime_data_result.is_err()); + + let path = "./tests/proc/uptime/nonexistent_uptime"; + let uptime_data_result = get_uptime_from_path(&path); + assert!(uptime_data_result.is_err()); + } } diff --git a/bottlecap/tests/proc/stat/invalid_stat_malformed_first_line b/bottlecap/tests/proc/stat/invalid_stat_malformed_first_line new file mode 100644 index 000000000..7071a126d --- /dev/null +++ b/bottlecap/tests/proc/stat/invalid_stat_malformed_first_line @@ -0,0 +1,2 @@ +cpu 2337 +... diff --git a/bottlecap/tests/proc/stat/invalid_stat_malformed_per_cpu_line b/bottlecap/tests/proc/stat/invalid_stat_malformed_per_cpu_line new file mode 100644 index 000000000..d4dd4badd --- /dev/null +++ b/bottlecap/tests/proc/stat/invalid_stat_malformed_per_cpu_line @@ -0,0 +1,10 @@ +cpu 2337 0 188 17838 8 0 16 181 0 0 +cpu0 1453 0 87 8649 2 0 10 85 0 0 +cpu1 884 0 ... +intr 67620 0 0 0 0 354 4356 233 1294 89 759 185 359 8 3 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +ctxt 135197 +btime 1716225108 +processes 1428 +procs_running 1 +procs_blocked 0 +softirq 27242 0 2425 2 696 6166 0 48 7838 0 10067 diff --git a/bottlecap/tests/proc/stat/invalid_stat_missing_cpun_data b/bottlecap/tests/proc/stat/invalid_stat_missing_cpun_data new file mode 100644 index 000000000..75119c03d --- /dev/null +++ b/bottlecap/tests/proc/stat/invalid_stat_missing_cpun_data @@ -0,0 +1,8 @@ +cpu 2337 0 188 17838 8 0 16 181 0 0 +intr 67620 0 0 0 0 354 4356 233 1294 89 759 185 359 8 3 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +ctxt 135197 +btime 1716225108 +processes 1428 +procs_running 1 +procs_blocked 0 +softirq 27242 0 2425 2 696 6166 0 48 7838 0 10067 diff --git a/bottlecap/tests/proc/stat/invalid_stat_non_numerical_value_1 b/bottlecap/tests/proc/stat/invalid_stat_non_numerical_value_1 new file mode 100644 index 000000000..d72287175 --- /dev/null +++ b/bottlecap/tests/proc/stat/invalid_stat_non_numerical_value_1 @@ -0,0 +1,2 @@ +cpu 2337 0 INVALID 17838 8 0 16 181 0 0 +... diff --git a/bottlecap/tests/proc/stat/invalid_stat_non_numerical_value_2 b/bottlecap/tests/proc/stat/invalid_stat_non_numerical_value_2 new file mode 100644 index 000000000..816ba9009 --- /dev/null +++ b/bottlecap/tests/proc/stat/invalid_stat_non_numerical_value_2 @@ -0,0 +1,2 @@ +cpu INVALID 0 188 17838 8 0 16 181 0 0 +... diff --git a/bottlecap/tests/proc/stat/valid_stat b/bottlecap/tests/proc/stat/valid_stat new file mode 100644 index 000000000..d0a082700 --- /dev/null +++ b/bottlecap/tests/proc/stat/valid_stat @@ -0,0 +1,10 @@ +cpu 2337 0 188 17838 8 0 16 181 0 0 +cpu0 884 0 100 9188 5 0 6 95 0 0 +cpu1 1453 0 87 8649 2 0 10 85 0 0 +intr 67620 0 0 0 0 354 4356 233 1294 89 759 185 359 8 3 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +ctxt 135197 +btime 1716225108 +processes 1428 +procs_running 1 +procs_blocked 0 +softirq 27242 0 2425 2 696 6166 0 48 7838 0 10067 diff --git a/bottlecap/tests/proc/uptime/invalid_data_uptime b/bottlecap/tests/proc/uptime/invalid_data_uptime new file mode 100644 index 000000000..7fc664612 --- /dev/null +++ b/bottlecap/tests/proc/uptime/invalid_data_uptime @@ -0,0 +1 @@ +3213103123 INVALID diff --git a/bottlecap/tests/proc/uptime/malformed_uptime b/bottlecap/tests/proc/uptime/malformed_uptime new file mode 100644 index 000000000..e75900cd7 --- /dev/null +++ b/bottlecap/tests/proc/uptime/malformed_uptime @@ -0,0 +1 @@ +3213103123 diff --git a/bottlecap/tests/proc/uptime/valid_uptime b/bottlecap/tests/proc/uptime/valid_uptime new file mode 100644 index 000000000..91c626c1b --- /dev/null +++ b/bottlecap/tests/proc/uptime/valid_uptime @@ -0,0 +1 @@ +3213103123 32131 From 84e36ae9942db8a25090f977e2b63929b0e5258b Mon Sep 17 00:00:00 2001 From: shreyamalpani Date: Mon, 28 Oct 2024 17:25:20 -0400 Subject: [PATCH 2/9] clippy fixes --- bottlecap/src/lifecycle/invocation/processor.rs | 8 ++++---- bottlecap/src/metrics/enhanced/lambda.rs | 11 ++++++----- bottlecap/src/proc/clock.rs | 1 + bottlecap/src/proc/mod.rs | 8 +++----- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/bottlecap/src/lifecycle/invocation/processor.rs b/bottlecap/src/lifecycle/invocation/processor.rs index 1476f2870..84dd4117b 100644 --- a/bottlecap/src/lifecycle/invocation/processor.rs +++ b/bottlecap/src/lifecycle/invocation/processor.rs @@ -85,9 +85,9 @@ impl Processor { let cpu_offset: Option = proc::get_cpu_data().ok(); let uptime_offset: Option = proc::get_uptime().ok(); let enhanced_metric_offsets = Some(EnhancedMetricData { - network_offset: network_offset, - cpu_offset: cpu_offset, - uptime_offset: uptime_offset, + network_offset, + cpu_offset, + uptime_offset, }); self.context_buffer .add_enhanced_metric_data(&request_id, enhanced_metric_offsets); @@ -140,7 +140,7 @@ impl Processor { // - trigger tags (from inferred spans) // - metrics tags (for asm) - enhanced_metric_data = context.enhanced_metric_data.clone(); + enhanced_metric_data.clone_from(&context.enhanced_metric_data); } self.inferrer.complete_inferred_span(&self.span); diff --git a/bottlecap/src/metrics/enhanced/lambda.rs b/bottlecap/src/metrics/enhanced/lambda.rs index 97cdbbc35..669cf93e3 100644 --- a/bottlecap/src/metrics/enhanced/lambda.rs +++ b/bottlecap/src/metrics/enhanced/lambda.rs @@ -204,10 +204,7 @@ impl Lambda { } } - pub fn set_cpu_time_enhanced_metrics( - &self, - cpu_offset: Option, - ) { + pub fn set_cpu_time_enhanced_metrics(&self, cpu_offset: Option) { if !self.config.enhanced_metrics { return; } @@ -682,7 +679,11 @@ mod tests { ); // the differences above and metric values below are from an invocation using the go agent to verify the calculations - assert_sketch(&metrics_aggr, constants::CPU_TOTAL_UTILIZATION_PCT_METRIC, 30.0); + assert_sketch( + &metrics_aggr, + constants::CPU_TOTAL_UTILIZATION_PCT_METRIC, + 30.0, + ); assert_sketch(&metrics_aggr, constants::CPU_TOTAL_UTILIZATION_METRIC, 0.6); assert_sketch(&metrics_aggr, constants::NUM_CORES_METRIC, 2.0); assert_sketch(&metrics_aggr, constants::CPU_MAX_UTILIZATION_METRIC, 30.0); diff --git a/bottlecap/src/proc/clock.rs b/bottlecap/src/proc/clock.rs index 0455554e5..ef3aaa26a 100644 --- a/bottlecap/src/proc/clock.rs +++ b/bottlecap/src/proc/clock.rs @@ -1,6 +1,7 @@ use libc; use std::io; +#[allow(clippy::cast_sign_loss)] #[cfg(not(target_os = "windows"))] pub fn get_clk_tck() -> Result { let clk_tck = unsafe { libc::sysconf(libc::_SC_CLK_TCK) }; diff --git a/bottlecap/src/proc/mod.rs b/bottlecap/src/proc/mod.rs index 2a2f894a1..8684019b8 100644 --- a/bottlecap/src/proc/mod.rs +++ b/bottlecap/src/proc/mod.rs @@ -7,9 +7,7 @@ use std::{ io::{self, BufRead}, }; -use constants::{ - LAMDBA_NETWORK_INTERFACE, PROC_NET_DEV_PATH, PROC_STAT_PATH, PROC_UPTIME_PATH, -}; +use constants::{LAMDBA_NETWORK_INTERFACE, PROC_NET_DEV_PATH, PROC_STAT_PATH, PROC_UPTIME_PATH}; #[derive(Copy, Clone, Debug, PartialEq)] pub struct NetworkData { @@ -114,8 +112,8 @@ fn get_cpu_data_from_path(path: &str) -> Result { )) } } - } else if label.starts_with("cpu") { // i.e. "cpu0", "cpu1" - // Parse per-core idle times + } else if label.starts_with("cpu") { + // Parse per core (i.e. "cpu0", "cpu1", etc.) idle times // Skip the first three values (user, nice, system) and get the 4th value (idle) let idle: Option = values.nth(3).and_then(|s| s.parse().ok()); From bd08c90a902e658a06f9322831440d5876483000 Mon Sep 17 00:00:00 2001 From: shreyamalpani Date: Tue, 29 Oct 2024 16:07:51 -0400 Subject: [PATCH 3/9] fixes --- bottlecap/src/bin/bottlecap/main.rs | 2 +- bottlecap/src/lifecycle/invocation/processor.rs | 6 ++++-- bottlecap/src/proc/clock.rs | 1 - bottlecap/src/proc/mod.rs | 3 ++- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/bottlecap/src/bin/bottlecap/main.rs b/bottlecap/src/bin/bottlecap/main.rs index e2e936aca..542609743 100644 --- a/bottlecap/src/bin/bottlecap/main.rs +++ b/bottlecap/src/bin/bottlecap/main.rs @@ -376,7 +376,7 @@ async fn extension_loop_active( ); lambda_enhanced_metrics.increment_invocation_metric(); let mut p = invocation_processor.lock().await; - p.on_invoke_event(request_id, lambda_enhanced_metrics.config.enhanced_metrics); + p.on_invoke_event(request_id); drop(p); } Ok(NextEventResponse::Shutdown { diff --git a/bottlecap/src/lifecycle/invocation/processor.rs b/bottlecap/src/lifecycle/invocation/processor.rs index 84dd4117b..5afde82b6 100644 --- a/bottlecap/src/lifecycle/invocation/processor.rs +++ b/bottlecap/src/lifecycle/invocation/processor.rs @@ -35,6 +35,7 @@ pub struct Processor { propagator: DatadogCompositePropagator, aws_config: AwsConfig, tracer_detected: bool, + collect_enhanced_data: bool, } impl Processor { @@ -74,13 +75,14 @@ impl Processor { propagator, aws_config: aws_config.clone(), tracer_detected: false, + collect_enhanced_data: config.enhanced_metrics, } } /// Given a `request_id`, creates the context and adds the enhanced metric offsets to the context buffer. /// - pub fn on_invoke_event(&mut self, request_id: String, collect_enhanced_data: bool) { - if collect_enhanced_data { + pub fn on_invoke_event(&mut self, request_id: String) { + if self.collect_enhanced_data { let network_offset: Option = proc::get_network_data().ok(); let cpu_offset: Option = proc::get_cpu_data().ok(); let uptime_offset: Option = proc::get_uptime().ok(); diff --git a/bottlecap/src/proc/clock.rs b/bottlecap/src/proc/clock.rs index ef3aaa26a..09c35c9bd 100644 --- a/bottlecap/src/proc/clock.rs +++ b/bottlecap/src/proc/clock.rs @@ -1,4 +1,3 @@ -use libc; use std::io; #[allow(clippy::cast_sign_loss)] diff --git a/bottlecap/src/proc/mod.rs b/bottlecap/src/proc/mod.rs index 8684019b8..c34f6e232 100644 --- a/bottlecap/src/proc/mod.rs +++ b/bottlecap/src/proc/mod.rs @@ -152,7 +152,8 @@ fn get_uptime_from_path(path: &str) -> Result { let file = File::open(path)?; let reader = io::BufReader::new(file); - if let Some(Ok(line)) = reader.lines().next() { + if let Some(line) = reader.lines().next() { + let line = line?; let mut values = line.split_whitespace(); let uptime: Option = values.next().and_then(|s| s.parse().ok()); From 50eb2d31c9b80884cc4196ed050f536c39b5bbc0 Mon Sep 17 00:00:00 2001 From: shreyamalpani Date: Thu, 31 Oct 2024 13:40:30 -0400 Subject: [PATCH 4/9] set utilization metrics before flushing & format fixes --- bottlecap/Cargo.toml | 2 +- bottlecap/src/bin/bottlecap/main.rs | 8 ++++---- bottlecap/src/lifecycle/invocation/context.rs | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/bottlecap/Cargo.toml b/bottlecap/Cargo.toml index 8fe260530..432171db8 100644 --- a/bottlecap/Cargo.toml +++ b/bottlecap/Cargo.toml @@ -19,7 +19,7 @@ dogstatsd = { git = "https://github.com/DataDog/libdatadog", branch = "main" } figment = { version = "0.10", default-features = false, features = ["yaml", "env"] } hyper = { version = "0.14", default-features = false, features = ["server"] } lazy_static = { version = "1.5", default-features = false } -libc = "0.2" +libc = { version = "0.2", default-features = false } log = { version = "0.4", default-features = false } protobuf = { version = "3.5", default-features = false } regex = { version = "1.10", default-features = false } diff --git a/bottlecap/src/bin/bottlecap/main.rs b/bottlecap/src/bin/bottlecap/main.rs index 542609743..2038d20a6 100644 --- a/bottlecap/src/bin/bottlecap/main.rs +++ b/bottlecap/src/bin/bottlecap/main.rs @@ -451,6 +451,10 @@ async fn extension_loop_active( request_id, status ); + if let Some(offsets) = enhanced_metric_data { + lambda_enhanced_metrics.set_cpu_utilization_enhanced_metrics(offsets.cpu_offset, offsets.uptime_offset); + } + // TODO(astuyve) it'll be easy to // pass the invocation deadline to // flush tasks here, so they can @@ -464,10 +468,6 @@ async fn extension_loop_active( ); } - if let Some(offsets) = enhanced_metric_data { - lambda_enhanced_metrics.set_cpu_utilization_enhanced_metrics(offsets.cpu_offset, offsets.uptime_offset); - } - break; } TelemetryRecord::PlatformReport { diff --git a/bottlecap/src/lifecycle/invocation/context.rs b/bottlecap/src/lifecycle/invocation/context.rs index 6621ccedf..4485cbe76 100644 --- a/bottlecap/src/lifecycle/invocation/context.rs +++ b/bottlecap/src/lifecycle/invocation/context.rs @@ -363,9 +363,9 @@ mod tests { let uptime_offset = Some(50.0); let enhanced_metric_data = Some(EnhancedMetricData { - network_offset: network_offset, - cpu_offset: cpu_offset, - uptime_offset: uptime_offset, + network_offset, + cpu_offset, + uptime_offset, }); buffer.add_enhanced_metric_data(&request_id, enhanced_metric_data.clone()); From 3423c577c934e68338e159f9aaabda3028b68fd3 Mon Sep 17 00:00:00 2001 From: shreyamalpani Date: Thu, 31 Oct 2024 14:16:11 -0400 Subject: [PATCH 5/9] added comment to explain utilization metrics calculation timing --- bottlecap/src/bin/bottlecap/main.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/bottlecap/src/bin/bottlecap/main.rs b/bottlecap/src/bin/bottlecap/main.rs index 2038d20a6..dcddf23b5 100644 --- a/bottlecap/src/bin/bottlecap/main.rs +++ b/bottlecap/src/bin/bottlecap/main.rs @@ -451,6 +451,7 @@ async fn extension_loop_active( request_id, status ); + // set cpu utilization metrics here to avoid accounting for extra idle time if let Some(offsets) = enhanced_metric_data { lambda_enhanced_metrics.set_cpu_utilization_enhanced_metrics(offsets.cpu_offset, offsets.uptime_offset); } From d4ad790360f06e4c054424a6d1918e6f279b5f38 Mon Sep 17 00:00:00 2001 From: shreyamalpani Date: Sat, 2 Nov 2024 10:03:25 -0400 Subject: [PATCH 6/9] use nix instead of libc to get system clock --- bottlecap/Cargo.lock | 31 ++++++++++++++++++++++++------- bottlecap/Cargo.toml | 2 +- bottlecap/src/proc/clock.rs | 15 ++++++++------- 3 files changed, 33 insertions(+), 15 deletions(-) diff --git a/bottlecap/Cargo.lock b/bottlecap/Cargo.lock index fbae75bbc..d9fdbfc14 100644 --- a/bottlecap/Cargo.lock +++ b/bottlecap/Cargo.lock @@ -344,7 +344,7 @@ version = "0.69.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0" dependencies = [ - "bitflags", + "bitflags 2.6.0", "cexpr", "clang-sys", "itertools 0.12.1", @@ -376,6 +376,12 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + [[package]] name = "bitflags" version = "2.6.0" @@ -426,8 +432,8 @@ dependencies = [ "httpmock", "hyper 0.14.30", "lazy_static", - "libc", "log", + "nix", "proptest", "protobuf", "rand", @@ -1576,7 +1582,7 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" dependencies = [ - "bitflags", + "bitflags 2.6.0", "libc", ] @@ -1671,6 +1677,17 @@ version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" +[[package]] +name = "nix" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "598beaf3cc6fdd9a5dfb1630c2800c7acd31df7aaf0f565796fba2b53ca1af1b" +dependencies = [ + "bitflags 1.3.2", + "cfg-if", + "libc", +] + [[package]] name = "nom" version = "7.1.3" @@ -1932,7 +1949,7 @@ checksum = "b4c2511913b88df1637da85cc8d96ec8e43a3f8bb8ccb71ee1ac240d6f3df58d" dependencies = [ "bit-set", "bit-vec", - "bitflags", + "bitflags 2.6.0", "lazy_static", "num-traits", "rand", @@ -2180,7 +2197,7 @@ version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b6dfecf2c74bce2466cabf93f6664d6998a69eb21e39f4207930065b27b771f" dependencies = [ - "bitflags", + "bitflags 2.6.0", ] [[package]] @@ -2367,7 +2384,7 @@ version = "0.38.37" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8acb788b847c24f28525660c4d7758620a7210875711f79e7f663cc152726811" dependencies = [ - "bitflags", + "bitflags 2.6.0", "errno", "libc", "linux-raw-sys", @@ -2554,7 +2571,7 @@ version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ - "bitflags", + "bitflags 2.6.0", "core-foundation", "core-foundation-sys", "libc", diff --git a/bottlecap/Cargo.toml b/bottlecap/Cargo.toml index 432171db8..bfbaef220 100644 --- a/bottlecap/Cargo.toml +++ b/bottlecap/Cargo.toml @@ -19,8 +19,8 @@ dogstatsd = { git = "https://github.com/DataDog/libdatadog", branch = "main" } figment = { version = "0.10", default-features = false, features = ["yaml", "env"] } hyper = { version = "0.14", default-features = false, features = ["server"] } lazy_static = { version = "1.5", default-features = false } -libc = { version = "0.2", default-features = false } log = { version = "0.4", default-features = false } +nix = { version = "0.26", default-features = false, features = ["feature"] } protobuf = { version = "3.5", default-features = false } regex = { version = "1.10", default-features = false } reqwest = { version = "0.12", features = ["json", "http2", "rustls-tls"], default-features = false } diff --git a/bottlecap/src/proc/clock.rs b/bottlecap/src/proc/clock.rs index 09c35c9bd..51e2b501d 100644 --- a/bottlecap/src/proc/clock.rs +++ b/bottlecap/src/proc/clock.rs @@ -1,16 +1,17 @@ +use nix::unistd::sysconf; +use nix::unistd::SysconfVar; use std::io; #[allow(clippy::cast_sign_loss)] #[cfg(not(target_os = "windows"))] pub fn get_clk_tck() -> Result { - let clk_tck = unsafe { libc::sysconf(libc::_SC_CLK_TCK) }; - if clk_tck == -1 { - return Err(io::Error::new( - io::ErrorKind::Other, - "Failed to get clock ticks per second", - )); + match sysconf(SysconfVar::CLK_TCK) { + Ok(Some(clk_tck)) if clk_tck > 0 => Ok(clk_tck as u64), + _ => Err(io::Error::new( + io::ErrorKind::NotFound, + "Could not find system clock ticks per second", + )), } - Ok(clk_tck as u64) } #[cfg(target_os = "windows")] From 16646ed19f398b0dcf03f8fe3f56dc14b489a58d Mon Sep 17 00:00:00 2001 From: shreyamalpani Date: Mon, 4 Nov 2024 09:13:45 -0500 Subject: [PATCH 7/9] update LICENSE-3rdparty.yml --- bottlecap/LICENSE-3rdparty.yml | 62 ++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/bottlecap/LICENSE-3rdparty.yml b/bottlecap/LICENSE-3rdparty.yml index d4e135179..ac1f5b110 100644 --- a/bottlecap/LICENSE-3rdparty.yml +++ b/bottlecap/LICENSE-3rdparty.yml @@ -2665,6 +2665,40 @@ third_party_libraries: THE SOFTWARE. - license: Apache-2.0 text: " Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\nTERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\nEND OF TERMS AND CONDITIONS\n\nAPPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\nCopyright [yyyy] [name of copyright owner]\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n\thttp://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n" +- package_name: bitflags + package_version: 1.3.2 + repository: https://github.com/bitflags/bitflags + license: MIT/Apache-2.0 + licenses: + - license: MIT + text: | + Copyright (c) 2014 The Rust Project Developers + + Permission is hereby granted, free of charge, to any + person obtaining a copy of this software and associated + documentation files (the "Software"), to deal in the + Software without restriction, including without + limitation the rights to use, copy, modify, merge, + publish, distribute, sublicense, and/or sell copies of + the Software, and to permit persons to whom the Software + is furnished to do so, subject to the following + conditions: + + The above copyright notice and this permission notice + shall be included in all copies or substantial portions + of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF + ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED + TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A + PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT + SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR + IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. + - license: Apache-2.0 + text: " Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\nTERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\nEND OF TERMS AND CONDITIONS\n\nAPPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\nCopyright [yyyy] [name of copyright owner]\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n\thttp://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n" - package_name: bitflags package_version: 2.6.0 repository: https://github.com/bitflags/bitflags @@ -10259,6 +10293,34 @@ third_party_libraries: licenses: - license: MIT text: NOT FOUND +- package_name: nix + package_version: 0.26.4 + repository: https://github.com/nix-rust/nix + license: MIT + licenses: + - license: MIT + text: | + The MIT License (MIT) + + Copyright (c) 2015 Carl Lerche + nix-rust Authors + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. - package_name: num-traits package_version: 0.2.19 repository: https://github.com/rust-num/num-traits From 66c6dee0d632853df2728a1c2200ae5ca41d6a0d Mon Sep 17 00:00:00 2001 From: shreyamalpani Date: Wed, 6 Nov 2024 10:54:42 -0500 Subject: [PATCH 8/9] added comments to explain calculations --- bottlecap/src/metrics/enhanced/lambda.rs | 6 ++++++ bottlecap/src/proc/clock.rs | 3 +-- bottlecap/src/proc/mod.rs | 11 +++++++---- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/bottlecap/src/metrics/enhanced/lambda.rs b/bottlecap/src/metrics/enhanced/lambda.rs index 669cf93e3..7a9ac4c1c 100644 --- a/bottlecap/src/metrics/enhanced/lambda.rs +++ b/bottlecap/src/metrics/enhanced/lambda.rs @@ -252,13 +252,19 @@ impl Lambda { } // Maximally utilized CPU is the one with the least time spent in the idle process + // Multiply by 100 to report as percentage let cpu_max_utilization = ((uptime - min_idle_time) / uptime) * 100.0; + // Minimally utilized CPU is the one with the most time spent in the idle process + // Multiply by 100 to report as percentage let cpu_min_utilization = ((uptime - max_idle_time) / uptime) * 100.0; + // CPU total utilization is the proportion of total non-idle time to the total uptime across all cores let cpu_total_utilization_decimal = ((uptime * num_cores) - total_idle_time) / (uptime * num_cores); + // Multiply by 100 to report as percentage let cpu_total_utilization_pct = cpu_total_utilization_decimal * 100.0; + // Multiply by num_cores to report in terms of cores let cpu_total_utilization = cpu_total_utilization_decimal * num_cores; let metric = Metric::new( diff --git a/bottlecap/src/proc/clock.rs b/bottlecap/src/proc/clock.rs index 51e2b501d..8c7c9b328 100644 --- a/bottlecap/src/proc/clock.rs +++ b/bottlecap/src/proc/clock.rs @@ -1,5 +1,4 @@ -use nix::unistd::sysconf; -use nix::unistd::SysconfVar; +use nix::unistd::{sysconf, SysconfVar}; use std::io; #[allow(clippy::cast_sign_loss)] diff --git a/bottlecap/src/proc/mod.rs b/bottlecap/src/proc/mod.rs index c34f6e232..1a74a2b9d 100644 --- a/bottlecap/src/proc/mod.rs +++ b/bottlecap/src/proc/mod.rs @@ -101,9 +101,10 @@ fn get_cpu_data_from_path(path: &str) -> Result { match (user, system, idle) { (Some(user_val), Some(system_val), Some(idle_val)) => { - cpu_data.total_user_time_ms = (1000.0 * user_val) / clktck; - cpu_data.total_system_time_ms = (1000.0 * system_val) / clktck; - cpu_data.total_idle_time_ms = (1000.0 * idle_val) / clktck; + // Divide values by clock tick to covert to seconds, then multiply by 1000 to convert to ms + cpu_data.total_user_time_ms = (user_val / clktck) * 1000.0; + cpu_data.total_system_time_ms = (system_val / clktck) * 1000.0; + cpu_data.total_idle_time_ms = (idle_val / clktck) * 1000.0; } (_, _, _) => { return Err(io::Error::new( @@ -119,9 +120,10 @@ fn get_cpu_data_from_path(path: &str) -> Result { match idle { Some(idle_val) => { + // Divide value by clock tick to covert to seconds, then multiply by 1000 to convert to ms cpu_data .individual_cpu_idle_times - .insert(label.to_string(), (1000.0 * idle_val) / clktck); + .insert(label.to_string(), (idle_val / clktck) * 1000.0); } None => { return Err(io::Error::new( @@ -161,6 +163,7 @@ fn get_uptime_from_path(path: &str) -> Result { match (uptime, idle) { // Check that the file is correctly formatted (i.e. has both values) + // Multiply val by 1000 to convert seconds to milliseconds (Some(uptime_val), Some(_idle_val)) => return Ok(uptime_val * 1000.0), (_, _) => { return Err(io::Error::new( From 73866e415c05362cbe5717b28d317ddabcd42d71 Mon Sep 17 00:00:00 2001 From: shreyamalpani Date: Wed, 6 Nov 2024 10:59:09 -0500 Subject: [PATCH 9/9] clippy --- bottlecap/src/metrics/enhanced/lambda.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bottlecap/src/metrics/enhanced/lambda.rs b/bottlecap/src/metrics/enhanced/lambda.rs index 7a9ac4c1c..13dd203a9 100644 --- a/bottlecap/src/metrics/enhanced/lambda.rs +++ b/bottlecap/src/metrics/enhanced/lambda.rs @@ -264,7 +264,7 @@ impl Lambda { ((uptime * num_cores) - total_idle_time) / (uptime * num_cores); // Multiply by 100 to report as percentage let cpu_total_utilization_pct = cpu_total_utilization_decimal * 100.0; - // Multiply by num_cores to report in terms of cores + // Multiply by num_cores to report in terms of cores let cpu_total_utilization = cpu_total_utilization_decimal * num_cores; let metric = Metric::new(