From 8b3ca100528a0ea1152f80f8813628e0e3c3651a Mon Sep 17 00:00:00 2001 From: Jan-Erik Rediger Date: Thu, 5 Jun 2025 12:18:29 +0200 Subject: [PATCH 01/30] Import sqlite storage as separate module This is a modified version of the kvstore/skv implementation: https://searchfox.org/firefox-main/rev/cced10961b53e0d29e22e635404fec37728b2644/toolkit/components/kvstore/src/skv/connection.rs Which itself is based on application-service's sql-support. It's stripped down to what we need in Glean: * A file-backed database * A schema set up on start, potentially applying migrations if we need that * A read-write connection, which is re-used for all access. --- glean-core/src/database/sqlite.rs | 442 +++++++++++++++++++ glean-core/src/database/sqlite/connection.rs | 111 +++++ glean-core/src/database/sqlite/schema.rs | 68 +++ 3 files changed, 621 insertions(+) create mode 100644 glean-core/src/database/sqlite.rs create mode 100644 glean-core/src/database/sqlite/connection.rs create mode 100644 glean-core/src/database/sqlite/schema.rs diff --git a/glean-core/src/database/sqlite.rs b/glean-core/src/database/sqlite.rs new file mode 100644 index 0000000000..59eddc11c6 --- /dev/null +++ b/glean-core/src/database/sqlite.rs @@ -0,0 +1,442 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::fs; +use std::num::NonZeroU64; +use std::path::Path; +use std::str; + +use rusqlite::params; +use rusqlite::types::FromSqlError; +use rusqlite::Transaction; + +use connection::Connection; +use schema::Schema; + +use crate::common_metric_data::CommonMetricDataInternal; +use crate::metrics::Metric; +use crate::Glean; +use crate::Lifetime; +use crate::Result; + +mod connection; +mod schema; + +pub struct Database { + /// The database connection. + conn: connection::Connection, +} + +impl std::fmt::Debug for Database { + fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result { + fmt.debug_struct("Database") + .field("conn", &self.conn) + .finish() + } +} + +const DEFAULT_DATABASE_FILE_NAME: &str = "glean.sqlite"; + +impl Database { + /// Initializes the data store. + /// + /// This opens the underlying SQLite store and creates + /// the underlying directory structure. + pub fn new(data_path: &Path, _delay_ping_lifetime_io: bool) -> Result { + let path = data_path.join("db"); + log::debug!("Database path: {:?}", path.display()); + + fs::create_dir_all(&path)?; + let store_path = path.join(DEFAULT_DATABASE_FILE_NAME); + let conn = Connection::new::(&store_path).unwrap(); + + let db = Self { conn }; + + Ok(db) + } + + /// Get the initial database file size. + pub fn file_size(&self) -> Option { + None + } + + /// Get the rkv load state. + pub fn rkv_load_state(&self) -> Option { + None + } + + /// Iterates with the provided transaction function + /// over the requested data from the given storage. + /// + /// * If the storage is unavailable, the transaction function is never invoked. + /// * If the read data cannot be deserialized it will be silently skipped. + /// + /// # Arguments + /// + /// * `lifetime` - The metric lifetime to iterate over. + /// * `storage_name` - The storage name to iterate over. + /// * `transaction_fn` - Called for each entry being iterated over. It is + /// passed two arguments: `(metric_id: &[u8], metric: &Metric)`. + /// + /// # Panics + /// + /// This function will **not** panic on database errors. + pub fn iter_store(&self, lifetime: Lifetime, storage_name: &str, mut transaction_fn: F) + where + F: FnMut(&[u8], &Metric), + { + let iter_sql = r#" + SELECT id, value + FROM telemetry + WHERE + lifetime = ?1 + AND ping = ?2 + "#; + + self.conn + .read(|conn| { + let mut stmt = conn.prepare_cached(iter_sql).unwrap(); + let rows = stmt + .query_map( + params![lifetime.as_str().to_string(), storage_name], + |row| { + let id: String = row.get(0)?; + let blob: Vec = row.get(1)?; + let blob: Metric = bincode::deserialize(&blob) + .map_err(|_| FromSqlError::InvalidType)?; + Ok((id, blob)) + }, + ) + .unwrap(); + + for row in rows { + let Ok((metric_id, metric)) = row else { + continue; + }; + transaction_fn(metric_id.as_bytes(), &metric); + } + + Result::<(), ()>::Ok(()) + }) + .unwrap() + } + + /// Determines if the storage has the given metric. + /// + /// If data cannot be read it is assumed that the storage does not have the metric. + /// + /// # Arguments + /// + /// * `lifetime` - The lifetime of the metric. + /// * `storage_name` - The storage name to look in. + /// * `metric_identifier` - The metric identifier. + /// + /// # Panics + /// + /// This function will **not** panic on database errors. + pub fn has_metric( + &self, + lifetime: Lifetime, + storage_name: &str, + metric_identifier: &str, + ) -> bool { + let has_metric_sql = r#" + SELECT id + FROM telemetry + WHERE + lifetime = ?1 + AND ping = ?2 + AND id = ?3 + "#; + + self.conn + .read(|conn| { + let Ok(mut stmt) = conn.prepare_cached(has_metric_sql) else { + return Ok(false); + }; + let Ok(mut metric_iter) = + stmt.query([lifetime.as_str(), storage_name, metric_identifier]) + else { + return Ok(false); + }; + + Result::::Ok(metric_iter.next().map(|m| m.is_some()).unwrap_or(false)) + }) + .unwrap_or(false) + } + + /// Records a metric in the underlying storage system. + pub fn record(&self, glean: &Glean, data: &CommonMetricDataInternal, value: &Metric) { + // If upload is disabled we don't want to record. + if !glean.is_upload_enabled() { + return; + } + + let name = data.identifier(glean); + + _ = self.conn.write(|tx| { + for ping_name in data.storage_names() { + if let Err(e) = + self.record_per_lifetime(tx, data.inner.lifetime, ping_name, &name, value) + { + log::error!( + "Failed to record metric '{}' into {}: {:?}", + data.base_identifier(), + ping_name, + e + ); + } + } + + Ok::<(), rusqlite::Error>(()) + }); + } + + /// Records a metric in the underlying storage system, for a single lifetime. + /// + /// # Returns + /// + /// If the storage is unavailable or the write fails, no data will be stored and an error will be returned. + /// + /// Otherwise `Ok(())` is returned. + /// + /// # Panics + /// + /// This function will **not** panic on database errors. + fn record_per_lifetime( + &self, + tx: &mut Transaction, + lifetime: Lifetime, + storage_name: &str, + key: &str, + metric: &Metric, + ) -> Result<()> { + let insert_sql = r#" + INSERT INTO + telemetry (id, ping, lifetime, value) + VALUES + (?1, ?2, ?3, ?4) + ON CONFLICT(id, ping) DO UPDATE SET + lifetime = excluded.lifetime, + value = excluded.value + "#; + + let mut stmt = tx.prepare_cached(insert_sql)?; + let encoded = bincode::serialize(&metric).expect("IMPOSSIBLE: Serializing metric failed"); + stmt.execute(params![key, storage_name, lifetime.as_str(), encoded])?; + + Ok(()) + } + + /// Records the provided value, with the given lifetime, + /// after applying a transformation function. + pub fn record_with(&self, glean: &Glean, data: &CommonMetricDataInternal, mut transform: F) + where + F: FnMut(Option) -> Metric, + { + // If upload is disabled we don't want to record. + if !glean.is_upload_enabled() { + return; + } + + _ = self.conn.write(|tx| { + let name = data.identifier(glean); + for ping_name in data.storage_names() { + if let Err(e) = self.record_per_lifetime_with( + tx, + data.inner.lifetime, + ping_name, + &name, + &mut transform, + ) { + log::error!( + "Failed to record metric '{}' into {}: {:?}", + data.base_identifier(), + ping_name, + e + ); + } + } + + Result::<(), rusqlite::Error>::Ok(()) + }); + } + + /// Records a metric in the underlying storage system, + /// after applying the given transformation function, for a single lifetime. + /// + /// # Returns + /// + /// If the storage is unavailable or the write fails, no data will be stored and an error will be returned. + /// + /// Otherwise `Ok(())` is returned. + /// + /// # Panics + /// + /// This function will **not** panic on database errors. + fn record_per_lifetime_with( + &self, + tx: &mut Transaction, + lifetime: Lifetime, + storage_name: &str, + key: &str, + mut transform: F, + ) -> Result<()> + where + F: FnMut(Option) -> Metric, + { + let find_sql = r#" + SELECT value + FROM telemetry + WHERE + lifetime = ?1 + AND ping = ?2 + AND id = ?3 + LIMIT 1 + "#; + + let new_value = { + let mut stmt = tx.prepare_cached(&find_sql)?; + let mut rows = stmt.query(params![lifetime.as_str().to_string(), storage_name, key])?; + + if let Ok(Some(row)) = rows.next() { + let blob: Vec = row.get(0)?; + let old_value = bincode::deserialize(&blob).ok(); + transform(old_value) + } else { + transform(None) + } + }; + + let insert_sql = r#" + INSERT INTO + telemetry (id, ping, lifetime, value) + VALUES + (?1, ?2, ?3, ?4) + ON CONFLICT(id, ping) DO UPDATE SET + lifetime = excluded.lifetime, + value = excluded.value + "#; + + { + let mut stmt = tx.prepare_cached(insert_sql)?; + let encoded = + bincode::serialize(&new_value).expect("IMPOSSIBLE: Serializing metric failed"); + stmt.execute(params![key, storage_name, lifetime.as_str(), encoded])?; + } + + Ok(()) + } + + /// Clears a storage (only Ping Lifetime). + /// + /// # Returns + /// + /// * If the storage is unavailable an error is returned. + /// * If any individual delete fails, an error is returned, but other deletions might have + /// happened. + /// + /// Otherwise `Ok(())` is returned. + /// + /// # Panics + /// + /// This function will **not** panic on database errors. + pub fn clear_ping_lifetime_storage(&self, storage_name: &str) -> Result<()> { + let clear_sql = "DELETE FROM telemetry WHERE lifetime = ?1 AND ping = ?2"; + self.conn.write(|tx| { + let mut stmt = tx.prepare_cached(clear_sql)?; + stmt.execute([Lifetime::Ping.as_str(), storage_name])?; + Ok(()) + }) + } + + /// Removes a single metric from the storage. + /// + /// # Arguments + /// + /// * `lifetime` - the lifetime of the storage in which to look for the metric. + /// * `storage_name` - the name of the storage to store/fetch data from. + /// * `metric_id` - the metric category + name. + /// + /// # Returns + /// + /// * If the storage is unavailable an error is returned. + /// * If the metric could not be deleted, an error is returned. + /// + /// Otherwise `Ok(())` is returned. + /// + /// # Panics + /// + /// This function will **not** panic on database errors. + pub fn remove_single_metric( + &self, + lifetime: Lifetime, + storage_name: &str, + metric_id: &str, + ) -> Result<()> { + let clear_sql = "DELETE FROM telemetry WHERE lifetime = ?1 AND ping = ?2 AND id = ?3"; + self.conn.write(|tx| { + let mut stmt = tx.prepare_cached(clear_sql)?; + stmt.execute([lifetime.as_str(), storage_name, metric_id])?; + Ok(()) + }) + } + + /// Clears all the metrics in the database, for the provided lifetime. + /// + /// Errors are logged. + /// + /// # Panics + /// + /// * This function will **not** panic on database errors. + pub fn clear_lifetime(&self, lifetime: Lifetime) { + let clear_sql = "DELETE FROM telemetry WHERE lifetime = ?1"; + _ = self.conn.write(|tx| { + let mut stmt = tx.prepare_cached(clear_sql)?; + let res = stmt.execute([lifetime.as_str()]); + + if let Err(e) = res { + log::warn!("Could not clear store for lifetime {:?}: {:?}", lifetime, e); + } + Ok::<(), rusqlite::Error>(()) + }); + } + + /// Clears all metrics in the database. + /// + /// Errors are logged. + /// + /// # Panics + /// + /// * This function will **not** panic on database errors. + pub fn clear_all(&self) { + let lifetimes = &[ + Lifetime::User.as_str(), + Lifetime::Ping.as_str(), + Lifetime::Application.as_str(), + ]; + let clear_sql = + "DELETE FROM telemetry WHERE lifetime = ?1 OR lifetime = ?2 OR lifetime = ?3"; + _ = self.conn.write(|tx| { + let mut stmt = tx.prepare_cached(clear_sql)?; + let res = stmt.execute(lifetimes); + + if let Err(e) = res { + log::warn!("Could not clear store for all lifetimes: {:?}", e); + } + Ok::<(), rusqlite::Error>(()) + }); + } + + /// Persists ping_lifetime_data to disk. + /// + /// Does nothing in case there is nothing to persist. + /// + /// # Panics + /// + /// * This function will **not** panic on database errors. + pub fn persist_ping_lifetime_data(&self) -> Result<()> { + Ok(()) + } +} diff --git a/glean-core/src/database/sqlite/connection.rs b/glean-core/src/database/sqlite/connection.rs new file mode 100644 index 0000000000..7c6d1cebeb --- /dev/null +++ b/glean-core/src/database/sqlite/connection.rs @@ -0,0 +1,111 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! Lower-level, generic SQLite connection management. +//! +//! This module is inspired by, and borrows concepts from, the +//! Application Services `sql-support` crate. + +use std::{fmt::Debug, num::NonZeroU32, path::Path, sync::Mutex}; + +use rusqlite::{OpenFlags, Transaction, TransactionBehavior}; + +/// Sets up an SQLite database connection, and either +/// initializes an empty physical database with the latest schema, or +/// upgrades an existing physical database to the latest schema. +pub trait ConnectionOpener { + /// The highest schema version that we support. + const MAX_SCHEMA_VERSION: u32; + + type Error: From; + + /// Sets up an opened connection for use. This is a good place to + /// set pragmas and configuration options, register functions, and + /// load extensions. + fn setup(_conn: &mut rusqlite::Connection) -> Result<(), Self::Error> { + Ok(()) + } + + /// Initializes an empty physical database with the latest schema. + fn create(tx: &mut Transaction<'_>) -> Result<(), Self::Error>; + + /// Upgrades an existing physical database to the schema with + /// the given version. + fn upgrade(tx: &mut Transaction<'_>, to_version: NonZeroU32) -> Result<(), Self::Error>; +} + +/// A thread-safe wrapper around a connection to a physical SQLite database. +pub struct Connection { + /// The inner connection. + conn: Mutex, +} + +impl Connection { + /// Opens a connection to a physical database at the given path. + pub fn new(path: &Path) -> Result + where + O: ConnectionOpener, + { + let flags = OpenFlags::SQLITE_OPEN_NO_MUTEX // Send/Sync is guaranteed by Rust already + | OpenFlags::SQLITE_OPEN_EXRESCODE // Extended result codes + | OpenFlags::SQLITE_OPEN_CREATE // Create if it doesn't exist + | OpenFlags::SQLITE_OPEN_READ_WRITE; // opened for reading and writing + + let mut conn = rusqlite::Connection::open_with_flags(path, flags)?; + O::setup(&mut conn)?; + + // On open upgrade the schema to the latest version. + let mut tx = conn.transaction_with_behavior(TransactionBehavior::Exclusive)?; + match tx.query_row_and_then("PRAGMA user_version", [], |row| row.get(0)) { + Ok(mut version @ 1..) => { + while version < O::MAX_SCHEMA_VERSION { + O::upgrade(&mut tx, NonZeroU32::new(version + 1).unwrap())?; + version += 1; + } + } + Ok(0) => O::create(&mut tx)?, + Err(err) => Err(err)?, + } + // Set the schema version to the highest that we support. + // If the current version is higher than ours, downgrade it, + // so that upgrading to it again in the future can fix up any + // invariants that our version might not uphold. + tx.execute_batch(&format!("PRAGMA user_version = {}", O::MAX_SCHEMA_VERSION))?; + tx.commit()?; + Ok(Self::with_connection(conn)) + } + + fn with_connection(conn: rusqlite::Connection) -> Self { + Self { + conn: Mutex::new(conn), + } + } + + /// Accesses the database for reading. + pub fn read( + &self, + f: impl FnOnce(&rusqlite::Connection) -> Result, + ) -> Result { + let conn = self.conn.lock().unwrap(); + f(&*conn) + } + + /// Accesses the database in a transaction for reading and writing. + pub fn write(&self, f: impl FnOnce(&mut Transaction<'_>) -> Result) -> Result + where + E: From, + { + let mut conn = self.conn.lock().unwrap(); + let mut tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + let result = f(&mut tx)?; + tx.commit()?; + Ok(result) + } +} + +impl Debug for Connection { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("Connection { .. }") + } +} diff --git a/glean-core/src/database/sqlite/schema.rs b/glean-core/src/database/sqlite/schema.rs new file mode 100644 index 0000000000..6bf919a074 --- /dev/null +++ b/glean-core/src/database/sqlite/schema.rs @@ -0,0 +1,68 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//! The SQLite database schema. + +use std::num::NonZeroU32; + +use rusqlite::{config::DbConfig, Transaction}; + +use super::connection::ConnectionOpener; + +/// The schema for a physical SQLite database that contains many +/// named logical databases. +#[derive(Debug)] +pub struct Schema; + +impl ConnectionOpener for Schema { + const MAX_SCHEMA_VERSION: u32 = 1; + + type Error = SchemaError; + + fn setup(conn: &mut rusqlite::Connection) -> Result<(), Self::Error> { + conn.execute_batch( + "PRAGMA journal_mode = WAL; + PRAGMA journal_size_limit = 512000; -- 512 KB. + PRAGMA temp_store = MEMORY; + PRAGMA auto_vacuum = INCREMENTAL; + ", + )?; + + // Set hardening flags. + conn.set_db_config(DbConfig::SQLITE_DBCONFIG_DEFENSIVE, true)?; + + // Turn off misfeatures: double-quoted strings and untrusted schemas. + conn.set_db_config(DbConfig::SQLITE_DBCONFIG_DQS_DML, false)?; + conn.set_db_config(DbConfig::SQLITE_DBCONFIG_DQS_DDL, false)?; + conn.set_db_config(DbConfig::SQLITE_DBCONFIG_TRUSTED_SCHEMA, true)?; + + Ok(()) + } + + fn create(tx: &mut Transaction<'_>) -> Result<(), Self::Error> { + tx.execute_batch( + "CREATE TABLE telemetry( + id TEXT NOT NULL, + ping TEXT NOT NULL, + lifetime TEXT NOT NULL, + labels TEXT NOT NULL, -- can't be null or ON CONFLICT won't work + value BLOB, + UNIQUE(id, ping, labels) + );", + )?; + Ok(()) + } + + fn upgrade(_: &mut Transaction<'_>, to_version: NonZeroU32) -> Result<(), Self::Error> { + Err(SchemaError::UnsupportedSchemaVersion(to_version.get())) + } +} + +#[derive(thiserror::Error, Debug)] +pub enum SchemaError { + #[error("unsupported schema version: {0}")] + UnsupportedSchemaVersion(u32), + #[error("sqlite: {0}")] + Sqlite(#[from] rusqlite::Error), +} From 15df53995519224b623e31d31587e99d97a320a7 Mon Sep 17 00:00:00 2001 From: Jan-Erik Rediger Date: Thu, 5 Jun 2025 12:18:29 +0200 Subject: [PATCH 02/30] Rust dependency: Add `rusqlite` --- Cargo.lock | 87 +++++++++++++++++++++++++++++---- glean-core/Cargo.toml | 1 + glean-core/benchmark/Cargo.lock | 76 +++++++++++++++++++++++++++- 3 files changed, 154 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 344b205ab7..77171672cc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -160,11 +160,11 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.4.1" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" +checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" dependencies = [ - "serde", + "serde_core", ] [[package]] @@ -230,11 +230,12 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.0.83" +version = "1.2.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" +checksum = "755d2fce177175ffca841e9a06afdb2c4ab0f593d53b4dee48147dfaade85932" dependencies = [ - "libc", + "find-msvc-tools", + "shlex", ] [[package]] @@ -477,6 +478,18 @@ dependencies = [ "libc", ] +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + [[package]] name = "fastrand" version = "2.3.0" @@ -494,6 +507,12 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "find-msvc-tools" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8591b0bcc8a98a64310a2fae1bb3e9b8564dd10e381e6e28010fde8e8e8568db" + [[package]] name = "flate2" version = "1.0.35" @@ -599,6 +618,7 @@ dependencies = [ "pulldown-cmark", "pulldown-cmark-to-cmark", "rkv", + "rusqlite", "serde", "serde_json", "tempfile", @@ -833,6 +853,17 @@ version = "0.2.176" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "58f929b4d672ea937a23a1ab494143d968337a5f47e56d0815df1e0890ddf174" +[[package]] +name = "libsqlite3-sys" +version = "0.35.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "133c182a6a2c87864fe97778797e46c7e999672690dc9fa3ee8e241aa4a9c13f" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "linux-raw-sys" version = "0.4.10" @@ -944,6 +975,12 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + [[package]] name = "plain" version = "0.2.3" @@ -1020,7 +1057,7 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e8bbe1a966bd2f362681a44f6edce3c2310ac21e4d5067a6e7ec396297a6ea0" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.10.0", "memchr", "unicase", ] @@ -1115,7 +1152,7 @@ checksum = "0f67a9dbc634fcd36a2d1d800ca818065dcf71a1d907dc35130c2d1552c6e1dc" dependencies = [ "arrayref", "bincode", - "bitflags 2.4.1", + "bitflags 2.10.0", "id-arena", "lazy_static", "log", @@ -1129,6 +1166,20 @@ dependencies = [ "wr_malloc_size_of", ] +[[package]] +name = "rusqlite" +version = "0.37.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "165ca6e57b20e1351573e3729b958bc62f0e48025386970b6e4d29e7a7e71f3f" +dependencies = [ + "bitflags 2.10.0", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", +] + [[package]] name = "rustc-hash" version = "2.1.1" @@ -1141,7 +1192,7 @@ version = "0.38.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67ce50cb2e16c2903e30d1cbccfd8387a74b9d4c938b6a4c5ec6cc7556f7a8a0" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.10.0", "errno", "libc", "linux-raw-sys", @@ -1261,12 +1312,24 @@ dependencies = [ "serde_core", ] +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "siphasher" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + [[package]] name = "smawk" version = "0.3.2" @@ -1601,6 +1664,12 @@ dependencies = [ "getrandom", ] +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "wait-timeout" version = "0.2.1" diff --git a/glean-core/Cargo.toml b/glean-core/Cargo.toml index 447fb4b67e..ba154d34c1 100644 --- a/glean-core/Cargo.toml +++ b/glean-core/Cargo.toml @@ -44,6 +44,7 @@ uniffi = { version = "0.31.0", default-features = false } env_logger = { version = "0.10.0", default-features = false, optional = true } malloc_size_of_derive = "0.1.3" malloc_size_of = { version = "0.2.2", package = "wr_malloc_size_of", default-features = false, features = ["once_cell"] } +rusqlite = { version = "0.37.0", features = ["bundled"] } [target.'cfg(target_os = "android")'.dependencies] android_logger = { version = "0.12.0", default-features = false } diff --git a/glean-core/benchmark/Cargo.lock b/glean-core/benchmark/Cargo.lock index 37be0ce779..ccdf5cd27b 100644 --- a/glean-core/benchmark/Cargo.lock +++ b/glean-core/benchmark/Cargo.lock @@ -477,6 +477,18 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + [[package]] name = "fastrand" version = "2.3.0" @@ -499,6 +511,12 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -543,6 +561,7 @@ dependencies = [ "once_cell", "oslog", "rkv", + "rusqlite", "serde", "serde_json", "thiserror", @@ -624,12 +643,30 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash", +] + [[package]] name = "hashbrown" version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +[[package]] +name = "hashlink" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" +dependencies = [ + "hashbrown 0.15.5", +] + [[package]] name = "heck" version = "0.5.0" @@ -781,7 +818,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" dependencies = [ "equivalent", - "hashbrown", + "hashbrown 0.16.1", "serde", "serde_core", ] @@ -833,6 +870,17 @@ dependencies = [ "windows-link", ] +[[package]] +name = "libsqlite3-sys" +version = "0.35.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "133c182a6a2c87864fe97778797e46c7e999672690dc9fa3ee8e241aa4a9c13f" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "linux-raw-sys" version = "0.11.0" @@ -961,6 +1009,12 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + [[package]] name = "plain" version = "0.2.3" @@ -1131,6 +1185,20 @@ dependencies = [ "wr_malloc_size_of", ] +[[package]] +name = "rusqlite" +version = "0.37.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "165ca6e57b20e1351573e3729b958bc62f0e48025386970b6e4d29e7a7e71f3f" +dependencies = [ + "bitflags", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", +] + [[package]] name = "rustc-hash" version = "2.1.1" @@ -1601,6 +1669,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "walkdir" version = "2.5.0" From be30dc5f181d67bfcefd702510a8ac2405b00594 Mon Sep 17 00:00:00 2001 From: Jan-Erik Rediger Date: Fri, 20 Feb 2026 16:34:03 +0100 Subject: [PATCH 03/30] Vetting newly imported crates --- supply-chain/audits.toml | 47 +++++++++ supply-chain/config.toml | 10 +- supply-chain/imports.lock | 210 +++++++++++++++++++++++++++++++++++--- 3 files changed, 254 insertions(+), 13 deletions(-) diff --git a/supply-chain/audits.toml b/supply-chain/audits.toml index efb37606f6..e050266a99 100644 --- a/supply-chain/audits.toml +++ b/supply-chain/audits.toml @@ -295,6 +295,11 @@ criteria = "safe-to-deploy" delta = "2.4.0 -> 2.4.1" notes = "Only allowing new clippy lints" +[[audits.bitflags]] +who = "Jan-Erik Rediger " +criteria = "safe-to-deploy" +delta = "2.9.4 -> 2.10.0" + [[audits.bstr]] who = "Jan-Erik Rediger " criteria = "safe-to-run" @@ -321,6 +326,11 @@ who = "Jan-Erik Rediger " criteria = "safe-to-deploy" delta = "1.0.78 -> 1.0.83" +[[audits.cc]] +who = "Jan-Erik Rediger " +criteria = "safe-to-deploy" +delta = "1.2.41 -> 1.2.53" + [[audits.chrono]] who = "Lars Eggert " criteria = "safe-to-deploy" @@ -381,12 +391,27 @@ who = "Lars Eggert " criteria = "safe-to-deploy" delta = "0.3.11 -> 0.4.0" +[[audits.fallible-iterator]] +who = "Jan-Erik Rediger " +criteria = "safe-to-deploy" +version = "0.2.0" + +[[audits.fallible-streaming-iterator]] +who = "Jan-Erik Rediger " +criteria = "safe-to-deploy" +version = "0.1.9" + [[audits.fd-lock]] who = "Jan-Erik Rediger " criteria = "safe-to-deploy" delta = "3.0.12 -> 3.0.13" notes = "Dependency updates only" +[[audits.find-msvc-tools]] +who = "Jan-Erik Rediger " +criteria = "safe-to-deploy" +delta = "0.1.4 -> 0.1.8" + [[audits.flate2]] who = "Jan-Erik Rediger " criteria = "safe-to-deploy" @@ -545,6 +570,16 @@ criteria = "safe-to-deploy" delta = "0.19.0 -> 0.20.0" notes = "Removed all LMDB-specific code, added malloc_size_of integration" +[[audits.rmp]] +who = "Jan-Erik Rediger " +criteria = "safe-to-deploy" +delta = "0.8.14 -> 0.8.15" + +[[audits.rmp-serde]] +who = "Jan-Erik Rediger " +criteria = "safe-to-deploy" +delta = "1.3.0 -> 1.3.1" + [[audits.rustversion]] who = "Jan-Erik Rediger " criteria = "safe-to-deploy" @@ -754,6 +789,18 @@ criteria = "safe-to-deploy" delta = "1.1.0 -> 1.2.0" notes = "Added a file lock on the created directory" +[[trusted.cc]] +criteria = "safe-to-deploy" +user-id = 55123 # rust-lang-owner +start = "2022-10-29" +end = "2027-02-20" + +[[trusted.find-msvc-tools]] +criteria = "safe-to-deploy" +user-id = 539 # Josh Stone (cuviper) +start = "2025-08-29" +end = "2027-02-20" + [[trusted.hashbrown]] criteria = "safe-to-deploy" user-id = 55123 # rust-lang-owner diff --git a/supply-chain/config.toml b/supply-chain/config.toml index c41551eb22..4d432db878 100644 --- a/supply-chain/config.toml +++ b/supply-chain/config.toml @@ -86,7 +86,7 @@ criteria = "safe-to-deploy" [[exemptions.hashlink]] version = "0.7.0" -criteria = "safe-to-run" +criteria = "safe-to-deploy" [[exemptions.hermit-abi]] version = "0.2.6" @@ -112,6 +112,10 @@ criteria = "safe-to-run" version = "0.2.139" criteria = "safe-to-deploy" +[[exemptions.libsqlite3-sys]] +version = "0.26.0" +criteria = "safe-to-deploy" + [[exemptions.memchr]] version = "2.5.0" criteria = "safe-to-deploy" @@ -172,6 +176,10 @@ criteria = "safe-to-run" version = "0.6.27" criteria = "safe-to-run" +[[exemptions.rusqlite]] +version = "0.27.0" +criteria = "safe-to-deploy" + [[exemptions.scroll]] version = "0.11.0" criteria = "safe-to-deploy" diff --git a/supply-chain/imports.lock b/supply-chain/imports.lock index 9b91ce80a7..cd4432f75e 100644 --- a/supply-chain/imports.lock +++ b/supply-chain/imports.lock @@ -8,6 +8,12 @@ user-id = 696 user-login = "fitzgen" user-name = "Nick Fitzgerald" +[[publisher.cc]] +version = "1.2.30" +when = "2025-07-18" +user-id = 55123 +user-login = "rust-lang-owner" + [[publisher.encoding_rs]] version = "0.8.35" when = "2024-10-24" @@ -15,6 +21,13 @@ user-id = 4484 user-login = "hsivonen" user-name = "Henri Sivonen" +[[publisher.find-msvc-tools]] +version = "0.1.0" +when = "2025-08-29" +user-id = 539 +user-login = "cuviper" +user-name = "Josh Stone" + [[publisher.hashbrown]] version = "0.15.4" when = "2025-06-07" @@ -352,6 +365,21 @@ Nothing outside the realm of what one would expect from a bitflags generator, all as expected. """ +[[audits.bytecode-alliance.audits.bitflags]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +delta = "2.4.1 -> 2.6.0" +notes = """ +Changes in how macros are invoked and various bits and pieces of macro-fu. +Otherwise no major changes and nothing dealing with `unsafe`. +""" + +[[audits.bytecode-alliance.audits.bitflags]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +delta = "2.7.0 -> 2.9.4" +notes = "Tweaks to the macro, nothing out of order." + [[audits.bytecode-alliance.audits.camino]] who = "Pat Hickey " criteria = "safe-to-deploy" @@ -384,8 +412,8 @@ notes = "Dependency updates and minor changes, nothing suspicious." [[audits.bytecode-alliance.audits.cc]] who = "Alex Crichton " criteria = "safe-to-deploy" -version = "1.0.73" -notes = "I am the author of this crate." +delta = "1.2.30 -> 1.2.41" +notes = "This is a trusted rust-lang/rust crate" [[audits.bytecode-alliance.audits.cfg-if]] who = "Alex Crichton " @@ -432,6 +460,16 @@ criteria = "safe-to-deploy" version = "0.1.2" notes = "This should be portable to any POSIX system and seems like it should be part of the libc crate, but at any rate it's safe as is." +[[audits.bytecode-alliance.audits.fallible-iterator]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +delta = "0.2.0 -> 0.3.0" +notes = """ +This major version update has a few minor breaking changes but everything +this crate has to do with iterators and `Result` and such. No `unsafe` or +anything like that, all looks good. +""" + [[audits.bytecode-alliance.audits.fastrand]] who = "Alex Crichton " criteria = "safe-to-deploy" @@ -465,6 +503,12 @@ criteria = "safe-to-deploy" delta = "3.0.10 -> 3.0.12" notes = "Just a dependency version bump" +[[audits.bytecode-alliance.audits.find-msvc-tools]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +delta = "0.1.0 -> 0.1.4" +notes = "Nothing out of the ordinary for a crate finding MSVC tooling." + [[audits.bytecode-alliance.audits.foldhash]] who = "Alex Crichton " criteria = "safe-to-deploy" @@ -611,6 +655,12 @@ criteria = "safe-to-deploy" delta = "0.7.1 -> 0.8.0" notes = "Minor updates, using new Rust features like `const`, no major changes." +[[audits.bytecode-alliance.audits.num-traits]] +who = "Andrew Brown " +criteria = "safe-to-deploy" +version = "0.2.19" +notes = "As advertised: a numeric library. The only `unsafe` is from some float-to-int conversions, which seems expected." + [[audits.bytecode-alliance.audits.percent-encoding]] who = "Alex Crichton " criteria = "safe-to-deploy" @@ -621,12 +671,44 @@ a few `unsafe` blocks related to utf-8 validation which are locally verifiable as correct and otherwise this crate is good to go. """ +[[audits.bytecode-alliance.audits.pkg-config]] +who = "Pat Hickey " +criteria = "safe-to-deploy" +version = "0.3.25" +notes = "This crate shells out to the pkg-config executable, but it appears to sanitize inputs reasonably." + +[[audits.bytecode-alliance.audits.pkg-config]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +delta = "0.3.26 -> 0.3.29" +notes = """ +No `unsafe` additions or anything outside of the purview of the crate in this +change. +""" + +[[audits.bytecode-alliance.audits.pkg-config]] +who = "Chris Fallin " +criteria = "safe-to-deploy" +delta = "0.3.29 -> 0.3.32" + [[audits.bytecode-alliance.audits.semver]] who = "Pat Hickey " criteria = "safe-to-deploy" version = "1.0.17" notes = "plenty of unsafe pointer and vec tricks, but in well-structured and commented code that appears to be correct" +[[audits.bytecode-alliance.audits.shlex]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +version = "1.1.0" +notes = "Only minor `unsafe` code blocks which look valid and otherwise does what it says on the tin." + +[[audits.bytecode-alliance.audits.smallvec]] +who = "Alex Crichton " +criteria = "safe-to-deploy" +delta = "1.13.2 -> 1.14.0" +notes = "Minor new feature, nothing out of the ordinary." + [[audits.bytecode-alliance.audits.static_assertions]] who = "Andrew Brown " criteria = "safe-to-deploy" @@ -693,6 +775,12 @@ is similar to what it once was back then. Skimming over the crate there is nothing suspicious and it's everything you'd expect a Rust URL parser to be. """ +[[audits.bytecode-alliance.audits.vcpkg]] +who = "Pat Hickey " +criteria = "safe-to-deploy" +version = "0.2.15" +notes = "no build.rs, no macros, no unsafe. It reads the filesystem and makes copies of DLLs into OUT_DIR." + [[audits.bytecode-alliance.audits.walkdir]] who = "Andrew Brown " criteria = "safe-to-deploy" @@ -1718,6 +1806,12 @@ delta = "1.0.218 -> 1.0.219" notes = "Minor changes (clippy tweaks, using `mem::take` instead of `mem::replace`)." aggregated-from = "https://chromium.googlesource.com/chromium/src/+/main/third_party/rust/chromium_crates_io/supply-chain/audits.toml?format=TEXT" +[[audits.google.audits.smallvec]] +who = "Manish Goregaokar " +criteria = "safe-to-deploy" +version = "1.13.2" +aggregated-from = "https://chromium.googlesource.com/chromium/src/+/main/third_party/rust/chromium_crates_io/supply-chain/audits.toml?format=TEXT" + [[audits.google.audits.synstructure]] who = "Manish Goregaokar " criteria = "safe-to-deploy" @@ -2113,10 +2207,13 @@ criteria = "safe-to-deploy" delta = "2.3.3 -> 2.4.0" aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" -[[audits.mozilla.audits.cc]] -who = "Mike Hommey " +[[audits.mozilla.audits.bitflags]] +who = [ + "Teodor Tanasoaia ", + "Erich Gubler ", +] criteria = "safe-to-deploy" -delta = "1.0.73 -> 1.0.78" +delta = "2.6.0 -> 2.7.0" aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" [[audits.mozilla.audits.chrono]] @@ -2272,6 +2369,30 @@ criteria = "safe-to-deploy" delta = "0.2.171 -> 0.2.176" aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" +[[audits.mozilla.audits.libsqlite3-sys]] +who = "Mark Hammond " +criteria = "safe-to-deploy" +delta = "0.26.0 -> 0.27.0" +aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" + +[[audits.mozilla.audits.libsqlite3-sys]] +who = "Mark Hammond " +criteria = "safe-to-deploy" +delta = "0.27.0 -> 0.28.0" +aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" + +[[audits.mozilla.audits.libsqlite3-sys]] +who = "Erich Gubler " +criteria = "safe-to-deploy" +delta = "0.28.0 -> 0.31.0" +aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" + +[[audits.mozilla.audits.libsqlite3-sys]] +who = "Mark Hammond " +criteria = "safe-to-deploy" +delta = "0.31.0 -> 0.35.0" +aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" + [[audits.mozilla.audits.malloc_size_of_derive]] who = "Bobby Holley " criteria = "safe-to-deploy" @@ -2283,13 +2404,6 @@ but convinced myself that any generated code will be entirely safe to deploy. """ aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" -[[audits.mozilla.audits.num-traits]] -who = "Josh Stone " -criteria = "safe-to-deploy" -version = "0.2.15" -notes = "All code written or reviewed by Josh Stone." -aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" - [[audits.mozilla.audits.once_cell]] who = "Erich Gubler " criteria = "safe-to-deploy" @@ -2309,6 +2423,12 @@ criteria = "safe-to-deploy" delta = "1.20.3 -> 1.21.1" aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" +[[audits.mozilla.audits.pkg-config]] +who = "Mike Hommey " +criteria = "safe-to-deploy" +delta = "0.3.25 -> 0.3.26" +aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" + [[audits.mozilla.audits.rayon]] who = "Josh Stone " criteria = "safe-to-deploy" @@ -2340,6 +2460,60 @@ criteria = "safe-to-deploy" version = "0.18.4" aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" +[[audits.mozilla.audits.rmp]] +who = "Ben Dean-Kawamura " +criteria = "safe-to-deploy" +version = "0.8.14" +notes = """ +Very popular crate. 1 instance of unsafe code, which is used to adjust a slice to work around +lifetime issues. No network or file access. +""" +aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" + +[[audits.mozilla.audits.rmp-serde]] +who = "Ben Dean-Kawamura " +criteria = "safe-to-deploy" +version = "1.3.0" +notes = "Very popular crate. No unsafe code, network or file access." +aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" + +[[audits.mozilla.audits.rusqlite]] +who = "Mike Hommey " +criteria = "safe-to-deploy" +delta = "0.27.0 -> 0.28.0" +aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" + +[[audits.mozilla.audits.rusqlite]] +who = "Ben Dean-Kawamura " +criteria = "safe-to-deploy" +delta = "0.28.0 -> 0.29.0" +aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" + +[[audits.mozilla.audits.rusqlite]] +who = "Mark Hammond " +criteria = "safe-to-deploy" +delta = "0.29.0 -> 0.30.0" +aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" + +[[audits.mozilla.audits.rusqlite]] +who = "Mark Hammond " +criteria = "safe-to-deploy" +delta = "0.30.0 -> 0.31.0" +notes = "Mostly build and dependency related changes, and bump to sqlite version" +aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" + +[[audits.mozilla.audits.rusqlite]] +who = "Erich Gubler " +criteria = "safe-to-deploy" +delta = "0.31.0 -> 0.33.0" +aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" + +[[audits.mozilla.audits.rusqlite]] +who = "Mark Hammond " +criteria = "safe-to-deploy" +delta = "0.33.0 -> 0.37.0" +aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" + [[audits.mozilla.audits.rustc-hash]] who = "Bobby Holley " criteria = "safe-to-deploy" @@ -2379,6 +2553,12 @@ version = "1.0.3" notes = "Relatively simple Serde trait implementations. No IO or unsafe code." aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" +[[audits.mozilla.audits.shlex]] +who = "Max Inden " +criteria = "safe-to-deploy" +delta = "1.1.0 -> 1.3.0" +aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" + [[audits.mozilla.audits.siphasher]] who = "Emilio Cobos Álvarez " criteria = "safe-to-deploy" @@ -2386,6 +2566,12 @@ delta = "0.3.11 -> 1.0.1" notes = "Only change to the crate source is adding documentation." aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" +[[audits.mozilla.audits.smallvec]] +who = "Erich Gubler " +criteria = "safe-to-deploy" +delta = "1.14.0 -> 1.15.1" +aggregated-from = "https://hg.mozilla.org/mozilla-central/raw-file/tip/supply-chain/audits.toml" + [[audits.mozilla.audits.tempfile]] who = "Mike Hommey " criteria = "safe-to-deploy" From 6e4578c8a11ed77c96982b7a4ca3d0284f79220a Mon Sep 17 00:00:00 2001 From: Jan-Erik Rediger Date: Thu, 5 Jun 2025 12:18:29 +0200 Subject: [PATCH 04/30] Use the sqlite module as the database layer This only integrates it into the module tree. It compiles, but not warning-free. It fully replaces the Rkv storage. No migration implemented. --- glean-core/src/core/mod.rs | 4 ++-- glean-core/src/database/mod.rs | 2 ++ glean-core/src/database/sqlite.rs | 20 ++++++++++++++++++- glean-core/src/error.rs | 12 +++++++++++ .../src/metrics/dual_labeled_counter.rs | 2 +- glean-core/src/metrics/labeled.rs | 2 +- glean-core/src/ping/mod.rs | 10 ---------- glean-core/src/storage/mod.rs | 14 ++++++------- 8 files changed, 44 insertions(+), 22 deletions(-) diff --git a/glean-core/src/core/mod.rs b/glean-core/src/core/mod.rs index e38798ffb4..3ab8ec7544 100644 --- a/glean-core/src/core/mod.rs +++ b/glean-core/src/core/mod.rs @@ -15,7 +15,7 @@ use malloc_size_of_derive::MallocSizeOf; use once_cell::sync::OnceCell; use uuid::Uuid; -use crate::database::Database; +use crate::database::sqlite::Database; use crate::debug::DebugOptions; use crate::error::ClientIdFileError; use crate::event_database::EventDatabase; @@ -341,7 +341,7 @@ impl Glean { { let data_store = glean.data_store.as_ref().unwrap(); - let file_size = data_store.file_size.map(|n| n.get()).unwrap_or(0); + let file_size = data_store.file_size().map(|n| n.get()).unwrap_or(0); // If we have a client ID on disk, we check the database if let Some(stored_client_id) = stored_client_id { diff --git a/glean-core/src/database/mod.rs b/glean-core/src/database/mod.rs index 8408aa1f18..8143b6c751 100644 --- a/glean-core/src/database/mod.rs +++ b/glean-core/src/database/mod.rs @@ -19,6 +19,8 @@ use crate::ErrorKind; use malloc_size_of::MallocSizeOf; use rkv::{StoreError, StoreOptions}; +pub mod sqlite; + /// Unwrap a `Result`s `Ok` value or do the specified action. /// /// This is an alternative to the question-mark operator (`?`), diff --git a/glean-core/src/database/sqlite.rs b/glean-core/src/database/sqlite.rs index 59eddc11c6..29a6aa713f 100644 --- a/glean-core/src/database/sqlite.rs +++ b/glean-core/src/database/sqlite.rs @@ -6,7 +6,9 @@ use std::fs; use std::num::NonZeroU64; use std::path::Path; use std::str; +use std::time::Duration; +use malloc_size_of::MallocSizeOf; use rusqlite::params; use rusqlite::types::FromSqlError; use rusqlite::Transaction; @@ -28,6 +30,13 @@ pub struct Database { conn: connection::Connection, } +impl MallocSizeOf for Database { + fn size_of(&self, _ops: &mut malloc_size_of::MallocSizeOfOps) -> usize { + // FIXME: Can we get the allocated size of the connection? + 0 + } +} + impl std::fmt::Debug for Database { fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result { fmt.debug_struct("Database") @@ -43,7 +52,12 @@ impl Database { /// /// This opens the underlying SQLite store and creates /// the underlying directory structure. - pub fn new(data_path: &Path, _delay_ping_lifetime_io: bool) -> Result { + pub fn new( + data_path: &Path, + _delay_ping_lifetime_io: bool, + _ping_lifetime_threshold: usize, + _ping_lifetime_max_time: Duration, + ) -> Result { let path = data_path.join("db"); log::debug!("Database path: {:?}", path.display()); @@ -351,6 +365,10 @@ impl Database { }) } + pub fn clear_lifetime_storage(&self, lifetime: Lifetime, storage_name: &str) -> Result<()> { + Ok(()) + } + /// Removes a single metric from the storage. /// /// # Arguments diff --git a/glean-core/src/error.rs b/glean-core/src/error.rs index 1694cae04e..51e9fa7a17 100644 --- a/glean-core/src/error.rs +++ b/glean-core/src/error.rs @@ -65,6 +65,9 @@ pub enum ErrorKind { /// Parsing a UUID from a string failed UuidError(uuid::Error), + + /// Database/SQLite error + SQLite(rusqlite::Error), } /// A specialized [`Error`] type for this crate's operations. @@ -121,6 +124,7 @@ impl Display for Error { s / 1024 ), UuidError(e) => write!(f, "Failed to parse UUID: {}", e), + SQLite(e) => write!(f, "SQLite error: {}", e), } } } @@ -155,6 +159,14 @@ impl From for Error { } } +impl From for Error { + fn from(error: rusqlite::Error) -> Error { + Error { + kind: ErrorKind::SQLite(error), + } + } +} + impl From for Error { fn from(error: OsString) -> Error { Error { diff --git a/glean-core/src/metrics/dual_labeled_counter.rs b/glean-core/src/metrics/dual_labeled_counter.rs index 597c129878..fa15dbbcbc 100644 --- a/glean-core/src/metrics/dual_labeled_counter.rs +++ b/glean-core/src/metrics/dual_labeled_counter.rs @@ -424,7 +424,7 @@ fn get_seen_keys_and_categories( for store in &meta.inner.send_in_pings { glean .storage() - .iter_store_from(lifetime, store, Some(&prefix), &mut snapshotter); + .iter_store_from(lifetime, store, &prefix, &mut snapshotter); } (seen_keys, seen_categories) diff --git a/glean-core/src/metrics/labeled.rs b/glean-core/src/metrics/labeled.rs index ba5d2855d2..ca13a438d0 100644 --- a/glean-core/src/metrics/labeled.rs +++ b/glean-core/src/metrics/labeled.rs @@ -431,7 +431,7 @@ pub fn validate_dynamic_label( for store in &meta.inner.send_in_pings { glean .storage() - .iter_store_from(lifetime, store, Some(prefix), &mut snapshotter); + .iter_store_from(lifetime, store, prefix, &mut snapshotter); } let label_count = labels.len(); diff --git a/glean-core/src/ping/mod.rs b/glean-core/src/ping/mod.rs index ce21b26e0d..ca59beb72a 100644 --- a/glean-core/src/ping/mod.rs +++ b/glean-core/src/ping/mod.rs @@ -284,16 +284,6 @@ impl PingMaker { info!("Collecting {}", ping.name()); let database = glean.storage(); - // HACK: Only for metrics pings we add the ping timings. - // But we want that to persist until the next metrics ping is actually sent. - let write_samples = database.write_timings.replace(Vec::with_capacity(64)); - if !write_samples.is_empty() { - glean - .database_metrics - .write_time - .accumulate_samples_sync(glean, &write_samples); - } - let mut metrics_data = StorageManager.snapshot_as_json(database, ping.name(), true); let events_data = glean diff --git a/glean-core/src/storage/mod.rs b/glean-core/src/storage/mod.rs index 14198f1fa2..f84aa868a4 100644 --- a/glean-core/src/storage/mod.rs +++ b/glean-core/src/storage/mod.rs @@ -10,7 +10,7 @@ use std::collections::HashMap; use serde_json::{json, Value as JsonValue}; -use crate::database::Database; +use crate::database::sqlite::Database; use crate::metrics::dual_labeled_counter::RECORD_SEPARATOR; use crate::metrics::Metric; use crate::Lifetime; @@ -132,13 +132,13 @@ impl StorageManager { } }; - storage.iter_store_from(Lifetime::Ping, store_name, None, &mut snapshotter); - storage.iter_store_from(Lifetime::Application, store_name, None, &mut snapshotter); - storage.iter_store_from(Lifetime::User, store_name, None, &mut snapshotter); + storage.iter_store(Lifetime::Ping, store_name, &mut snapshotter); + storage.iter_store(Lifetime::Application, store_name, &mut snapshotter); + storage.iter_store(Lifetime::User, store_name, &mut snapshotter); // Add send in all pings client.annotations if store_name != "glean_client_info" { - storage.iter_store_from(Lifetime::Application, "all-pings", None, snapshotter); + storage.iter_store(Lifetime::Application, "all-pings", snapshotter); } if clear_store { @@ -181,7 +181,7 @@ impl StorageManager { } }; - storage.iter_store_from(metric_lifetime, store_name, None, &mut snapshotter); + storage.iter_store(metric_lifetime, store_name, &mut snapshotter); snapshot } @@ -260,7 +260,7 @@ impl StorageManager { } }; - storage.iter_store_from(Lifetime::Application, store_name, None, &mut snapshotter); + storage.iter_store(Lifetime::Application, store_name, &mut snapshotter); if snapshot.is_empty() { None From 17bfdf27a729d79bf0cf61f7477b067a0f08b14c Mon Sep 17 00:00:00 2001 From: Jan-Erik Rediger Date: Wed, 21 Jan 2026 15:28:34 +0100 Subject: [PATCH 05/30] Serialize values using MessagePack instead of bincode The bincode crate isn't maintained anymore. While it's been stable and without issues for us for years, switching to anotherformat is easy while we're switching the database anyway. MessagePack can be even smaller than bincode for the same data (just a couple of bytes here and there). Whether it's actually faster has not been benchmarked. Compared to everything else the (de)serialization overhead is probably a small fraction of the whole thing. Why do we need serialization anyway? Ping assembly does not have any knowledge of metrics. It only knows what's in the database. So in order to put in in the right place in the ping payload we need to know the type of the stored data. That data needs to be somewhere. By serializing the whole value (the `Metric` enum) we can deserialize it into that enum and the serde part takes care of "knowing" the type. --- Cargo.lock | 24 ++++++++++++++++++++++-- glean-core/Cargo.toml | 1 + glean-core/benchmark/Cargo.lock | 20 ++++++++++++++++++++ glean-core/src/database/sqlite.rs | 8 ++++---- 4 files changed, 47 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 77171672cc..eddf0ceea9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -618,6 +618,7 @@ dependencies = [ "pulldown-cmark", "pulldown-cmark-to-cmark", "rkv", + "rmp-serde", "rusqlite", "serde", "serde_json", @@ -920,9 +921,9 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.15" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", ] @@ -1166,6 +1167,25 @@ dependencies = [ "wr_malloc_size_of", ] +[[package]] +name = "rmp" +version = "0.8.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ba8be72d372b2c9b35542551678538b562e7cf86c3315773cae48dfbfe7790c" +dependencies = [ + "num-traits", +] + +[[package]] +name = "rmp-serde" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f81bee8c8ef9b577d1681a70ebbc962c232461e397b22c208c43c04b67a155" +dependencies = [ + "rmp", + "serde", +] + [[package]] name = "rusqlite" version = "0.37.0" diff --git a/glean-core/Cargo.toml b/glean-core/Cargo.toml index ba154d34c1..71f5bdd897 100644 --- a/glean-core/Cargo.toml +++ b/glean-core/Cargo.toml @@ -45,6 +45,7 @@ env_logger = { version = "0.10.0", default-features = false, optional = true } malloc_size_of_derive = "0.1.3" malloc_size_of = { version = "0.2.2", package = "wr_malloc_size_of", default-features = false, features = ["once_cell"] } rusqlite = { version = "0.37.0", features = ["bundled"] } +rmp-serde = "1.3.1" [target.'cfg(target_os = "android")'.dependencies] android_logger = { version = "0.12.0", default-features = false } diff --git a/glean-core/benchmark/Cargo.lock b/glean-core/benchmark/Cargo.lock index ccdf5cd27b..a11de53917 100644 --- a/glean-core/benchmark/Cargo.lock +++ b/glean-core/benchmark/Cargo.lock @@ -561,6 +561,7 @@ dependencies = [ "once_cell", "oslog", "rkv", + "rmp-serde", "rusqlite", "serde", "serde_json", @@ -1185,6 +1186,25 @@ dependencies = [ "wr_malloc_size_of", ] +[[package]] +name = "rmp" +version = "0.8.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ba8be72d372b2c9b35542551678538b562e7cf86c3315773cae48dfbfe7790c" +dependencies = [ + "num-traits", +] + +[[package]] +name = "rmp-serde" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f81bee8c8ef9b577d1681a70ebbc962c232461e397b22c208c43c04b67a155" +dependencies = [ + "rmp", + "serde", +] + [[package]] name = "rusqlite" version = "0.37.0" diff --git a/glean-core/src/database/sqlite.rs b/glean-core/src/database/sqlite.rs index 29a6aa713f..6ce1ece75e 100644 --- a/glean-core/src/database/sqlite.rs +++ b/glean-core/src/database/sqlite.rs @@ -117,7 +117,7 @@ impl Database { |row| { let id: String = row.get(0)?; let blob: Vec = row.get(1)?; - let blob: Metric = bincode::deserialize(&blob) + let blob: Metric = rmp_serde::from_slice(&blob) .map_err(|_| FromSqlError::InvalidType)?; Ok((id, blob)) }, @@ -237,7 +237,7 @@ impl Database { "#; let mut stmt = tx.prepare_cached(insert_sql)?; - let encoded = bincode::serialize(&metric).expect("IMPOSSIBLE: Serializing metric failed"); + let encoded = rmp_serde::to_vec(&metric).expect("IMPOSSIBLE: Serializing metric failed"); stmt.execute(params![key, storage_name, lifetime.as_str(), encoded])?; Ok(()) @@ -316,7 +316,7 @@ impl Database { if let Ok(Some(row)) = rows.next() { let blob: Vec = row.get(0)?; - let old_value = bincode::deserialize(&blob).ok(); + let old_value = rmp_serde::from_slice(&blob).ok(); transform(old_value) } else { transform(None) @@ -336,7 +336,7 @@ impl Database { { let mut stmt = tx.prepare_cached(insert_sql)?; let encoded = - bincode::serialize(&new_value).expect("IMPOSSIBLE: Serializing metric failed"); + rmp_serde::to_vec(&new_value).expect("IMPOSSIBLE: Serializing metric failed"); stmt.execute(params![key, storage_name, lifetime.as_str(), encoded])?; } From b095a0e2f4881a67ccdc27fa235ad2c07ce47e33 Mon Sep 17 00:00:00 2001 From: Jan-Erik Rediger Date: Wed, 28 Jan 2026 16:43:07 +0100 Subject: [PATCH 06/30] Implement the label verifier against sqlite storage It's now easier to do: query the column and count. There's some complications when we get to dual-labeled metrics, but that comes later. --- glean-core/src/common_metric_data.rs | 22 +++++++- glean-core/src/database/sqlite.rs | 75 +++++++++++++++++++++------- glean-core/src/metrics/labeled.rs | 49 ++++++++++++++++++ 3 files changed, 127 insertions(+), 19 deletions(-) diff --git a/glean-core/src/common_metric_data.rs b/glean-core/src/common_metric_data.rs index 6e9c1dc596..5a4e360552 100644 --- a/glean-core/src/common_metric_data.rs +++ b/glean-core/src/common_metric_data.rs @@ -6,10 +6,11 @@ use std::ops::Deref; use std::sync::atomic::{AtomicU8, Ordering}; use malloc_size_of_derive::MallocSizeOf; +use rusqlite::Transaction; use crate::error::{Error, ErrorKind}; use crate::metrics::dual_labeled_counter::validate_dynamic_key_and_or_category; -use crate::metrics::labeled::validate_dynamic_label; +use crate::metrics::labeled::{validate_dynamic_label, validate_dynamic_label_sqlite}; use crate::Glean; use serde::{Deserialize, Serialize}; @@ -165,6 +166,25 @@ impl CommonMetricDataInternal { } } + /// TODO + /// + /// If `category` is empty, it's ommitted. + /// Otherwise, it's the combination of the metric's `category`, `name` and `label`. + pub(crate) fn check_labels(&self, tx: &mut Transaction<'_>) -> Option { + let base_identifier = self.base_identifier(); + + if let Some(label) = &self.inner.dynamic_label { + match label { + DynamicLabelType::Label(label) => { + validate_dynamic_label_sqlite(tx, &base_identifier, label) + } + _ => todo!(), + } + } else { + None + } + } + /// The metric's unique identifier, including the category, name and label. /// /// If `category` is empty, it's ommitted. diff --git a/glean-core/src/database/sqlite.rs b/glean-core/src/database/sqlite.rs index 6ce1ece75e..6a4a567f6b 100644 --- a/glean-core/src/database/sqlite.rs +++ b/glean-core/src/database/sqlite.rs @@ -17,6 +17,7 @@ use connection::Connection; use schema::Schema; use crate::common_metric_data::CommonMetricDataInternal; +use crate::metrics::labeled::strip_label; use crate::metrics::Metric; use crate::Glean; use crate::Lifetime; @@ -187,13 +188,24 @@ impl Database { return; } - let name = data.identifier(glean); + let base_identifer = data.base_identifier(); + let name = strip_label(&base_identifer); _ = self.conn.write(|tx| { + let mut labels = String::from(""); + if let Some(checked_labels) = data.check_labels(tx) { + labels = checked_labels; + } + for ping_name in data.storage_names() { - if let Err(e) = - self.record_per_lifetime(tx, data.inner.lifetime, ping_name, &name, value) - { + if let Err(e) = self.record_per_lifetime( + tx, + data.inner.lifetime, + ping_name, + &name, + &labels, + value, + ) { log::error!( "Failed to record metric '{}' into {}: {:?}", data.base_identifier(), @@ -224,21 +236,28 @@ impl Database { lifetime: Lifetime, storage_name: &str, key: &str, + labels: &str, metric: &Metric, ) -> Result<()> { let insert_sql = r#" INSERT INTO - telemetry (id, ping, lifetime, value) + telemetry (id, ping, lifetime, labels, value) VALUES - (?1, ?2, ?3, ?4) - ON CONFLICT(id, ping) DO UPDATE SET + (?1, ?2, ?3, ?4, ?5) + ON CONFLICT(id, ping, labels) DO UPDATE SET lifetime = excluded.lifetime, value = excluded.value "#; let mut stmt = tx.prepare_cached(insert_sql)?; let encoded = rmp_serde::to_vec(&metric).expect("IMPOSSIBLE: Serializing metric failed"); - stmt.execute(params![key, storage_name, lifetime.as_str(), encoded])?; + stmt.execute(params![ + key, + storage_name, + lifetime.as_str(), + labels, + encoded + ])?; Ok(()) } @@ -254,14 +273,21 @@ impl Database { return; } + let base_identifer = data.base_identifier(); + let name = strip_label(&base_identifer); + _ = self.conn.write(|tx| { - let name = data.identifier(glean); + let mut labels = String::from(""); + if let Some(checked_labels) = data.check_labels(tx) { + labels = checked_labels; + } for ping_name in data.storage_names() { if let Err(e) = self.record_per_lifetime_with( tx, data.inner.lifetime, ping_name, &name, + &labels, &mut transform, ) { log::error!( @@ -295,24 +321,31 @@ impl Database { lifetime: Lifetime, storage_name: &str, key: &str, + labels: &str, mut transform: F, ) -> Result<()> where F: FnMut(Option) -> Metric, { - let find_sql = r#" + let value_sql = r#" SELECT value FROM telemetry WHERE - lifetime = ?1 + id = ?1 AND ping = ?2 - AND id = ?3 + AND lifetime = ?3 + AND labels = ?4 LIMIT 1 "#; let new_value = { - let mut stmt = tx.prepare_cached(&find_sql)?; - let mut rows = stmt.query(params![lifetime.as_str().to_string(), storage_name, key])?; + let mut stmt = tx.prepare_cached(value_sql)?; + let mut rows = stmt.query(params![ + key, + storage_name, + lifetime.as_str().to_string(), + labels + ])?; if let Ok(Some(row)) = rows.next() { let blob: Vec = row.get(0)?; @@ -325,10 +358,10 @@ impl Database { let insert_sql = r#" INSERT INTO - telemetry (id, ping, lifetime, value) + telemetry (id, ping, lifetime, labels, value) VALUES - (?1, ?2, ?3, ?4) - ON CONFLICT(id, ping) DO UPDATE SET + (?1, ?2, ?3, ?4, ?5) + ON CONFLICT(id, ping, labels) DO UPDATE SET lifetime = excluded.lifetime, value = excluded.value "#; @@ -337,7 +370,13 @@ impl Database { let mut stmt = tx.prepare_cached(insert_sql)?; let encoded = rmp_serde::to_vec(&new_value).expect("IMPOSSIBLE: Serializing metric failed"); - stmt.execute(params![key, storage_name, lifetime.as_str(), encoded])?; + stmt.execute(params![ + key, + storage_name, + lifetime.as_str(), + labels, + encoded + ])?; } Ok(()) diff --git a/glean-core/src/metrics/labeled.rs b/glean-core/src/metrics/labeled.rs index ca13a438d0..990b2b6795 100644 --- a/glean-core/src/metrics/labeled.rs +++ b/glean-core/src/metrics/labeled.rs @@ -10,6 +10,7 @@ use std::mem; use std::sync::{Arc, Mutex}; use malloc_size_of::MallocSizeOf; +use rusqlite::{params, Transaction}; use crate::common_metric_data::{CommonMetricData, CommonMetricDataInternal, DynamicLabelType}; use crate::error_recording::{record_error, test_get_num_recorded_errors, ErrorType}; @@ -455,3 +456,51 @@ pub fn validate_dynamic_label( key } } + +pub fn validate_dynamic_label_sqlite( + tx: &mut Transaction, + base_identifier: &str, + label: &str, +) -> Option { + let existing_labels_sql = "SELECT DISTINCT labels FROM telemetry WHERE id = ?1"; + + let Ok(mut stmt) = tx.prepare_cached(&existing_labels_sql) else { + // If we can't fetch from the database, assume the label is ok to use + return Some(label.to_string()); + }; + + let Ok(mut rows) = stmt.query(params![base_identifier]) else { + // If we can't fetch from the database, assume the label is ok to use + return Some(label.to_string()); + }; + + let mut label_already_used = false; + let mut label_count = 0; + while let Ok(Some(row)) = rows.next() { + let existing_label: String = row.get(0).unwrap(); + + label_count += 1; + if existing_label == label { + label_already_used = true; + break; + } + } + + if label_already_used || label_count < MAX_LABELS { + return Some(label.to_string()); + } + + /* + } else if label.len() > MAX_LABEL_LENGTH { + let msg = format!( + "label length {} exceeds maximum of {}", + label.len(), + MAX_LABEL_LENGTH + ); + record_error(glean, meta, ErrorType::InvalidLabel, msg, None); + true + } else { + */ + + Some(String::from(OTHER_LABEL)) +} From da7e06f9d238f8aeb5f05a34968a63536ab987f9 Mon Sep 17 00:00:00 2001 From: Jan-Erik Rediger Date: Thu, 29 Jan 2026 15:52:55 +0100 Subject: [PATCH 07/30] Correctly assemble payload for labeled metrics --- glean-core/src/database/sqlite.rs | 15 +++++--- glean-core/src/storage/mod.rs | 58 ++++++++++++++++++------------- 2 files changed, 43 insertions(+), 30 deletions(-) diff --git a/glean-core/src/database/sqlite.rs b/glean-core/src/database/sqlite.rs index 6a4a567f6b..d819149e19 100644 --- a/glean-core/src/database/sqlite.rs +++ b/glean-core/src/database/sqlite.rs @@ -99,10 +99,13 @@ impl Database { /// This function will **not** panic on database errors. pub fn iter_store(&self, lifetime: Lifetime, storage_name: &str, mut transaction_fn: F) where - F: FnMut(&[u8], &Metric), + F: FnMut(&[u8], &[&str], &Metric), { let iter_sql = r#" - SELECT id, value + SELECT + id, + value, + labels FROM telemetry WHERE lifetime = ?1 @@ -118,18 +121,20 @@ impl Database { |row| { let id: String = row.get(0)?; let blob: Vec = row.get(1)?; + let labels: String = row.get(2)?; let blob: Metric = rmp_serde::from_slice(&blob) .map_err(|_| FromSqlError::InvalidType)?; - Ok((id, blob)) + Ok((id, labels, blob)) }, ) .unwrap(); for row in rows { - let Ok((metric_id, metric)) = row else { + let Ok((metric_id, labels, metric)) = row else { continue; }; - transaction_fn(metric_id.as_bytes(), &metric); + let labels = labels.split(',').collect::>(); + transaction_fn(metric_id.as_bytes(), &labels, &metric); } Result::<(), ()>::Ok(()) diff --git a/glean-core/src/storage/mod.rs b/glean-core/src/storage/mod.rs index f84aa868a4..325352b07e 100644 --- a/glean-core/src/storage/mod.rs +++ b/glean-core/src/storage/mod.rs @@ -29,6 +29,7 @@ pub struct StorageManager; fn snapshot_labeled_metrics( snapshot: &mut HashMap>, metric_id: &str, + label: &str, metric: &Metric, ) { // Explicit match for supported labeled metrics, avoiding the formatting string @@ -45,9 +46,6 @@ fn snapshot_labeled_metrics( }; let map = snapshot.entry(ping_section).or_default(); - // Safe unwrap, the function is only called when the id does contain a '/' - let (metric_id, label) = metric_id.split_once('/').unwrap(); - let obj = map.entry(metric_id.into()).or_insert_with(|| json!({})); let obj = obj.as_object_mut().unwrap(); // safe unwrap, we constructed the object above obj.insert(label.into(), metric.as_json()); @@ -61,20 +59,21 @@ fn snapshot_labeled_metrics( fn snapshot_dual_labeled_metrics( snapshot: &mut HashMap>, metric_id: &str, + label1: &str, + label2: &str, metric: &Metric, ) { let ping_section = format!("dual_labeled_{}", metric.ping_section()); let map = snapshot.entry(ping_section).or_default(); - let parts = metric_id.split(RECORD_SEPARATOR).collect::>(); let obj = map - .entry(parts[0].into()) + .entry(metric_id.into()) .or_insert_with(|| json!({})) .as_object_mut() .unwrap(); // safe unwrap, we constructed the object above - let key_obj = obj.entry(parts[1].to_string()).or_insert_with(|| json!({})); + let key_obj = obj.entry(label1).or_insert_with(|| json!({})); let key_obj = key_obj.as_object_mut().unwrap(); - key_obj.insert(parts[2].into(), metric.as_json()); + key_obj.insert(label2.into(), metric.as_json()); } impl StorageManager { @@ -120,15 +119,26 @@ impl StorageManager { ) -> Option { let mut snapshot: HashMap> = HashMap::new(); - let mut snapshotter = |metric_id: &[u8], metric: &Metric| { + let mut snapshotter = |metric_id: &[u8], labels: &[&str], metric: &Metric| { let metric_id = String::from_utf8_lossy(metric_id).into_owned(); - if metric_id.contains('/') { - snapshot_labeled_metrics(&mut snapshot, &metric_id, metric); - } else if metric_id.split(RECORD_SEPARATOR).count() == 3 { - snapshot_dual_labeled_metrics(&mut snapshot, &metric_id, metric); - } else { - let map = snapshot.entry(metric.ping_section().into()).or_default(); - map.insert(metric_id, metric.as_json()); + match labels { + [] | [""] => { + let map = snapshot.entry(metric.ping_section().into()).or_default(); + map.insert(metric_id, metric.as_json()); + } + [label] => { + snapshot_labeled_metrics(&mut snapshot, &metric_id, label, metric); + } + [label1, label2] => { + snapshot_dual_labeled_metrics( + &mut snapshot, + &metric_id, + label1, + label2, + metric, + ); + } + _ => panic!("uh wat?"), } }; @@ -174,7 +184,7 @@ impl StorageManager { ) -> Option { let mut snapshot: Option = None; - let mut snapshotter = |id: &[u8], metric: &Metric| { + let mut snapshotter = |id: &[u8], labels: &[&str], metric: &Metric| { let id = String::from_utf8_lossy(id).into_owned(); if id == metric_id { snapshot = Some(metric.clone()) @@ -207,17 +217,15 @@ impl StorageManager { ) -> Vec { let mut labels = Vec::new(); - let mut snapshotter = |id: &[u8], _metric: &Metric| { - let id = String::from_utf8_lossy(id).into_owned(); - if let Some((base_id, label)) = id.split_once('/') { - if base_id == metric_id { - labels.push(label.to_owned()); - } + let mut snapshotter = |id: &[u8], found_labels: &[&str], _metric: &Metric| { + let id = String::from_utf8_lossy(id); + // Not doing this for dual-labeled metrics. + if id == metric_id && found_labels.len() == 1 { + labels.push(found_labels[0].to_string()); } }; - storage.iter_store_from(metric_lifetime, store_name, None, &mut snapshotter); - + _ = storage.iter_store(metric_lifetime, store_name, &mut snapshotter); labels } @@ -252,7 +260,7 @@ impl StorageManager { ) -> Option { let mut snapshot: HashMap = HashMap::new(); - let mut snapshotter = |metric_id: &[u8], metric: &Metric| { + let mut snapshotter = |metric_id: &[u8], labels: &[&str], metric: &Metric| { let metric_id = String::from_utf8_lossy(metric_id).into_owned(); if metric_id.ends_with("#experiment") { let (name, _) = metric_id.split_once('#').unwrap(); // safe unwrap, we ensured there's a `#` in the string From d9198ec365b3b403bd26a61bdcde924564876576 Mon Sep 17 00:00:00 2001 From: Jan-Erik Rediger Date: Fri, 30 Jan 2026 12:21:55 +0100 Subject: [PATCH 08/30] Simplify getting metrics out again --- glean-core/src/common_metric_data.rs | 2 +- glean-core/src/database/sqlite.rs | 39 ++++++++++++++++++++ glean-core/src/database/sqlite/connection.rs | 12 +++--- glean-core/src/error_recording.rs | 5 ++- glean-core/src/metrics/labeled.rs | 2 +- 5 files changed, 50 insertions(+), 10 deletions(-) diff --git a/glean-core/src/common_metric_data.rs b/glean-core/src/common_metric_data.rs index 5a4e360552..1bd1adcf25 100644 --- a/glean-core/src/common_metric_data.rs +++ b/glean-core/src/common_metric_data.rs @@ -170,7 +170,7 @@ impl CommonMetricDataInternal { /// /// If `category` is empty, it's ommitted. /// Otherwise, it's the combination of the metric's `category`, `name` and `label`. - pub(crate) fn check_labels(&self, tx: &mut Transaction<'_>) -> Option { + pub(crate) fn check_labels(&self, tx: &Transaction<'_>) -> Option { let base_identifier = self.base_identifier(); if let Some(label) = &self.inner.dynamic_label { diff --git a/glean-core/src/database/sqlite.rs b/glean-core/src/database/sqlite.rs index d819149e19..4c160a338b 100644 --- a/glean-core/src/database/sqlite.rs +++ b/glean-core/src/database/sqlite.rs @@ -11,6 +11,7 @@ use std::time::Duration; use malloc_size_of::MallocSizeOf; use rusqlite::params; use rusqlite::types::FromSqlError; +use rusqlite::OptionalExtension; use rusqlite::Transaction; use connection::Connection; @@ -142,6 +143,44 @@ impl Database { .unwrap() } + /// TODO + pub fn get_metric( + &self, + data: &CommonMetricDataInternal, + storage_name: &str, + ) -> Option { + let get_metric_sql = r#" + SELECT + value + FROM telemetry + WHERE + id = ?1 + AND ping = ?2 + AND labels = ?3 + LIMIT 1 + "#; + + let metric_identifier = &data.base_identifier(); + + self.conn + .read(|tx| { + let mut labels = String::from(""); + if let Some(checked_labels) = data.check_labels(tx) { + labels = checked_labels; + } + + let mut stmt = tx.prepare_cached(get_metric_sql)?; + stmt.query_one([metric_identifier, storage_name, &labels], |row| { + let blob: Vec = row.get(0)?; + let blob: Metric = + rmp_serde::from_slice(&blob).map_err(|_| FromSqlError::InvalidType)?; + Ok(blob) + }) + .optional() + }) + .unwrap_or(None) + } + /// Determines if the storage has the given metric. /// /// If data cannot be read it is assumed that the storage does not have the metric. diff --git a/glean-core/src/database/sqlite/connection.rs b/glean-core/src/database/sqlite/connection.rs index 7c6d1cebeb..2e8aa54869 100644 --- a/glean-core/src/database/sqlite/connection.rs +++ b/glean-core/src/database/sqlite/connection.rs @@ -83,12 +83,12 @@ impl Connection { } /// Accesses the database for reading. - pub fn read( - &self, - f: impl FnOnce(&rusqlite::Connection) -> Result, - ) -> Result { - let conn = self.conn.lock().unwrap(); - f(&*conn) + pub fn read(&self, f: impl FnOnce(&Transaction<'_>) -> Result) -> Result { + let mut conn = self.conn.lock().unwrap(); + let tx = conn + .transaction_with_behavior(TransactionBehavior::Immediate) + .unwrap(); + f(&tx) } /// Accesses the database in a transaction for reading and writing. diff --git a/glean-core/src/error_recording.rs b/glean-core/src/error_recording.rs index 24dcedd3e8..65f7a9d481 100644 --- a/glean-core/src/error_recording.rs +++ b/glean-core/src/error_recording.rs @@ -18,9 +18,9 @@ use crate::common_metric_data::CommonMetricDataInternal; use crate::error::{Error, ErrorKind}; use crate::metrics::labeled::{combine_base_identifier_and_label, strip_label}; use crate::metrics::CounterMetric; -use crate::CommonMetricData; use crate::Glean; use crate::Lifetime; +use crate::{CommonMetricData, DynamicLabelType}; /// The possible error types for metric recording. /// @@ -103,10 +103,11 @@ fn get_error_metric_for_metric(meta: &CommonMetricDataInternal, error: ErrorType } CounterMetric::new(CommonMetricData { - name: combine_base_identifier_and_label(error.as_str(), name), + name: error.as_str().to_string(), category: "glean.error".into(), lifetime: Lifetime::Ping, send_in_pings, + dynamic_label: Some(DynamicLabelType::Label(name.to_string())), ..Default::default() }) } diff --git a/glean-core/src/metrics/labeled.rs b/glean-core/src/metrics/labeled.rs index 990b2b6795..78fd859d3c 100644 --- a/glean-core/src/metrics/labeled.rs +++ b/glean-core/src/metrics/labeled.rs @@ -458,7 +458,7 @@ pub fn validate_dynamic_label( } pub fn validate_dynamic_label_sqlite( - tx: &mut Transaction, + tx: &Transaction, base_identifier: &str, label: &str, ) -> Option { From 1bc9f9abe106e9f7150afa066e667078275c2a0a Mon Sep 17 00:00:00 2001 From: Jan-Erik Rediger Date: Fri, 30 Jan 2026 14:27:11 +0100 Subject: [PATCH 09/30] sqlite: Implement clearing of lifetime storage Now that it's just another column this becomes straight-forward to do. --- glean-core/src/database/sqlite.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/glean-core/src/database/sqlite.rs b/glean-core/src/database/sqlite.rs index 4c160a338b..287a9d4aca 100644 --- a/glean-core/src/database/sqlite.rs +++ b/glean-core/src/database/sqlite.rs @@ -449,7 +449,12 @@ impl Database { } pub fn clear_lifetime_storage(&self, lifetime: Lifetime, storage_name: &str) -> Result<()> { - Ok(()) + let clear_sql = "DELETE FROM telemetry WHERE lifetime = ?1 AND ping = ?2"; + self.conn.write(|tx| { + let mut stmt = tx.prepare_cached(clear_sql)?; + stmt.execute([lifetime.as_str(), storage_name])?; + Ok(()) + }) } /// Removes a single metric from the storage. From fe7f105cf7481b33a8a7e8ee0a3b42ab36b3319d Mon Sep 17 00:00:00 2001 From: Jan-Erik Rediger Date: Fri, 30 Jan 2026 14:39:27 +0100 Subject: [PATCH 10/30] sqlite: Measure database on-disk size by scanning the directory Same way this was done on Rkv: we just some up the size of all files in the database directory. --- glean-core/src/database/sqlite.rs | 38 +++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/glean-core/src/database/sqlite.rs b/glean-core/src/database/sqlite.rs index 287a9d4aca..8090017cc1 100644 --- a/glean-core/src/database/sqlite.rs +++ b/glean-core/src/database/sqlite.rs @@ -30,6 +30,9 @@ mod schema; pub struct Database { /// The database connection. conn: connection::Connection, + + /// Initial file size when opening the database. + pub(crate) file_size: Option, } impl MallocSizeOf for Database { @@ -49,6 +52,36 @@ impl std::fmt::Debug for Database { const DEFAULT_DATABASE_FILE_NAME: &str = "glean.sqlite"; +/// Calculate the database size from all the files in the directory. +/// +/// # Arguments +/// +/// *`path` - The path to the directory +/// +/// # Returns +/// +/// Returns the non-zero combined size of all files in a directory, +/// or `None` on error or if the size is `0`. +fn database_size(dir: &Path) -> Option { + let mut total_size = 0; + if let Ok(entries) = fs::read_dir(dir) { + for entry in entries.flatten() { + if let Ok(file_type) = entry.file_type() { + if file_type.is_file() { + let path = entry.path(); + if let Ok(metadata) = fs::metadata(path) { + total_size += metadata.len(); + } else { + continue; + } + } + } + } + } + + NonZeroU64::new(total_size) +} + impl Database { /// Initializes the data store. /// @@ -62,19 +95,20 @@ impl Database { ) -> Result { let path = data_path.join("db"); log::debug!("Database path: {:?}", path.display()); + let file_size = database_size(&path); fs::create_dir_all(&path)?; let store_path = path.join(DEFAULT_DATABASE_FILE_NAME); let conn = Connection::new::(&store_path).unwrap(); - let db = Self { conn }; + let db = Self { conn, file_size }; Ok(db) } /// Get the initial database file size. pub fn file_size(&self) -> Option { - None + self.file_size } /// Get the rkv load state. From 8e20a22043b9129b0e1111de62c1a8f5e0a4b673 Mon Sep 17 00:00:00 2001 From: Jan-Erik Rediger Date: Mon, 2 Feb 2026 10:12:25 +0100 Subject: [PATCH 11/30] Handle static labels like dynamic ones: stored along with the metric This will unify label check code: all cases are handled through the same code paths, just that for the static label variant we don't need to do any more checks. --- glean-core/src/common_metric_data.rs | 4 ++++ glean-core/src/metrics/boolean.rs | 2 +- glean-core/src/metrics/counter.rs | 2 +- glean-core/src/metrics/custom_distribution.rs | 2 +- glean-core/src/metrics/dual_labeled_counter.rs | 18 ++++++++---------- glean-core/src/metrics/labeled.rs | 11 ++++------- glean-core/src/metrics/memory_distribution.rs | 2 +- glean-core/src/metrics/mod.rs | 2 +- glean-core/src/metrics/quantity.rs | 2 +- glean-core/src/metrics/string.rs | 2 +- glean-core/src/metrics/text.rs | 2 +- glean-core/src/metrics/timing_distribution.rs | 2 +- 12 files changed, 25 insertions(+), 26 deletions(-) diff --git a/glean-core/src/common_metric_data.rs b/glean-core/src/common_metric_data.rs index 1bd1adcf25..9fb13fb537 100644 --- a/glean-core/src/common_metric_data.rs +++ b/glean-core/src/common_metric_data.rs @@ -82,6 +82,8 @@ pub struct CommonMetricData { /// the necessary validation to be performed. #[derive(Debug, Clone, Deserialize, Serialize, MallocSizeOf, uniffi::Enum)] pub enum DynamicLabelType { + /// Static Label -- no validation required + Static(String), /// A dynamic label applied from a `LabeledMetric` Label(String), /// A label applied by a `DualLabeledCounter` that contains a dynamic key @@ -103,6 +105,7 @@ impl Deref for DynamicLabelType { fn deref(&self) -> &Self::Target { match self { + DynamicLabelType::Static(label) => todo!(), DynamicLabelType::Label(label) => label, DynamicLabelType::KeyOnly(key) => key, DynamicLabelType::CategoryOnly(category) => category, @@ -175,6 +178,7 @@ impl CommonMetricDataInternal { if let Some(label) = &self.inner.dynamic_label { match label { + DynamicLabelType::Static(label) => Some(label.to_string()), DynamicLabelType::Label(label) => { validate_dynamic_label_sqlite(tx, &base_identifier, label) } diff --git a/glean-core/src/metrics/boolean.rs b/glean-core/src/metrics/boolean.rs index 504f3347a5..3982547b9e 100644 --- a/glean-core/src/metrics/boolean.rs +++ b/glean-core/src/metrics/boolean.rs @@ -33,7 +33,7 @@ impl MetricType for BooleanMetric { } } - fn with_dynamic_label(&self, label: DynamicLabelType) -> Self { + fn with_label(&self, label: DynamicLabelType) -> Self { let mut meta = (*self.meta).clone(); meta.inner.dynamic_label = Some(label); Self { diff --git a/glean-core/src/metrics/counter.rs b/glean-core/src/metrics/counter.rs index 300a088472..7dacf48740 100644 --- a/glean-core/src/metrics/counter.rs +++ b/glean-core/src/metrics/counter.rs @@ -35,7 +35,7 @@ impl MetricType for CounterMetric { } } - fn with_dynamic_label(&self, label: DynamicLabelType) -> Self { + fn with_label(&self, label: DynamicLabelType) -> Self { let mut meta = (*self.meta).clone(); meta.inner.dynamic_label = Some(label); Self { diff --git a/glean-core/src/metrics/custom_distribution.rs b/glean-core/src/metrics/custom_distribution.rs index a94af73a08..f5e2313623 100644 --- a/glean-core/src/metrics/custom_distribution.rs +++ b/glean-core/src/metrics/custom_distribution.rs @@ -55,7 +55,7 @@ impl MetricType for CustomDistributionMetric { } } - fn with_dynamic_label(&self, label: DynamicLabelType) -> Self { + fn with_label(&self, label: DynamicLabelType) -> Self { let mut meta = (*self.meta).clone(); meta.inner.dynamic_label = Some(label); Self { diff --git a/glean-core/src/metrics/dual_labeled_counter.rs b/glean-core/src/metrics/dual_labeled_counter.rs index fa15dbbcbc..5d2d7fbd70 100644 --- a/glean-core/src/metrics/dual_labeled_counter.rs +++ b/glean-core/src/metrics/dual_labeled_counter.rs @@ -108,23 +108,20 @@ impl DualLabeledCounterMetric { /// the static or dynamic labels where needed. fn new_counter_metric(&self, key: &str, category: &str) -> CounterMetric { match (&self.keys, &self.categories) { - (None, None) => self - .counter - .with_dynamic_label(DynamicLabelType::KeyAndCategory( - make_label_from_key_and_category(key, category), - )), + (None, None) => self.counter.with_label(DynamicLabelType::KeyAndCategory( + make_label_from_key_and_category(key, category), + )), (None, _) => { let static_category = self.static_category(category); - self.counter.with_dynamic_label(DynamicLabelType::KeyOnly( + self.counter.with_label(DynamicLabelType::KeyOnly( make_label_from_key_and_category(key, static_category), )) } (_, None) => { let static_key = self.static_key(key); - self.counter - .with_dynamic_label(DynamicLabelType::CategoryOnly( - make_label_from_key_and_category(static_key, category), - )) + self.counter.with_label(DynamicLabelType::CategoryOnly( + make_label_from_key_and_category(static_key, category), + )) } (_, _) => { // Both labels are static and can be validated now @@ -327,6 +324,7 @@ pub fn validate_dynamic_key_and_or_category( // one(s) to check based on the label variant. let (seen_keys, seen_categories) = get_seen_keys_and_categories(meta, glean); match label { + DynamicLabelType::Static(_) => todo!(), DynamicLabelType::Label(ref label) => { record_error( glean, diff --git a/glean-core/src/metrics/labeled.rs b/glean-core/src/metrics/labeled.rs index 78fd859d3c..e8f9864be1 100644 --- a/glean-core/src/metrics/labeled.rs +++ b/glean-core/src/metrics/labeled.rs @@ -254,8 +254,8 @@ where /// Creates a new metric with a specific label. /// /// This is used for static labels where we can just set the name to be `name/label`. - fn new_metric_with_name(&self, name: String) -> T { - self.submetric.with_name(name) + fn new_metric_with_label(&self, label: DynamicLabelType) -> T { + self.submetric.with_label(label) } /// Creates a new metric with a specific label. @@ -263,7 +263,7 @@ where /// This is used for dynamic labels where we have to actually validate and correct the /// label later when we have a Glean object. fn new_metric_with_dynamic_label(&self, label: DynamicLabelType) -> T { - self.submetric.with_dynamic_label(label) + self.submetric.with_label(label) } /// Creates a static label. @@ -321,10 +321,7 @@ where let metric = match self.labels { Some(_) => { let label = self.static_label(label); - self.new_metric_with_name(combine_base_identifier_and_label( - &self.submetric.meta().inner.name, - label, - )) + self.new_metric_with_label(DynamicLabelType::Static(label.to_string())) } None => self .new_metric_with_dynamic_label(DynamicLabelType::Label(label.to_string())), diff --git a/glean-core/src/metrics/memory_distribution.rs b/glean-core/src/metrics/memory_distribution.rs index 3513dabf3d..ee6108534f 100644 --- a/glean-core/src/metrics/memory_distribution.rs +++ b/glean-core/src/metrics/memory_distribution.rs @@ -64,7 +64,7 @@ impl MetricType for MemoryDistributionMetric { } } - fn with_dynamic_label(&self, label: DynamicLabelType) -> Self { + fn with_label(&self, label: DynamicLabelType) -> Self { let mut meta = (*self.meta).clone(); meta.inner.dynamic_label = Some(label); Self { diff --git a/glean-core/src/metrics/mod.rs b/glean-core/src/metrics/mod.rs index a916d017d5..cfb86ac57b 100644 --- a/glean-core/src/metrics/mod.rs +++ b/glean-core/src/metrics/mod.rs @@ -196,7 +196,7 @@ pub trait MetricType { } /// Create a new metric from this with a specific label. - fn with_dynamic_label(&self, _label: DynamicLabelType) -> Self + fn with_label(&self, _label: DynamicLabelType) -> Self where Self: Sized, { diff --git a/glean-core/src/metrics/quantity.rs b/glean-core/src/metrics/quantity.rs index 137569b53e..d19271e0c3 100644 --- a/glean-core/src/metrics/quantity.rs +++ b/glean-core/src/metrics/quantity.rs @@ -33,7 +33,7 @@ impl MetricType for QuantityMetric { } } - fn with_dynamic_label(&self, label: DynamicLabelType) -> Self { + fn with_label(&self, label: DynamicLabelType) -> Self { let mut meta = (*self.meta).clone(); meta.inner.dynamic_label = Some(label); Self { diff --git a/glean-core/src/metrics/string.rs b/glean-core/src/metrics/string.rs index 2e33068686..42e062d0e0 100644 --- a/glean-core/src/metrics/string.rs +++ b/glean-core/src/metrics/string.rs @@ -37,7 +37,7 @@ impl MetricType for StringMetric { } } - fn with_dynamic_label(&self, label: DynamicLabelType) -> Self { + fn with_label(&self, label: DynamicLabelType) -> Self { let mut meta = (*self.meta).clone(); meta.inner.dynamic_label = Some(label); Self { diff --git a/glean-core/src/metrics/text.rs b/glean-core/src/metrics/text.rs index ae47fd34bc..be707a7f37 100644 --- a/glean-core/src/metrics/text.rs +++ b/glean-core/src/metrics/text.rs @@ -39,7 +39,7 @@ impl MetricType for TextMetric { } } - fn with_dynamic_label(&self, label: DynamicLabelType) -> Self { + fn with_label(&self, label: DynamicLabelType) -> Self { let mut meta = (*self.meta).clone(); meta.inner.dynamic_label = Some(label); Self { diff --git a/glean-core/src/metrics/timing_distribution.rs b/glean-core/src/metrics/timing_distribution.rs index 5620cd3a1e..90bb7a4952 100644 --- a/glean-core/src/metrics/timing_distribution.rs +++ b/glean-core/src/metrics/timing_distribution.rs @@ -111,7 +111,7 @@ impl MetricType for TimingDistributionMetric { } } - fn with_dynamic_label(&self, label: DynamicLabelType) -> Self { + fn with_label(&self, label: DynamicLabelType) -> Self { let mut meta = (*self.meta).clone(); meta.inner.dynamic_label = Some(label); Self { From 1d81dbb6588bca91c8f370e875b21d58b50f4818 Mon Sep 17 00:00:00 2001 From: Jan-Erik Rediger Date: Mon, 2 Feb 2026 12:38:58 +0100 Subject: [PATCH 12/30] Copy over is_ping_enabled check instead of is_upload_enabled --- glean-core/src/database/sqlite.rs | 66 ++++++++++++++----------------- 1 file changed, 30 insertions(+), 36 deletions(-) diff --git a/glean-core/src/database/sqlite.rs b/glean-core/src/database/sqlite.rs index 8090017cc1..f7318be734 100644 --- a/glean-core/src/database/sqlite.rs +++ b/glean-core/src/database/sqlite.rs @@ -261,11 +261,6 @@ impl Database { /// Records a metric in the underlying storage system. pub fn record(&self, glean: &Glean, data: &CommonMetricDataInternal, value: &Metric) { - // If upload is disabled we don't want to record. - if !glean.is_upload_enabled() { - return; - } - let base_identifer = data.base_identifier(); let name = strip_label(&base_identifer); @@ -276,20 +271,22 @@ impl Database { } for ping_name in data.storage_names() { - if let Err(e) = self.record_per_lifetime( - tx, - data.inner.lifetime, - ping_name, - &name, - &labels, - value, - ) { - log::error!( - "Failed to record metric '{}' into {}: {:?}", - data.base_identifier(), + if glean.is_ping_enabled(ping_name) { + if let Err(e) = self.record_per_lifetime( + tx, + data.inner.lifetime, ping_name, - e - ); + name, + &labels, + value, + ) { + log::error!( + "Failed to record metric '{}' into {}: {:?}", + data.base_identifier(), + ping_name, + e + ); + } } } @@ -346,11 +343,6 @@ impl Database { where F: FnMut(Option) -> Metric, { - // If upload is disabled we don't want to record. - if !glean.is_upload_enabled() { - return; - } - let base_identifer = data.base_identifier(); let name = strip_label(&base_identifer); @@ -360,20 +352,22 @@ impl Database { labels = checked_labels; } for ping_name in data.storage_names() { - if let Err(e) = self.record_per_lifetime_with( - tx, - data.inner.lifetime, - ping_name, - &name, - &labels, - &mut transform, - ) { - log::error!( - "Failed to record metric '{}' into {}: {:?}", - data.base_identifier(), + if glean.is_ping_enabled(ping_name) { + if let Err(e) = self.record_per_lifetime_with( + tx, + data.inner.lifetime, ping_name, - e - ); + name, + &labels, + &mut transform, + ) { + log::error!( + "Failed to record metric '{}' into {}: {:?}", + data.base_identifier(), + ping_name, + e + ); + } } } From 6e5f26dad63e3b84da3de1c88f3ef0ab6c3420b2 Mon Sep 17 00:00:00 2001 From: Jan-Erik Rediger Date: Mon, 2 Feb 2026 13:01:44 +0100 Subject: [PATCH 13/30] Rework how dynamic labels are validated --- glean-core/src/common_metric_data.rs | 45 +++--- glean-core/src/database/sqlite.rs | 11 +- glean-core/src/error_recording.rs | 85 ++++++++++- .../src/metrics/dual_labeled_counter.rs | 143 ++++++++---------- glean-core/src/metrics/labeled.rs | 69 +++++---- glean-core/src/metrics/mod.rs | 3 +- 6 files changed, 213 insertions(+), 143 deletions(-) diff --git a/glean-core/src/common_metric_data.rs b/glean-core/src/common_metric_data.rs index 9fb13fb537..a2cb6ed843 100644 --- a/glean-core/src/common_metric_data.rs +++ b/glean-core/src/common_metric_data.rs @@ -9,7 +9,9 @@ use malloc_size_of_derive::MallocSizeOf; use rusqlite::Transaction; use crate::error::{Error, ErrorKind}; -use crate::metrics::dual_labeled_counter::validate_dynamic_key_and_or_category; +use crate::metrics::dual_labeled_counter::{ + validate_dual_label_sqlite, validate_dynamic_key_and_or_category, +}; use crate::metrics::labeled::{validate_dynamic_label, validate_dynamic_label_sqlite}; use crate::Glean; use serde::{Deserialize, Serialize}; @@ -87,11 +89,11 @@ pub enum DynamicLabelType { /// A dynamic label applied from a `LabeledMetric` Label(String), /// A label applied by a `DualLabeledCounter` that contains a dynamic key - KeyOnly(String), + KeyOnly(String, String), /// A label applied by a `DualLabeledCounter` that contains a dynamic category - CategoryOnly(String), + CategoryOnly(String, String), /// A label applied by a `DualLabeledCounter` that contains a dynamic key and category - KeyAndCategory(String), + KeyAndCategory(String, String), } impl Default for DynamicLabelType { @@ -100,20 +102,6 @@ impl Default for DynamicLabelType { } } -impl Deref for DynamicLabelType { - type Target = str; - - fn deref(&self) -> &Self::Target { - match self { - DynamicLabelType::Static(label) => todo!(), - DynamicLabelType::Label(label) => label, - DynamicLabelType::KeyOnly(key) => key, - DynamicLabelType::CategoryOnly(category) => category, - DynamicLabelType::KeyAndCategory(key_and_category) => key_and_category, - } - } -} - #[derive(Default, Debug, MallocSizeOf)] pub struct CommonMetricDataInternal { pub inner: CommonMetricData, @@ -173,16 +161,27 @@ impl CommonMetricDataInternal { /// /// If `category` is empty, it's ommitted. /// Otherwise, it's the combination of the metric's `category`, `name` and `label`. - pub(crate) fn check_labels(&self, tx: &Transaction<'_>) -> Option { + pub(crate) fn check_labels(&self, tx: &mut Transaction<'_>) -> Option { let base_identifier = self.base_identifier(); if let Some(label) = &self.inner.dynamic_label { match label { DynamicLabelType::Static(label) => Some(label.to_string()), - DynamicLabelType::Label(label) => { - validate_dynamic_label_sqlite(tx, &base_identifier, label) - } - _ => todo!(), + DynamicLabelType::Label(label) => validate_dynamic_label_sqlite( + tx, + &base_identifier, + label, + &self.inner.send_in_pings, + ), + DynamicLabelType::KeyOnly(..) => todo!(), + DynamicLabelType::CategoryOnly(..) => todo!(), + DynamicLabelType::KeyAndCategory(key, category) => validate_dual_label_sqlite( + tx, + &base_identifier, + key, + category, + &self.inner.send_in_pings, + ), } } else { None diff --git a/glean-core/src/database/sqlite.rs b/glean-core/src/database/sqlite.rs index f7318be734..3eb46182ec 100644 --- a/glean-core/src/database/sqlite.rs +++ b/glean-core/src/database/sqlite.rs @@ -18,6 +18,7 @@ use connection::Connection; use schema::Schema; use crate::common_metric_data::CommonMetricDataInternal; +use crate::metrics::dual_labeled_counter::RECORD_SEPARATOR; use crate::metrics::labeled::strip_label; use crate::metrics::Metric; use crate::Glean; @@ -168,7 +169,7 @@ impl Database { let Ok((metric_id, labels, metric)) = row else { continue; }; - let labels = labels.split(',').collect::>(); + let labels = labels.split(RECORD_SEPARATOR).collect::>(); transaction_fn(metric_id.as_bytes(), &labels, &metric); } @@ -261,8 +262,8 @@ impl Database { /// Records a metric in the underlying storage system. pub fn record(&self, glean: &Glean, data: &CommonMetricDataInternal, value: &Metric) { - let base_identifer = data.base_identifier(); - let name = strip_label(&base_identifer); + let base_identifier = data.base_identifier(); + let name = strip_label(&base_identifier); _ = self.conn.write(|tx| { let mut labels = String::from(""); @@ -343,8 +344,8 @@ impl Database { where F: FnMut(Option) -> Metric, { - let base_identifer = data.base_identifier(); - let name = strip_label(&base_identifer); + let base_identifier = data.base_identifier(); + let name = strip_label(&base_identifier); _ = self.conn.write(|tx| { let mut labels = String::from(""); diff --git a/glean-core/src/error_recording.rs b/glean-core/src/error_recording.rs index 65f7a9d481..11e00aee2c 100644 --- a/glean-core/src/error_recording.rs +++ b/glean-core/src/error_recording.rs @@ -14,10 +14,13 @@ use std::fmt::Display; +use rusqlite::params; +use rusqlite::Transaction; + use crate::common_metric_data::CommonMetricDataInternal; use crate::error::{Error, ErrorKind}; use crate::metrics::labeled::{combine_base_identifier_and_label, strip_label}; -use crate::metrics::CounterMetric; +use crate::metrics::{CounterMetric, Metric}; use crate::Glean; use crate::Lifetime; use crate::{CommonMetricData, DynamicLabelType}; @@ -143,6 +146,86 @@ pub fn record_error>>( metric.add_sync(glean, to_report); } +pub fn record_error_sqlite( + tx: &mut Transaction, + metric_name: &str, + send_in_pings: &[String], + error: ErrorType, + num_errors: i32, +) { + assert!(num_errors > 0); + + let ping_name = "metrics".to_string(); + let need_metrics = !send_in_pings.contains(&ping_name); + + let full_id = format!("glean.error.{}", error.as_str()); + let lifetime = Lifetime::Ping; + let transform = |old_value| match old_value { + Some(Metric::Counter(old_value)) => Metric::Counter(old_value.saturating_add(num_errors)), + _ => Metric::Counter(num_errors), + }; + + let value_sql = r#" + SELECT value + FROM telemetry + WHERE + id = ?1 + AND ping = ?2 + AND lifetime = ?3 + AND labels = ?4 + LIMIT 1 + "#; + + let insert_sql = r#" + INSERT INTO + telemetry (id, ping, lifetime, labels, value) + VALUES + (?1, ?2, ?3, ?4, ?5) + ON CONFLICT(id, ping, labels) DO UPDATE SET + lifetime = excluded.lifetime, + value = excluded.value + "#; + + for ping in send_in_pings + .iter() + .chain(need_metrics.then_some(&ping_name)) + { + let new_value = { + let mut stmt = tx.prepare_cached(value_sql).unwrap(); + let mut rows = stmt + .query(params![ + full_id, + ping, + lifetime.as_str().to_string(), + metric_name + ]) + .unwrap(); + + if let Ok(Some(row)) = rows.next() { + let blob: Vec = row.get(0).unwrap(); + let old_value = rmp_serde::from_slice(&blob).ok(); + transform(old_value) + } else { + transform(None) + } + }; + + { + let mut stmt = tx.prepare_cached(insert_sql).unwrap(); + let encoded = + rmp_serde::to_vec(&new_value).expect("IMPOSSIBLE: Serializing metric failed"); + stmt.execute(params![ + full_id, + ping, + lifetime.as_str(), + metric_name, + encoded + ]) + .unwrap(); + } + } +} + /// Gets the number of recorded errors for the given metric and error type. /// /// *Notes: This is a **test-only** API, but we need to expose it to be used in integration tests. diff --git a/glean-core/src/metrics/dual_labeled_counter.rs b/glean-core/src/metrics/dual_labeled_counter.rs index 5d2d7fbd70..8d9a58845d 100644 --- a/glean-core/src/metrics/dual_labeled_counter.rs +++ b/glean-core/src/metrics/dual_labeled_counter.rs @@ -8,6 +8,8 @@ use std::collections::{HashMap, HashSet}; use std::mem; use std::sync::{Arc, Mutex}; +use rusqlite::{params, Transaction}; + use crate::common_metric_data::{CommonMetricData, CommonMetricDataInternal, DynamicLabelType}; use crate::error_recording::{record_error, test_get_num_recorded_errors, ErrorType}; use crate::metrics::{CounterMetric, Metric, MetricType}; @@ -109,30 +111,29 @@ impl DualLabeledCounterMetric { fn new_counter_metric(&self, key: &str, category: &str) -> CounterMetric { match (&self.keys, &self.categories) { (None, None) => self.counter.with_label(DynamicLabelType::KeyAndCategory( - make_label_from_key_and_category(key, category), + key.into(), + category.into(), )), (None, _) => { let static_category = self.static_category(category); self.counter.with_label(DynamicLabelType::KeyOnly( - make_label_from_key_and_category(key, static_category), + key.into(), + static_category.into(), )) } (_, None) => { let static_key = self.static_key(key); self.counter.with_label(DynamicLabelType::CategoryOnly( - make_label_from_key_and_category(static_key, category), + static_key.into(), + category.into(), )) } (_, _) => { // Both labels are static and can be validated now let static_key = self.static_key(key); let static_category = self.static_category(category); - let name = combine_base_identifier_and_labels( - self.counter.meta().inner.name.as_str(), - static_key, - static_category, - ); - self.counter.with_name(name) + let label = format!("{static_key}{RECORD_SEPARATOR}{static_category}"); + self.counter.with_label(DynamicLabelType::Static(label)) } } } @@ -250,13 +251,13 @@ impl TestGetValue for DualLabeledCounterMetric { /// Combines a metric's base identifier and label pub fn combine_base_identifier_and_labels( - base_identifer: &str, + base_identifier: &str, key: &str, category: &str, ) -> String { format!( "{}{}", - base_identifer, + base_identifier, make_label_from_key_and_category(key, category) ) } @@ -299,81 +300,59 @@ pub fn validate_dynamic_key_and_or_category( base_identifier: &str, label: DynamicLabelType, ) -> String { - // We should have exactly 3 elements when splitting by `RECORD_SEPARATOR`, since the label should begin with one and - // then the key and category are separated by one. Split should contain an empty string, the key, and the category. - // If we have more than 3 elements, then the consuming app must have used this character as part of a label and we - // cannot determine whether it was the key or the category at this point, so we record an `InvalidLabel` error and - // return `OTHER_LABEL` for both key and category. - if label.split(RECORD_SEPARATOR).count() != 3 { - let msg = "Label cannot contain the ASCII record separator character (0x1E)".to_string(); - record_error(glean, meta, ErrorType::InvalidLabel, msg, None); - return combine_base_identifier_and_labels(base_identifier, OTHER_LABEL, OTHER_LABEL); - } + panic!("not validating dual labeled like this anymore"); +} - // Pick out the key and category from the supplied label - if let Some((mut key, mut category)) = separate_label_into_key_and_category(&label) { - // Loop through the stores we expect to find this metric in, and if we - // find it then just return the full metric identifier that was found - for store in &meta.inner.send_in_pings { - if glean.storage().has_metric(meta.inner.lifetime, store, key) { - return combine_base_identifier_and_labels(base_identifier, key, category); - } - } +pub fn validate_dual_label_sqlite( + tx: &mut Transaction, + base_identifier: &str, + key: &str, + category: &str, + send_in_pings: &[String], +) -> Option { + let existing_labels_sql = "SELECT DISTINCT labels FROM telemetry WHERE id = ?1"; + + let mut existing_keys = HashSet::new(); + let mut existing_categories = HashSet::new(); + 'checkdb: { + let Ok(mut stmt) = tx.prepare(existing_labels_sql) else { + // If we can't fetch from the database, assume the label is ok to use + break 'checkdb; + }; - // Count the number of distinct keys and categories already recorded, we can figure out which - // one(s) to check based on the label variant. - let (seen_keys, seen_categories) = get_seen_keys_and_categories(meta, glean); - match label { - DynamicLabelType::Static(_) => todo!(), - DynamicLabelType::Label(ref label) => { - record_error( - glean, - meta, - ErrorType::InvalidLabel, - format!("Invalid `DualLabeledCounter` label format: {label:?}"), - None, - ); - key = OTHER_LABEL; - category = OTHER_LABEL; - } - DynamicLabelType::KeyOnly(_) => { - if (!seen_keys.contains(key) && seen_keys.len() >= MAX_LABELS) - || !label_is_valid(key, glean, meta) - { - key = OTHER_LABEL; - } - } - DynamicLabelType::CategoryOnly(_) => { - if (!seen_categories.contains(category) && seen_categories.len() >= MAX_LABELS) - || !label_is_valid(category, glean, meta) - { - category = OTHER_LABEL; - } - } - DynamicLabelType::KeyAndCategory(_) => { - if (!seen_keys.contains(key) && seen_keys.len() >= MAX_LABELS) - || !label_is_valid(key, glean, meta) - { - key = OTHER_LABEL; - } - if (!seen_categories.contains(category) && seen_categories.len() >= MAX_LABELS) - || !label_is_valid(category, glean, meta) - { - category = OTHER_LABEL; - } - } + let Ok(mut rows) = stmt.query(params![base_identifier]) else { + // If we can't fetch from the database, assume the label is ok to use + break 'checkdb; + }; + + while let Ok(Some(row)) = rows.next() { + let existing_labels: String = row.get(0).unwrap(); + let Some((existing_key, existing_category)) = + existing_labels.split_once(RECORD_SEPARATOR) + else { + log::debug!("Database contains invalid dual-label: {existing_labels:?}"); + continue; + }; + + existing_keys.insert(existing_key.to_string()); + existing_categories.insert(existing_category.to_string()); } - combine_base_identifier_and_labels(base_identifier, key, category) - } else { - record_error( - glean, - meta, - ErrorType::InvalidLabel, - "Invalid `DualLabeledCounter` label format, unable to determine key and/or category", - None, - ); - combine_base_identifier_and_labels(base_identifier, OTHER_LABEL, OTHER_LABEL) } + + let new_key = if existing_keys.contains(key) || existing_keys.len() < MAX_LABELS { + key + } else { + OTHER_LABEL + }; + + let new_category = + if existing_categories.contains(category) || existing_categories.len() < MAX_LABELS { + category + } else { + OTHER_LABEL + }; + + Some(format!("{new_key}{RECORD_SEPARATOR}{new_category}")) } fn label_is_valid(label: &str, glean: &Glean, meta: &CommonMetricDataInternal) -> bool { diff --git a/glean-core/src/metrics/labeled.rs b/glean-core/src/metrics/labeled.rs index e8f9864be1..b7de942244 100644 --- a/glean-core/src/metrics/labeled.rs +++ b/glean-core/src/metrics/labeled.rs @@ -13,7 +13,9 @@ use malloc_size_of::MallocSizeOf; use rusqlite::{params, Transaction}; use crate::common_metric_data::{CommonMetricData, CommonMetricDataInternal, DynamicLabelType}; -use crate::error_recording::{record_error, test_get_num_recorded_errors, ErrorType}; +use crate::error_recording::{ + record_error, record_error_sqlite, test_get_num_recorded_errors, ErrorType, +}; use crate::histogram::HistogramType; use crate::metrics::{ BooleanMetric, CounterMetric, CustomDistributionMetric, MemoryDistributionMetric, MemoryUnit, @@ -384,8 +386,8 @@ where } /// Combines a metric's base identifier and label -pub fn combine_base_identifier_and_label(base_identifer: &str, label: &str) -> String { - format!("{}/{}", base_identifer, label) +pub fn combine_base_identifier_and_label(base_identifier: &str, label: &str) -> String { + format!("{}/{}", base_identifier, label) } /// Strips the label off of a complete identifier @@ -455,49 +457,54 @@ pub fn validate_dynamic_label( } pub fn validate_dynamic_label_sqlite( - tx: &Transaction, + tx: &mut Transaction, base_identifier: &str, label: &str, + send_in_pings: &[String], ) -> Option { let existing_labels_sql = "SELECT DISTINCT labels FROM telemetry WHERE id = ?1"; - let Ok(mut stmt) = tx.prepare_cached(&existing_labels_sql) else { - // If we can't fetch from the database, assume the label is ok to use - return Some(label.to_string()); - }; - - let Ok(mut rows) = stmt.query(params![base_identifier]) else { - // If we can't fetch from the database, assume the label is ok to use - return Some(label.to_string()); - }; - let mut label_already_used = false; let mut label_count = 0; - while let Ok(Some(row)) = rows.next() { - let existing_label: String = row.get(0).unwrap(); + { + let Ok(mut stmt) = tx.prepare(existing_labels_sql) else { + // If we can't fetch from the database, assume the label is ok to use + return Some(label.to_string()); + }; - label_count += 1; - if existing_label == label { - label_already_used = true; - break; - } - } + let Ok(mut rows) = stmt.query(params![base_identifier]) else { + // If we can't fetch from the database, assume the label is ok to use + return Some(label.to_string()); + }; - if label_already_used || label_count < MAX_LABELS { - return Some(label.to_string()); + while let Ok(Some(row)) = rows.next() { + let existing_label: String = row.get(0).unwrap(); + + label_count += 1; + if existing_label == label { + label_already_used = true; + break; + } + } } - /* + if !label_already_used && label_count >= MAX_LABELS { + Some(String::from(OTHER_LABEL)) } else if label.len() > MAX_LABEL_LENGTH { - let msg = format!( + log::warn!( "label length {} exceeds maximum of {}", label.len(), MAX_LABEL_LENGTH ); - record_error(glean, meta, ErrorType::InvalidLabel, msg, None); - true + record_error_sqlite( + tx, + base_identifier, + send_in_pings, + ErrorType::InvalidLabel, + 1, + ); + Some(String::from(OTHER_LABEL)) } else { - */ - - Some(String::from(OTHER_LABEL)) + Some(label.to_string()) + } } diff --git a/glean-core/src/metrics/mod.rs b/glean-core/src/metrics/mod.rs index cfb86ac57b..2693df9a60 100644 --- a/glean-core/src/metrics/mod.rs +++ b/glean-core/src/metrics/mod.rs @@ -299,7 +299,8 @@ where { fn get_identifiers(&'a self) -> (&'a str, &'a str, Option<&'a str>) { let meta = &self.meta().inner; - (&meta.category, &meta.name, meta.dynamic_label.as_deref()) + todo!() + //(&meta.category, &meta.name, meta.dynamic_label.as_deref()) } } From e75e1887381d7b3c5b0fe93f5e54f9fa14fe2ad4 Mon Sep 17 00:00:00 2001 From: Jan-Erik Rediger Date: Mon, 2 Feb 2026 13:01:44 +0100 Subject: [PATCH 14/30] Correctly validate usage of key-only/category-only dual-labels --- glean-core/src/common_metric_data.rs | 19 +++++- .../src/metrics/dual_labeled_counter.rs | 61 ++++++++++++++++++- 2 files changed, 75 insertions(+), 5 deletions(-) diff --git a/glean-core/src/common_metric_data.rs b/glean-core/src/common_metric_data.rs index a2cb6ed843..b12723f97d 100644 --- a/glean-core/src/common_metric_data.rs +++ b/glean-core/src/common_metric_data.rs @@ -9,6 +9,7 @@ use malloc_size_of_derive::MallocSizeOf; use rusqlite::Transaction; use crate::error::{Error, ErrorKind}; +use crate::metrics::dual_labeled_counter::RECORD_SEPARATOR; use crate::metrics::dual_labeled_counter::{ validate_dual_label_sqlite, validate_dynamic_key_and_or_category, }; @@ -173,8 +174,22 @@ impl CommonMetricDataInternal { label, &self.inner.send_in_pings, ), - DynamicLabelType::KeyOnly(..) => todo!(), - DynamicLabelType::CategoryOnly(..) => todo!(), + DynamicLabelType::KeyOnly(key, static_category) => validate_dual_label_sqlite( + tx, + &base_identifier, + key, + "", + &self.inner.send_in_pings, + ) + .map(|key| format!("{key}{static_category}")), + DynamicLabelType::CategoryOnly(static_key, category) => validate_dual_label_sqlite( + tx, + &base_identifier, + "", + category, + &self.inner.send_in_pings, + ) + .map(|category| format!("{static_key}{category}")), DynamicLabelType::KeyAndCategory(key, category) => validate_dual_label_sqlite( tx, &base_identifier, diff --git a/glean-core/src/metrics/dual_labeled_counter.rs b/glean-core/src/metrics/dual_labeled_counter.rs index 8d9a58845d..51f5df8721 100644 --- a/glean-core/src/metrics/dual_labeled_counter.rs +++ b/glean-core/src/metrics/dual_labeled_counter.rs @@ -11,7 +11,9 @@ use std::sync::{Arc, Mutex}; use rusqlite::{params, Transaction}; use crate::common_metric_data::{CommonMetricData, CommonMetricDataInternal, DynamicLabelType}; -use crate::error_recording::{record_error, test_get_num_recorded_errors, ErrorType}; +use crate::error_recording::{ + record_error, record_error_sqlite, test_get_num_recorded_errors, ErrorType, +}; use crate::metrics::{CounterMetric, Metric, MetricType}; use crate::{Glean, TestGetValue}; @@ -312,6 +314,22 @@ pub fn validate_dual_label_sqlite( ) -> Option { let existing_labels_sql = "SELECT DISTINCT labels FROM telemetry WHERE id = ?1"; + // TODO: We can now detect if _either_ key or category contains `RECORD_SEPARATOR` and thus keep + // the other potentially valid label. + // This needs adjustement of the test `labels_containing_a_record_separator_record_an_error`. + if key.contains(RECORD_SEPARATOR) || category.contains(RECORD_SEPARATOR) { + let msg = "Label cannot contain the ASCII record separator character (0x1E)".to_string(); + record_error_sqlite( + tx, + base_identifier, + send_in_pings, + ErrorType::InvalidLabel, + msg, + 1, + ); + return Some(format!("{OTHER_LABEL}{RECORD_SEPARATOR}{OTHER_LABEL}")); + } + let mut existing_keys = HashSet::new(); let mut existing_categories = HashSet::new(); 'checkdb: { @@ -340,14 +358,14 @@ pub fn validate_dual_label_sqlite( } let new_key = if existing_keys.contains(key) || existing_keys.len() < MAX_LABELS { - key + label_is_valid_sqlite(key, tx, base_identifier, send_in_pings) } else { OTHER_LABEL }; let new_category = if existing_categories.contains(category) || existing_categories.len() < MAX_LABELS { - category + label_is_valid_sqlite(category, tx, base_identifier, send_in_pings) } else { OTHER_LABEL }; @@ -355,6 +373,43 @@ pub fn validate_dual_label_sqlite( Some(format!("{new_key}{RECORD_SEPARATOR}{new_category}")) } +fn label_is_valid_sqlite<'a>( + label: &'a str, + tx: &mut Transaction, + base_identifier: &str, + send_in_pings: &[String], +) -> &'a str { + if label.len() > MAX_LABEL_LENGTH { + let msg = format!( + "label length {} exceeds maximum of {}", + label.len(), + MAX_LABEL_LENGTH + ); + record_error_sqlite( + tx, + base_identifier, + send_in_pings, + ErrorType::InvalidLabel, + msg, + 1, + ); + OTHER_LABEL + } else if label.contains(RECORD_SEPARATOR) { + let msg = "Label cannot contain the ASCII record separator character (0x1E)".to_string(); + record_error_sqlite( + tx, + base_identifier, + send_in_pings, + ErrorType::InvalidLabel, + msg, + 1, + ); + OTHER_LABEL + } else { + label + } +} + fn label_is_valid(label: &str, glean: &Glean, meta: &CommonMetricDataInternal) -> bool { if label.len() > MAX_LABEL_LENGTH { let msg = format!( From 588e805389c650a504c9b64bb8c725a435320d7b Mon Sep 17 00:00:00 2001 From: Jan-Erik Rediger Date: Wed, 4 Feb 2026 14:52:19 +0100 Subject: [PATCH 15/30] Remove now-unused database code Basically anything that assumes the database layout of rkv, now that it has been reimplemented with sqlite. --- glean-core/src/common_metric_data.rs | 2 - glean-core/src/database/mod.rs | 1620 ----------------- glean-core/src/error_recording.rs | 4 +- glean-core/src/metrics/boolean.rs | 1 - glean-core/src/metrics/counter.rs | 1 - .../src/metrics/dual_labeled_counter.rs | 90 +- glean-core/src/metrics/labeled.rs | 62 +- glean-core/src/metrics/mod.rs | 1 - glean-core/src/metrics/quantity.rs | 1 - glean-core/src/metrics/string.rs | 1 - glean-core/src/storage/mod.rs | 5 +- 11 files changed, 12 insertions(+), 1776 deletions(-) diff --git a/glean-core/src/common_metric_data.rs b/glean-core/src/common_metric_data.rs index b12723f97d..051cccbd80 100644 --- a/glean-core/src/common_metric_data.rs +++ b/glean-core/src/common_metric_data.rs @@ -2,14 +2,12 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -use std::ops::Deref; use std::sync::atomic::{AtomicU8, Ordering}; use malloc_size_of_derive::MallocSizeOf; use rusqlite::Transaction; use crate::error::{Error, ErrorKind}; -use crate::metrics::dual_labeled_counter::RECORD_SEPARATOR; use crate::metrics::dual_labeled_counter::{ validate_dual_label_sqlite, validate_dynamic_key_and_or_category, }; diff --git a/glean-core/src/database/mod.rs b/glean-core/src/database/mod.rs index 8143b6c751..190dbc6849 100644 --- a/glean-core/src/database/mod.rs +++ b/glean-core/src/database/mod.rs @@ -2,1624 +2,4 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -use std::cell::{Cell, RefCell}; -use std::collections::btree_map::Entry; -use std::collections::BTreeMap; -use std::fs; -use std::io; -use std::num::NonZeroU64; -use std::path::Path; -use std::str; -use std::sync::atomic::{AtomicUsize, Ordering}; -use std::sync::RwLock; -use std::time::{Duration, Instant}; - -use crate::ErrorKind; - -use malloc_size_of::MallocSizeOf; -use rkv::{StoreError, StoreOptions}; - pub mod sqlite; - -/// Unwrap a `Result`s `Ok` value or do the specified action. -/// -/// This is an alternative to the question-mark operator (`?`), -/// when the other action should not be to return the error. -macro_rules! unwrap_or { - ($expr:expr, $or:expr) => { - match $expr { - Ok(x) => x, - Err(_) => { - $or; - } - } - }; -} - -macro_rules! measure_commit { - ($this:ident, $expr:expr) => {{ - let now = ::std::time::Instant::now(); - let res = $expr; - let elapsed = now.elapsed(); - if let Ok(elapsed) = elapsed.as_micros().try_into() { - let mut samples = $this.write_timings.borrow_mut(); - samples.push(elapsed); - } - res - }}; -} - -/// cbindgen:ignore -pub type Rkv = rkv::Rkv; -/// cbindgen:ignore -pub type SingleStore = rkv::SingleStore; -/// cbindgen:ignore -pub type Writer<'t> = rkv::Writer>; - -#[derive(Debug)] -pub enum RkvLoadState { - Ok, - Err(rkv::StoreError), -} - -pub fn rkv_new(path: &Path) -> std::result::Result<(Rkv, RkvLoadState), rkv::StoreError> { - match Rkv::new::(path) { - // An invalid file can mean: - // 1. An empty file. - // 2. A corrupted file. - // - // In both instances there's not much we can do. - // Drop the data by removing the file, and start over. - Err(rkv::StoreError::FileInvalid) => { - log::debug!("rkv failed: invalid file. starting from scratch."); - let safebin = path.join("data.safe.bin"); - fs::remove_file(safebin).map_err(|_| rkv::StoreError::FileInvalid)?; - // Now try again, we only handle that error once. - let rkv = Rkv::new::(path)?; - Ok((rkv, RkvLoadState::Err(rkv::StoreError::FileInvalid))) - } - Err(rkv::StoreError::DatabaseCorrupted) => { - log::debug!("rkv failed: database corrupted. starting from scratch."); - let safebin = path.join("data.safe.bin"); - fs::remove_file(safebin).map_err(|_| rkv::StoreError::DatabaseCorrupted)?; - // Try again, only allowing the error once. - let rkv = Rkv::new::(path)?; - Ok((rkv, RkvLoadState::Err(rkv::StoreError::DatabaseCorrupted))) - } - other => { - let rkv = other?; - Ok((rkv, RkvLoadState::Ok)) - } - } -} - -use crate::common_metric_data::CommonMetricDataInternal; -use crate::metrics::Metric; -use crate::Glean; -use crate::Lifetime; -use crate::Result; - -pub struct Database { - /// Handle to the database environment. - rkv: Rkv, - - /// Handles to the "lifetime" stores. - /// - /// A "store" is a handle to the underlying database. - /// We keep them open for fast and frequent access. - user_store: SingleStore, - ping_store: SingleStore, - application_store: SingleStore, - - /// If the `delay_ping_lifetime_io` Glean config option is `true`, - /// we will save metrics with 'ping' lifetime data in a map temporarily - /// so as to persist them to disk using rkv in bulk on demand. - ping_lifetime_data: Option>>, - - /// A count of how many database writes have been done since the last ping-lifetime flush. - /// - /// A ping-lifetime flush is automatically done after `ping_lifetime_threshold` writes. - /// - /// Only relevant if `delay_ping_lifetime_io` is set to `true`, - ping_lifetime_count: AtomicUsize, - - /// Write-count threshold when to auto-flush. `0` disables it. - ping_lifetime_threshold: usize, - - /// The last time the `lifetime=ping` data was flushed to disk. - /// - /// Data is flushed to disk automatically when the last flush was more than - /// `ping_lifetime_max_time` ago. - /// - /// Only relevant if `delay_ping_lifetime_io` is set to `true`, - ping_lifetime_store_ts: Cell, - - /// After what time to auto-flush. 0 disables it. - ping_lifetime_max_time: Duration, - - /// Initial file size when opening the database. - pub(crate) file_size: Option, - - /// RKV load state - rkv_load_state: RkvLoadState, - - /// Times an Rkv write-commit took. - /// Re-applied as samples in a timing distribution later. - pub(crate) write_timings: RefCell>, -} - -impl MallocSizeOf for Database { - fn size_of(&self, ops: &mut malloc_size_of::MallocSizeOfOps) -> usize { - // TODO(bug 1960592): Fill in gaps. - - let mut n = 0; - - n += self.rkv.size_of(ops); - n += self.user_store.size_of(ops); - n += self.ping_store.size_of(ops); - n += self.application_store.size_of(ops); - - n += self - .ping_lifetime_data - .as_ref() - .map(|data| { - // TODO(bug 1960592): servo's malloc_size_of implements it for BTreeMap. - let lock = data.read().unwrap(); - (*lock).size_of(ops) - }) - .unwrap_or(0); - - n - } -} - -impl std::fmt::Debug for Database { - fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result { - fmt.debug_struct("Database") - .field("rkv", &self.rkv) - .field("user_store", &"SingleStore") - .field("ping_store", &"SingleStore") - .field("application_store", &"SingleStore") - .field("ping_lifetime_data", &self.ping_lifetime_data) - .finish() - } -} - -/// Calculate the database size from all the files in the directory. -/// -/// # Arguments -/// -/// *`path` - The path to the directory -/// -/// # Returns -/// -/// Returns the non-zero combined size of all files in a directory, -/// or `None` on error or if the size is `0`. -fn database_size(dir: &Path) -> Option { - let mut total_size = 0; - if let Ok(entries) = fs::read_dir(dir) { - for entry in entries.flatten() { - if let Ok(file_type) = entry.file_type() { - if file_type.is_file() { - let path = entry.path(); - if let Ok(metadata) = fs::metadata(path) { - total_size += metadata.len(); - } else { - continue; - } - } - } - } - } - - NonZeroU64::new(total_size) -} - -impl Database { - /// Initializes the data store. - /// - /// This opens the underlying rkv store and creates - /// the underlying directory structure. - /// - /// It also loads any Lifetime::Ping data that might be - /// persisted, in case `delay_ping_lifetime_io` is set. - pub fn new( - data_path: &Path, - delay_ping_lifetime_io: bool, - ping_lifetime_threshold: usize, - ping_lifetime_max_time: Duration, - ) -> Result { - let path = data_path.join("db"); - log::debug!("Database path: {:?}", path.display()); - let file_size = database_size(&path); - - let (rkv, rkv_load_state) = Self::open_rkv(&path)?; - let user_store = rkv.open_single(Lifetime::User.as_str(), StoreOptions::create())?; - let ping_store = rkv.open_single(Lifetime::Ping.as_str(), StoreOptions::create())?; - let application_store = - rkv.open_single(Lifetime::Application.as_str(), StoreOptions::create())?; - let ping_lifetime_data = if delay_ping_lifetime_io { - Some(RwLock::new(BTreeMap::new())) - } else { - None - }; - - // We are gonna write, so we allocate some capacity upfront. - // The value was chosen at random. - let write_timings = RefCell::new(Vec::with_capacity(64)); - - let now = Instant::now(); - - let db = Self { - rkv, - user_store, - ping_store, - application_store, - ping_lifetime_data, - ping_lifetime_count: AtomicUsize::new(0), - ping_lifetime_threshold, - ping_lifetime_store_ts: Cell::new(now), - ping_lifetime_max_time, - file_size, - rkv_load_state, - write_timings, - }; - - db.load_ping_lifetime_data(); - - Ok(db) - } - - /// Get the initial database file size. - pub fn file_size(&self) -> Option { - self.file_size - } - - /// Get the rkv load state. - pub fn rkv_load_state(&self) -> Option { - if let RkvLoadState::Err(e) = &self.rkv_load_state { - Some(e.to_string()) - } else { - None - } - } - - fn get_store(&self, lifetime: Lifetime) -> &SingleStore { - match lifetime { - Lifetime::User => &self.user_store, - Lifetime::Ping => &self.ping_store, - Lifetime::Application => &self.application_store, - } - } - - /// Creates the storage directories and inits rkv. - fn open_rkv(path: &Path) -> Result<(Rkv, RkvLoadState)> { - fs::create_dir_all(path)?; - - let (rkv, load_state) = rkv_new(path)?; - - log::info!("Database initialized"); - Ok((rkv, load_state)) - } - - /// Build the key of the final location of the data in the database. - /// Such location is built using the storage name and the metric - /// key/name (if available). - /// - /// # Arguments - /// - /// * `storage_name` - the name of the storage to store/fetch data from. - /// * `metric_key` - the optional metric key/name. - /// - /// # Returns - /// - /// A string representing the location in the database. - fn get_storage_key(storage_name: &str, metric_key: Option<&str>) -> String { - match metric_key { - Some(k) => format!("{}#{}", storage_name, k), - None => format!("{}#", storage_name), - } - } - - /// Loads Lifetime::Ping data from rkv to memory, - /// if `delay_ping_lifetime_io` is set to true. - /// - /// Does nothing if it isn't or if there is not data to load. - fn load_ping_lifetime_data(&self) { - if let Some(ping_lifetime_data) = &self.ping_lifetime_data { - let mut data = ping_lifetime_data - .write() - .expect("Can't read ping lifetime data"); - - let reader = unwrap_or!(self.rkv.read(), return); - let store = self.get_store(Lifetime::Ping); - let mut iter = unwrap_or!(store.iter_start(&reader), return); - - while let Some(Ok((metric_id, value))) = iter.next() { - let metric_id = match str::from_utf8(metric_id) { - Ok(metric_id) => metric_id.to_string(), - _ => continue, - }; - let metric: Metric = match value { - rkv::Value::Blob(blob) => unwrap_or!(bincode::deserialize(blob), continue), - _ => continue, - }; - - data.insert(metric_id, metric); - } - } - } - - /// Iterates with the provided transaction function - /// over the requested data from the given storage. - /// - /// * If the storage is unavailable, the transaction function is never invoked. - /// * If the read data cannot be deserialized it will be silently skipped. - /// - /// # Arguments - /// - /// * `lifetime` - The metric lifetime to iterate over. - /// * `storage_name` - The storage name to iterate over. - /// * `metric_key` - The metric key to iterate over. All metrics iterated over - /// will have this prefix. For example, if `metric_key` is of the form `{category}.`, - /// it will iterate over all metrics in the given category. If the `metric_key` is of the - /// form `{category}.{name}/`, the iterator will iterate over all specific metrics for - /// a given labeled metric. If not provided, the entire storage for the given lifetime - /// will be iterated over. - /// * `transaction_fn` - Called for each entry being iterated over. It is - /// passed two arguments: `(metric_id: &[u8], metric: &Metric)`. - /// - /// # Panics - /// - /// This function will **not** panic on database errors. - pub fn iter_store_from( - &self, - lifetime: Lifetime, - storage_name: &str, - metric_key: Option<&str>, - mut transaction_fn: F, - ) where - F: FnMut(&[u8], &Metric), - { - let iter_start = Self::get_storage_key(storage_name, metric_key); - let len = iter_start.len(); - - // Lifetime::Ping data is not immediately persisted to disk if - // Glean has `delay_ping_lifetime_io` set to true - if lifetime == Lifetime::Ping { - if let Some(ping_lifetime_data) = &self.ping_lifetime_data { - let data = ping_lifetime_data - .read() - .expect("Can't read ping lifetime data"); - for (key, value) in data.iter() { - if key.starts_with(&iter_start) { - let key = &key[len..]; - transaction_fn(key.as_bytes(), value); - } - } - return; - } - } - - let reader = unwrap_or!(self.rkv.read(), return); - let mut iter = unwrap_or!( - self.get_store(lifetime).iter_from(&reader, &iter_start), - return - ); - - while let Some(Ok((metric_id, value))) = iter.next() { - if !metric_id.starts_with(iter_start.as_bytes()) { - break; - } - - let metric_id = &metric_id[len..]; - let metric: Metric = match value { - rkv::Value::Blob(blob) => unwrap_or!(bincode::deserialize(blob), continue), - _ => continue, - }; - transaction_fn(metric_id, &metric); - } - } - - /// Determines if the storage has the given metric. - /// - /// If data cannot be read it is assumed that the storage does not have the metric. - /// - /// # Arguments - /// - /// * `lifetime` - The lifetime of the metric. - /// * `storage_name` - The storage name to look in. - /// * `metric_identifier` - The metric identifier. - /// - /// # Panics - /// - /// This function will **not** panic on database errors. - pub fn has_metric( - &self, - lifetime: Lifetime, - storage_name: &str, - metric_identifier: &str, - ) -> bool { - let key = Self::get_storage_key(storage_name, Some(metric_identifier)); - - // Lifetime::Ping data is not persisted to disk if - // Glean has `delay_ping_lifetime_io` set to true - if lifetime == Lifetime::Ping { - if let Some(ping_lifetime_data) = &self.ping_lifetime_data { - return ping_lifetime_data - .read() - .map(|data| data.contains_key(&key)) - .unwrap_or(false); - } - } - - let reader = unwrap_or!(self.rkv.read(), return false); - self.get_store(lifetime) - .get(&reader, &key) - .unwrap_or(None) - .is_some() - } - - /// Writes to the specified storage with the provided transaction function. - /// - /// If the storage is unavailable, it will return an error. - /// - /// # Panics - /// - /// * This function will **not** panic on database errors. - fn write_with_store(&self, store_name: Lifetime, mut transaction_fn: F) -> Result<()> - where - F: FnMut(Writer, &SingleStore) -> Result<()>, - { - let writer = self.rkv.write().unwrap(); - let store = self.get_store(store_name); - transaction_fn(writer, store) - } - - /// Records a metric in the underlying storage system. - pub fn record(&self, glean: &Glean, data: &CommonMetricDataInternal, value: &Metric) { - let name = data.identifier(glean); - for ping_name in data.storage_names() { - if glean.is_ping_enabled(ping_name) { - if let Err(e) = - self.record_per_lifetime(data.inner.lifetime, ping_name, &name, value) - { - log::info!( - "Failed to record metric '{}' into {}: {:?}", - data.base_identifier(), - ping_name, - e - ); - } - } - } - } - - /// Records a metric in the underlying storage system, for a single lifetime. - /// - /// # Returns - /// - /// If the storage is unavailable or the write fails, no data will be stored and an error will be returned. - /// - /// Otherwise `Ok(())` is returned. - /// - /// # Panics - /// - /// This function will **not** panic on database errors. - fn record_per_lifetime( - &self, - lifetime: Lifetime, - storage_name: &str, - key: &str, - metric: &Metric, - ) -> Result<()> { - let final_key = Self::get_storage_key(storage_name, Some(key)); - - // Lifetime::Ping data is not immediately persisted to disk if - // Glean has `delay_ping_lifetime_io` set to true - if lifetime == Lifetime::Ping { - if let Some(ping_lifetime_data) = &self.ping_lifetime_data { - let mut data = ping_lifetime_data - .write() - .expect("Can't read ping lifetime data"); - data.insert(final_key, metric.clone()); - - // flush ping lifetime - self.persist_ping_lifetime_data_if_full(&data)?; - return Ok(()); - } - } - - let encoded = bincode::serialize(&metric).expect("IMPOSSIBLE: Serializing metric failed"); - let value = rkv::Value::Blob(&encoded); - - let mut writer = self.rkv.write()?; - self.get_store(lifetime) - .put(&mut writer, final_key, &value)?; - measure_commit!(self, writer.commit())?; - Ok(()) - } - - /// Records the provided value, with the given lifetime, - /// after applying a transformation function. - pub fn record_with(&self, glean: &Glean, data: &CommonMetricDataInternal, mut transform: F) - where - F: FnMut(Option) -> Metric, - { - let name = data.identifier(glean); - for ping_name in data.storage_names() { - if glean.is_ping_enabled(ping_name) { - if let Err(e) = self.record_per_lifetime_with( - data.inner.lifetime, - ping_name, - &name, - &mut transform, - ) { - log::info!( - "Failed to record metric '{}' into {}: {:?}", - data.base_identifier(), - ping_name, - e - ); - } - } - } - } - - /// Records a metric in the underlying storage system, - /// after applying the given transformation function, for a single lifetime. - /// - /// # Returns - /// - /// If the storage is unavailable or the write fails, no data will be stored and an error will be returned. - /// - /// Otherwise `Ok(())` is returned. - /// - /// # Panics - /// - /// This function will **not** panic on database errors. - fn record_per_lifetime_with( - &self, - lifetime: Lifetime, - storage_name: &str, - key: &str, - mut transform: F, - ) -> Result<()> - where - F: FnMut(Option) -> Metric, - { - let final_key = Self::get_storage_key(storage_name, Some(key)); - - // Lifetime::Ping data is not persisted to disk if - // Glean has `delay_ping_lifetime_io` set to true - if lifetime == Lifetime::Ping { - if let Some(ping_lifetime_data) = &self.ping_lifetime_data { - let mut data = ping_lifetime_data - .write() - .expect("Can't access ping lifetime data as writable"); - let entry = data.entry(final_key); - match entry { - Entry::Vacant(entry) => { - entry.insert(transform(None)); - } - Entry::Occupied(mut entry) => { - let old_value = entry.get().clone(); - entry.insert(transform(Some(old_value))); - } - } - - // flush ping lifetime - self.persist_ping_lifetime_data_if_full(&data)?; - return Ok(()); - } - } - - let mut writer = self.rkv.write()?; - let store = self.get_store(lifetime); - let new_value: Metric = { - let old_value = store.get(&writer, &final_key)?; - - match old_value { - Some(rkv::Value::Blob(blob)) => { - let old_value = bincode::deserialize(blob).ok(); - transform(old_value) - } - _ => transform(None), - } - }; - - let encoded = - bincode::serialize(&new_value).expect("IMPOSSIBLE: Serializing metric failed"); - let value = rkv::Value::Blob(&encoded); - store.put(&mut writer, final_key, &value)?; - measure_commit!(self, writer.commit())?; - Ok(()) - } - - /// Clears a storage (only Ping Lifetime). - /// - /// # Returns - /// - /// * If the storage is unavailable an error is returned. - /// * If any individual delete fails, an error is returned, but other deletions might have - /// happened. - /// - /// Otherwise `Ok(())` is returned. - /// - /// # Panics - /// - /// This function will **not** panic on database errors. - pub fn clear_ping_lifetime_storage(&self, storage_name: &str) -> Result<()> { - // Lifetime::Ping data will be saved to `ping_lifetime_data` - // in case `delay_ping_lifetime_io` is set to true - if let Some(ping_lifetime_data) = &self.ping_lifetime_data { - ping_lifetime_data - .write() - .expect("Can't access ping lifetime data as writable") - .retain(|metric_id, _| !metric_id.starts_with(storage_name)); - } - - self.write_with_store(Lifetime::Ping, |mut writer, store| { - let mut metrics = Vec::new(); - { - let mut iter = store.iter_from(&writer, storage_name)?; - while let Some(Ok((metric_id, _))) = iter.next() { - if let Ok(metric_id) = std::str::from_utf8(metric_id) { - if !metric_id.starts_with(storage_name) { - break; - } - metrics.push(metric_id.to_owned()); - } - } - } - - let mut res = Ok(()); - for to_delete in metrics { - if let Err(e) = store.delete(&mut writer, to_delete) { - log::warn!("Can't delete from store: {:?}", e); - res = Err(e); - } - } - - measure_commit!(self, writer.commit())?; - Ok(res?) - }) - } - - pub fn clear_lifetime_storage(&self, lifetime: Lifetime, storage_name: &str) -> Result<()> { - self.write_with_store(lifetime, |mut writer, store| { - let mut metrics = Vec::new(); - { - let mut iter = store.iter_from(&writer, storage_name)?; - while let Some(Ok((metric_id, _))) = iter.next() { - if let Ok(metric_id) = std::str::from_utf8(metric_id) { - if !metric_id.starts_with(storage_name) { - break; - } - metrics.push(metric_id.to_owned()); - } - } - } - - let mut res = Ok(()); - for to_delete in metrics { - if let Err(e) = store.delete(&mut writer, to_delete) { - log::warn!("Can't delete from store: {:?}", e); - res = Err(e); - } - } - - measure_commit!(self, writer.commit())?; - Ok(res?) - }) - } - - /// Removes a single metric from the storage. - /// - /// # Arguments - /// - /// * `lifetime` - the lifetime of the storage in which to look for the metric. - /// * `storage_name` - the name of the storage to store/fetch data from. - /// * `metric_id` - the metric category + name. - /// - /// # Returns - /// - /// * If the storage is unavailable an error is returned. - /// * If the metric could not be deleted, an error is returned. - /// - /// Otherwise `Ok(())` is returned. - /// - /// # Panics - /// - /// This function will **not** panic on database errors. - pub fn remove_single_metric( - &self, - lifetime: Lifetime, - storage_name: &str, - metric_id: &str, - ) -> Result<()> { - let final_key = Self::get_storage_key(storage_name, Some(metric_id)); - - // Lifetime::Ping data is not persisted to disk if - // Glean has `delay_ping_lifetime_io` set to true - if lifetime == Lifetime::Ping { - if let Some(ping_lifetime_data) = &self.ping_lifetime_data { - let mut data = ping_lifetime_data - .write() - .expect("Can't access app lifetime data as writable"); - data.remove(&final_key); - } - } - - self.write_with_store(lifetime, |mut writer, store| { - if let Err(e) = store.delete(&mut writer, final_key.clone()) { - if self.ping_lifetime_data.is_some() { - // If ping_lifetime_data exists, it might be - // that data is in memory, but not yet in rkv. - return Ok(()); - } - return Err(e.into()); - } - measure_commit!(self, writer.commit())?; - Ok(()) - }) - } - - /// Clears all the metrics in the database, for the provided lifetime. - /// - /// Errors are logged. - /// - /// # Panics - /// - /// * This function will **not** panic on database errors. - pub fn clear_lifetime(&self, lifetime: Lifetime) { - let res = self.write_with_store(lifetime, |mut writer, store| { - store.clear(&mut writer)?; - measure_commit!(self, writer.commit())?; - Ok(()) - }); - - if let Err(e) = res { - // We try to clear everything. - // If there was no data to begin with we encounter a `NotFound` error. - // There's no point in logging that. - if let ErrorKind::Rkv(StoreError::IoError(ioerr)) = e.kind() { - if let io::ErrorKind::NotFound = ioerr.kind() { - log::debug!( - "Could not clear store for lifetime {:?}: {:?}", - lifetime, - ioerr - ); - return; - } - } - - log::warn!("Could not clear store for lifetime {:?}: {:?}", lifetime, e); - } - } - - /// Clears all metrics in the database. - /// - /// Errors are logged. - /// - /// # Panics - /// - /// * This function will **not** panic on database errors. - pub fn clear_all(&self) { - if let Some(ping_lifetime_data) = &self.ping_lifetime_data { - ping_lifetime_data - .write() - .expect("Can't access ping lifetime data as writable") - .clear(); - } - - for lifetime in [Lifetime::User, Lifetime::Ping, Lifetime::Application].iter() { - self.clear_lifetime(*lifetime); - } - } - - /// Persists ping_lifetime_data to disk. - /// - /// Does nothing in case there is nothing to persist. - /// - /// # Panics - /// - /// * This function will **not** panic on database errors. - pub fn persist_ping_lifetime_data(&self) -> Result<()> { - if let Some(ping_lifetime_data) = &self.ping_lifetime_data { - let data = ping_lifetime_data - .read() - .expect("Can't read ping lifetime data"); - - // We can reset the write-counter. Current data has been persisted. - self.ping_lifetime_count.store(0, Ordering::Release); - self.ping_lifetime_store_ts.replace(Instant::now()); - - self.write_with_store(Lifetime::Ping, |mut writer, store| { - for (key, value) in data.iter() { - let encoded = - bincode::serialize(&value).expect("IMPOSSIBLE: Serializing metric failed"); - // There is no need for `get_storage_key` here because - // the key is already formatted from when it was saved - // to ping_lifetime_data. - store.put(&mut writer, key, &rkv::Value::Blob(&encoded))?; - } - measure_commit!(self, writer.commit())?; - Ok(()) - })?; - } - Ok(()) - } - - pub fn persist_ping_lifetime_data_if_full( - &self, - data: &BTreeMap, - ) -> Result<()> { - if self.ping_lifetime_threshold == 0 && self.ping_lifetime_max_time.is_zero() { - return Ok(()); - } - - let write_count = self.ping_lifetime_count.fetch_add(1, Ordering::Release) + 1; - let last_write = self.ping_lifetime_store_ts.get(); - let elapsed = last_write.elapsed(); - - if (self.ping_lifetime_threshold == 0 || write_count < self.ping_lifetime_threshold) - && (self.ping_lifetime_max_time.is_zero() || elapsed < self.ping_lifetime_max_time) - { - log::trace!( - "Not flushing. write_count={} (threshold={}), elapsed={:?} (max={:?})", - write_count, - self.ping_lifetime_threshold, - elapsed, - self.ping_lifetime_max_time - ); - return Ok(()); - } - - if self.ping_lifetime_threshold > 0 && write_count >= self.ping_lifetime_threshold { - log::debug!( - "Flushing database due to threshold of {} reached.", - self.ping_lifetime_threshold - ) - } else if !self.ping_lifetime_max_time.is_zero() && elapsed >= self.ping_lifetime_max_time { - log::debug!( - "Flushing database due to last write more than {:?} ago", - self.ping_lifetime_max_time - ); - } - - self.ping_lifetime_count.store(0, Ordering::Release); - self.ping_lifetime_store_ts.replace(Instant::now()); - self.write_with_store(Lifetime::Ping, |mut writer, store| { - for (key, value) in data.iter() { - let encoded = - bincode::serialize(&value).expect("IMPOSSIBLE: Serializing metric failed"); - // There is no need for `get_storage_key` here because - // the key is already formatted from when it was saved - // to ping_lifetime_data. - store.put(&mut writer, key, &rkv::Value::Blob(&encoded))?; - } - writer.commit()?; - Ok(()) - }) - } -} - -#[cfg(test)] -mod test { - use super::*; - use crate::tests::new_glean; - use std::collections::HashMap; - use tempfile::tempdir; - - #[test] - fn test_panicks_if_fails_dir_creation() { - let path = Path::new("/!#\"'@#°ç"); - assert!(Database::new(path, false, 0, Duration::ZERO).is_err()); - } - - #[test] - #[cfg(windows)] - fn windows_invalid_utf16_panicfree() { - use std::ffi::OsString; - use std::os::windows::prelude::*; - - // Here the values 0x0066 and 0x006f correspond to 'f' and 'o' - // respectively. The value 0xD800 is a lone surrogate half, invalid - // in a UTF-16 sequence. - let source = [0x0066, 0x006f, 0xD800, 0x006f]; - let os_string = OsString::from_wide(&source[..]); - let os_str = os_string.as_os_str(); - let dir = tempdir().unwrap(); - let path = dir.path().join(os_str); - - let res = Database::new(&path, false, 0, Duration::ZERO); - - assert!( - res.is_ok(), - "Database should succeed at {}: {:?}", - path.display(), - res - ); - } - - #[test] - #[cfg(target_os = "linux")] - fn linux_invalid_utf8_panicfree() { - use std::ffi::OsStr; - use std::os::unix::ffi::OsStrExt; - - // Here, the values 0x66 and 0x6f correspond to 'f' and 'o' - // respectively. The value 0x80 is a lone continuation byte, invalid - // in a UTF-8 sequence. - let source = [0x66, 0x6f, 0x80, 0x6f]; - let os_str = OsStr::from_bytes(&source[..]); - let dir = tempdir().unwrap(); - let path = dir.path().join(os_str); - - let res = Database::new(&path, false, 0, Duration::ZERO); - assert!( - res.is_ok(), - "Database should not fail at {}: {:?}", - path.display(), - res - ); - } - - #[test] - #[cfg(target_os = "macos")] - fn macos_invalid_utf8_panicfree() { - use std::ffi::OsStr; - use std::os::unix::ffi::OsStrExt; - - // Here, the values 0x66 and 0x6f correspond to 'f' and 'o' - // respectively. The value 0x80 is a lone continuation byte, invalid - // in a UTF-8 sequence. - let source = [0x66, 0x6f, 0x80, 0x6f]; - let os_str = OsStr::from_bytes(&source[..]); - let dir = tempdir().unwrap(); - let path = dir.path().join(os_str); - - let res = Database::new(&path, false, 0, Duration::ZERO); - assert!( - res.is_err(), - "Database should not fail at {}: {:?}", - path.display(), - res - ); - } - - #[test] - fn test_data_dir_rkv_inits() { - let dir = tempdir().unwrap(); - Database::new(dir.path(), false, 0, Duration::ZERO).unwrap(); - - assert!(dir.path().exists()); - } - - #[test] - fn test_ping_lifetime_metric_recorded() { - // Init the database in a temporary directory. - let dir = tempdir().unwrap(); - let db = Database::new(dir.path(), false, 0, Duration::ZERO).unwrap(); - - assert!(db.ping_lifetime_data.is_none()); - - // Attempt to record a known value. - let test_value = "test-value"; - let test_storage = "test-storage"; - let test_metric_id = "telemetry_test.test_name"; - db.record_per_lifetime( - Lifetime::Ping, - test_storage, - test_metric_id, - &Metric::String(test_value.to_string()), - ) - .unwrap(); - - // Verify that the data is correctly recorded. - let mut found_metrics = 0; - let mut snapshotter = |metric_id: &[u8], metric: &Metric| { - found_metrics += 1; - let metric_id = String::from_utf8_lossy(metric_id).into_owned(); - assert_eq!(test_metric_id, metric_id); - match metric { - Metric::String(s) => assert_eq!(test_value, s), - _ => panic!("Unexpected data found"), - } - }; - - db.iter_store_from(Lifetime::Ping, test_storage, None, &mut snapshotter); - assert_eq!(1, found_metrics, "We only expect 1 Lifetime.Ping metric."); - } - - #[test] - fn test_application_lifetime_metric_recorded() { - // Init the database in a temporary directory. - let dir = tempdir().unwrap(); - let db = Database::new(dir.path(), false, 0, Duration::ZERO).unwrap(); - - // Attempt to record a known value. - let test_value = "test-value"; - let test_storage = "test-storage1"; - let test_metric_id = "telemetry_test.test_name"; - db.record_per_lifetime( - Lifetime::Application, - test_storage, - test_metric_id, - &Metric::String(test_value.to_string()), - ) - .unwrap(); - - // Verify that the data is correctly recorded. - let mut found_metrics = 0; - let mut snapshotter = |metric_id: &[u8], metric: &Metric| { - found_metrics += 1; - let metric_id = String::from_utf8_lossy(metric_id).into_owned(); - assert_eq!(test_metric_id, metric_id); - match metric { - Metric::String(s) => assert_eq!(test_value, s), - _ => panic!("Unexpected data found"), - } - }; - - db.iter_store_from(Lifetime::Application, test_storage, None, &mut snapshotter); - assert_eq!( - 1, found_metrics, - "We only expect 1 Lifetime.Application metric." - ); - } - - #[test] - fn test_user_lifetime_metric_recorded() { - // Init the database in a temporary directory. - let dir = tempdir().unwrap(); - let db = Database::new(dir.path(), false, 0, Duration::ZERO).unwrap(); - - // Attempt to record a known value. - let test_value = "test-value"; - let test_storage = "test-storage2"; - let test_metric_id = "telemetry_test.test_name"; - db.record_per_lifetime( - Lifetime::User, - test_storage, - test_metric_id, - &Metric::String(test_value.to_string()), - ) - .unwrap(); - - // Verify that the data is correctly recorded. - let mut found_metrics = 0; - let mut snapshotter = |metric_id: &[u8], metric: &Metric| { - found_metrics += 1; - let metric_id = String::from_utf8_lossy(metric_id).into_owned(); - assert_eq!(test_metric_id, metric_id); - match metric { - Metric::String(s) => assert_eq!(test_value, s), - _ => panic!("Unexpected data found"), - } - }; - - db.iter_store_from(Lifetime::User, test_storage, None, &mut snapshotter); - assert_eq!(1, found_metrics, "We only expect 1 Lifetime.User metric."); - } - - #[test] - fn test_clear_ping_storage() { - // Init the database in a temporary directory. - let dir = tempdir().unwrap(); - let db = Database::new(dir.path(), false, 0, Duration::ZERO).unwrap(); - - // Attempt to record a known value for every single lifetime. - let test_storage = "test-storage"; - db.record_per_lifetime( - Lifetime::User, - test_storage, - "telemetry_test.test_name_user", - &Metric::String("test-value-user".to_string()), - ) - .unwrap(); - db.record_per_lifetime( - Lifetime::Ping, - test_storage, - "telemetry_test.test_name_ping", - &Metric::String("test-value-ping".to_string()), - ) - .unwrap(); - db.record_per_lifetime( - Lifetime::Application, - test_storage, - "telemetry_test.test_name_application", - &Metric::String("test-value-application".to_string()), - ) - .unwrap(); - - // Take a snapshot for the data, all the lifetimes. - { - let mut snapshot: HashMap = HashMap::new(); - let mut snapshotter = |metric_id: &[u8], metric: &Metric| { - let metric_id = String::from_utf8_lossy(metric_id).into_owned(); - match metric { - Metric::String(s) => snapshot.insert(metric_id, s.to_string()), - _ => panic!("Unexpected data found"), - }; - }; - - db.iter_store_from(Lifetime::User, test_storage, None, &mut snapshotter); - db.iter_store_from(Lifetime::Ping, test_storage, None, &mut snapshotter); - db.iter_store_from(Lifetime::Application, test_storage, None, &mut snapshotter); - - assert_eq!(3, snapshot.len(), "We expect all lifetimes to be present."); - assert!(snapshot.contains_key("telemetry_test.test_name_user")); - assert!(snapshot.contains_key("telemetry_test.test_name_ping")); - assert!(snapshot.contains_key("telemetry_test.test_name_application")); - } - - // Clear the Ping lifetime. - db.clear_ping_lifetime_storage(test_storage).unwrap(); - - // Take a snapshot again and check that we're only clearing the Ping lifetime. - { - let mut snapshot: HashMap = HashMap::new(); - let mut snapshotter = |metric_id: &[u8], metric: &Metric| { - let metric_id = String::from_utf8_lossy(metric_id).into_owned(); - match metric { - Metric::String(s) => snapshot.insert(metric_id, s.to_string()), - _ => panic!("Unexpected data found"), - }; - }; - - db.iter_store_from(Lifetime::User, test_storage, None, &mut snapshotter); - db.iter_store_from(Lifetime::Ping, test_storage, None, &mut snapshotter); - db.iter_store_from(Lifetime::Application, test_storage, None, &mut snapshotter); - - assert_eq!(2, snapshot.len(), "We only expect 2 metrics to be left."); - assert!(snapshot.contains_key("telemetry_test.test_name_user")); - assert!(snapshot.contains_key("telemetry_test.test_name_application")); - } - } - - #[test] - fn test_remove_single_metric() { - // Init the database in a temporary directory. - let dir = tempdir().unwrap(); - let db = Database::new(dir.path(), false, 0, Duration::ZERO).unwrap(); - - let test_storage = "test-storage-single-lifetime"; - let metric_id_pattern = "telemetry_test.single_metric"; - - // Write sample metrics to the database. - let lifetimes = [Lifetime::User, Lifetime::Ping, Lifetime::Application]; - - for lifetime in lifetimes.iter() { - for value in &["retain", "delete"] { - db.record_per_lifetime( - *lifetime, - test_storage, - &format!("{}_{}", metric_id_pattern, value), - &Metric::String((*value).to_string()), - ) - .unwrap(); - } - } - - // Remove "telemetry_test.single_metric_delete" from each lifetime. - for lifetime in lifetimes.iter() { - db.remove_single_metric( - *lifetime, - test_storage, - &format!("{}_delete", metric_id_pattern), - ) - .unwrap(); - } - - // Verify that "telemetry_test.single_metric_retain" is still around for all lifetimes. - for lifetime in lifetimes.iter() { - let mut found_metrics = 0; - let mut snapshotter = |metric_id: &[u8], metric: &Metric| { - found_metrics += 1; - let metric_id = String::from_utf8_lossy(metric_id).into_owned(); - assert_eq!(format!("{}_retain", metric_id_pattern), metric_id); - match metric { - Metric::String(s) => assert_eq!("retain", s), - _ => panic!("Unexpected data found"), - } - }; - - // Check the User lifetime. - db.iter_store_from(*lifetime, test_storage, None, &mut snapshotter); - assert_eq!( - 1, found_metrics, - "We only expect 1 metric for this lifetime." - ); - } - } - - #[test] - fn test_delayed_ping_lifetime_persistence() { - // Init the database in a temporary directory. - let dir = tempdir().unwrap(); - let db = Database::new(dir.path(), true, 0, Duration::ZERO).unwrap(); - let test_storage = "test-storage"; - - assert!(db.ping_lifetime_data.is_some()); - - // Attempt to record a known value. - let test_value1 = "test-value1"; - let test_metric_id1 = "telemetry_test.test_name1"; - db.record_per_lifetime( - Lifetime::Ping, - test_storage, - test_metric_id1, - &Metric::String(test_value1.to_string()), - ) - .unwrap(); - - // Attempt to persist data. - db.persist_ping_lifetime_data().unwrap(); - - // Attempt to record another known value. - let test_value2 = "test-value2"; - let test_metric_id2 = "telemetry_test.test_name2"; - db.record_per_lifetime( - Lifetime::Ping, - test_storage, - test_metric_id2, - &Metric::String(test_value2.to_string()), - ) - .unwrap(); - - { - // At this stage we expect `test_value1` to be persisted and in memory, - // since it was recorded before calling `persist_ping_lifetime_data`, - // and `test_value2` to be only in memory, since it was recorded after. - let store: SingleStore = db - .rkv - .open_single(Lifetime::Ping.as_str(), StoreOptions::create()) - .unwrap(); - let reader = db.rkv.read().unwrap(); - - // Verify that test_value1 is in rkv. - assert!(store - .get(&reader, format!("{}#{}", test_storage, test_metric_id1)) - .unwrap_or(None) - .is_some()); - // Verifiy that test_value2 is **not** in rkv. - assert!(store - .get(&reader, format!("{}#{}", test_storage, test_metric_id2)) - .unwrap_or(None) - .is_none()); - - let data = match &db.ping_lifetime_data { - Some(ping_lifetime_data) => ping_lifetime_data, - None => panic!("Expected `ping_lifetime_data` to exist here!"), - }; - let data = data.read().unwrap(); - // Verify that test_value1 is also in memory. - assert!(data - .get(&format!("{}#{}", test_storage, test_metric_id1)) - .is_some()); - // Verify that test_value2 is in memory. - assert!(data - .get(&format!("{}#{}", test_storage, test_metric_id2)) - .is_some()); - } - - // Attempt to persist data again. - db.persist_ping_lifetime_data().unwrap(); - - { - // At this stage we expect `test_value1` and `test_value2` to - // be persisted, since both were created before a call to - // `persist_ping_lifetime_data`. - let store: SingleStore = db - .rkv - .open_single(Lifetime::Ping.as_str(), StoreOptions::create()) - .unwrap(); - let reader = db.rkv.read().unwrap(); - - // Verify that test_value1 is in rkv. - assert!(store - .get(&reader, format!("{}#{}", test_storage, test_metric_id1)) - .unwrap_or(None) - .is_some()); - // Verifiy that test_value2 is also in rkv. - assert!(store - .get(&reader, format!("{}#{}", test_storage, test_metric_id2)) - .unwrap_or(None) - .is_some()); - - let data = match &db.ping_lifetime_data { - Some(ping_lifetime_data) => ping_lifetime_data, - None => panic!("Expected `ping_lifetime_data` to exist here!"), - }; - let data = data.read().unwrap(); - // Verify that test_value1 is also in memory. - assert!(data - .get(&format!("{}#{}", test_storage, test_metric_id1)) - .is_some()); - // Verify that test_value2 is also in memory. - assert!(data - .get(&format!("{}#{}", test_storage, test_metric_id2)) - .is_some()); - } - } - - #[test] - fn test_load_ping_lifetime_data_from_memory() { - // Init the database in a temporary directory. - let dir = tempdir().unwrap(); - - let test_storage = "test-storage"; - let test_value = "test-value"; - let test_metric_id = "telemetry_test.test_name"; - - { - let db = Database::new(dir.path(), true, 0, Duration::ZERO).unwrap(); - - // Attempt to record a known value. - db.record_per_lifetime( - Lifetime::Ping, - test_storage, - test_metric_id, - &Metric::String(test_value.to_string()), - ) - .unwrap(); - - // Verify that test_value is in memory. - let data = match &db.ping_lifetime_data { - Some(ping_lifetime_data) => ping_lifetime_data, - None => panic!("Expected `ping_lifetime_data` to exist here!"), - }; - let data = data.read().unwrap(); - assert!(data - .get(&format!("{}#{}", test_storage, test_metric_id)) - .is_some()); - - // Attempt to persist data. - db.persist_ping_lifetime_data().unwrap(); - - // Verify that test_value is now in rkv. - let store: SingleStore = db - .rkv - .open_single(Lifetime::Ping.as_str(), StoreOptions::create()) - .unwrap(); - let reader = db.rkv.read().unwrap(); - assert!(store - .get(&reader, format!("{}#{}", test_storage, test_metric_id)) - .unwrap_or(None) - .is_some()); - } - - // Now create a new instace of the db and check if data was - // correctly loaded from rkv to memory. - { - let db = Database::new(dir.path(), true, 0, Duration::ZERO).unwrap(); - - // Verify that test_value is in memory. - let data = match &db.ping_lifetime_data { - Some(ping_lifetime_data) => ping_lifetime_data, - None => panic!("Expected `ping_lifetime_data` to exist here!"), - }; - let data = data.read().unwrap(); - assert!(data - .get(&format!("{}#{}", test_storage, test_metric_id)) - .is_some()); - - // Verify that test_value is also in rkv. - let store: SingleStore = db - .rkv - .open_single(Lifetime::Ping.as_str(), StoreOptions::create()) - .unwrap(); - let reader = db.rkv.read().unwrap(); - assert!(store - .get(&reader, format!("{}#{}", test_storage, test_metric_id)) - .unwrap_or(None) - .is_some()); - } - } - - #[test] - fn test_delayed_ping_lifetime_clear() { - // Init the database in a temporary directory. - let dir = tempdir().unwrap(); - let db = Database::new(dir.path(), true, 0, Duration::ZERO).unwrap(); - let test_storage = "test-storage"; - - assert!(db.ping_lifetime_data.is_some()); - - // Attempt to record a known value. - let test_value1 = "test-value1"; - let test_metric_id1 = "telemetry_test.test_name1"; - db.record_per_lifetime( - Lifetime::Ping, - test_storage, - test_metric_id1, - &Metric::String(test_value1.to_string()), - ) - .unwrap(); - - { - let data = match &db.ping_lifetime_data { - Some(ping_lifetime_data) => ping_lifetime_data, - None => panic!("Expected `ping_lifetime_data` to exist here!"), - }; - let data = data.read().unwrap(); - // Verify that test_value1 is in memory. - assert!(data - .get(&format!("{}#{}", test_storage, test_metric_id1)) - .is_some()); - } - - // Clear ping lifetime storage for a storage that isn't test_storage. - // Doesn't matter what it's called, just that it isn't test_storage. - db.clear_ping_lifetime_storage(&(test_storage.to_owned() + "x")) - .unwrap(); - - { - let data = match &db.ping_lifetime_data { - Some(ping_lifetime_data) => ping_lifetime_data, - None => panic!("Expected `ping_lifetime_data` to exist here!"), - }; - let data = data.read().unwrap(); - // Verify that test_value1 is still in memory. - assert!(data - .get(&format!("{}#{}", test_storage, test_metric_id1)) - .is_some()); - } - - // Clear test_storage's ping lifetime storage. - db.clear_ping_lifetime_storage(test_storage).unwrap(); - - { - let data = match &db.ping_lifetime_data { - Some(ping_lifetime_data) => ping_lifetime_data, - None => panic!("Expected `ping_lifetime_data` to exist here!"), - }; - let data = data.read().unwrap(); - // Verify that test_value1 is no longer in memory. - assert!(data - .get(&format!("{}#{}", test_storage, test_metric_id1)) - .is_none()); - } - } - - #[test] - fn doesnt_record_when_upload_is_disabled() { - let (mut glean, dir) = new_glean(None); - - // Init the database in a temporary directory. - - let test_storage = "test-storage"; - let test_data = CommonMetricDataInternal::new("category", "name", test_storage); - let test_metric_id = test_data.identifier(&glean); - - // Attempt to record metric with the record and record_with functions, - // this should work since upload is enabled. - let db = Database::new(dir.path(), true, 0, Duration::ZERO).unwrap(); - db.record(&glean, &test_data, &Metric::String("record".to_owned())); - db.iter_store_from( - Lifetime::Ping, - test_storage, - None, - &mut |metric_id: &[u8], metric: &Metric| { - assert_eq!( - String::from_utf8_lossy(metric_id).into_owned(), - test_metric_id - ); - match metric { - Metric::String(v) => assert_eq!("record", *v), - _ => panic!("Unexpected data found"), - } - }, - ); - - db.record_with(&glean, &test_data, |_| { - Metric::String("record_with".to_owned()) - }); - db.iter_store_from( - Lifetime::Ping, - test_storage, - None, - &mut |metric_id: &[u8], metric: &Metric| { - assert_eq!( - String::from_utf8_lossy(metric_id).into_owned(), - test_metric_id - ); - match metric { - Metric::String(v) => assert_eq!("record_with", *v), - _ => panic!("Unexpected data found"), - } - }, - ); - - // Disable upload - glean.set_upload_enabled(false); - - // Attempt to record metric with the record and record_with functions, - // this should work since upload is now **disabled**. - db.record(&glean, &test_data, &Metric::String("record_nop".to_owned())); - db.iter_store_from( - Lifetime::Ping, - test_storage, - None, - &mut |metric_id: &[u8], metric: &Metric| { - assert_eq!( - String::from_utf8_lossy(metric_id).into_owned(), - test_metric_id - ); - match metric { - Metric::String(v) => assert_eq!("record_with", *v), - _ => panic!("Unexpected data found"), - } - }, - ); - db.record_with(&glean, &test_data, |_| { - Metric::String("record_with_nop".to_owned()) - }); - db.iter_store_from( - Lifetime::Ping, - test_storage, - None, - &mut |metric_id: &[u8], metric: &Metric| { - assert_eq!( - String::from_utf8_lossy(metric_id).into_owned(), - test_metric_id - ); - match metric { - Metric::String(v) => assert_eq!("record_with", *v), - _ => panic!("Unexpected data found"), - } - }, - ); - } - - mod safe_mode { - use std::fs::File; - - use super::*; - - #[test] - fn empty_data_file() { - let dir = tempdir().unwrap(); - - // Create database directory structure. - let database_dir = dir.path().join("db"); - fs::create_dir_all(&database_dir).expect("create database dir"); - - // Create empty database file. - let safebin = database_dir.join("data.safe.bin"); - let f = File::create(safebin).expect("create database file"); - drop(f); - - let db = Database::new(dir.path(), false, 0, Duration::ZERO).unwrap(); - - assert!(dir.path().exists()); - assert!( - matches!(db.rkv_load_state, RkvLoadState::Err(_)), - "Load error recorded" - ); - } - - #[test] - fn corrupted_data_file() { - let dir = tempdir().unwrap(); - - // Create database directory structure. - let database_dir = dir.path().join("db"); - fs::create_dir_all(&database_dir).expect("create database dir"); - - // Create empty database file. - let safebin = database_dir.join("data.safe.bin"); - fs::write(safebin, "").expect("write to database file"); - - let db = Database::new(dir.path(), false, 0, Duration::ZERO).unwrap(); - - assert!(dir.path().exists()); - assert!( - matches!(db.rkv_load_state, RkvLoadState::Err(_)), - "Load error recorded" - ); - } - } -} diff --git a/glean-core/src/error_recording.rs b/glean-core/src/error_recording.rs index 11e00aee2c..c843ca8acb 100644 --- a/glean-core/src/error_recording.rs +++ b/glean-core/src/error_recording.rs @@ -19,7 +19,6 @@ use rusqlite::Transaction; use crate::common_metric_data::CommonMetricDataInternal; use crate::error::{Error, ErrorKind}; -use crate::metrics::labeled::{combine_base_identifier_and_label, strip_label}; use crate::metrics::{CounterMetric, Metric}; use crate::Glean; use crate::Lifetime; @@ -95,8 +94,7 @@ impl TryFrom for ErrorType { fn get_error_metric_for_metric(meta: &CommonMetricDataInternal, error: ErrorType) -> CounterMetric { // Can't use meta.identifier here, since that might cause infinite recursion // if the label on this metric needs to report an error. - let identifier = meta.base_identifier(); - let name = strip_label(&identifier); + let name = meta.base_identifier(); // Record errors in the pings the metric is in, as well as the metrics ping. let mut send_in_pings = meta.inner.send_in_pings.clone(); diff --git a/glean-core/src/metrics/boolean.rs b/glean-core/src/metrics/boolean.rs index 3982547b9e..95ce3dc92d 100644 --- a/glean-core/src/metrics/boolean.rs +++ b/glean-core/src/metrics/boolean.rs @@ -8,7 +8,6 @@ use crate::common_metric_data::{CommonMetricDataInternal, DynamicLabelType}; use crate::error_recording::{test_get_num_recorded_errors, ErrorType}; use crate::metrics::MetricType; use crate::metrics::{Metric, TestGetValue}; -use crate::storage::StorageManager; use crate::CommonMetricData; use crate::Glean; diff --git a/glean-core/src/metrics/counter.rs b/glean-core/src/metrics/counter.rs index 7dacf48740..1605e195c3 100644 --- a/glean-core/src/metrics/counter.rs +++ b/glean-core/src/metrics/counter.rs @@ -9,7 +9,6 @@ use crate::common_metric_data::{CommonMetricDataInternal, DynamicLabelType}; use crate::error_recording::{record_error, test_get_num_recorded_errors, ErrorType}; use crate::metrics::Metric; use crate::metrics::MetricType; -use crate::storage::StorageManager; use crate::Glean; use crate::{CommonMetricData, TestGetValue}; diff --git a/glean-core/src/metrics/dual_labeled_counter.rs b/glean-core/src/metrics/dual_labeled_counter.rs index 51f5df8721..5c9f4a17de 100644 --- a/glean-core/src/metrics/dual_labeled_counter.rs +++ b/glean-core/src/metrics/dual_labeled_counter.rs @@ -11,10 +11,8 @@ use std::sync::{Arc, Mutex}; use rusqlite::{params, Transaction}; use crate::common_metric_data::{CommonMetricData, CommonMetricDataInternal, DynamicLabelType}; -use crate::error_recording::{ - record_error, record_error_sqlite, test_get_num_recorded_errors, ErrorType, -}; -use crate::metrics::{CounterMetric, Metric, MetricType}; +use crate::error_recording::{record_error_sqlite, test_get_num_recorded_errors, ErrorType}; +use crate::metrics::{CounterMetric, MetricType}; use crate::{Glean, TestGetValue}; const MAX_LABELS: usize = 16; @@ -251,38 +249,6 @@ impl TestGetValue for DualLabeledCounterMetric { } } -/// Combines a metric's base identifier and label -pub fn combine_base_identifier_and_labels( - base_identifier: &str, - key: &str, - category: &str, -) -> String { - format!( - "{}{}", - base_identifier, - make_label_from_key_and_category(key, category) - ) -} - -/// Separate label into key and category components. -/// Must validate the label format before calling this to ensure it doesn't contain -/// any ASCII record separator characters aside from the one's we put there. -pub fn separate_label_into_key_and_category(label: &str) -> Option<(&str, &str)> { - label - .strip_prefix(RECORD_SEPARATOR) - .unwrap_or(label) - .split_once(RECORD_SEPARATOR) -} - -/// Construct and return a label from a given key and category with the RECORD_SEPARATOR -/// characters in the format: `` -pub fn make_label_from_key_and_category(key: &str, category: &str) -> String { - format!( - "{}{}{}{}", - RECORD_SEPARATOR, key, RECORD_SEPARATOR, category - ) -} - /// Validates a dynamic label, changing it to `OTHER_LABEL` if it's invalid. /// /// Checks the requested label against limitations, such as the label length and allowed @@ -409,55 +375,3 @@ fn label_is_valid_sqlite<'a>( label } } - -fn label_is_valid(label: &str, glean: &Glean, meta: &CommonMetricDataInternal) -> bool { - if label.len() > MAX_LABEL_LENGTH { - let msg = format!( - "label length {} exceeds maximum of {}", - label.len(), - MAX_LABEL_LENGTH - ); - record_error(glean, meta, ErrorType::InvalidLabel, msg, None); - false - } else { - true - } -} - -fn get_seen_keys_and_categories( - meta: &CommonMetricDataInternal, - glean: &Glean, -) -> (HashSet, HashSet) { - let base_identifier = &meta.base_identifier(); - let prefix = format!("{base_identifier}{RECORD_SEPARATOR}"); - let mut seen_keys: HashSet = HashSet::new(); - let mut seen_categories: HashSet = HashSet::new(); - let mut snapshotter = |metric_id: &[u8], _: &Metric| { - let metric_id_str = String::from_utf8_lossy(metric_id); - - // Split full identifier on the ASCII Record Separator (\x1e) - let parts: Vec<&str> = metric_id_str.split(RECORD_SEPARATOR).collect(); - - if parts.len() == 2 { - seen_keys.insert(parts[0].into()); - seen_categories.insert(parts[1].into()); - } else { - record_error( - glean, - meta, - ErrorType::InvalidLabel, - "Dual Labeled Counter label doesn't contain exactly 2 parts".to_string(), - None, - ); - } - }; - - let lifetime = meta.inner.lifetime; - for store in &meta.inner.send_in_pings { - glean - .storage() - .iter_store_from(lifetime, store, &prefix, &mut snapshotter); - } - - (seen_keys, seen_categories) -} diff --git a/glean-core/src/metrics/labeled.rs b/glean-core/src/metrics/labeled.rs index b7de942244..9d8eaa0a53 100644 --- a/glean-core/src/metrics/labeled.rs +++ b/glean-core/src/metrics/labeled.rs @@ -4,7 +4,6 @@ use std::any::Any; use std::borrow::Cow; -use std::collections::HashSet; use std::collections::{hash_map::Entry, HashMap}; use std::mem; use std::sync::{Arc, Mutex}; @@ -13,14 +12,11 @@ use malloc_size_of::MallocSizeOf; use rusqlite::{params, Transaction}; use crate::common_metric_data::{CommonMetricData, CommonMetricDataInternal, DynamicLabelType}; -use crate::error_recording::{ - record_error, record_error_sqlite, test_get_num_recorded_errors, ErrorType, -}; +use crate::error_recording::{record_error_sqlite, test_get_num_recorded_errors, ErrorType}; use crate::histogram::HistogramType; use crate::metrics::{ BooleanMetric, CounterMetric, CustomDistributionMetric, MemoryDistributionMetric, MemoryUnit, - Metric, MetricType, QuantityMetric, StringMetric, TestGetValue, TimeUnit, - TimingDistributionMetric, + MetricType, QuantityMetric, StringMetric, TestGetValue, TimeUnit, TimingDistributionMetric, }; use crate::storage::StorageManager; use crate::Glean; @@ -385,11 +381,6 @@ where } } -/// Combines a metric's base identifier and label -pub fn combine_base_identifier_and_label(base_identifier: &str, label: &str) -> String { - format!("{}/{}", base_identifier, label) -} - /// Strips the label off of a complete identifier pub fn strip_label(identifier: &str) -> &str { identifier.split_once('/').map_or(identifier, |s| s.0) @@ -409,51 +400,12 @@ pub fn strip_label(identifier: &str) -> &str { /// The entire identifier for the metric, including the base identifier and the corrected label. /// The errors are logged. pub fn validate_dynamic_label( - glean: &Glean, - meta: &CommonMetricDataInternal, - base_identifier: &str, - label: &str, + _glean: &Glean, + _meta: &CommonMetricDataInternal, + _base_identifier: &str, + _label: &str, ) -> String { - let key = combine_base_identifier_and_label(base_identifier, label); - for store in &meta.inner.send_in_pings { - if glean.storage().has_metric(meta.inner.lifetime, store, &key) { - return key; - } - } - - let mut labels = HashSet::new(); - let prefix = &key[..=base_identifier.len()]; - let mut snapshotter = |metric_id: &[u8], _: &Metric| { - labels.insert(metric_id.to_vec()); - }; - - let lifetime = meta.inner.lifetime; - for store in &meta.inner.send_in_pings { - glean - .storage() - .iter_store_from(lifetime, store, prefix, &mut snapshotter); - } - - let label_count = labels.len(); - let error = if label_count >= MAX_LABELS { - true - } else if label.len() > MAX_LABEL_LENGTH { - let msg = format!( - "label length {} exceeds maximum of {}", - label.len(), - MAX_LABEL_LENGTH - ); - record_error(glean, meta, ErrorType::InvalidLabel, msg, None); - true - } else { - false - }; - - if error { - combine_base_identifier_and_label(base_identifier, OTHER_LABEL) - } else { - key - } + panic!("not validating labels like this anymore"); } pub fn validate_dynamic_label_sqlite( diff --git a/glean-core/src/metrics/mod.rs b/glean-core/src/metrics/mod.rs index 2693df9a60..fc455be553 100644 --- a/glean-core/src/metrics/mod.rs +++ b/glean-core/src/metrics/mod.rs @@ -298,7 +298,6 @@ where T: MetricType, { fn get_identifiers(&'a self) -> (&'a str, &'a str, Option<&'a str>) { - let meta = &self.meta().inner; todo!() //(&meta.category, &meta.name, meta.dynamic_label.as_deref()) } diff --git a/glean-core/src/metrics/quantity.rs b/glean-core/src/metrics/quantity.rs index d19271e0c3..59fe259af2 100644 --- a/glean-core/src/metrics/quantity.rs +++ b/glean-core/src/metrics/quantity.rs @@ -8,7 +8,6 @@ use crate::common_metric_data::{CommonMetricDataInternal, DynamicLabelType}; use crate::error_recording::{record_error, test_get_num_recorded_errors, ErrorType}; use crate::metrics::Metric; use crate::metrics::MetricType; -use crate::storage::StorageManager; use crate::Glean; use crate::{CommonMetricData, TestGetValue}; diff --git a/glean-core/src/metrics/string.rs b/glean-core/src/metrics/string.rs index 42e062d0e0..f9397c0226 100644 --- a/glean-core/src/metrics/string.rs +++ b/glean-core/src/metrics/string.rs @@ -8,7 +8,6 @@ use crate::common_metric_data::{CommonMetricDataInternal, DynamicLabelType}; use crate::error_recording::{test_get_num_recorded_errors, ErrorType}; use crate::metrics::Metric; use crate::metrics::MetricType; -use crate::storage::StorageManager; use crate::util::truncate_string_at_boundary_with_error; use crate::Glean; use crate::{CommonMetricData, TestGetValue}; diff --git a/glean-core/src/storage/mod.rs b/glean-core/src/storage/mod.rs index 325352b07e..3c4285adfb 100644 --- a/glean-core/src/storage/mod.rs +++ b/glean-core/src/storage/mod.rs @@ -11,7 +11,6 @@ use std::collections::HashMap; use serde_json::{json, Value as JsonValue}; use crate::database::sqlite::Database; -use crate::metrics::dual_labeled_counter::RECORD_SEPARATOR; use crate::metrics::Metric; use crate::Lifetime; @@ -184,7 +183,7 @@ impl StorageManager { ) -> Option { let mut snapshot: Option = None; - let mut snapshotter = |id: &[u8], labels: &[&str], metric: &Metric| { + let mut snapshotter = |id: &[u8], _labels: &[&str], metric: &Metric| { let id = String::from_utf8_lossy(id).into_owned(); if id == metric_id { snapshot = Some(metric.clone()) @@ -260,7 +259,7 @@ impl StorageManager { ) -> Option { let mut snapshot: HashMap = HashMap::new(); - let mut snapshotter = |metric_id: &[u8], labels: &[&str], metric: &Metric| { + let mut snapshotter = |metric_id: &[u8], _labels: &[&str], metric: &Metric| { let metric_id = String::from_utf8_lossy(metric_id).into_owned(); if metric_id.ends_with("#experiment") { let (name, _) = metric_id.split_once('#').unwrap(); // safe unwrap, we ensured there's a `#` in the string From b8f1973ec31841e0cee9d3ec6f153bde5a8e5457 Mon Sep 17 00:00:00 2001 From: Jan-Erik Rediger Date: Wed, 4 Feb 2026 15:00:29 +0100 Subject: [PATCH 16/30] Switch to get_metric everywhere --- glean-core/src/common_metric_data.rs | 31 ++----------------- glean-core/src/core/mod.rs | 10 +++--- glean-core/src/metrics/boolean.rs | 7 +---- glean-core/src/metrics/counter.rs | 7 +---- glean-core/src/metrics/custom_distribution.rs | 9 +----- glean-core/src/metrics/datetime.rs | 8 +---- glean-core/src/metrics/denominator.rs | 8 +---- .../src/metrics/dual_labeled_counter.rs | 24 +------------- glean-core/src/metrics/experiment.rs | 9 ++---- glean-core/src/metrics/labeled.rs | 27 ++-------------- glean-core/src/metrics/memory_distribution.rs | 8 +---- glean-core/src/metrics/object.rs | 8 +---- glean-core/src/metrics/quantity.rs | 7 +---- glean-core/src/metrics/rate.rs | 8 +---- glean-core/src/metrics/string.rs | 7 +---- glean-core/src/metrics/string_list.rs | 8 +---- glean-core/src/metrics/text.rs | 8 +---- glean-core/src/metrics/timespan.rs | 8 +---- glean-core/src/metrics/timing_distribution.rs | 8 +---- glean-core/src/metrics/url.rs | 8 +---- glean-core/src/metrics/uuid.rs | 8 +---- glean-core/src/ping/mod.rs | 7 +---- glean-core/src/storage/mod.rs | 2 +- 23 files changed, 29 insertions(+), 206 deletions(-) diff --git a/glean-core/src/common_metric_data.rs b/glean-core/src/common_metric_data.rs index 051cccbd80..7096d6c93c 100644 --- a/glean-core/src/common_metric_data.rs +++ b/glean-core/src/common_metric_data.rs @@ -8,11 +8,8 @@ use malloc_size_of_derive::MallocSizeOf; use rusqlite::Transaction; use crate::error::{Error, ErrorKind}; -use crate::metrics::dual_labeled_counter::{ - validate_dual_label_sqlite, validate_dynamic_key_and_or_category, -}; -use crate::metrics::labeled::{validate_dynamic_label, validate_dynamic_label_sqlite}; -use crate::Glean; +use crate::metrics::dual_labeled_counter::validate_dual_label_sqlite; +use crate::metrics::labeled::validate_dynamic_label_sqlite; use serde::{Deserialize, Serialize}; /// The supported metrics' lifetimes. @@ -201,30 +198,6 @@ impl CommonMetricDataInternal { } } - /// The metric's unique identifier, including the category, name and label. - /// - /// If `category` is empty, it's ommitted. - /// Otherwise, it's the combination of the metric's `category`, `name` and `label`. - pub(crate) fn identifier(&self, glean: &Glean) -> String { - let base_identifier = self.base_identifier(); - - if let Some(label) = &self.inner.dynamic_label { - match label { - DynamicLabelType::Label(label) => { - validate_dynamic_label(glean, self, &base_identifier, label) - } - _ => validate_dynamic_key_and_or_category( - glean, - self, - &base_identifier, - label.clone(), - ), - } - } else { - base_identifier - } - } - /// The list of storages this metric should be recorded into. pub fn storage_names(&self) -> &[String] { &self.inner.send_in_pings diff --git a/glean-core/src/core/mod.rs b/glean-core/src/core/mod.rs index 3ab8ec7544..bc2f8d7e2e 100644 --- a/glean-core/src/core/mod.rs +++ b/glean-core/src/core/mod.rs @@ -1289,12 +1289,10 @@ impl Glean { /// Checks the stored value of the "dirty flag". pub fn is_dirty_flag_set(&self) -> bool { let dirty_bit_metric = self.get_dirty_bit_metric(); - match StorageManager.snapshot_metric( - self.storage(), - INTERNAL_STORAGE, - &dirty_bit_metric.meta().identifier(self), - dirty_bit_metric.meta().inner.lifetime, - ) { + match self + .storage() + .get_metric(dirty_bit_metric.meta(), INTERNAL_STORAGE) + { Some(Metric::Boolean(b)) => b, _ => false, } diff --git a/glean-core/src/metrics/boolean.rs b/glean-core/src/metrics/boolean.rs index 95ce3dc92d..39a770de25 100644 --- a/glean-core/src/metrics/boolean.rs +++ b/glean-core/src/metrics/boolean.rs @@ -88,12 +88,7 @@ impl BooleanMetric { pub fn get_value(&self, glean: &Glean, ping_name: Option<&str>) -> Option { let queried_ping_name = ping_name.unwrap_or_else(|| &self.meta().inner.send_in_pings[0]); - match StorageManager.snapshot_metric( - glean.storage(), - queried_ping_name, - &self.meta.identifier(glean), - self.meta.inner.lifetime, - ) { + match glean.storage().get_metric(self.meta(), queried_ping_name) { Some(Metric::Boolean(b)) => Some(b), _ => None, } diff --git a/glean-core/src/metrics/counter.rs b/glean-core/src/metrics/counter.rs index 1605e195c3..9bad5d6756 100644 --- a/glean-core/src/metrics/counter.rs +++ b/glean-core/src/metrics/counter.rs @@ -125,12 +125,7 @@ impl CounterMetric { .into() .unwrap_or_else(|| &self.meta().inner.send_in_pings[0]); - match StorageManager.snapshot_metric( - glean.storage(), - queried_ping_name, - &self.meta.identifier(glean), - self.meta.inner.lifetime, - ) { + match glean.storage().get_metric(self.meta(), queried_ping_name) { Some(Metric::Counter(i)) => Some(i), _ => None, } diff --git a/glean-core/src/metrics/custom_distribution.rs b/glean-core/src/metrics/custom_distribution.rs index f5e2313623..60806ed620 100644 --- a/glean-core/src/metrics/custom_distribution.rs +++ b/glean-core/src/metrics/custom_distribution.rs @@ -9,7 +9,6 @@ use crate::common_metric_data::{CommonMetricDataInternal, DynamicLabelType}; use crate::error_recording::{record_error, test_get_num_recorded_errors, ErrorType}; use crate::histogram::{Bucketing, Histogram, HistogramType, LinearOrExponential}; use crate::metrics::{DistributionData, Metric, MetricType}; -use crate::storage::StorageManager; use crate::Glean; use crate::{CommonMetricData, TestGetValue}; @@ -218,13 +217,7 @@ impl CustomDistributionMetric { .into() .unwrap_or_else(|| &self.meta().inner.send_in_pings[0]); - match StorageManager.snapshot_metric( - glean.storage(), - queried_ping_name, - &self.meta.identifier(glean), - self.meta.inner.lifetime, - ) { - // Boxing the value, in order to return either of the possible buckets + match glean.storage().get_metric(self.meta(), queried_ping_name) { Some(Metric::CustomDistributionExponential(hist)) => Some(snapshot(&hist)), Some(Metric::CustomDistributionLinear(hist)) => Some(snapshot(&hist)), _ => None, diff --git a/glean-core/src/metrics/datetime.rs b/glean-core/src/metrics/datetime.rs index b6b39a4909..82b9642c40 100644 --- a/glean-core/src/metrics/datetime.rs +++ b/glean-core/src/metrics/datetime.rs @@ -10,7 +10,6 @@ use crate::error_recording::{record_error, test_get_num_recorded_errors, ErrorTy use crate::metrics::time_unit::TimeUnit; use crate::metrics::Metric; use crate::metrics::MetricType; -use crate::storage::StorageManager; use crate::util::{get_iso_time_string, local_now_with_offset}; use crate::Glean; use crate::{CommonMetricData, TestGetValue}; @@ -218,12 +217,7 @@ impl DatetimeMetric { ) -> Option<(ChronoDatetime, TimeUnit)> { let queried_ping_name = ping_name.unwrap_or_else(|| &self.meta().inner.send_in_pings[0]); - match StorageManager.snapshot_metric( - glean.storage(), - queried_ping_name, - &self.meta.identifier(glean), - self.meta.inner.lifetime, - ) { + match glean.storage().get_metric(self.meta(), queried_ping_name) { Some(Metric::Datetime(d, tu)) => Some((d, tu)), _ => None, } diff --git a/glean-core/src/metrics/denominator.rs b/glean-core/src/metrics/denominator.rs index 1d343e7d40..c25d00d4c4 100644 --- a/glean-core/src/metrics/denominator.rs +++ b/glean-core/src/metrics/denominator.rs @@ -8,7 +8,6 @@ use crate::metrics::CounterMetric; use crate::metrics::Metric; use crate::metrics::MetricType; use crate::metrics::RateMetric; -use crate::storage::StorageManager; use crate::Glean; use crate::{CommonMetricData, TestGetValue}; @@ -96,12 +95,7 @@ impl DenominatorMetric { .into() .unwrap_or_else(|| &self.meta().inner.send_in_pings[0]); - match StorageManager.snapshot_metric( - glean.storage(), - queried_ping_name, - &self.meta().identifier(glean), - self.meta().inner.lifetime, - ) { + match glean.storage().get_metric(self.meta(), queried_ping_name) { Some(Metric::Counter(i)) => Some(i), _ => None, } diff --git a/glean-core/src/metrics/dual_labeled_counter.rs b/glean-core/src/metrics/dual_labeled_counter.rs index 5c9f4a17de..bb7838a1f3 100644 --- a/glean-core/src/metrics/dual_labeled_counter.rs +++ b/glean-core/src/metrics/dual_labeled_counter.rs @@ -13,7 +13,7 @@ use rusqlite::{params, Transaction}; use crate::common_metric_data::{CommonMetricData, CommonMetricDataInternal, DynamicLabelType}; use crate::error_recording::{record_error_sqlite, test_get_num_recorded_errors, ErrorType}; use crate::metrics::{CounterMetric, MetricType}; -use crate::{Glean, TestGetValue}; +use crate::TestGetValue; const MAX_LABELS: usize = 16; const OTHER_LABEL: &str = "__other__"; @@ -249,28 +249,6 @@ impl TestGetValue for DualLabeledCounterMetric { } } -/// Validates a dynamic label, changing it to `OTHER_LABEL` if it's invalid. -/// -/// Checks the requested label against limitations, such as the label length and allowed -/// characters. -/// -/// # Arguments -/// -/// * `label` - The requested label -/// -/// # Returns -/// -/// The entire identifier for the metric, including the base identifier and the corrected label. -/// The errors are logged. -pub fn validate_dynamic_key_and_or_category( - glean: &Glean, - meta: &CommonMetricDataInternal, - base_identifier: &str, - label: DynamicLabelType, -) -> String { - panic!("not validating dual labeled like this anymore"); -} - pub fn validate_dual_label_sqlite( tx: &mut Transaction, base_identifier: &str, diff --git a/glean-core/src/metrics/experiment.rs b/glean-core/src/metrics/experiment.rs index 334c85e2e1..2d49bc91d7 100644 --- a/glean-core/src/metrics/experiment.rs +++ b/glean-core/src/metrics/experiment.rs @@ -9,7 +9,7 @@ use std::sync::atomic::AtomicU8; use crate::common_metric_data::CommonMetricDataInternal; use crate::error_recording::{record_error, ErrorType}; use crate::metrics::{Metric, MetricType, RecordedExperiment}; -use crate::storage::{StorageManager, INTERNAL_STORAGE}; +use crate::storage::INTERNAL_STORAGE; use crate::util::{truncate_string_at_boundary, truncate_string_at_boundary_with_error}; use crate::Lifetime; use crate::{CommonMetricData, Glean}; @@ -205,12 +205,7 @@ impl ExperimentMetric { /// /// The stored value or `None` if nothing stored. pub fn test_get_value(&self, glean: &Glean) -> Option { - match StorageManager.snapshot_metric( - glean.storage(), - INTERNAL_STORAGE, - &self.meta.identifier(glean), - self.meta.inner.lifetime, - ) { + match glean.storage().get_metric(self.meta(), INTERNAL_STORAGE) { Some(Metric::Experiment(e)) => Some(e), _ => None, } diff --git a/glean-core/src/metrics/labeled.rs b/glean-core/src/metrics/labeled.rs index 9d8eaa0a53..430b324e22 100644 --- a/glean-core/src/metrics/labeled.rs +++ b/glean-core/src/metrics/labeled.rs @@ -11,7 +11,7 @@ use std::sync::{Arc, Mutex}; use malloc_size_of::MallocSizeOf; use rusqlite::{params, Transaction}; -use crate::common_metric_data::{CommonMetricData, CommonMetricDataInternal, DynamicLabelType}; +use crate::common_metric_data::{CommonMetricData, DynamicLabelType}; use crate::error_recording::{record_error_sqlite, test_get_num_recorded_errors, ErrorType}; use crate::histogram::HistogramType; use crate::metrics::{ @@ -19,7 +19,6 @@ use crate::metrics::{ MetricType, QuantityMetric, StringMetric, TestGetValue, TimeUnit, TimingDistributionMetric, }; use crate::storage::StorageManager; -use crate::Glean; const MAX_LABELS: usize = 16; const OTHER_LABEL: &str = "__other__"; @@ -367,7 +366,7 @@ where StorageManager.snapshot_labels( glean.storage(), queried_ping_name, - &self.submetric.meta().identifier(glean), + &self.submetric.meta().base_identifier(), self.submetric.meta().inner.lifetime, ) }); @@ -386,28 +385,6 @@ pub fn strip_label(identifier: &str) -> &str { identifier.split_once('/').map_or(identifier, |s| s.0) } -/// Validates a dynamic label, changing it to `OTHER_LABEL` if it's invalid. -/// -/// Checks the requested label against limitations, such as the label length and allowed -/// characters. -/// -/// # Arguments -/// -/// * `label` - The requested label -/// -/// # Returns -/// -/// The entire identifier for the metric, including the base identifier and the corrected label. -/// The errors are logged. -pub fn validate_dynamic_label( - _glean: &Glean, - _meta: &CommonMetricDataInternal, - _base_identifier: &str, - _label: &str, -) -> String { - panic!("not validating labels like this anymore"); -} - pub fn validate_dynamic_label_sqlite( tx: &mut Transaction, base_identifier: &str, diff --git a/glean-core/src/metrics/memory_distribution.rs b/glean-core/src/metrics/memory_distribution.rs index ee6108534f..3d7d87693e 100644 --- a/glean-core/src/metrics/memory_distribution.rs +++ b/glean-core/src/metrics/memory_distribution.rs @@ -10,7 +10,6 @@ use crate::error_recording::{record_error, test_get_num_recorded_errors, ErrorTy use crate::histogram::{Functional, Histogram}; use crate::metrics::memory_unit::MemoryUnit; use crate::metrics::{DistributionData, Metric, MetricType}; -use crate::storage::StorageManager; use crate::Glean; use crate::{CommonMetricData, TestGetValue}; @@ -257,12 +256,7 @@ impl MemoryDistributionMetric { .into() .unwrap_or_else(|| &self.meta().inner.send_in_pings[0]); - match StorageManager.snapshot_metric( - glean.storage(), - queried_ping_name, - &self.meta.identifier(glean), - self.meta.inner.lifetime, - ) { + match glean.storage().get_metric(self.meta(), queried_ping_name) { Some(Metric::MemoryDistribution(hist)) => Some(snapshot(&hist)), _ => None, } diff --git a/glean-core/src/metrics/object.rs b/glean-core/src/metrics/object.rs index cbb5af36c7..9739f6e06d 100644 --- a/glean-core/src/metrics/object.rs +++ b/glean-core/src/metrics/object.rs @@ -9,7 +9,6 @@ use crate::error_recording::{record_error, test_get_num_recorded_errors, ErrorTy use crate::metrics::JsonValue; use crate::metrics::Metric; use crate::metrics::MetricType; -use crate::storage::StorageManager; use crate::Glean; use crate::{CommonMetricData, TestGetValue}; @@ -118,12 +117,7 @@ impl ObjectMetric { .into() .unwrap_or_else(|| &self.meta().inner.send_in_pings[0]); - match StorageManager.snapshot_metric( - glean.storage(), - queried_ping_name, - &self.meta.identifier(glean), - self.meta.inner.lifetime, - ) { + match glean.storage().get_metric(self.meta(), queried_ping_name) { Some(Metric::Object(o)) => Some(o), _ => None, } diff --git a/glean-core/src/metrics/quantity.rs b/glean-core/src/metrics/quantity.rs index 59fe259af2..82a73d8809 100644 --- a/glean-core/src/metrics/quantity.rs +++ b/glean-core/src/metrics/quantity.rs @@ -101,12 +101,7 @@ impl QuantityMetric { .into() .unwrap_or_else(|| &self.meta().inner.send_in_pings[0]); - match StorageManager.snapshot_metric( - glean.storage(), - queried_ping_name, - &self.meta.identifier(glean), - self.meta.inner.lifetime, - ) { + match glean.storage().get_metric(self.meta(), queried_ping_name) { Some(Metric::Quantity(i)) => Some(i), _ => None, } diff --git a/glean-core/src/metrics/rate.rs b/glean-core/src/metrics/rate.rs index 40e920fd2a..37fc66d268 100644 --- a/glean-core/src/metrics/rate.rs +++ b/glean-core/src/metrics/rate.rs @@ -6,7 +6,6 @@ use crate::common_metric_data::CommonMetricDataInternal; use crate::error_recording::{record_error, test_get_num_recorded_errors, ErrorType}; use crate::metrics::Metric; use crate::metrics::MetricType; -use crate::storage::StorageManager; use crate::Glean; use crate::{CommonMetricData, TestGetValue}; @@ -147,12 +146,7 @@ impl RateMetric { .into() .unwrap_or_else(|| &self.meta().inner.send_in_pings[0]); - match StorageManager.snapshot_metric( - glean.storage(), - queried_ping_name, - &self.meta.identifier(glean), - self.meta.inner.lifetime, - ) { + match glean.storage().get_metric(self.meta(), queried_ping_name) { Some(Metric::Rate(n, d)) => Some((n, d).into()), _ => None, } diff --git a/glean-core/src/metrics/string.rs b/glean-core/src/metrics/string.rs index f9397c0226..1f696bc653 100644 --- a/glean-core/src/metrics/string.rs +++ b/glean-core/src/metrics/string.rs @@ -95,12 +95,7 @@ impl StringMetric { .into() .unwrap_or_else(|| &self.meta().inner.send_in_pings[0]); - match StorageManager.snapshot_metric( - glean.storage(), - queried_ping_name, - &self.meta.identifier(glean), - self.meta.inner.lifetime, - ) { + match glean.storage().get_metric(self.meta(), queried_ping_name) { Some(Metric::String(s)) => Some(s), _ => None, } diff --git a/glean-core/src/metrics/string_list.rs b/glean-core/src/metrics/string_list.rs index ad2a1b6244..257196c567 100644 --- a/glean-core/src/metrics/string_list.rs +++ b/glean-core/src/metrics/string_list.rs @@ -8,7 +8,6 @@ use crate::common_metric_data::CommonMetricDataInternal; use crate::error_recording::{record_error, test_get_num_recorded_errors, ErrorType}; use crate::metrics::Metric; use crate::metrics::MetricType; -use crate::storage::StorageManager; use crate::util::truncate_string_at_boundary_with_error; use crate::Glean; use crate::{CommonMetricData, TestGetValue}; @@ -155,12 +154,7 @@ impl StringListMetric { .into() .unwrap_or_else(|| &self.meta().inner.send_in_pings[0]); - match StorageManager.snapshot_metric( - glean.storage(), - queried_ping_name, - &self.meta.identifier(glean), - self.meta.inner.lifetime, - ) { + match glean.storage().get_metric(self.meta(), queried_ping_name) { Some(Metric::StringList(values)) => Some(values), _ => None, } diff --git a/glean-core/src/metrics/text.rs b/glean-core/src/metrics/text.rs index be707a7f37..8f111bf650 100644 --- a/glean-core/src/metrics/text.rs +++ b/glean-core/src/metrics/text.rs @@ -8,7 +8,6 @@ use crate::common_metric_data::{CommonMetricDataInternal, DynamicLabelType}; use crate::error_recording::{test_get_num_recorded_errors, ErrorType}; use crate::metrics::Metric; use crate::metrics::MetricType; -use crate::storage::StorageManager; use crate::util::truncate_string_at_boundary_with_error; use crate::Glean; use crate::{CommonMetricData, TestGetValue}; @@ -100,12 +99,7 @@ impl TextMetric { .into() .unwrap_or_else(|| &self.meta().inner.send_in_pings[0]); - match StorageManager.snapshot_metric( - glean.storage(), - queried_ping_name, - &self.meta.identifier(glean), - self.meta.inner.lifetime, - ) { + match glean.storage().get_metric(self.meta(), queried_ping_name) { Some(Metric::Text(s)) => Some(s), _ => None, } diff --git a/glean-core/src/metrics/timespan.rs b/glean-core/src/metrics/timespan.rs index 5ab5f769a4..4f3d6fddd1 100644 --- a/glean-core/src/metrics/timespan.rs +++ b/glean-core/src/metrics/timespan.rs @@ -10,7 +10,6 @@ use crate::error_recording::{record_error, test_get_num_recorded_errors, ErrorTy use crate::metrics::time_unit::TimeUnit; use crate::metrics::Metric; use crate::metrics::MetricType; -use crate::storage::StorageManager; use crate::Glean; use crate::{CommonMetricData, TestGetValue}; @@ -254,12 +253,7 @@ impl TimespanMetric { .into() .unwrap_or_else(|| &self.meta().inner.send_in_pings[0]); - match StorageManager.snapshot_metric( - glean.storage(), - queried_ping_name, - &self.meta.identifier(glean), - self.meta.inner.lifetime, - ) { + match glean.storage().get_metric(self.meta(), queried_ping_name) { Some(Metric::Timespan(time, time_unit)) => Some(time_unit.duration_convert(time)), _ => None, } diff --git a/glean-core/src/metrics/timing_distribution.rs b/glean-core/src/metrics/timing_distribution.rs index 90bb7a4952..8a1df39c42 100644 --- a/glean-core/src/metrics/timing_distribution.rs +++ b/glean-core/src/metrics/timing_distribution.rs @@ -15,7 +15,6 @@ use crate::error_recording::{record_error, test_get_num_recorded_errors, ErrorTy use crate::histogram::{Functional, Histogram}; use crate::metrics::time_unit::TimeUnit; use crate::metrics::{DistributionData, Metric, MetricType}; -use crate::storage::StorageManager; use crate::Glean; use crate::{CommonMetricData, TestGetValue}; @@ -542,12 +541,7 @@ impl TimingDistributionMetric { .into() .unwrap_or_else(|| &self.meta().inner.send_in_pings[0]); - match StorageManager.snapshot_metric( - glean.storage(), - queried_ping_name, - &self.meta.identifier(glean), - self.meta.inner.lifetime, - ) { + match glean.storage().get_metric(self.meta(), queried_ping_name) { Some(Metric::TimingDistribution(hist)) => Some(snapshot(&hist)), _ => None, } diff --git a/glean-core/src/metrics/url.rs b/glean-core/src/metrics/url.rs index d9d0b46525..ab1e3bce02 100644 --- a/glean-core/src/metrics/url.rs +++ b/glean-core/src/metrics/url.rs @@ -8,7 +8,6 @@ use crate::common_metric_data::CommonMetricDataInternal; use crate::error_recording::{record_error, test_get_num_recorded_errors, ErrorType}; use crate::metrics::Metric; use crate::metrics::MetricType; -use crate::storage::StorageManager; use crate::util::truncate_string_at_boundary_with_error; use crate::Glean; use crate::{CommonMetricData, TestGetValue}; @@ -115,12 +114,7 @@ impl UrlMetric { .into() .unwrap_or_else(|| &self.meta().inner.send_in_pings[0]); - match StorageManager.snapshot_metric( - glean.storage(), - queried_ping_name, - &self.meta.identifier(glean), - self.meta.inner.lifetime, - ) { + match glean.storage().get_metric(self.meta(), queried_ping_name) { Some(Metric::Url(s)) => Some(s), _ => None, } diff --git a/glean-core/src/metrics/uuid.rs b/glean-core/src/metrics/uuid.rs index 30ed56eb46..291938a57f 100644 --- a/glean-core/src/metrics/uuid.rs +++ b/glean-core/src/metrics/uuid.rs @@ -10,7 +10,6 @@ use crate::common_metric_data::CommonMetricDataInternal; use crate::error_recording::{record_error, test_get_num_recorded_errors, ErrorType}; use crate::metrics::Metric; use crate::metrics::MetricType; -use crate::storage::StorageManager; use crate::Glean; use crate::{CommonMetricData, TestGetValue}; @@ -114,12 +113,7 @@ impl UuidMetric { .into() .unwrap_or_else(|| &self.meta().inner.send_in_pings[0]); - match StorageManager.snapshot_metric( - glean.storage(), - queried_ping_name, - &self.meta.identifier(glean), - self.meta.inner.lifetime, - ) { + match glean.storage().get_metric(self.meta(), queried_ping_name) { Some(Metric::Uuid(uuid)) => Uuid::parse_str(&uuid).ok(), _ => None, } diff --git a/glean-core/src/ping/mod.rs b/glean-core/src/ping/mod.rs index ca59beb72a..35761b63a7 100644 --- a/glean-core/src/ping/mod.rs +++ b/glean-core/src/ping/mod.rs @@ -83,12 +83,7 @@ impl PingMaker { ..Default::default() }); - let current_seq = match StorageManager.snapshot_metric( - glean.storage(), - INTERNAL_STORAGE, - &seq.meta().identifier(glean), - seq.meta().inner.lifetime, - ) { + let current_seq = match glean.storage().get_metric(seq.meta(), INTERNAL_STORAGE) { Some(Metric::Counter(i)) => i, _ => 0, }; diff --git a/glean-core/src/storage/mod.rs b/glean-core/src/storage/mod.rs index 3c4285adfb..c602a5a4e4 100644 --- a/glean-core/src/storage/mod.rs +++ b/glean-core/src/storage/mod.rs @@ -174,7 +174,7 @@ impl StorageManager { /// # Returns /// /// The decoded metric or `None` if no data is found. - pub fn snapshot_metric( + pub fn _snapshot_metric( &self, storage: &Database, store_name: &str, From 8cdb98de27ed4cdfd43e978f06d7c352ccf9730e Mon Sep 17 00:00:00 2001 From: Jan-Erik Rediger Date: Thu, 5 Feb 2026 12:38:23 +0100 Subject: [PATCH 17/30] Return a specific LabelCheck that knows how to report an error at a later point downside: slightly worse error messages, but maybe we can inline them --- glean-core/src/common_metric_data.rs | 92 ++++++++++++------- glean-core/src/database/sqlite.rs | 24 ++--- .../src/metrics/dual_labeled_counter.rs | 86 +++++++---------- glean-core/src/metrics/labeled.rs | 26 ++---- 4 files changed, 113 insertions(+), 115 deletions(-) diff --git a/glean-core/src/common_metric_data.rs b/glean-core/src/common_metric_data.rs index 7096d6c93c..fd73499566 100644 --- a/glean-core/src/common_metric_data.rs +++ b/glean-core/src/common_metric_data.rs @@ -8,8 +8,10 @@ use malloc_size_of_derive::MallocSizeOf; use rusqlite::Transaction; use crate::error::{Error, ErrorKind}; +use crate::error_recording::record_error_sqlite; use crate::metrics::dual_labeled_counter::validate_dual_label_sqlite; use crate::metrics::labeled::validate_dynamic_label_sqlite; +use crate::ErrorType; use serde::{Deserialize, Serialize}; /// The supported metrics' lifetimes. @@ -123,6 +125,47 @@ impl From for CommonMetricDataInternal { } } +pub enum LabelCheck { + NoLabel, + Label(String), + Error(String, i32), +} + +impl LabelCheck { + pub fn label(&self) -> &str { + use LabelCheck::*; + match self { + NoLabel => "", + Label(label) | Error(label, _) => label, + } + } + + pub fn record_error(&self, tx: &mut Transaction, metric_name: &str, send_in_pings: &[String]) { + let LabelCheck::Error(_, count) = self else { + return; + }; + + record_error_sqlite( + tx, + metric_name, + send_in_pings, + ErrorType::InvalidLabel, + *count, + ); + } + + fn map(self, mut f: impl FnMut(String) -> String) -> Self { + use LabelCheck::*; + + match self { + NoLabel => NoLabel, + Label(s) => Label(f(s)), + // shoud use `MAX_LABEL_LENGTH`, but we can't const-format this + Error(s, cnt) => Error(f(s), cnt), + } + } +} + impl CommonMetricDataInternal { /// Creates a new metadata object. pub fn new, B: Into, C: Into>( @@ -157,44 +200,29 @@ impl CommonMetricDataInternal { /// /// If `category` is empty, it's ommitted. /// Otherwise, it's the combination of the metric's `category`, `name` and `label`. - pub(crate) fn check_labels(&self, tx: &mut Transaction<'_>) -> Option { + pub(crate) fn check_labels(&self, tx: &Transaction<'_>) -> LabelCheck { let base_identifier = self.base_identifier(); if let Some(label) = &self.inner.dynamic_label { match label { - DynamicLabelType::Static(label) => Some(label.to_string()), - DynamicLabelType::Label(label) => validate_dynamic_label_sqlite( - tx, - &base_identifier, - label, - &self.inner.send_in_pings, - ), - DynamicLabelType::KeyOnly(key, static_category) => validate_dual_label_sqlite( - tx, - &base_identifier, - key, - "", - &self.inner.send_in_pings, - ) - .map(|key| format!("{key}{static_category}")), - DynamicLabelType::CategoryOnly(static_key, category) => validate_dual_label_sqlite( - tx, - &base_identifier, - "", - category, - &self.inner.send_in_pings, - ) - .map(|category| format!("{static_key}{category}")), - DynamicLabelType::KeyAndCategory(key, category) => validate_dual_label_sqlite( - tx, - &base_identifier, - key, - category, - &self.inner.send_in_pings, - ), + DynamicLabelType::Static(label) => LabelCheck::Label(label.to_string()), + DynamicLabelType::Label(label) => { + validate_dynamic_label_sqlite(tx, &base_identifier, label) + } + DynamicLabelType::KeyOnly(key, static_category) => { + validate_dual_label_sqlite(tx, &base_identifier, key, "") + .map(|key| format!("{key}{static_category}")) + } + DynamicLabelType::CategoryOnly(static_key, category) => { + validate_dual_label_sqlite(tx, &base_identifier, "", category) + .map(|category| format!("{static_key}{category}")) + } + DynamicLabelType::KeyAndCategory(key, category) => { + validate_dual_label_sqlite(tx, &base_identifier, key, category) + } } } else { - None + LabelCheck::NoLabel } } diff --git a/glean-core/src/database/sqlite.rs b/glean-core/src/database/sqlite.rs index 3eb46182ec..113bf58139 100644 --- a/glean-core/src/database/sqlite.rs +++ b/glean-core/src/database/sqlite.rs @@ -199,13 +199,10 @@ impl Database { self.conn .read(|tx| { - let mut labels = String::from(""); - if let Some(checked_labels) = data.check_labels(tx) { - labels = checked_labels; - } + let labels = data.check_labels(tx); let mut stmt = tx.prepare_cached(get_metric_sql)?; - stmt.query_one([metric_identifier, storage_name, &labels], |row| { + stmt.query_one([metric_identifier, storage_name, labels.label()], |row| { let blob: Vec = row.get(0)?; let blob: Metric = rmp_serde::from_slice(&blob).map_err(|_| FromSqlError::InvalidType)?; @@ -266,10 +263,8 @@ impl Database { let name = strip_label(&base_identifier); _ = self.conn.write(|tx| { - let mut labels = String::from(""); - if let Some(checked_labels) = data.check_labels(tx) { - labels = checked_labels; - } + let labels = data.check_labels(tx); + labels.record_error(tx, name, data.storage_names()); for ping_name in data.storage_names() { if glean.is_ping_enabled(ping_name) { @@ -278,7 +273,7 @@ impl Database { data.inner.lifetime, ping_name, name, - &labels, + labels.label(), value, ) { log::error!( @@ -348,10 +343,9 @@ impl Database { let name = strip_label(&base_identifier); _ = self.conn.write(|tx| { - let mut labels = String::from(""); - if let Some(checked_labels) = data.check_labels(tx) { - labels = checked_labels; - } + let labels = data.check_labels(tx); + labels.record_error(tx, name, data.storage_names()); + for ping_name in data.storage_names() { if glean.is_ping_enabled(ping_name) { if let Err(e) = self.record_per_lifetime_with( @@ -359,7 +353,7 @@ impl Database { data.inner.lifetime, ping_name, name, - &labels, + labels.label(), &mut transform, ) { log::error!( diff --git a/glean-core/src/metrics/dual_labeled_counter.rs b/glean-core/src/metrics/dual_labeled_counter.rs index bb7838a1f3..0bb6d1f3bf 100644 --- a/glean-core/src/metrics/dual_labeled_counter.rs +++ b/glean-core/src/metrics/dual_labeled_counter.rs @@ -10,8 +10,10 @@ use std::sync::{Arc, Mutex}; use rusqlite::{params, Transaction}; -use crate::common_metric_data::{CommonMetricData, CommonMetricDataInternal, DynamicLabelType}; -use crate::error_recording::{record_error_sqlite, test_get_num_recorded_errors, ErrorType}; +use crate::common_metric_data::{ + CommonMetricData, CommonMetricDataInternal, DynamicLabelType, LabelCheck, +}; +use crate::error_recording::{test_get_num_recorded_errors, ErrorType}; use crate::metrics::{CounterMetric, MetricType}; use crate::TestGetValue; @@ -250,28 +252,19 @@ impl TestGetValue for DualLabeledCounterMetric { } pub fn validate_dual_label_sqlite( - tx: &mut Transaction, + tx: &Transaction, base_identifier: &str, key: &str, category: &str, - send_in_pings: &[String], -) -> Option { +) -> LabelCheck { let existing_labels_sql = "SELECT DISTINCT labels FROM telemetry WHERE id = ?1"; // TODO: We can now detect if _either_ key or category contains `RECORD_SEPARATOR` and thus keep // the other potentially valid label. // This needs adjustement of the test `labels_containing_a_record_separator_record_an_error`. if key.contains(RECORD_SEPARATOR) || category.contains(RECORD_SEPARATOR) { - let msg = "Label cannot contain the ASCII record separator character (0x1E)".to_string(); - record_error_sqlite( - tx, - base_identifier, - send_in_pings, - ErrorType::InvalidLabel, - msg, - 1, - ); - return Some(format!("{OTHER_LABEL}{RECORD_SEPARATOR}{OTHER_LABEL}")); + log::warn!("Label cannot contain the ASCII record separator character (0x1E)"); + return LabelCheck::Error(format!("{OTHER_LABEL}{RECORD_SEPARATOR}{OTHER_LABEL}"), 1); } let mut existing_keys = HashSet::new(); @@ -301,55 +294,46 @@ pub fn validate_dual_label_sqlite( } } - let new_key = if existing_keys.contains(key) || existing_keys.len() < MAX_LABELS { - label_is_valid_sqlite(key, tx, base_identifier, send_in_pings) + let mut errors = 0; + let new_key = if (existing_keys.contains(key) || existing_keys.len() < MAX_LABELS) + && label_is_valid(key) + { + key } else { + errors += 1; OTHER_LABEL }; - let new_category = - if existing_categories.contains(category) || existing_categories.len() < MAX_LABELS { - label_is_valid_sqlite(category, tx, base_identifier, send_in_pings) - } else { - OTHER_LABEL - }; + let new_category = if (existing_categories.contains(category) + || existing_categories.len() < MAX_LABELS) + && label_is_valid(category) + { + category + } else { + errors += 1; + OTHER_LABEL + }; - Some(format!("{new_key}{RECORD_SEPARATOR}{new_category}")) + let label = format!("{new_key}{RECORD_SEPARATOR}{new_category}"); + if errors == 0 { + LabelCheck::Label(label) + } else { + LabelCheck::Error(label, errors) + } } -fn label_is_valid_sqlite<'a>( - label: &'a str, - tx: &mut Transaction, - base_identifier: &str, - send_in_pings: &[String], -) -> &'a str { +fn label_is_valid(label: &str) -> bool { if label.len() > MAX_LABEL_LENGTH { - let msg = format!( + log::warn!( "label length {} exceeds maximum of {}", label.len(), MAX_LABEL_LENGTH ); - record_error_sqlite( - tx, - base_identifier, - send_in_pings, - ErrorType::InvalidLabel, - msg, - 1, - ); - OTHER_LABEL + false } else if label.contains(RECORD_SEPARATOR) { - let msg = "Label cannot contain the ASCII record separator character (0x1E)".to_string(); - record_error_sqlite( - tx, - base_identifier, - send_in_pings, - ErrorType::InvalidLabel, - msg, - 1, - ); - OTHER_LABEL + log::warn!("Label cannot contain the ASCII record separator character (0x1E)"); + false } else { - label + true } } diff --git a/glean-core/src/metrics/labeled.rs b/glean-core/src/metrics/labeled.rs index 430b324e22..923ae49ad7 100644 --- a/glean-core/src/metrics/labeled.rs +++ b/glean-core/src/metrics/labeled.rs @@ -11,8 +11,8 @@ use std::sync::{Arc, Mutex}; use malloc_size_of::MallocSizeOf; use rusqlite::{params, Transaction}; -use crate::common_metric_data::{CommonMetricData, DynamicLabelType}; -use crate::error_recording::{record_error_sqlite, test_get_num_recorded_errors, ErrorType}; +use crate::common_metric_data::{CommonMetricData, DynamicLabelType, LabelCheck}; +use crate::error_recording::{test_get_num_recorded_errors, ErrorType}; use crate::histogram::HistogramType; use crate::metrics::{ BooleanMetric, CounterMetric, CustomDistributionMetric, MemoryDistributionMetric, MemoryUnit, @@ -386,11 +386,10 @@ pub fn strip_label(identifier: &str) -> &str { } pub fn validate_dynamic_label_sqlite( - tx: &mut Transaction, + tx: &Transaction, base_identifier: &str, label: &str, - send_in_pings: &[String], -) -> Option { +) -> LabelCheck { let existing_labels_sql = "SELECT DISTINCT labels FROM telemetry WHERE id = ?1"; let mut label_already_used = false; @@ -398,12 +397,12 @@ pub fn validate_dynamic_label_sqlite( { let Ok(mut stmt) = tx.prepare(existing_labels_sql) else { // If we can't fetch from the database, assume the label is ok to use - return Some(label.to_string()); + return LabelCheck::Label(label.to_string()); }; let Ok(mut rows) = stmt.query(params![base_identifier]) else { // If we can't fetch from the database, assume the label is ok to use - return Some(label.to_string()); + return LabelCheck::Label(label.to_string()); }; while let Ok(Some(row)) = rows.next() { @@ -418,22 +417,15 @@ pub fn validate_dynamic_label_sqlite( } if !label_already_used && label_count >= MAX_LABELS { - Some(String::from(OTHER_LABEL)) + LabelCheck::Label(String::from(OTHER_LABEL)) } else if label.len() > MAX_LABEL_LENGTH { log::warn!( "label length {} exceeds maximum of {}", label.len(), MAX_LABEL_LENGTH ); - record_error_sqlite( - tx, - base_identifier, - send_in_pings, - ErrorType::InvalidLabel, - 1, - ); - Some(String::from(OTHER_LABEL)) + LabelCheck::Error(String::from(OTHER_LABEL), 1) } else { - Some(label.to_string()) + LabelCheck::Label(label.to_string()) } } From 4138c1332683a5fa7d298bcfd1b1efd651b20c8e Mon Sep 17 00:00:00 2001 From: Jan-Erik Rediger Date: Thu, 5 Feb 2026 14:35:45 +0100 Subject: [PATCH 18/30] Remove strip_label usage now that labels are stored separately --- glean-core/src/database/sqlite.rs | 15 ++++++--------- glean-core/src/metrics/labeled.rs | 5 ----- glean-core/src/metrics/mod.rs | 8 ++------ 3 files changed, 8 insertions(+), 20 deletions(-) diff --git a/glean-core/src/database/sqlite.rs b/glean-core/src/database/sqlite.rs index 113bf58139..e7ac4c0fa8 100644 --- a/glean-core/src/database/sqlite.rs +++ b/glean-core/src/database/sqlite.rs @@ -19,7 +19,6 @@ use schema::Schema; use crate::common_metric_data::CommonMetricDataInternal; use crate::metrics::dual_labeled_counter::RECORD_SEPARATOR; -use crate::metrics::labeled::strip_label; use crate::metrics::Metric; use crate::Glean; use crate::Lifetime; @@ -259,12 +258,11 @@ impl Database { /// Records a metric in the underlying storage system. pub fn record(&self, glean: &Glean, data: &CommonMetricDataInternal, value: &Metric) { - let base_identifier = data.base_identifier(); - let name = strip_label(&base_identifier); + let name = data.base_identifier(); _ = self.conn.write(|tx| { let labels = data.check_labels(tx); - labels.record_error(tx, name, data.storage_names()); + labels.record_error(tx, &name, data.storage_names()); for ping_name in data.storage_names() { if glean.is_ping_enabled(ping_name) { @@ -272,7 +270,7 @@ impl Database { tx, data.inner.lifetime, ping_name, - name, + &name, labels.label(), value, ) { @@ -339,12 +337,11 @@ impl Database { where F: FnMut(Option) -> Metric, { - let base_identifier = data.base_identifier(); - let name = strip_label(&base_identifier); + let name = data.base_identifier(); _ = self.conn.write(|tx| { let labels = data.check_labels(tx); - labels.record_error(tx, name, data.storage_names()); + labels.record_error(tx, &name, data.storage_names()); for ping_name in data.storage_names() { if glean.is_ping_enabled(ping_name) { @@ -352,7 +349,7 @@ impl Database { tx, data.inner.lifetime, ping_name, - name, + &name, labels.label(), &mut transform, ) { diff --git a/glean-core/src/metrics/labeled.rs b/glean-core/src/metrics/labeled.rs index 923ae49ad7..3559278ef9 100644 --- a/glean-core/src/metrics/labeled.rs +++ b/glean-core/src/metrics/labeled.rs @@ -380,11 +380,6 @@ where } } -/// Strips the label off of a complete identifier -pub fn strip_label(identifier: &str) -> &str { - identifier.split_once('/').map_or(identifier, |s| s.0) -} - pub fn validate_dynamic_label_sqlite( tx: &Transaction, base_identifier: &str, diff --git a/glean-core/src/metrics/mod.rs b/glean-core/src/metrics/mod.rs index fc455be553..de223bf629 100644 --- a/glean-core/src/metrics/mod.rs +++ b/glean-core/src/metrics/mod.rs @@ -233,17 +233,13 @@ pub trait MetricType { let remote_settings_config = &glean.remote_settings_config.lock().unwrap(); // Get the value from the remote configuration if it is there, otherwise return the default value. let current_disabled = { - let base_id = self.meta().base_identifier(); - let identifier = base_id - .split_once('/') - .map(|split| split.0) - .unwrap_or(&base_id); + let identifier = self.meta().base_identifier(); // NOTE: The `!` preceding the `*is_enabled` is important for inverting the logic since the // underlying property in the metrics.yaml is `disabled` and the outward API is treating it as // if it were `enabled` to make it easier to understand. if !remote_settings_config.metrics_enabled.is_empty() { - if let Some(is_enabled) = remote_settings_config.metrics_enabled.get(identifier) { + if let Some(is_enabled) = remote_settings_config.metrics_enabled.get(&identifier) { u8::from(!*is_enabled) } else { u8::from(self.meta().inner.disabled) From 907989711243b9f882f38b3bb9ec97138da34d21 Mon Sep 17 00:00:00 2001 From: Jan-Erik Rediger Date: Fri, 6 Feb 2026 12:23:42 +0100 Subject: [PATCH 19/30] Implement get_identifiers based on the new label storage. This is another BREAKING CHANGE in the return type. We can't return references to the labels anymore, we need owned values. --- glean-core/rlb/src/private/event.rs | 2 +- glean-core/rlb/src/private/object.rs | 2 +- glean-core/rlb/tests/metric_metadata.rs | 4 ++-- glean-core/src/common_metric_data.rs | 20 ++++++++++++++++++++ glean-core/src/metrics/mod.rs | 12 ++++++++---- 5 files changed, 32 insertions(+), 8 deletions(-) diff --git a/glean-core/rlb/src/private/event.rs b/glean-core/rlb/src/private/event.rs index 3ad16c11d5..378f7bad66 100644 --- a/glean-core/rlb/src/private/event.rs +++ b/glean-core/rlb/src/private/event.rs @@ -34,7 +34,7 @@ impl MallocSizeOf for EventMetric { } impl<'a, K> MetricIdentifier<'a> for EventMetric { - fn get_identifiers(&'a self) -> (&'a str, &'a str, Option<&'a str>) { + fn get_identifiers(&'a self) -> (&'a str, &'a str, Option) { self.inner.get_identifiers() } } diff --git a/glean-core/rlb/src/private/object.rs b/glean-core/rlb/src/private/object.rs index 214e4175a8..5356a4a486 100644 --- a/glean-core/rlb/src/private/object.rs +++ b/glean-core/rlb/src/private/object.rs @@ -34,7 +34,7 @@ impl MallocSizeOf for ObjectMetric { } impl<'a, K> MetricIdentifier<'a> for ObjectMetric { - fn get_identifiers(&'a self) -> (&'a str, &'a str, Option<&'a str>) { + fn get_identifiers(&'a self) -> (&'a str, &'a str, Option) { self.inner.get_identifiers() } } diff --git a/glean-core/rlb/tests/metric_metadata.rs b/glean-core/rlb/tests/metric_metadata.rs index 803290bb03..39e2c88065 100644 --- a/glean-core/rlb/tests/metric_metadata.rs +++ b/glean-core/rlb/tests/metric_metadata.rs @@ -87,14 +87,14 @@ fn check_metadata() { let (category, name, label) = metrics::countit.get_identifiers(); assert_eq!(category, "sesame"); assert_eq!(name, "count_von_count"); - assert_eq!(label, Some("ah_ah_ah")); + assert_eq!(label, Some("ah_ah_ah".to_string())); // Events and Objects have MetricIdentifier implemented explicitly, as // they wrap the glean-core Event and Object types let (category, name, label) = metrics::event.get_identifiers(); assert_eq!(category, "shire"); assert_eq!(name, "birthday"); - assert_eq!(label, Some("111th")); + assert_eq!(label, Some("111th".to_string())); let (category, name, label) = metrics::object.get_identifiers(); assert_eq!(category, "court"); diff --git a/glean-core/src/common_metric_data.rs b/glean-core/src/common_metric_data.rs index fd73499566..86ec2ce5de 100644 --- a/glean-core/src/common_metric_data.rs +++ b/glean-core/src/common_metric_data.rs @@ -2,6 +2,7 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. +use std::fmt::Display; use std::sync::atomic::{AtomicU8, Ordering}; use malloc_size_of_derive::MallocSizeOf; @@ -100,6 +101,25 @@ impl Default for DynamicLabelType { } } +impl Display for DynamicLabelType { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + use crate::metrics::dual_labeled_counter::RECORD_SEPARATOR; + match self { + DynamicLabelType::Static(label) => write!(f, "{label}"), + DynamicLabelType::Label(label) => write!(f, "{label}"), + DynamicLabelType::KeyOnly(key, category) => { + write!(f, "{key}{RECORD_SEPARATOR}{category}") + } + DynamicLabelType::CategoryOnly(key, category) => { + write!(f, "{key}{RECORD_SEPARATOR}{category}") + } + DynamicLabelType::KeyAndCategory(key, category) => { + write!(f, "{key}{RECORD_SEPARATOR}{category}") + } + } + } +} + #[derive(Default, Debug, MallocSizeOf)] pub struct CommonMetricDataInternal { pub inner: CommonMetricData, diff --git a/glean-core/src/metrics/mod.rs b/glean-core/src/metrics/mod.rs index de223bf629..b0f1e57ccd 100644 --- a/glean-core/src/metrics/mod.rs +++ b/glean-core/src/metrics/mod.rs @@ -262,7 +262,7 @@ pub trait MetricType { /// identifier (category, name, label) for a metric pub trait MetricIdentifier<'a> { /// Retrieve the category, name and (maybe) label of the metric - fn get_identifiers(&'a self) -> (&'a str, &'a str, Option<&'a str>); + fn get_identifiers(&'a self) -> (&'a str, &'a str, Option); } /// [`TestGetValue`] describes an interface for retrieving the value for a given metric @@ -293,9 +293,13 @@ impl<'a, T> MetricIdentifier<'a> for T where T: MetricType, { - fn get_identifiers(&'a self) -> (&'a str, &'a str, Option<&'a str>) { - todo!() - //(&meta.category, &meta.name, meta.dynamic_label.as_deref()) + fn get_identifiers(&'a self) -> (&'a str, &'a str, Option) { + let meta = &self.meta().inner; + ( + &meta.category, + &meta.name, + meta.dynamic_label.as_ref().map(|label| label.to_string()), + ) } } From 4d49c82dd8b5f8cd2ded371d2ae8c3db607b5d96 Mon Sep 17 00:00:00 2001 From: Jan-Erik Rediger Date: Fri, 6 Feb 2026 15:43:35 +0100 Subject: [PATCH 20/30] sqlite: Set synchronous=NORMAL to sync less often, but at the critical moments See all details: https://sqlite.org/pragma.html#pragma_synchronous The default (FULL) syncs on every write. That's slightly higher guarantees, but also costly. We're already using WAL (write-ahead log). It's safe from corruption in NORMAL mode and consistent. It does lose durability, that means data might roll back following a power loss or system crash. Note: `rkv` does NOT sync at all. It only writes to disk (and moves files around). That's strictly worse than WAL in `NORMAL` mode. --- glean-core/src/database/sqlite/schema.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/glean-core/src/database/sqlite/schema.rs b/glean-core/src/database/sqlite/schema.rs index 6bf919a074..d9082779a2 100644 --- a/glean-core/src/database/sqlite/schema.rs +++ b/glean-core/src/database/sqlite/schema.rs @@ -23,6 +23,7 @@ impl ConnectionOpener for Schema { fn setup(conn: &mut rusqlite::Connection) -> Result<(), Self::Error> { conn.execute_batch( "PRAGMA journal_mode = WAL; + PRAGMA synchronous = NORMAL; PRAGMA journal_size_limit = 512000; -- 512 KB. PRAGMA temp_store = MEMORY; PRAGMA auto_vacuum = INCREMENTAL; From 60d0e22b4b6e46c814edd89fabfae0c970423f39 Mon Sep 17 00:00:00 2001 From: Jan-Erik Rediger Date: Fri, 13 Feb 2026 14:09:37 +0100 Subject: [PATCH 21/30] Rename to `MetricLabel` now that it's always used to store the label --- glean-core/python/glean/_loader.py | 4 +- .../python/tests/metrics/test_boolean.py | 8 +-- .../python/tests/metrics/test_counter.py | 10 +-- .../python/tests/metrics/test_datetime.py | 4 +- glean-core/python/tests/metrics/test_event.py | 18 ++--- .../python/tests/metrics/test_labeled.py | 20 +++--- .../tests/metrics/test_memory_distribution.py | 10 +-- .../python/tests/metrics/test_object.py | 10 +-- .../python/tests/metrics/test_quantity.py | 10 +-- glean-core/python/tests/metrics/test_rate.py | 10 +-- .../python/tests/metrics/test_string.py | 10 +-- .../python/tests/metrics/test_string_list.py | 12 ++-- glean-core/python/tests/metrics/test_text.py | 4 +- .../python/tests/metrics/test_timespan.py | 26 +++---- .../tests/metrics/test_timing_distribution.py | 16 ++--- glean-core/python/tests/metrics/test_url.py | 10 +-- glean-core/python/tests/metrics/test_uuid.py | 18 ++--- .../python/tests/test_collection_enabled.py | 4 +- glean-core/python/tests/test_glean.py | 28 ++++---- glean-core/python/tests/test_network.py | 4 +- glean-core/rlb/src/test.rs | 12 ++-- glean-core/rlb/tests/metric_metadata.rs | 6 +- glean-core/rlb/tests/upload_timing.rs | 6 +- glean-core/src/common_metric_data.rs | 45 ++++++------ glean-core/src/error_recording.rs | 4 +- glean-core/src/glean.udl | 10 +-- glean-core/src/internal_metrics.rs | 68 +++++++++---------- glean-core/src/lib.rs | 2 +- glean-core/src/metrics/boolean.rs | 6 +- glean-core/src/metrics/counter.rs | 6 +- glean-core/src/metrics/custom_distribution.rs | 6 +- .../src/metrics/dual_labeled_counter.rs | 19 +++--- glean-core/src/metrics/labeled.rs | 13 ++-- glean-core/src/metrics/memory_distribution.rs | 6 +- glean-core/src/metrics/mod.rs | 6 +- glean-core/src/metrics/quantity.rs | 6 +- glean-core/src/metrics/string.rs | 8 +-- glean-core/src/metrics/text.rs | 8 +-- glean-core/src/metrics/timing_distribution.rs | 6 +- glean-core/src/metrics/url.rs | 8 +-- glean-core/tests/clientid_textfile.rs | 2 +- glean-core/tests/ping.rs | 2 +- 42 files changed, 245 insertions(+), 246 deletions(-) diff --git a/glean-core/python/glean/_loader.py b/glean-core/python/glean/_loader.py index 00cb48c9ec..ca50f388cb 100644 --- a/glean-core/python/glean/_loader.py +++ b/glean-core/python/glean/_loader.py @@ -284,8 +284,8 @@ def _get_metric_objects( glean_metric = metrics.ObjectMetricType(metrics.CommonMetricData(**args), obj_cls) # type: ignore else: # Hack for the time being. - if "dynamic_label" not in args: - args["dynamic_label"] = None + if "label" not in args: + args["label"] = None meta_args, rest = _split_ctor_args(args) if getattr(metric, "labeled", False): glean_metric = metric_type( diff --git a/glean-core/python/tests/metrics/test_boolean.py b/glean-core/python/tests/metrics/test_boolean.py index 9e3ed68f8e..ba0733e024 100644 --- a/glean-core/python/tests/metrics/test_boolean.py +++ b/glean-core/python/tests/metrics/test_boolean.py @@ -14,7 +14,7 @@ def test_the_api_saves_to_its_storage_engine(): lifetime=Lifetime.APPLICATION, name="boolean_metric", send_in_pings=["store1"], - dynamic_label=None, + label=None, ) ) @@ -35,7 +35,7 @@ def test_disabled_booleans_must_not_record_data(): lifetime=Lifetime.APPLICATION, name="boolean_metric", send_in_pings=["store1"], - dynamic_label=None, + label=None, ) ) @@ -52,7 +52,7 @@ def test_get_value_throws_if_nothing_is_stored(): lifetime=Lifetime.APPLICATION, name="boolean_metric", send_in_pings=["store1"], - dynamic_label=None, + label=None, ) ) @@ -67,7 +67,7 @@ def test_the_api_saves_to_secondary_pings(): lifetime=Lifetime.APPLICATION, name="boolean_metric", send_in_pings=["store1", "store2"], - dynamic_label=None, + label=None, ) ) diff --git a/glean-core/python/tests/metrics/test_counter.py b/glean-core/python/tests/metrics/test_counter.py index a8231bd26e..fd6a2f9d24 100644 --- a/glean-core/python/tests/metrics/test_counter.py +++ b/glean-core/python/tests/metrics/test_counter.py @@ -16,7 +16,7 @@ def test_the_api_saves_to_its_storage_engine(): lifetime=Lifetime.APPLICATION, name="counter_metric", send_in_pings=["store1"], - dynamic_label=None, + label=None, ) ) @@ -43,7 +43,7 @@ def test_disabled_counters_must_not_record_data(): lifetime=Lifetime.APPLICATION, name="counter_metric", send_in_pings=["store1"], - dynamic_label=None, + label=None, ) ) @@ -62,7 +62,7 @@ def test_get_value_throws_value_error_if_nothing_is_stored(): lifetime=Lifetime.APPLICATION, name="counter_metric", send_in_pings=["store1"], - dynamic_label=None, + label=None, ) ) @@ -78,7 +78,7 @@ def test_api_saves_to_secondary_pings(): lifetime=Lifetime.APPLICATION, name="counter_metric", send_in_pings=["store1", "store2"], - dynamic_label=None, + label=None, ) ) @@ -104,7 +104,7 @@ def test_negative_values_are_not_counted(): lifetime=Lifetime.APPLICATION, name="counter_metric", send_in_pings=["store1"], - dynamic_label=None, + label=None, ) ) diff --git a/glean-core/python/tests/metrics/test_datetime.py b/glean-core/python/tests/metrics/test_datetime.py index b58d6ccf87..d8f0d72694 100644 --- a/glean-core/python/tests/metrics/test_datetime.py +++ b/glean-core/python/tests/metrics/test_datetime.py @@ -17,7 +17,7 @@ def test_the_api_saves_to_its_storage_engine(): lifetime=Lifetime.APPLICATION, name="datetime_metric", send_in_pings=["store1"], - dynamic_label=None, + label=None, ), time_unit=metrics.TimeUnit.MINUTE, ) @@ -69,7 +69,7 @@ def test_disabled_datetimes_must_not_record_data(): lifetime=Lifetime.APPLICATION, name="datetime_metric", send_in_pings=["store1"], - dynamic_label=None, + label=None, ), time_unit=metrics.TimeUnit.MINUTE, ) diff --git a/glean-core/python/tests/metrics/test_event.py b/glean-core/python/tests/metrics/test_event.py index 398861e362..f6b5c35564 100644 --- a/glean-core/python/tests/metrics/test_event.py +++ b/glean-core/python/tests/metrics/test_event.py @@ -46,7 +46,7 @@ def test_the_api_records_to_its_storage_engine(): lifetime=Lifetime.PING, name="click", send_in_pings=["store1"], - dynamic_label=None, + label=None, ), allowed_extra_keys=["object_id", "other"], ) @@ -84,7 +84,7 @@ def test_the_api_records_to_its_storage_engine_when_category_is_empty(): lifetime=Lifetime.PING, name="click", send_in_pings=["store1"], - dynamic_label=None, + label=None, ), allowed_extra_keys=["object_id"], ) @@ -117,7 +117,7 @@ def test_disabled_events_must_not_record_data(): lifetime=Lifetime.PING, name="click", send_in_pings=["store1"], - dynamic_label=None, + label=None, ), allowed_extra_keys=["object_id", "other"], ) @@ -137,7 +137,7 @@ def test_test_get_value_throws_valueerror_if_nothing_is_stored(): lifetime=Lifetime.PING, name="click", send_in_pings=["store1"], - dynamic_label=None, + label=None, ), allowed_extra_keys=["object_id", "other"], ) @@ -153,7 +153,7 @@ def test_the_api_records_to_secondary_pings(): lifetime=Lifetime.PING, name="click", send_in_pings=["store1", "store2"], - dynamic_label=None, + label=None, ), allowed_extra_keys=["object_id"], ) @@ -191,7 +191,7 @@ class EventKeys(enum.Enum): lifetime=Lifetime.PING, name="click", send_in_pings=["store1"], - dynamic_label=None, + label=None, ), allowed_extra_keys=["test_name"], ) @@ -226,7 +226,7 @@ class EventKeys(enum.Enum): lifetime=Lifetime.PING, name="test_event", send_in_pings=["events"], - dynamic_label=None, + label=None, ), allowed_extra_keys=["some_extra"], ) @@ -255,7 +255,7 @@ def test_long_extra_values_record_an_error(): lifetime=Lifetime.PING, name="click", send_in_pings=["store1"], - dynamic_label=None, + label=None, ), allowed_extra_keys=["object_id", "other"], ) @@ -304,7 +304,7 @@ def test_the_convenient_extrakeys_api(): lifetime=Lifetime.PING, name="click", send_in_pings=["store1"], - dynamic_label=None, + label=None, ), allowed_extra_keys=["object_id", "other"], ) diff --git a/glean-core/python/tests/metrics/test_labeled.py b/glean-core/python/tests/metrics/test_labeled.py index eac757826f..3ba30baec3 100644 --- a/glean-core/python/tests/metrics/test_labeled.py +++ b/glean-core/python/tests/metrics/test_labeled.py @@ -18,7 +18,7 @@ def test_labeled_counter_type(): lifetime=Lifetime.APPLICATION, name="labeled_counter_metric", send_in_pings=["metrics"], - dynamic_label=None, + label=None, ) ) ) @@ -40,7 +40,7 @@ def test_labeled_counter_type_test_get_metrics(): lifetime=Lifetime.APPLICATION, name="labeled_counter_metric", send_in_pings=["metrics"], - dynamic_label=None, + label=None, ) ) ) @@ -64,7 +64,7 @@ def test_labeled_boolean_type(): lifetime=Lifetime.APPLICATION, name="labeled_boolean_metric", send_in_pings=["metrics"], - dynamic_label=None, + label=None, ) ) ) @@ -86,7 +86,7 @@ def test_labeled_string_type(): lifetime=Lifetime.APPLICATION, name="labeled_string_metric", send_in_pings=["metrics"], - dynamic_label=None, + label=None, ) ) ) @@ -108,7 +108,7 @@ def test_labeled_quantity_type(): lifetime=Lifetime.APPLICATION, name="labeled_quantity_metric", send_in_pings=["metrics"], - dynamic_label=None, + label=None, ) ) ) @@ -129,7 +129,7 @@ def test_other_label_with_predefined_labels(): lifetime=Lifetime.APPLICATION, name="labeled_counter_metric", send_in_pings=["metrics"], - dynamic_label=None, + label=None, ) ), labels=["foo", "bar", "baz"], @@ -157,7 +157,7 @@ def test_other_label_without_predefined_labels(): lifetime=Lifetime.APPLICATION, name="labeled_counter_metric", send_in_pings=["metrics"], - dynamic_label=None, + label=None, ) ) ) @@ -182,7 +182,7 @@ def test_other_label_without_predefined_labels_before_glean_init(): lifetime=Lifetime.APPLICATION, name="labeled_counter_metric", send_in_pings=["metrics"], - dynamic_label=None, + label=None, ) ) ) @@ -214,7 +214,7 @@ def test_invalid_labels_go_to_other(): lifetime=Lifetime.APPLICATION, name="labeled_counter_metric", send_in_pings=["metrics"], - dynamic_label=None, + label=None, ) ) ) @@ -251,7 +251,7 @@ def test_rapidly_recreating_labeled_metrics_does_not_crash(): send_in_pings=["metrics"], lifetime=Lifetime.APPLICATION, disabled=False, - dynamic_label=None, + label=None, ) ), labels=["foo"], diff --git a/glean-core/python/tests/metrics/test_memory_distribution.py b/glean-core/python/tests/metrics/test_memory_distribution.py index a4b4d4dc52..b346027039 100644 --- a/glean-core/python/tests/metrics/test_memory_distribution.py +++ b/glean-core/python/tests/metrics/test_memory_distribution.py @@ -15,7 +15,7 @@ def test_the_api_saves_to_its_storage_engine(): lifetime=Lifetime.APPLICATION, name="memory_distribution", send_in_pings=["store1"], - dynamic_label=None, + label=None, ), memory_unit=MemoryUnit.KILOBYTE, ) @@ -40,7 +40,7 @@ def test_values_are_truncated_to_1tb(): lifetime=Lifetime.APPLICATION, name="memory_distribution", send_in_pings=["store1"], - dynamic_label=None, + label=None, ), memory_unit=MemoryUnit.GIGABYTE, ) @@ -61,7 +61,7 @@ def test_disabled_memory_distributions_must_not_record_data(): lifetime=Lifetime.APPLICATION, name="memory_distribution", send_in_pings=["store1"], - dynamic_label=None, + label=None, ), memory_unit=MemoryUnit.KILOBYTE, ) @@ -79,7 +79,7 @@ def test_get_value_throws_if_nothing_is_stored(): lifetime=Lifetime.APPLICATION, name="memory_distribution", send_in_pings=["store1"], - dynamic_label=None, + label=None, ), memory_unit=MemoryUnit.KILOBYTE, ) @@ -95,7 +95,7 @@ def test_the_api_saves_to_secondary_pings(): lifetime=Lifetime.APPLICATION, name="memory_distribution", send_in_pings=["store1", "store2", "store3"], - dynamic_label=None, + label=None, ), memory_unit=MemoryUnit.KILOBYTE, ) diff --git a/glean-core/python/tests/metrics/test_object.py b/glean-core/python/tests/metrics/test_object.py index 15ee6a8556..b83432ee45 100644 --- a/glean-core/python/tests/metrics/test_object.py +++ b/glean-core/python/tests/metrics/test_object.py @@ -37,7 +37,7 @@ def test_the_api_records_to_its_storage_engine(): name="baloon", lifetime=Lifetime.PING, send_in_pings=["store1"], - dynamic_label=None, + label=None, disabled=False, ), BalloonsObject, @@ -62,7 +62,7 @@ def test_object_must_not_record_if_disabled(): name="baloon", lifetime=Lifetime.PING, send_in_pings=["store1"], - dynamic_label=None, + label=None, disabled=True, ), BalloonsObject, @@ -82,7 +82,7 @@ def test_object_get_value_returns_nil_if_nothing_is_stored(): name="baloon", lifetime=Lifetime.PING, send_in_pings=["store1"], - dynamic_label=None, + label=None, disabled=True, ), BalloonsObject, @@ -98,7 +98,7 @@ def test_object_saves_to_secondary_pings(): name="baloon", lifetime=Lifetime.PING, send_in_pings=["store1", "store2"], - dynamic_label=None, + label=None, disabled=False, ), BalloonsObject, @@ -125,7 +125,7 @@ def test_wrong_object_records_an_error(): name="baloon", lifetime=Lifetime.PING, send_in_pings=["store1"], - dynamic_label=None, + label=None, disabled=False, ), BalloonsObject, diff --git a/glean-core/python/tests/metrics/test_quantity.py b/glean-core/python/tests/metrics/test_quantity.py index 80a49841b5..84ba6b54c9 100644 --- a/glean-core/python/tests/metrics/test_quantity.py +++ b/glean-core/python/tests/metrics/test_quantity.py @@ -16,7 +16,7 @@ def test_the_api_saves_to_its_storage_engine(): lifetime=Lifetime.APPLICATION, name="quantity_metric", send_in_pings=["store1"], - dynamic_label=None, + label=None, ) ) @@ -42,7 +42,7 @@ def test_disabled_quantities_must_not_record_data(): lifetime=Lifetime.APPLICATION, name="quantity_metric", send_in_pings=["store1"], - dynamic_label=None, + label=None, ) ) @@ -61,7 +61,7 @@ def test_get_value_throws_value_error_if_nothing_is_stored(): lifetime=Lifetime.APPLICATION, name="quantity_metric", send_in_pings=["store1"], - dynamic_label=None, + label=None, ) ) @@ -77,7 +77,7 @@ def test_api_saves_to_secondary_pings(): lifetime=Lifetime.APPLICATION, name="quantity_metric", send_in_pings=["store1", "store2"], - dynamic_label=None, + label=None, ) ) @@ -101,7 +101,7 @@ def test_negative_values_are_not_counted(): lifetime=Lifetime.APPLICATION, name="quantity_metric", send_in_pings=["store1"], - dynamic_label=None, + label=None, ) ) diff --git a/glean-core/python/tests/metrics/test_rate.py b/glean-core/python/tests/metrics/test_rate.py index 770cddcbe8..3650d5ed9e 100644 --- a/glean-core/python/tests/metrics/test_rate.py +++ b/glean-core/python/tests/metrics/test_rate.py @@ -19,7 +19,7 @@ def test_rate_smoke(): name="rate", lifetime=Lifetime.PING, send_in_pings=["store1"], - dynamic_label=None, + label=None, disabled=False, ), ) @@ -55,7 +55,7 @@ def test_numerator_smoke(): name="numerator", lifetime=Lifetime.PING, send_in_pings=["store1"], - dynamic_label=None, + label=None, disabled=False, ), ) @@ -87,7 +87,7 @@ def test_denominator_smoke(): name="rate1", lifetime=Lifetime.PING, send_in_pings=["store1"], - dynamic_label=None, + label=None, disabled=False, ) @@ -96,7 +96,7 @@ def test_denominator_smoke(): name="rate2", lifetime=Lifetime.PING, send_in_pings=["store1"], - dynamic_label=None, + label=None, disabled=False, ) @@ -107,7 +107,7 @@ def test_denominator_smoke(): name="counter", lifetime=Lifetime.PING, send_in_pings=["store1"], - dynamic_label=None, + label=None, disabled=False, ), [meta1, meta2], diff --git a/glean-core/python/tests/metrics/test_string.py b/glean-core/python/tests/metrics/test_string.py index c5bc118454..d985e0c438 100644 --- a/glean-core/python/tests/metrics/test_string.py +++ b/glean-core/python/tests/metrics/test_string.py @@ -16,7 +16,7 @@ def test_the_api_saves_to_its_storage_engine(): lifetime=Lifetime.APPLICATION, name="string_metric", send_in_pings=["store1"], - dynamic_label=None, + label=None, ) ) @@ -37,7 +37,7 @@ def test_disabled_strings_must_not_record_data(): lifetime=Lifetime.APPLICATION, name="string_metric", send_in_pings=["store1"], - dynamic_label=None, + label=None, ) ) @@ -54,7 +54,7 @@ def test_the_api_saves_to_secondary_pings(): lifetime=Lifetime.APPLICATION, name="string_metric", send_in_pings=["store1", "store2"], - dynamic_label=None, + label=None, ) ) @@ -75,7 +75,7 @@ def test_setting_a_long_string_records_an_error(): lifetime=Lifetime.APPLICATION, name="string_metric", send_in_pings=["store1", "store2"], - dynamic_label=None, + label=None, ) ) @@ -92,7 +92,7 @@ def test_setting_a_string_as_none(): lifetime=Lifetime.APPLICATION, name="string_metric", send_in_pings=["store1", "store2"], - dynamic_label=None, + label=None, ) ) diff --git a/glean-core/python/tests/metrics/test_string_list.py b/glean-core/python/tests/metrics/test_string_list.py index 1309100cc0..102bab37ae 100644 --- a/glean-core/python/tests/metrics/test_string_list.py +++ b/glean-core/python/tests/metrics/test_string_list.py @@ -15,7 +15,7 @@ def test_the_api_saves_to_its_storage_engine_by_first_adding_then_setting(): lifetime=Lifetime.APPLICATION, name="string_list_metric", send_in_pings=["store1"], - dynamic_label=None, + label=None, ) ) @@ -40,7 +40,7 @@ def test_the_api_saves_to_its_storage_engine_by_first_setting_then_adding(): lifetime=Lifetime.APPLICATION, name="string_list_metric", send_in_pings=["store1"], - dynamic_label=None, + label=None, ) ) @@ -63,7 +63,7 @@ def test_disabled_lists_must_not_record_data(): lifetime=Lifetime.APPLICATION, name="string_list_metric", send_in_pings=["store1"], - dynamic_label=None, + label=None, ) ) @@ -84,7 +84,7 @@ def test_test_get_value_throws(): lifetime=Lifetime.APPLICATION, name="string_list_metric", send_in_pings=["store1"], - dynamic_label=None, + label=None, ) ) @@ -99,7 +99,7 @@ def test_api_saves_to_secondary_pings(): lifetime=Lifetime.APPLICATION, name="string_list_metric", send_in_pings=["store1", "store2"], - dynamic_label=None, + label=None, ) ) @@ -124,7 +124,7 @@ def test_long_string_lists_are_truncated(): lifetime=Lifetime.APPLICATION, name="string_list_metric", send_in_pings=["store1"], - dynamic_label=None, + label=None, ) ) diff --git a/glean-core/python/tests/metrics/test_text.py b/glean-core/python/tests/metrics/test_text.py index 8eb52f07e7..41b1fd0a6c 100644 --- a/glean-core/python/tests/metrics/test_text.py +++ b/glean-core/python/tests/metrics/test_text.py @@ -19,7 +19,7 @@ def test_text_smoke(): name="text", lifetime=Lifetime.PING, send_in_pings=["store1"], - dynamic_label=None, + label=None, disabled=False, ), ) @@ -37,7 +37,7 @@ def test_text_truncation(): name="text", lifetime=Lifetime.PING, send_in_pings=["store1"], - dynamic_label=None, + label=None, disabled=False, ), ) diff --git a/glean-core/python/tests/metrics/test_timespan.py b/glean-core/python/tests/metrics/test_timespan.py index 59f8b9a196..aa0d4e4e7c 100644 --- a/glean-core/python/tests/metrics/test_timespan.py +++ b/glean-core/python/tests/metrics/test_timespan.py @@ -22,7 +22,7 @@ def test_the_api_saves_to_its_storage_engine(): lifetime=Lifetime.APPLICATION, name="timespan_metric", send_in_pings=["store1"], - dynamic_label=None, + label=None, ), time_unit=TimeUnit.MILLISECOND, ) @@ -41,7 +41,7 @@ def test_disabled_timespans_must_not_record_data(): lifetime=Lifetime.APPLICATION, name="timespan_metric", send_in_pings=["store1"], - dynamic_label=None, + label=None, ), time_unit=TimeUnit.MILLISECOND, ) @@ -60,7 +60,7 @@ def test_the_api_must_correctly_cancel(): lifetime=Lifetime.APPLICATION, name="timespan_metric", send_in_pings=["store1"], - dynamic_label=None, + label=None, ), time_unit=TimeUnit.MILLISECOND, ) @@ -81,7 +81,7 @@ def test_get_value_throws_if_nothing_is_stored(): lifetime=Lifetime.APPLICATION, name="timespan_metric", send_in_pings=["store1"], - dynamic_label=None, + label=None, ), time_unit=TimeUnit.MILLISECOND, ) @@ -97,7 +97,7 @@ def test_the_api_saves_to_secondary_pings(): lifetime=Lifetime.APPLICATION, name="timespan_metric", send_in_pings=["store1", "store2"], - dynamic_label=None, + label=None, ), time_unit=TimeUnit.MILLISECOND, ) @@ -116,7 +116,7 @@ def test_records_an_error_if_started_twice(): lifetime=Lifetime.APPLICATION, name="timespan_metric", send_in_pings=["store1", "store2"], - dynamic_label=None, + label=None, ), time_unit=TimeUnit.MILLISECOND, ) @@ -137,7 +137,7 @@ def test_value_unchanged_if_stopped_twice(): lifetime=Lifetime.APPLICATION, name="timespan_metric", send_in_pings=["store1"], - dynamic_label=None, + label=None, ), time_unit=TimeUnit.MILLISECOND, ) @@ -163,7 +163,7 @@ def test_set_raw_nanos(): lifetime=Lifetime.APPLICATION, name="timespan_metric", send_in_pings=["store1"], - dynamic_label=None, + label=None, ), time_unit=TimeUnit.SECOND, ) @@ -182,7 +182,7 @@ def test_set_raw_nanos_followed_by_other_api(): lifetime=Lifetime.APPLICATION, name="timespan_metric", send_in_pings=["store1"], - dynamic_label=None, + label=None, ), time_unit=TimeUnit.SECOND, ) @@ -205,7 +205,7 @@ def test_set_raw_nanos_does_not_overwrite_value(): lifetime=Lifetime.APPLICATION, name="timespan_metric", send_in_pings=["store1"], - dynamic_label=None, + label=None, ), time_unit=TimeUnit.SECOND, ) @@ -228,7 +228,7 @@ def test_set_raw_nanos_does_nothing_when_timer_is_running(): lifetime=Lifetime.APPLICATION, name="timespan_metric", send_in_pings=["store1"], - dynamic_label=None, + label=None, ), time_unit=TimeUnit.SECOND, ) @@ -248,7 +248,7 @@ def test_measure(): lifetime=Lifetime.APPLICATION, name="timespan_metric", send_in_pings=["store1"], - dynamic_label=None, + label=None, ), time_unit=TimeUnit.NANOSECOND, ) @@ -267,7 +267,7 @@ def test_measure_exception(): lifetime=Lifetime.APPLICATION, name="timespan_metric", send_in_pings=["store1"], - dynamic_label=None, + label=None, ), time_unit=TimeUnit.NANOSECOND, ) diff --git a/glean-core/python/tests/metrics/test_timing_distribution.py b/glean-core/python/tests/metrics/test_timing_distribution.py index 7ce9f37993..74a05b5afe 100644 --- a/glean-core/python/tests/metrics/test_timing_distribution.py +++ b/glean-core/python/tests/metrics/test_timing_distribution.py @@ -20,7 +20,7 @@ def test_the_api_saves_to_its_storage_engine(): lifetime=Lifetime.APPLICATION, name="timing_distribution", send_in_pings=["store1"], - dynamic_label=None, + label=None, ), time_unit=TimeUnit.NANOSECOND, ) @@ -42,7 +42,7 @@ def test_disabled_timing_distributions_must_not_record_data(): lifetime=Lifetime.APPLICATION, name="timing_distribution", send_in_pings=["store1"], - dynamic_label=None, + label=None, ), time_unit=TimeUnit.NANOSECOND, ) @@ -61,7 +61,7 @@ def test_get_value_throws(): lifetime=Lifetime.APPLICATION, name="timing_distribution", send_in_pings=["store1"], - dynamic_label=None, + label=None, ), time_unit=TimeUnit.NANOSECOND, ) @@ -77,7 +77,7 @@ def test_api_saves_to_secondary_pings(): lifetime=Lifetime.APPLICATION, name="timing_distribution", send_in_pings=["store1", "store2", "store3"], - dynamic_label=None, + label=None, ), time_unit=TimeUnit.NANOSECOND, ) @@ -100,7 +100,7 @@ def test_stopping_a_non_existent_timer_records_an_error(): lifetime=Lifetime.APPLICATION, name="timing_distribution", send_in_pings=["store1", "store2", "store3"], - dynamic_label=None, + label=None, ), time_unit=TimeUnit.NANOSECOND, ) @@ -120,7 +120,7 @@ def test_measure(): lifetime=Lifetime.APPLICATION, name="timing_distribution", send_in_pings=["baseline"], - dynamic_label=None, + label=None, ), time_unit=TimeUnit.NANOSECOND, ) @@ -143,7 +143,7 @@ def test_measure_exception(): lifetime=Lifetime.APPLICATION, name="timing_distribution", send_in_pings=["baseline"], - dynamic_label=None, + label=None, ), time_unit=TimeUnit.NANOSECOND, ) @@ -163,7 +163,7 @@ def test_the_accumulate_apis_record_data(): lifetime=Lifetime.APPLICATION, name="timing_distribution", send_in_pings=["store1"], - dynamic_label=None, + label=None, ), time_unit=TimeUnit.NANOSECOND, ) diff --git a/glean-core/python/tests/metrics/test_url.py b/glean-core/python/tests/metrics/test_url.py index c9cf82210a..fe2d87bc47 100644 --- a/glean-core/python/tests/metrics/test_url.py +++ b/glean-core/python/tests/metrics/test_url.py @@ -16,7 +16,7 @@ def test_the_api_saves_to_its_storage_engine(): lifetime=Lifetime.APPLICATION, name="url_metric", send_in_pings=["store1"], - dynamic_label=None, + label=None, ) ) @@ -37,7 +37,7 @@ def test_disabled_urls_must_not_record_data(): lifetime=Lifetime.APPLICATION, name="url_metric", send_in_pings=["store1"], - dynamic_label=None, + label=None, ) ) @@ -54,7 +54,7 @@ def test_the_api_saves_to_secondary_pings(): lifetime=Lifetime.APPLICATION, name="url_metric", send_in_pings=["store1", "store2"], - dynamic_label=None, + label=None, ) ) @@ -75,7 +75,7 @@ def test_setting_a_long_url_records_an_error(): lifetime=Lifetime.APPLICATION, name="url_metric", send_in_pings=["store1", "store2"], - dynamic_label=None, + label=None, ) ) @@ -108,7 +108,7 @@ def test_setting_a_url_as_none(): lifetime=Lifetime.APPLICATION, name="url_metric", send_in_pings=["store1", "store2"], - dynamic_label=None, + label=None, ) ) diff --git a/glean-core/python/tests/metrics/test_uuid.py b/glean-core/python/tests/metrics/test_uuid.py index 071c16e9de..9402264869 100644 --- a/glean-core/python/tests/metrics/test_uuid.py +++ b/glean-core/python/tests/metrics/test_uuid.py @@ -24,7 +24,7 @@ def test_the_api_saves_to_its_storage_engine(): lifetime=Lifetime.APPLICATION, name="uuid_metric", send_in_pings=["store1"], - dynamic_label=None, + label=None, ) ) @@ -52,7 +52,7 @@ def test_disabled_uuids_must_not_record_data(): lifetime=Lifetime.PING, name="uuid_metric", send_in_pings=["store1"], - dynamic_label=None, + label=None, ) ) @@ -68,7 +68,7 @@ def test_test_get_value_throws_exception_if_nothing_is_stored(): lifetime=Lifetime.PING, name="uuid_metric", send_in_pings=["store1"], - dynamic_label=None, + label=None, ) ) @@ -83,7 +83,7 @@ def test_the_api_saves_to_secondary_pings(): lifetime=Lifetime.PING, name="uuid_metric", send_in_pings=["store1", "store2"], - dynamic_label=None, + label=None, ) ) @@ -107,7 +107,7 @@ def test_invalid_uuid_must_not_crash(): lifetime=Lifetime.PING, name="uuid_metric", send_in_pings=["store1"], - dynamic_label=None, + label=None, ) ) @@ -126,7 +126,7 @@ def test_invalid_uuid_string(): lifetime=Lifetime.PING, name="uuid_metric", send_in_pings=["store1"], - dynamic_label=None, + label=None, ) ) @@ -163,7 +163,7 @@ def test_what_looks_like_it_might_be_uuid(tmpdir, helpers): lifetime=Lifetime.PING, name="chksum", send_in_pings=["metrics"], - dynamic_label=None, + label=None, ) ) @@ -174,7 +174,7 @@ def test_what_looks_like_it_might_be_uuid(tmpdir, helpers): lifetime=Lifetime.PING, name="random", send_in_pings=["metrics"], - dynamic_label=None, + label=None, ) ) @@ -185,7 +185,7 @@ def test_what_looks_like_it_might_be_uuid(tmpdir, helpers): lifetime=Lifetime.PING, name="valid", send_in_pings=["metrics"], - dynamic_label=None, + label=None, ) ) diff --git a/glean-core/python/tests/test_collection_enabled.py b/glean-core/python/tests/test_collection_enabled.py index a1dbb0470e..5925290597 100644 --- a/glean-core/python/tests/test_collection_enabled.py +++ b/glean-core/python/tests/test_collection_enabled.py @@ -56,7 +56,7 @@ def test_pings_with_follows_false_follow_their_own_setting(tmpdir, helpers): lifetime=Lifetime.PING, name="counter", send_in_pings=["nofollows"], - dynamic_label=None, + label=None, ) ) @@ -93,7 +93,7 @@ def test_loader_sets_flags(tmpdir, helpers): lifetime=Lifetime.PING, name="counter", send_in_pings=["nofollows-defined"], - dynamic_label=None, + label=None, ) ) diff --git a/glean-core/python/tests/test_glean.py b/glean-core/python/tests/test_glean.py index 65013eab23..860d303721 100644 --- a/glean-core/python/tests/test_glean.py +++ b/glean-core/python/tests/test_glean.py @@ -76,7 +76,7 @@ def test_submit_a_ping(safe_httpserver): lifetime=Lifetime.APPLICATION, name="counter_metric", send_in_pings=["baseline"], - dynamic_label=None, + label=None, ) ) @@ -105,7 +105,7 @@ def test_disabling_upload_should_disable_metrics_recording(): lifetime=Lifetime.APPLICATION, name="counter_metric", send_in_pings=["store1"], - dynamic_label=None, + label=None, ) ) @@ -198,7 +198,7 @@ def test_queued_recorded_metrics_correctly_during_init(): lifetime=Lifetime.APPLICATION, name="counter_metric", send_in_pings=["store1"], - dynamic_label=None, + label=None, ) ) @@ -236,7 +236,7 @@ def test_dont_schedule_pings_if_metrics_disabled(safe_httpserver): lifetime=Lifetime.APPLICATION, name="counter_metric", send_in_pings=["store1"], - dynamic_label=None, + label=None, ) ) @@ -331,7 +331,7 @@ def test_ping_collection_must_happen_after_currently_scheduled_metrics_recording lifetime=Lifetime.PING, name="string_metric", send_in_pings=[ping_name], - dynamic_label=None, + label=None, ) ) @@ -368,7 +368,7 @@ def test_basic_metrics_should_be_cleared_when_disabling_uploading(): lifetime=Lifetime.APPLICATION, name="counter_metric", send_in_pings=["store1"], - dynamic_label=None, + label=None, ) ) @@ -503,7 +503,7 @@ def test_configuration_property(safe_httpserver): lifetime=Lifetime.APPLICATION, name="counter_metric", send_in_pings=["baseline"], - dynamic_label=None, + label=None, ) ) @@ -676,7 +676,7 @@ def test_clear_application_lifetime_metrics(tmpdir): lifetime=Lifetime.APPLICATION, name="lifetime_reset", send_in_pings=["store1"], - dynamic_label=None, + label=None, ) ) @@ -881,7 +881,7 @@ def test_sending_of_custom_pings(safe_httpserver): lifetime=Lifetime.APPLICATION, name="counter_metric", send_in_pings=["store1"], - dynamic_label=None, + label=None, ) ) @@ -952,7 +952,7 @@ def test_max_events_overflow(tmpdir, helpers): lifetime=Lifetime.APPLICATION, name="event", send_in_pings=["events"], - dynamic_label=None, + label=None, ), allowed_extra_keys=[], ) @@ -1001,7 +1001,7 @@ def test_glean_shutdown(safe_httpserver): send_in_pings=["custom"], lifetime=Lifetime.APPLICATION, disabled=False, - dynamic_label=None, + label=None, ) ) @@ -1056,7 +1056,7 @@ def test_uploader_capabilities_reported(tmpdir, helpers): lifetime=Lifetime.APPLICATION, name="counter_metric", send_in_pings=["store1"], - dynamic_label=None, + label=None, ) ) @@ -1107,7 +1107,7 @@ def test_uploader_capabilities_empty_not_reported(tmpdir, helpers): lifetime=Lifetime.PING, name="stringlist_metric", send_in_pings=["store1"], - dynamic_label=None, + label=None, ) ) @@ -1243,7 +1243,7 @@ def test_uploader_capabilities_in_events_ping(tmpdir, helpers): name="custom", lifetime=Lifetime.PING, send_in_pings=["custom-events"], - dynamic_label=None, + label=None, ), allowed_extra_keys=[], ) diff --git a/glean-core/python/tests/test_network.py b/glean-core/python/tests/test_network.py index 89bd5d750c..d84d7fea0d 100644 --- a/glean-core/python/tests/test_network.py +++ b/glean-core/python/tests/test_network.py @@ -35,7 +35,7 @@ def get_upload_failure_metric(): name="ping_upload_failure", category="glean.upload", lifetime=metrics.Lifetime.PING, - dynamic_label=None, + label=None, ) ), labels=[ @@ -90,7 +90,7 @@ def test_recording_upload_errors_doesnt_clobber_database(tmpdir, safe_httpserver lifetime=Lifetime.PING, name="counter_metric", send_in_pings=["baseline"], - dynamic_label=None, + label=None, ) ) counter_metric.add(10) diff --git a/glean-core/rlb/src/test.rs b/glean-core/rlb/src/test.rs index 76e88fd678..651e9e9401 100644 --- a/glean-core/rlb/src/test.rs +++ b/glean-core/rlb/src/test.rs @@ -8,7 +8,7 @@ use std::time::{Duration, Instant}; use crossbeam_channel::RecvTimeoutError; use flate2::read::GzDecoder; -use glean_core::{glean_test_get_experimentation_id, DynamicLabelType, LabeledCounter}; +use glean_core::{glean_test_get_experimentation_id, LabeledCounter, MetricLabel}; use serde_json::Value as JsonValue; use crate::private::PingType; @@ -142,7 +142,7 @@ fn disabling_upload_disables_metrics_recording() { send_in_pings: vec!["store1".into()], lifetime: Lifetime::Application, disabled: false, - dynamic_label: None, + label: None, }); crate::set_upload_enabled(false); @@ -437,7 +437,7 @@ fn queued_recorded_metrics_correctly_record_during_init() { send_in_pings: vec!["store1".into()], lifetime: Lifetime::Application, disabled: false, - dynamic_label: None, + label: None, }); // This will queue 3 tasks that will add to the metric value once Glean is initialized @@ -1258,7 +1258,7 @@ fn test_a_ping_before_submission() { send_in_pings: vec!["custom1".into()], lifetime: Lifetime::Application, disabled: false, - dynamic_label: None, + label: None, }); metric.add(1); @@ -1288,7 +1288,7 @@ fn test_boolean_get_num_errors() { send_in_pings: vec!["custom1".into()], lifetime: Lifetime::Application, disabled: false, - dynamic_label: Some(DynamicLabelType::Label(str::to_string("asdf"))), + label: Some(MetricLabel::Label(str::to_string("asdf"))), }); // Check specifically for an invalid label @@ -1374,7 +1374,7 @@ fn test_text_can_hold_long_string() { send_in_pings: vec!["custom1".into()], lifetime: Lifetime::Application, disabled: false, - dynamic_label: Some(DynamicLabelType::Label(str::to_string("text"))), + label: Some(MetricLabel::Label(str::to_string("text"))), }); // 216 characters, which would overflow StringMetric diff --git a/glean-core/rlb/tests/metric_metadata.rs b/glean-core/rlb/tests/metric_metadata.rs index 39e2c88065..c0cba456a8 100644 --- a/glean-core/rlb/tests/metric_metadata.rs +++ b/glean-core/rlb/tests/metric_metadata.rs @@ -15,7 +15,7 @@ mod metrics { use glean::private::*; use glean::traits; use glean::CommonMetricData; - use glean_core::DynamicLabelType; + use glean_core::MetricLabel; use once_cell::sync::Lazy; use std::collections::HashMap; @@ -45,7 +45,7 @@ mod metrics { name: "count_von_count".into(), category: "sesame".into(), send_in_pings: vec!["validation".into()], - dynamic_label: Some(DynamicLabelType::Label("ah_ah_ah".into())), + label: Some(MetricLabel::Label("ah_ah_ah".into())), ..Default::default() }) }); @@ -56,7 +56,7 @@ mod metrics { name: "birthday".into(), category: "shire".into(), send_in_pings: vec!["validation".into()], - dynamic_label: Some(DynamicLabelType::Label("111th".into())), + label: Some(MetricLabel::Label("111th".into())), ..Default::default() }) }); diff --git a/glean-core/rlb/tests/upload_timing.rs b/glean-core/rlb/tests/upload_timing.rs index 6fa77cbaa0..5fd07d55a3 100644 --- a/glean-core/rlb/tests/upload_timing.rs +++ b/glean-core/rlb/tests/upload_timing.rs @@ -55,7 +55,7 @@ pub mod metrics { send_in_pings: vec!["metrics".into()], lifetime: Lifetime::Ping, disabled: false, - dynamic_label: None, + label: None, }, TimeUnit::Millisecond, ) @@ -70,7 +70,7 @@ pub mod metrics { send_in_pings: vec!["metrics".into()], lifetime: Lifetime::Ping, disabled: false, - dynamic_label: None, + label: None, }, TimeUnit::Millisecond, ) @@ -85,7 +85,7 @@ pub mod metrics { send_in_pings: vec!["metrics".into()], lifetime: Lifetime::Ping, disabled: false, - dynamic_label: None, + label: None, }, TimeUnit::Millisecond, ) diff --git a/glean-core/src/common_metric_data.rs b/glean-core/src/common_metric_data.rs index 86ec2ce5de..7507cab0a7 100644 --- a/glean-core/src/common_metric_data.rs +++ b/glean-core/src/common_metric_data.rs @@ -70,50 +70,51 @@ pub struct CommonMetricData { /// /// Disabled metrics are never recorded. pub disabled: bool, - /// Dynamic label. + /// Label for this metric. /// - /// When a [`LabeledMetric`](crate::metrics::LabeledMetric) factory creates the specific - /// metric to be recorded to, dynamic labels are stored in the specific - /// label so that we can validate them when the Glean singleton is - /// available. - pub dynamic_label: Option, + /// When a [`LabeledMetric`](crate::metrics::LabeledMetric) factory + /// or [`DualLabeledCounterMetric`](crate::metrics::DualLabeledCounterMetric) + /// creates the specific metric to be recorded to, + /// labels are stored in the metric data + /// so that it can validated against the database later. + pub label: Option, } /// The type of dynamic label applied to a base metric. Used to help identify /// the necessary validation to be performed. #[derive(Debug, Clone, Deserialize, Serialize, MallocSizeOf, uniffi::Enum)] -pub enum DynamicLabelType { +pub enum MetricLabel { /// Static Label -- no validation required Static(String), /// A dynamic label applied from a `LabeledMetric` Label(String), - /// A label applied by a `DualLabeledCounter` that contains a dynamic key + /// A label applied by a `DualLabeledCounter` that contains a dynamic key and static category KeyOnly(String, String), - /// A label applied by a `DualLabeledCounter` that contains a dynamic category + /// A label applied by a `DualLabeledCounter` that contains a static key and dynamic category CategoryOnly(String, String), /// A label applied by a `DualLabeledCounter` that contains a dynamic key and category KeyAndCategory(String, String), } -impl Default for DynamicLabelType { +impl Default for MetricLabel { fn default() -> Self { Self::Label(String::new()) } } -impl Display for DynamicLabelType { +impl Display for MetricLabel { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { use crate::metrics::dual_labeled_counter::RECORD_SEPARATOR; match self { - DynamicLabelType::Static(label) => write!(f, "{label}"), - DynamicLabelType::Label(label) => write!(f, "{label}"), - DynamicLabelType::KeyOnly(key, category) => { + MetricLabel::Static(label) => write!(f, "{label}"), + MetricLabel::Label(label) => write!(f, "{label}"), + MetricLabel::KeyOnly(key, category) => { write!(f, "{key}{RECORD_SEPARATOR}{category}") } - DynamicLabelType::CategoryOnly(key, category) => { + MetricLabel::CategoryOnly(key, category) => { write!(f, "{key}{RECORD_SEPARATOR}{category}") } - DynamicLabelType::KeyAndCategory(key, category) => { + MetricLabel::KeyAndCategory(key, category) => { write!(f, "{key}{RECORD_SEPARATOR}{category}") } } @@ -223,21 +224,21 @@ impl CommonMetricDataInternal { pub(crate) fn check_labels(&self, tx: &Transaction<'_>) -> LabelCheck { let base_identifier = self.base_identifier(); - if let Some(label) = &self.inner.dynamic_label { + if let Some(label) = &self.inner.label { match label { - DynamicLabelType::Static(label) => LabelCheck::Label(label.to_string()), - DynamicLabelType::Label(label) => { + MetricLabel::Static(label) => LabelCheck::Label(label.to_string()), + MetricLabel::Label(label) => { validate_dynamic_label_sqlite(tx, &base_identifier, label) } - DynamicLabelType::KeyOnly(key, static_category) => { + MetricLabel::KeyOnly(key, static_category) => { validate_dual_label_sqlite(tx, &base_identifier, key, "") .map(|key| format!("{key}{static_category}")) } - DynamicLabelType::CategoryOnly(static_key, category) => { + MetricLabel::CategoryOnly(static_key, category) => { validate_dual_label_sqlite(tx, &base_identifier, "", category) .map(|category| format!("{static_key}{category}")) } - DynamicLabelType::KeyAndCategory(key, category) => { + MetricLabel::KeyAndCategory(key, category) => { validate_dual_label_sqlite(tx, &base_identifier, key, category) } } diff --git a/glean-core/src/error_recording.rs b/glean-core/src/error_recording.rs index c843ca8acb..a090e40472 100644 --- a/glean-core/src/error_recording.rs +++ b/glean-core/src/error_recording.rs @@ -22,7 +22,7 @@ use crate::error::{Error, ErrorKind}; use crate::metrics::{CounterMetric, Metric}; use crate::Glean; use crate::Lifetime; -use crate::{CommonMetricData, DynamicLabelType}; +use crate::{CommonMetricData, MetricLabel}; /// The possible error types for metric recording. /// @@ -108,7 +108,7 @@ fn get_error_metric_for_metric(meta: &CommonMetricDataInternal, error: ErrorType category: "glean.error".into(), lifetime: Lifetime::Ping, send_in_pings, - dynamic_label: Some(DynamicLabelType::Label(name.to_string())), + label: Some(MetricLabel::Label(name.to_string())), ..Default::default() }) } diff --git a/glean-core/src/glean.udl b/glean-core/src/glean.udl index f63c714088..b1aafbe874 100644 --- a/glean-core/src/glean.udl +++ b/glean-core/src/glean.udl @@ -354,7 +354,7 @@ interface PingType { void set_enabled(boolean enabled); }; -typedef enum DynamicLabelType; +typedef enum MetricLabel; // The common set of data shared across all different metric types. dictionary CommonMetricData { @@ -373,12 +373,12 @@ dictionary CommonMetricData { // Disabled metrics are never recorded. boolean disabled; - // Dynamic label. + // Label for this metric. // // When a labeled metric factory creates the specific metric to be recorded to, - // dynamic labels are stored in the specific label so that - // we can validate them when the Glean singleton is available. - DynamicLabelType? dynamic_label = null; + // labels are stored in the metric data + // so that it can validated against the database later. + MetricLabel? label = null; }; interface CounterMetric { diff --git a/glean-core/src/internal_metrics.rs b/glean-core/src/internal_metrics.rs index fd46c642b1..00704f4b6f 100644 --- a/glean-core/src/internal_metrics.rs +++ b/glean-core/src/internal_metrics.rs @@ -58,7 +58,7 @@ impl CoreMetrics { send_in_pings: vec!["glean_client_info".into()], lifetime: Lifetime::User, disabled: false, - dynamic_label: None, + label: None, }), first_run_date: DatetimeMetric::new( @@ -68,7 +68,7 @@ impl CoreMetrics { send_in_pings: vec!["glean_client_info".into()], lifetime: Lifetime::User, disabled: false, - dynamic_label: None, + label: None, }, TimeUnit::Day, ), @@ -79,7 +79,7 @@ impl CoreMetrics { send_in_pings: vec!["glean_client_info".into()], lifetime: Lifetime::Application, disabled: false, - dynamic_label: None, + label: None, }), attribution_source: StringMetric::new(CommonMetricData { @@ -88,7 +88,7 @@ impl CoreMetrics { send_in_pings: vec!["glean_client_info".into()], lifetime: Lifetime::User, disabled: false, - dynamic_label: None, + label: None, }), attribution_medium: StringMetric::new(CommonMetricData { @@ -97,7 +97,7 @@ impl CoreMetrics { send_in_pings: vec!["glean_client_info".into()], lifetime: Lifetime::User, disabled: false, - dynamic_label: None, + label: None, }), attribution_campaign: StringMetric::new(CommonMetricData { @@ -106,7 +106,7 @@ impl CoreMetrics { send_in_pings: vec!["glean_client_info".into()], lifetime: Lifetime::User, disabled: false, - dynamic_label: None, + label: None, }), attribution_term: StringMetric::new(CommonMetricData { @@ -115,7 +115,7 @@ impl CoreMetrics { send_in_pings: vec!["glean_client_info".into()], lifetime: Lifetime::User, disabled: false, - dynamic_label: None, + label: None, }), attribution_content: StringMetric::new(CommonMetricData { @@ -124,7 +124,7 @@ impl CoreMetrics { send_in_pings: vec!["glean_client_info".into()], lifetime: Lifetime::User, disabled: false, - dynamic_label: None, + label: None, }), distribution_name: StringMetric::new(CommonMetricData { @@ -133,7 +133,7 @@ impl CoreMetrics { send_in_pings: vec!["glean_client_info".into()], lifetime: Lifetime::User, disabled: false, - dynamic_label: None, + label: None, }), } } @@ -148,7 +148,7 @@ impl AdditionalMetrics { send_in_pings: vec!["metrics".into(), "health".into()], lifetime: Lifetime::Ping, disabled: false, - dynamic_label: None, + label: None, }), pings_submitted: LabeledMetric::::new( @@ -159,7 +159,7 @@ impl AdditionalMetrics { send_in_pings: vec!["metrics".into(), "baseline".into(), "health".into()], lifetime: Lifetime::Ping, disabled: false, - dynamic_label: None, + label: None, }, }, None, @@ -172,7 +172,7 @@ impl AdditionalMetrics { send_in_pings: vec!["metrics".into(), "health".into()], lifetime: Lifetime::Ping, disabled: false, - dynamic_label: None, + label: None, }, TimeUnit::Millisecond, ), @@ -184,7 +184,7 @@ impl AdditionalMetrics { send_in_pings: vec!["metrics".into(), "health".into()], lifetime: Lifetime::Ping, disabled: false, - dynamic_label: None, + label: None, }, TimeUnit::Millisecond, ), @@ -203,7 +203,7 @@ impl AdditionalMetrics { send_in_pings: vec!["all-pings".into()], lifetime: Lifetime::Application, disabled: false, - dynamic_label: None, + label: None, }), event_timestamp_clamped: CounterMetric::new(CommonMetricData { @@ -212,7 +212,7 @@ impl AdditionalMetrics { send_in_pings: vec!["health".into()], lifetime: Lifetime::Ping, disabled: false, - dynamic_label: None, + label: None, }), server_knobs_config: ObjectMetric::new(CommonMetricData { @@ -221,7 +221,7 @@ impl AdditionalMetrics { send_in_pings: vec!["glean_internal_info".into()], lifetime: Lifetime::Application, disabled: false, - dynamic_label: None, + label: None, }), } } @@ -251,7 +251,7 @@ impl UploadMetrics { send_in_pings: vec!["metrics".into(), "health".into()], lifetime: Lifetime::Ping, disabled: false, - dynamic_label: None, + label: None, }, }, Some(vec![ @@ -271,7 +271,7 @@ impl UploadMetrics { send_in_pings: vec!["metrics".into(), "health".into()], lifetime: Lifetime::Ping, disabled: false, - dynamic_label: None, + label: None, }, MemoryUnit::Kilobyte, ), @@ -283,7 +283,7 @@ impl UploadMetrics { send_in_pings: vec!["metrics".into(), "health".into()], lifetime: Lifetime::Ping, disabled: false, - dynamic_label: None, + label: None, }, MemoryUnit::Kilobyte, ), @@ -294,7 +294,7 @@ impl UploadMetrics { send_in_pings: vec!["metrics".into(), "health".into()], lifetime: Lifetime::Ping, disabled: false, - dynamic_label: None, + label: None, }), pending_pings: CounterMetric::new(CommonMetricData { @@ -303,7 +303,7 @@ impl UploadMetrics { send_in_pings: vec!["metrics".into(), "health".into()], lifetime: Lifetime::Ping, disabled: false, - dynamic_label: None, + label: None, }), send_success: TimingDistributionMetric::new( @@ -313,7 +313,7 @@ impl UploadMetrics { send_in_pings: vec!["metrics".into(), "health".into()], lifetime: Lifetime::Ping, disabled: false, - dynamic_label: None, + label: None, }, TimeUnit::Millisecond, ), @@ -325,7 +325,7 @@ impl UploadMetrics { send_in_pings: vec!["metrics".into(), "health".into()], lifetime: Lifetime::Ping, disabled: false, - dynamic_label: None, + label: None, }, TimeUnit::Millisecond, ), @@ -336,7 +336,7 @@ impl UploadMetrics { send_in_pings: vec!["metrics".into(), "health".into()], lifetime: Lifetime::Ping, disabled: false, - dynamic_label: None, + label: None, }), missing_send_ids: CounterMetric::new(CommonMetricData { @@ -345,7 +345,7 @@ impl UploadMetrics { send_in_pings: vec!["metrics".into(), "health".into()], lifetime: Lifetime::Ping, disabled: false, - dynamic_label: None, + label: None, }), } } @@ -372,7 +372,7 @@ impl DatabaseMetrics { send_in_pings: vec!["metrics".into(), "health".into()], lifetime: Lifetime::Ping, disabled: false, - dynamic_label: None, + label: None, }, MemoryUnit::Byte, ), @@ -383,7 +383,7 @@ impl DatabaseMetrics { send_in_pings: vec!["metrics".into(), "health".into()], lifetime: Lifetime::Ping, disabled: false, - dynamic_label: None, + label: None, }), write_time: TimingDistributionMetric::new( @@ -393,7 +393,7 @@ impl DatabaseMetrics { send_in_pings: vec!["metrics".into(), "health".into()], lifetime: Lifetime::Ping, disabled: true, - dynamic_label: None, + label: None, }, TimeUnit::Microsecond, ), @@ -450,7 +450,7 @@ impl HealthMetrics { send_in_pings: vec!["metrics".into(), "health".into()], lifetime: Lifetime::Ping, disabled: false, - dynamic_label: None, + label: None, }), init_count: CounterMetric::new(CommonMetricData { name: "init_count".into(), @@ -458,7 +458,7 @@ impl HealthMetrics { send_in_pings: vec!["health".into()], lifetime: Lifetime::User, disabled: false, - dynamic_label: None, + label: None, }), exception_state: StringMetric::new(CommonMetricData { name: "exception_state".into(), @@ -466,7 +466,7 @@ impl HealthMetrics { send_in_pings: vec!["health".into()], lifetime: Lifetime::Ping, disabled: false, - dynamic_label: None, + label: None, }), recovered_client_id: UuidMetric::new(CommonMetricData { name: "recovered_client_id".into(), @@ -474,7 +474,7 @@ impl HealthMetrics { send_in_pings: vec!["health".into()], lifetime: Lifetime::Ping, disabled: false, - dynamic_label: None, + label: None, }), file_read_error: LabeledMetric::::new( LabeledMetricData::Common { @@ -484,7 +484,7 @@ impl HealthMetrics { send_in_pings: vec!["health".into()], lifetime: Lifetime::Ping, disabled: false, - dynamic_label: None, + label: None, }, }, Some(vec![ @@ -503,7 +503,7 @@ impl HealthMetrics { send_in_pings: vec!["health".into()], lifetime: Lifetime::Ping, disabled: false, - dynamic_label: None, + label: None, }, }, Some(vec![ diff --git a/glean-core/src/lib.rs b/glean-core/src/lib.rs index 7f47159a23..b835465b15 100644 --- a/glean-core/src/lib.rs +++ b/glean-core/src/lib.rs @@ -63,7 +63,7 @@ mod util; #[cfg(all(not(target_os = "android"), not(target_os = "ios")))] mod fd_logger; -pub use crate::common_metric_data::{CommonMetricData, DynamicLabelType, Lifetime}; +pub use crate::common_metric_data::{CommonMetricData, Lifetime, MetricLabel}; pub use crate::core::Glean; pub use crate::core_metrics::{AttributionMetrics, ClientInfoMetrics, DistributionMetrics}; use crate::dispatcher::is_test_mode; diff --git a/glean-core/src/metrics/boolean.rs b/glean-core/src/metrics/boolean.rs index 39a770de25..2614ba1f3a 100644 --- a/glean-core/src/metrics/boolean.rs +++ b/glean-core/src/metrics/boolean.rs @@ -4,7 +4,7 @@ use std::sync::Arc; -use crate::common_metric_data::{CommonMetricDataInternal, DynamicLabelType}; +use crate::common_metric_data::{CommonMetricDataInternal, MetricLabel}; use crate::error_recording::{test_get_num_recorded_errors, ErrorType}; use crate::metrics::MetricType; use crate::metrics::{Metric, TestGetValue}; @@ -32,9 +32,9 @@ impl MetricType for BooleanMetric { } } - fn with_label(&self, label: DynamicLabelType) -> Self { + fn with_label(&self, label: MetricLabel) -> Self { let mut meta = (*self.meta).clone(); - meta.inner.dynamic_label = Some(label); + meta.inner.label = Some(label); Self { meta: Arc::new(meta), } diff --git a/glean-core/src/metrics/counter.rs b/glean-core/src/metrics/counter.rs index 9bad5d6756..407426a66b 100644 --- a/glean-core/src/metrics/counter.rs +++ b/glean-core/src/metrics/counter.rs @@ -5,7 +5,7 @@ use std::cmp::Ordering; use std::sync::Arc; -use crate::common_metric_data::{CommonMetricDataInternal, DynamicLabelType}; +use crate::common_metric_data::{CommonMetricDataInternal, MetricLabel}; use crate::error_recording::{record_error, test_get_num_recorded_errors, ErrorType}; use crate::metrics::Metric; use crate::metrics::MetricType; @@ -34,9 +34,9 @@ impl MetricType for CounterMetric { } } - fn with_label(&self, label: DynamicLabelType) -> Self { + fn with_label(&self, label: MetricLabel) -> Self { let mut meta = (*self.meta).clone(); - meta.inner.dynamic_label = Some(label); + meta.inner.label = Some(label); Self { meta: Arc::new(meta), } diff --git a/glean-core/src/metrics/custom_distribution.rs b/glean-core/src/metrics/custom_distribution.rs index 60806ed620..418c06b2d7 100644 --- a/glean-core/src/metrics/custom_distribution.rs +++ b/glean-core/src/metrics/custom_distribution.rs @@ -5,7 +5,7 @@ use std::mem; use std::sync::Arc; -use crate::common_metric_data::{CommonMetricDataInternal, DynamicLabelType}; +use crate::common_metric_data::{CommonMetricDataInternal, MetricLabel}; use crate::error_recording::{record_error, test_get_num_recorded_errors, ErrorType}; use crate::histogram::{Bucketing, Histogram, HistogramType, LinearOrExponential}; use crate::metrics::{DistributionData, Metric, MetricType}; @@ -54,9 +54,9 @@ impl MetricType for CustomDistributionMetric { } } - fn with_label(&self, label: DynamicLabelType) -> Self { + fn with_label(&self, label: MetricLabel) -> Self { let mut meta = (*self.meta).clone(); - meta.inner.dynamic_label = Some(label); + meta.inner.label = Some(label); Self { meta: Arc::new(meta), range_min: self.range_min, diff --git a/glean-core/src/metrics/dual_labeled_counter.rs b/glean-core/src/metrics/dual_labeled_counter.rs index 0bb6d1f3bf..970b88158c 100644 --- a/glean-core/src/metrics/dual_labeled_counter.rs +++ b/glean-core/src/metrics/dual_labeled_counter.rs @@ -11,7 +11,7 @@ use std::sync::{Arc, Mutex}; use rusqlite::{params, Transaction}; use crate::common_metric_data::{ - CommonMetricData, CommonMetricDataInternal, DynamicLabelType, LabelCheck, + CommonMetricData, CommonMetricDataInternal, LabelCheck, MetricLabel, }; use crate::error_recording::{test_get_num_recorded_errors, ErrorType}; use crate::metrics::{CounterMetric, MetricType}; @@ -112,20 +112,17 @@ impl DualLabeledCounterMetric { /// the static or dynamic labels where needed. fn new_counter_metric(&self, key: &str, category: &str) -> CounterMetric { match (&self.keys, &self.categories) { - (None, None) => self.counter.with_label(DynamicLabelType::KeyAndCategory( - key.into(), - category.into(), - )), + (None, None) => self + .counter + .with_label(MetricLabel::KeyAndCategory(key.into(), category.into())), (None, _) => { let static_category = self.static_category(category); - self.counter.with_label(DynamicLabelType::KeyOnly( - key.into(), - static_category.into(), - )) + self.counter + .with_label(MetricLabel::KeyOnly(key.into(), static_category.into())) } (_, None) => { let static_key = self.static_key(key); - self.counter.with_label(DynamicLabelType::CategoryOnly( + self.counter.with_label(MetricLabel::CategoryOnly( static_key.into(), category.into(), )) @@ -135,7 +132,7 @@ impl DualLabeledCounterMetric { let static_key = self.static_key(key); let static_category = self.static_category(category); let label = format!("{static_key}{RECORD_SEPARATOR}{static_category}"); - self.counter.with_label(DynamicLabelType::Static(label)) + self.counter.with_label(MetricLabel::Static(label)) } } } diff --git a/glean-core/src/metrics/labeled.rs b/glean-core/src/metrics/labeled.rs index 3559278ef9..deeaf2158f 100644 --- a/glean-core/src/metrics/labeled.rs +++ b/glean-core/src/metrics/labeled.rs @@ -11,7 +11,7 @@ use std::sync::{Arc, Mutex}; use malloc_size_of::MallocSizeOf; use rusqlite::{params, Transaction}; -use crate::common_metric_data::{CommonMetricData, DynamicLabelType, LabelCheck}; +use crate::common_metric_data::{CommonMetricData, LabelCheck, MetricLabel}; use crate::error_recording::{test_get_num_recorded_errors, ErrorType}; use crate::histogram::HistogramType; use crate::metrics::{ @@ -251,7 +251,7 @@ where /// Creates a new metric with a specific label. /// /// This is used for static labels where we can just set the name to be `name/label`. - fn new_metric_with_label(&self, label: DynamicLabelType) -> T { + fn new_metric_with_label(&self, label: MetricLabel) -> T { self.submetric.with_label(label) } @@ -259,7 +259,7 @@ where /// /// This is used for dynamic labels where we have to actually validate and correct the /// label later when we have a Glean object. - fn new_metric_with_dynamic_label(&self, label: DynamicLabelType) -> T { + fn new_metric_with_dynamic_label(&self, label: MetricLabel) -> T { self.submetric.with_label(label) } @@ -318,10 +318,11 @@ where let metric = match self.labels { Some(_) => { let label = self.static_label(label); - self.new_metric_with_label(DynamicLabelType::Static(label.to_string())) + self.new_metric_with_label(MetricLabel::Static(label.to_string())) + } + None => { + self.new_metric_with_dynamic_label(MetricLabel::Label(label.to_string())) } - None => self - .new_metric_with_dynamic_label(DynamicLabelType::Label(label.to_string())), }; let metric = Arc::new(metric); entry.insert(Arc::clone(&metric)); diff --git a/glean-core/src/metrics/memory_distribution.rs b/glean-core/src/metrics/memory_distribution.rs index 3d7d87693e..9349537cf4 100644 --- a/glean-core/src/metrics/memory_distribution.rs +++ b/glean-core/src/metrics/memory_distribution.rs @@ -5,7 +5,7 @@ use std::mem; use std::sync::Arc; -use crate::common_metric_data::{CommonMetricDataInternal, DynamicLabelType}; +use crate::common_metric_data::{CommonMetricDataInternal, MetricLabel}; use crate::error_recording::{record_error, test_get_num_recorded_errors, ErrorType}; use crate::histogram::{Functional, Histogram}; use crate::metrics::memory_unit::MemoryUnit; @@ -63,9 +63,9 @@ impl MetricType for MemoryDistributionMetric { } } - fn with_label(&self, label: DynamicLabelType) -> Self { + fn with_label(&self, label: MetricLabel) -> Self { let mut meta = (*self.meta).clone(); - meta.inner.dynamic_label = Some(label); + meta.inner.label = Some(label); Self { meta: Arc::new(meta), memory_unit: self.memory_unit, diff --git a/glean-core/src/metrics/mod.rs b/glean-core/src/metrics/mod.rs index b0f1e57ccd..ddb1bee98c 100644 --- a/glean-core/src/metrics/mod.rs +++ b/glean-core/src/metrics/mod.rs @@ -41,7 +41,7 @@ mod url; mod uuid; use crate::common_metric_data::CommonMetricDataInternal; -pub use crate::common_metric_data::DynamicLabelType; +pub use crate::common_metric_data::MetricLabel; pub use crate::event_database::RecordedEvent; use crate::histogram::{Functional, Histogram, PrecomputedExponential, PrecomputedLinear}; pub use crate::metrics::datetime::Datetime; @@ -196,7 +196,7 @@ pub trait MetricType { } /// Create a new metric from this with a specific label. - fn with_label(&self, _label: DynamicLabelType) -> Self + fn with_label(&self, _label: MetricLabel) -> Self where Self: Sized, { @@ -298,7 +298,7 @@ where ( &meta.category, &meta.name, - meta.dynamic_label.as_ref().map(|label| label.to_string()), + meta.label.as_ref().map(|label| label.to_string()), ) } } diff --git a/glean-core/src/metrics/quantity.rs b/glean-core/src/metrics/quantity.rs index 82a73d8809..d20f6ddc90 100644 --- a/glean-core/src/metrics/quantity.rs +++ b/glean-core/src/metrics/quantity.rs @@ -4,7 +4,7 @@ use std::sync::Arc; -use crate::common_metric_data::{CommonMetricDataInternal, DynamicLabelType}; +use crate::common_metric_data::{CommonMetricDataInternal, MetricLabel}; use crate::error_recording::{record_error, test_get_num_recorded_errors, ErrorType}; use crate::metrics::Metric; use crate::metrics::MetricType; @@ -32,9 +32,9 @@ impl MetricType for QuantityMetric { } } - fn with_label(&self, label: DynamicLabelType) -> Self { + fn with_label(&self, label: MetricLabel) -> Self { let mut meta = (*self.meta).clone(); - meta.inner.dynamic_label = Some(label); + meta.inner.label = Some(label); Self { meta: Arc::new(meta), } diff --git a/glean-core/src/metrics/string.rs b/glean-core/src/metrics/string.rs index 1f696bc653..0bbbe645d4 100644 --- a/glean-core/src/metrics/string.rs +++ b/glean-core/src/metrics/string.rs @@ -4,7 +4,7 @@ use std::sync::Arc; -use crate::common_metric_data::{CommonMetricDataInternal, DynamicLabelType}; +use crate::common_metric_data::{CommonMetricDataInternal, MetricLabel}; use crate::error_recording::{test_get_num_recorded_errors, ErrorType}; use crate::metrics::Metric; use crate::metrics::MetricType; @@ -36,9 +36,9 @@ impl MetricType for StringMetric { } } - fn with_label(&self, label: DynamicLabelType) -> Self { + fn with_label(&self, label: MetricLabel) -> Self { let mut meta = (*self.meta).clone(); - meta.inner.dynamic_label = Some(label); + meta.inner.label = Some(label); Self { meta: Arc::new(meta), } @@ -161,7 +161,7 @@ mod test { send_in_pings: vec!["store1".into()], lifetime: Lifetime::Application, disabled: false, - dynamic_label: None, + label: None, }); let sample_string = "0123456789".repeat(26); diff --git a/glean-core/src/metrics/text.rs b/glean-core/src/metrics/text.rs index 8f111bf650..06900dbd33 100644 --- a/glean-core/src/metrics/text.rs +++ b/glean-core/src/metrics/text.rs @@ -4,7 +4,7 @@ use std::sync::Arc; -use crate::common_metric_data::{CommonMetricDataInternal, DynamicLabelType}; +use crate::common_metric_data::{CommonMetricDataInternal, MetricLabel}; use crate::error_recording::{test_get_num_recorded_errors, ErrorType}; use crate::metrics::Metric; use crate::metrics::MetricType; @@ -38,9 +38,9 @@ impl MetricType for TextMetric { } } - fn with_label(&self, label: DynamicLabelType) -> Self { + fn with_label(&self, label: MetricLabel) -> Self { let mut meta = (*self.meta).clone(); - meta.inner.dynamic_label = Some(label); + meta.inner.label = Some(label); Self { meta: Arc::new(meta), } @@ -165,7 +165,7 @@ mod test { send_in_pings: vec!["store1".into()], lifetime: Lifetime::Application, disabled: false, - dynamic_label: None, + label: None, }); let sample_string = "0123456789".repeat(200 * 1024); diff --git a/glean-core/src/metrics/timing_distribution.rs b/glean-core/src/metrics/timing_distribution.rs index 8a1df39c42..d7ed5735bb 100644 --- a/glean-core/src/metrics/timing_distribution.rs +++ b/glean-core/src/metrics/timing_distribution.rs @@ -10,7 +10,7 @@ use std::time::Duration; use malloc_size_of_derive::MallocSizeOf; -use crate::common_metric_data::{CommonMetricDataInternal, DynamicLabelType}; +use crate::common_metric_data::{CommonMetricDataInternal, MetricLabel}; use crate::error_recording::{record_error, test_get_num_recorded_errors, ErrorType}; use crate::histogram::{Functional, Histogram}; use crate::metrics::time_unit::TimeUnit; @@ -110,9 +110,9 @@ impl MetricType for TimingDistributionMetric { } } - fn with_label(&self, label: DynamicLabelType) -> Self { + fn with_label(&self, label: MetricLabel) -> Self { let mut meta = (*self.meta).clone(); - meta.inner.dynamic_label = Some(label); + meta.inner.label = Some(label); Self { meta: Arc::new(meta), time_unit: self.time_unit, diff --git a/glean-core/src/metrics/url.rs b/glean-core/src/metrics/url.rs index ab1e3bce02..58a1cf674e 100644 --- a/glean-core/src/metrics/url.rs +++ b/glean-core/src/metrics/url.rs @@ -179,7 +179,7 @@ mod test { send_in_pings: vec!["store1".into()], lifetime: Lifetime::Application, disabled: false, - dynamic_label: None, + label: None, }); let sample_url = "glean://test".to_string(); @@ -197,7 +197,7 @@ mod test { send_in_pings: vec!["store1".into()], lifetime: Lifetime::Application, disabled: false, - dynamic_label: None, + label: None, }); // Whenever the URL is longer than our MAX_URL_LENGTH, we truncate the URL to the @@ -235,7 +235,7 @@ mod test { send_in_pings: vec!["store1".into()], lifetime: Lifetime::Application, disabled: false, - dynamic_label: None, + label: None, }); let test_url = "data:application/json"; @@ -259,7 +259,7 @@ mod test { send_in_pings: vec!["store1".into()], lifetime: Lifetime::Application, disabled: false, - dynamic_label: None, + label: None, }); let incorrects = vec![ diff --git a/glean-core/tests/clientid_textfile.rs b/glean-core/tests/clientid_textfile.rs index 41b9a34d5d..454050e969 100644 --- a/glean-core/tests/clientid_textfile.rs +++ b/glean-core/tests/clientid_textfile.rs @@ -23,7 +23,7 @@ fn clientid_metric() -> UuidMetric { send_in_pings: vec!["glean_client_info".into()], lifetime: Lifetime::User, disabled: false, - dynamic_label: None, + label: None, }) } diff --git a/glean-core/tests/ping.rs b/glean-core/tests/ping.rs index cb63092160..d1ad8408c0 100644 --- a/glean-core/tests/ping.rs +++ b/glean-core/tests/ping.rs @@ -136,7 +136,7 @@ fn test_pings_submitted_metric() { send_in_pings: vec!["metrics".into(), "baseline".into()], lifetime: Lifetime::Ping, disabled: false, - dynamic_label: None, + label: None, }, }, None, From ba6b882515dc425364c4ec2b9e3c3f18ffe141ab Mon Sep 17 00:00:00 2001 From: Jan-Erik Rediger Date: Fri, 13 Feb 2026 15:51:38 +0100 Subject: [PATCH 22/30] Implement rkv->sqlite migration It will be applied at start if (1) no sqlite database is detected, and (2) an Rkv database is detected. Migration works by iterating through all data in the rkv "safe-mode" database and inserting it into the new database. The Rkv database will be kept on disk. This will allow for a rollback if any problems are detected in production and we can implement a recovery step then. --- glean-core/src/database/migration.rs | 332 ++++++++++++++++++ glean-core/src/database/mod.rs | 1 + glean-core/src/database/sqlite.rs | 12 +- ...a0472-5124-4f6b-971d-4a2a928fb158.safe.bin | Bin 0 -> 1155 bytes glean-core/tests/sqlite_migration.rs | 87 +++++ 5 files changed, 430 insertions(+), 2 deletions(-) create mode 100644 glean-core/src/database/migration.rs create mode 100644 glean-core/tests/77ca0472-5124-4f6b-971d-4a2a928fb158.safe.bin create mode 100644 glean-core/tests/sqlite_migration.rs diff --git a/glean-core/src/database/migration.rs b/glean-core/src/database/migration.rs new file mode 100644 index 0000000000..9550dc6c0c --- /dev/null +++ b/glean-core/src/database/migration.rs @@ -0,0 +1,332 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::fs; +use std::path::Path; +use std::str; + +use crate::metrics::Metric; +use crate::Error; +use crate::Lifetime; +use crate::Result; + +use rkv::StoreOptions; +use rusqlite::Transaction; + +use super::sqlite; + +pub type Rkv = rkv::Rkv; +pub type SingleStore = rkv::SingleStore; + +pub(crate) const RECORD_SEPARATOR: char = '\x1E'; + +pub fn rkv_new(path: &Path) -> std::result::Result { + match Rkv::new::(path) { + // An invalid file can mean: + // 1. An empty file. + // 2. A corrupted file. + // + // In both instances there's not much we can do. + // Drop the data by removing the file. + Err(rkv::StoreError::FileInvalid) => { + log::debug!("rkv failed: invalid file. starting from scratch."); + let safebin = path.join("data.safe.bin"); + fs::remove_file(safebin).map_err(|_| rkv::StoreError::FileInvalid)?; + Err(rkv::StoreError::FileInvalid) + } + Err(rkv::StoreError::DatabaseCorrupted) => { + log::debug!("rkv failed: database corrupted. starting from scratch."); + let safebin = path.join("data.safe.bin"); + fs::remove_file(safebin).map_err(|_| rkv::StoreError::DatabaseCorrupted)?; + Err(rkv::StoreError::DatabaseCorrupted) + } + other => { + let rkv = other?; + Ok(rkv) + } + } +} + +pub struct Database { + /// Handle to the database environment. + rkv: Rkv, + + /// Handles to the "lifetime" stores. + /// + /// A "store" is a handle to the underlying database. + /// We keep them open for fast and frequent access. + user_store: SingleStore, + ping_store: SingleStore, + application_store: SingleStore, +} + +impl Database { + /// Open the Rkv database and the embbedded stores. + pub fn new(data_path: &Path) -> Result { + log::debug!("Rkv database path: {:?}", data_path.display()); + + let rkv = Self::open_rkv(data_path)?; + let user_store = rkv.open_single(Lifetime::User.as_str(), StoreOptions::create())?; + let ping_store = rkv.open_single(Lifetime::Ping.as_str(), StoreOptions::create())?; + let application_store = + rkv.open_single(Lifetime::Application.as_str(), StoreOptions::create())?; + + let db = Self { + rkv, + user_store, + ping_store, + application_store, + }; + + Ok(db) + } + + fn open_rkv(path: &Path) -> Result { + let rkv = rkv_new(path)?; + Ok(rkv) + } + + fn get_store(&self, lifetime: Lifetime) -> &SingleStore { + match lifetime { + Lifetime::User => &self.user_store, + Lifetime::Ping => &self.ping_store, + Lifetime::Application => &self.application_store, + } + } + + /// Iterates with the provided transaction function + /// over the requested data from the given storage. + /// + /// * If the storage is unavailable, the transaction function is never invoked. + /// * If the read data cannot be deserialized it will be silently skipped. + /// + /// # Arguments + /// + /// * `lifetime` - The metric lifetime to iterate over. + /// * `transaction_fn` - Called for each entry being iterated over. + /// It is passed two arguments: `(key: String, metric: &Metric)`. + pub fn iter_store(&self, lifetime: Lifetime, mut transaction_fn: F) + where + F: FnMut(String, &Metric), + { + let Ok(reader) = self.rkv.read() else { return }; + let Ok(mut iter) = self.get_store(lifetime).iter_start(&reader) else { + log::debug!("No store for {lifetime:?}"); + return; + }; + + while let Some(Ok((key, value))) = iter.next() { + let Ok(key) = String::from_utf8(key.to_vec()) else { + log::debug!("Key is not valid UTF-8: {key:?}"); + continue; + }; + let metric: Metric = match value { + rkv::Value::Blob(blob) => { + let Ok(value) = bincode::deserialize(blob) else { + log::debug!("Value for key {key:?} could not be deserialized"); + continue; + }; + value + } + _ => { + log::debug!("Blob for key {key:?} is not a valid blob"); + continue; + } + }; + transaction_fn(key, &metric); + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct MetricKey<'a> { + ping: &'a str, + id: &'a str, + label: Option, +} + +/// Split a database key into its metric key parts. +fn split_key(key: &str) -> Option> { + let (ping, rest) = key.split_once('#')?; + if ping.is_empty() || rest.is_empty() { + return None; + } + + let (id, labels) = match rest.split_once(|c| ['/', RECORD_SEPARATOR].contains(&c)) { + Some((id, labels)) => { + if labels.is_empty() { + return None; + } + (id, labels) + } + _ => (rest, ""), + }; + if id.is_empty() { + return None; + } + + let label = if labels.is_empty() { + // No label at all + None + } else if labels.contains(RECORD_SEPARATOR) { + // Label separated by + let (key, category) = labels.split_once(RECORD_SEPARATOR)?; + + if key.is_empty() || category.is_empty() { + return None; + } + + Some(String::from(labels)) + } else { + Some(String::from(labels)) + }; + + Some(MetricKey { ping, id, label }) +} + +/// Migrate the `rkv` database to SQL. +/// +/// Returns the number of migrated metrics. +fn migrate(rkv: &Database, sql_db: &sqlite::Database, tx: &mut Transaction) -> usize { + let mut migrated_metrics = 0; + let mut migrate_metric = |lifetime: Lifetime, key: String, metric: &Metric| { + let Some(metric_id) = split_key(&key) else { + log::debug!("Invalid metric key: {key:?}"); + return; + }; + let label = metric_id.label.as_deref().unwrap_or(""); + _ = sql_db.record_per_lifetime(tx, lifetime, metric_id.ping, metric_id.id, label, metric); + migrated_metrics += 1; + }; + + let snapshotter_user = + |key: String, metric: &Metric| migrate_metric(Lifetime::User, key, metric); + rkv.iter_store(Lifetime::User, snapshotter_user); + + let snapshotter_app = + |key: String, metric: &Metric| migrate_metric(Lifetime::Application, key, metric); + rkv.iter_store(Lifetime::Application, snapshotter_app); + + let snapshotter_ping = + |key: String, metric: &Metric| migrate_metric(Lifetime::Ping, key, metric); + rkv.iter_store(Lifetime::Ping, snapshotter_ping); + + migrated_metrics +} + +pub fn try_migrate(data_path: &Path, db: &sqlite::Database) -> Result<()> { + use super::migration::{self, Database as RkvDatabase}; + + let rkv_file = data_path.join("data.safe.bin"); + log::debug!( + "Trying to migrate. Data path: {}, expected file: {}", + data_path.display(), + rkv_file.display() + ); + + if !rkv_file.exists() { + log::debug!("No rkv file. No migration."); + return Ok(()); + } + + let Ok(rkv) = RkvDatabase::new(data_path) else { + log::debug!("Can't open rkv database. No migration."); + return Ok(()); + }; + let count = db.conn.write(|tx| { + let count = migration::migrate(&rkv, db, tx); + Ok::<_, Error>(count) + })?; + + log::info!("{count} metrics migrated to sqlite"); + + log::debug!( + "Data migrated. Would be removing Rkv database at {}", + rkv_file.display() + ); + + Ok(()) +} + +#[cfg(test)] +mod test { + use super::*; + + impl<'a> MetricKey<'a> { + fn new<'b>(ping: &'a str, id: &'a str, label: impl Into>) -> Self { + Self { + ping, + id, + label: label.into().map(|s| s.to_string()), + } + } + } + + #[test] + fn splitting_key() { + let matches = &[ + (MetricKey::new("metrics", "name", None), "metrics#name"), + ( + MetricKey::new("metrics", "cat.name", None), + "metrics#cat.name", + ), + ( + MetricKey::new("metrics", "cat1.cat2.name", None), + "metrics#cat1.cat2.name", + ), + ( + MetricKey::new("metrics", "cat1.cat2.name", "label"), + "metrics#cat1.cat2.name/label", + ), + ( + // This currently works. We do allow slashes in labels. + // Maybe we shouldn't have. + MetricKey::new("metrics", "cat1.cat2.name", "label1/label2"), + "metrics#cat1.cat2.name/label1/label2", + ), + ( + // This currently works. We do allow slashes in labels. + // Maybe we shouldn't have. + MetricKey::new("metrics", "cat.name", "label//"), + "metrics#cat.name/label//", + ), + ( + MetricKey::new("metrics", "cat1.cat2.name", "label1\x1Elabel2"), + "metrics#cat1.cat2.name\x1elabel1\x1elabel2", + ), + ( + MetricKey::new("glean_internal_info", "baseline#sequence", None), + "glean_internal_info#baseline#sequence", + ), + ]; + + for (exp, key) in matches { + let m = split_key(key).unwrap_or_else(|| panic!("{key:?} should be splittable")); + assert_eq!(*exp, m, "did not split correctly: {key:?}"); + } + } + + #[test] + fn splitting_key_fails() { + let matches = &[ + "", + "metrics", + "metrics#", + "#cat", + "#cat.name", + "metrics#/", + "metrics#//", + "metrics#/label", + "metrics#cat.name/", + "metrics#cat.name\x1e", + "metrics#cat.name\x1e\x1e", + "metrics#cat.name\x1elabel1\x1e", + "metrics#cat.name\x1e\x1elabel2", + ]; + + for key in matches { + assert_eq!(None, split_key(key), "should not split: {key:?}"); + } + } +} diff --git a/glean-core/src/database/mod.rs b/glean-core/src/database/mod.rs index 190dbc6849..3c1fee301a 100644 --- a/glean-core/src/database/mod.rs +++ b/glean-core/src/database/mod.rs @@ -2,4 +2,5 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. +pub mod migration; pub mod sqlite; diff --git a/glean-core/src/database/sqlite.rs b/glean-core/src/database/sqlite.rs index e7ac4c0fa8..1e0104850a 100644 --- a/glean-core/src/database/sqlite.rs +++ b/glean-core/src/database/sqlite.rs @@ -18,6 +18,7 @@ use connection::Connection; use schema::Schema; use crate::common_metric_data::CommonMetricDataInternal; +use crate::database::migration; use crate::metrics::dual_labeled_counter::RECORD_SEPARATOR; use crate::metrics::Metric; use crate::Glean; @@ -29,7 +30,7 @@ mod schema; pub struct Database { /// The database connection. - conn: connection::Connection, + pub(crate) conn: connection::Connection, /// Initial file size when opening the database. pub(crate) file_size: Option, @@ -99,10 +100,17 @@ impl Database { fs::create_dir_all(&path)?; let store_path = path.join(DEFAULT_DATABASE_FILE_NAME); + let sqlite_exists = store_path.exists(); let conn = Connection::new::(&store_path).unwrap(); let db = Self { conn, file_size }; + if sqlite_exists { + log::debug!("SQLite database already exists. Not trying to migrate Rkv"); + } else { + _ = migration::try_migrate(&path, &db); + } + Ok(db) } @@ -299,7 +307,7 @@ impl Database { /// # Panics /// /// This function will **not** panic on database errors. - fn record_per_lifetime( + pub(crate) fn record_per_lifetime( &self, tx: &mut Transaction, lifetime: Lifetime, diff --git a/glean-core/tests/77ca0472-5124-4f6b-971d-4a2a928fb158.safe.bin b/glean-core/tests/77ca0472-5124-4f6b-971d-4a2a928fb158.safe.bin new file mode 100644 index 0000000000000000000000000000000000000000..86ebf4231c9c5fd527a375acbef3fd3adfda408d GIT binary patch literal 1155 zcma)5%T9za6rJ(S#>B@;jWHP?3j=9O8HBZ;;K~i@K*wo<9c+unzqf^3A3?`yAm|lOhafi zjnLRpb*t4}IrQ!O*daZC!d2${?Fvwjqrjq??i{F{9%3+u?Yqy!? zD9LFO;g}MV_HAz1VmB+uJ~uk5nH5#L>$&Ewyk0w@5zl;W{$IzG5Q2LCPN>geY=nG* H(j9yNUA)yg literal 0 HcmV?d00001 diff --git a/glean-core/tests/sqlite_migration.rs b/glean-core/tests/sqlite_migration.rs new file mode 100644 index 0000000000..9d6697c590 --- /dev/null +++ b/glean-core/tests/sqlite_migration.rs @@ -0,0 +1,87 @@ +mod common; +use std::fs; + +use crate::common::*; + +use glean_core::metrics::*; +use glean_core::CommonMetricData; +use glean_core::Lifetime; +use uuid::uuid; + +fn clientid_metric() -> UuidMetric { + UuidMetric::new(CommonMetricData { + name: "client_id".into(), + category: "".into(), + send_in_pings: vec!["glean_client_info".into()], + lifetime: Lifetime::User, + disabled: false, + label: None, + }) +} + +#[test] +fn migration_succeeds() { + let temp = tempfile::tempdir().unwrap(); + let db_path = temp.path().join("db"); + fs::create_dir_all(&db_path).unwrap(); + + let safe_bin = db_path.join("data.safe.bin"); + // File has been generated from essentially: + // + // ```rust + // let tmpname = PathBuf::new("/tmp/glean-fc"); + // let cfg = ConfigurationBuilder::new(true, tmpname.clone(), "glean-fc") + // .with_server_endpoint("invalid-test-host") + // .with_use_core_mps(false) + // .build(); + // glean::initialize(cfg, client_info); + // glean::shutdown(); + // ``` + // + // All ping-specific metrics have been removed. + // Only client_info metrics are migrated, including the client ID. + fs::write( + safe_bin, + include_bytes!("77ca0472-5124-4f6b-971d-4a2a928fb158.safe.bin"), + ) + .unwrap(); + let exp_client_id = uuid!("77ca0472-5124-4f6b-971d-4a2a928fb158"); + + let (glean, _temp) = new_glean(Some(temp)); + + let client_id = clientid_metric().get_value(&glean, None).unwrap(); + assert_eq!(exp_client_id, client_id); + + // TODO: validate migration metrics +} + +#[test] +fn migration_skipped_if_database_exists() { + let (first_client_id, temp) = { + let (glean, temp) = new_glean(None); + let client_id = clientid_metric().get_value(&glean, None).unwrap(); + drop(glean); + (client_id, temp) + }; + + let safe_bin = temp.path().join("db").join("data.safe.bin"); + fs::write( + &safe_bin, + include_bytes!("77ca0472-5124-4f6b-971d-4a2a928fb158.safe.bin"), + ) + .unwrap(); + let rkv_client_id = uuid!("77ca0472-5124-4f6b-971d-4a2a928fb158"); + + let (glean, _temp) = new_glean(Some(temp)); + + let client_id = clientid_metric().get_value(&glean, None).unwrap(); + assert_eq!( + first_client_id, client_id, + "Client ID should be the one first generated" + ); + assert_ne!( + rkv_client_id, client_id, + "Client ID should not be one from the Rkv database" + ); + assert!(safe_bin.exists(), "Rkv file should not have been deleted"); +} From 880a6b7562ce4618bd20021c791d2dd9a60eeb40 Mon Sep 17 00:00:00 2001 From: Jan-Erik Rediger Date: Mon, 16 Feb 2026 12:33:53 +0100 Subject: [PATCH 23/30] Re-enable client_id tests These tests were disabled because they are very rkv-specific: Manually opening and writing to an Rkv database in the format that Glean expects. Then testing Glean behaves accordingly. We now do the same, but do it in SQL. --- glean-core/tests/clientid_textfile.rs | 37 +++++++++------------------ 1 file changed, 12 insertions(+), 25 deletions(-) diff --git a/glean-core/tests/clientid_textfile.rs b/glean-core/tests/clientid_textfile.rs index 454050e969..3dbbc51180 100644 --- a/glean-core/tests/clientid_textfile.rs +++ b/glean-core/tests/clientid_textfile.rs @@ -8,8 +8,7 @@ use crate::common::*; use std::fs; use std::path::Path; -use rkv::Rkv; -use rkv::StoreOptions; +use rusqlite::params; use uuid::Uuid; use glean_core::metrics::*; @@ -126,16 +125,9 @@ fn clientid_regen_issue_with_existing_db() { // We modify the database and ONLY clear out the client id. { - let path = temp.path().join("db"); - let rkv = Rkv::new::(&path).unwrap(); - let user_store = rkv.open_single("user", StoreOptions::create()).unwrap(); - - // We know this. - let client_id_key = "glean_client_info#client_id"; - - let mut writer = rkv.write().unwrap(); - user_store.delete(&mut writer, client_id_key).unwrap(); - writer.commit().unwrap(); + let path = temp.path().join("db").join("glean.sqlite"); + let conn = rusqlite::Connection::open(path).unwrap(); + _ = conn.execute("DELETE FROM telemetry WHERE id = 'client_id'", ()); } let (glean, temp) = new_glean(Some(temp)); @@ -201,23 +193,18 @@ fn c0ffee_in_db_gets_overwritten_by_stored_client_id() { // We modify the database and ONLY set the client id to c0ffee. { - let path = temp.path().join("db"); - let rkv = Rkv::new::(&path).unwrap(); - let user_store = rkv.open_single("user", StoreOptions::create()).unwrap(); - - // We know this. - let client_id_key = "glean_client_info#client_id"; + let path = temp.path().join("db").join("glean.sqlite"); + let conn = rusqlite::Connection::open(path).unwrap(); - let mut writer = rkv.write().unwrap(); - let encoded = bincode::serialize(&Metric::Uuid(String::from( + let encoded = rmp_serde::to_vec(&Metric::Uuid(String::from( "c0ffeec0-ffee-c0ff-eec0-ffeec0ffeec0", ))) .unwrap(); - let known_client_id = rkv::Value::Blob(&encoded); - user_store - .put(&mut writer, client_id_key, &known_client_id) - .unwrap(); - writer.commit().unwrap(); + + _ = conn.execute( + "UPDATE telemetry SET value = ? WHERE id = 'client_id'", + params![encoded], + ); } let (glean, temp) = new_glean(Some(temp)); From ed8a7e03471f68ca34c6c6c985680808ff71959d Mon Sep 17 00:00:00 2001 From: Jan-Erik Rediger Date: Mon, 16 Feb 2026 11:43:43 +0100 Subject: [PATCH 24/30] Handle error recording in a single place again The previous refactoring duplicated some of the logic between different parts. Now we unify them again. --- glean-core/src/common_metric_data.rs | 11 +++- glean-core/src/database/sqlite.rs | 61 +++++++++++-------- glean-core/src/error_recording.rs | 91 +++++++++------------------- 3 files changed, 74 insertions(+), 89 deletions(-) diff --git a/glean-core/src/common_metric_data.rs b/glean-core/src/common_metric_data.rs index 7507cab0a7..c99f7b4d18 100644 --- a/glean-core/src/common_metric_data.rs +++ b/glean-core/src/common_metric_data.rs @@ -12,7 +12,7 @@ use crate::error::{Error, ErrorKind}; use crate::error_recording::record_error_sqlite; use crate::metrics::dual_labeled_counter::validate_dual_label_sqlite; use crate::metrics::labeled::validate_dynamic_label_sqlite; -use crate::ErrorType; +use crate::{ErrorType, Glean}; use serde::{Deserialize, Serialize}; /// The supported metrics' lifetimes. @@ -161,12 +161,19 @@ impl LabelCheck { } } - pub fn record_error(&self, tx: &mut Transaction, metric_name: &str, send_in_pings: &[String]) { + pub fn record_error( + &self, + glean: &Glean, + tx: &mut Transaction, + metric_name: &str, + send_in_pings: &[String], + ) { let LabelCheck::Error(_, count) = self else { return; }; record_error_sqlite( + glean, tx, metric_name, send_in_pings, diff --git a/glean-core/src/database/sqlite.rs b/glean-core/src/database/sqlite.rs index 1e0104850a..2b64ade12e 100644 --- a/glean-core/src/database/sqlite.rs +++ b/glean-core/src/database/sqlite.rs @@ -270,7 +270,7 @@ impl Database { _ = self.conn.write(|tx| { let labels = data.check_labels(tx); - labels.record_error(tx, &name, data.storage_names()); + labels.record_error(glean, tx, &name, data.storage_names()); for ping_name in data.storage_names() { if glean.is_ping_enabled(ping_name) { @@ -341,38 +341,51 @@ impl Database { /// Records the provided value, with the given lifetime, /// after applying a transformation function. - pub fn record_with(&self, glean: &Glean, data: &CommonMetricDataInternal, mut transform: F) + pub fn record_with(&self, glean: &Glean, data: &CommonMetricDataInternal, transform: F) where F: FnMut(Option) -> Metric, { - let name = data.base_identifier(); + _ = self + .conn + .write(|tx| self.record_with_transaction(glean, tx, data, transform)); + } - _ = self.conn.write(|tx| { - let labels = data.check_labels(tx); - labels.record_error(tx, &name, data.storage_names()); + pub fn record_with_transaction( + &self, + glean: &Glean, + tx: &mut Transaction, + data: &CommonMetricDataInternal, + mut transform: F, + ) -> Result<()> + where + F: FnMut(Option) -> Metric, + { + let name = data.base_identifier(); - for ping_name in data.storage_names() { - if glean.is_ping_enabled(ping_name) { - if let Err(e) = self.record_per_lifetime_with( - tx, - data.inner.lifetime, + let labels = data.check_labels(tx); + labels.record_error(glean, tx, &name, data.storage_names()); + + for ping_name in data.storage_names() { + if glean.is_ping_enabled(ping_name) { + if let Err(e) = self.record_per_lifetime_with( + tx, + data.inner.lifetime, + ping_name, + &name, + labels.label(), + &mut transform, + ) { + log::error!( + "Failed to record metric '{}' into {}: {:?}", + data.base_identifier(), ping_name, - &name, - labels.label(), - &mut transform, - ) { - log::error!( - "Failed to record metric '{}' into {}: {:?}", - data.base_identifier(), - ping_name, - e - ); - } + e + ); } } + } - Result::<(), rusqlite::Error>::Ok(()) - }); + Ok(()) } /// Records a metric in the underlying storage system, diff --git a/glean-core/src/error_recording.rs b/glean-core/src/error_recording.rs index a090e40472..3488e3bbad 100644 --- a/glean-core/src/error_recording.rs +++ b/glean-core/src/error_recording.rs @@ -13,8 +13,8 @@ //! not some constant value that we could define in `metrics.yaml`. use std::fmt::Display; +use std::sync::atomic::AtomicU8; -use rusqlite::params; use rusqlite::Transaction; use crate::common_metric_data::CommonMetricDataInternal; @@ -145,6 +145,7 @@ pub fn record_error>>( } pub fn record_error_sqlite( + glean: &Glean, tx: &mut Transaction, metric_name: &str, send_in_pings: &[String], @@ -153,75 +154,39 @@ pub fn record_error_sqlite( ) { assert!(num_errors > 0); - let ping_name = "metrics".to_string(); - let need_metrics = !send_in_pings.contains(&ping_name); + // We explicitly don't use the `Counter` metric directly here. + // + // * This is called from within the recording functions in `sqlite.rs` + // * That means a transaction is already opened. We can't open a new one. + // * We can avoid some allocations by constructing only what we need and what we already have + + let ping_name = String::from("metrics"); + let mut send_in_pings = send_in_pings.to_vec(); + if !send_in_pings.contains(&ping_name) { + send_in_pings.push(ping_name); + } - let full_id = format!("glean.error.{}", error.as_str()); let lifetime = Lifetime::Ping; let transform = |old_value| match old_value { Some(Metric::Counter(old_value)) => Metric::Counter(old_value.saturating_add(num_errors)), _ => Metric::Counter(num_errors), }; - let value_sql = r#" - SELECT value - FROM telemetry - WHERE - id = ?1 - AND ping = ?2 - AND lifetime = ?3 - AND labels = ?4 - LIMIT 1 - "#; - - let insert_sql = r#" - INSERT INTO - telemetry (id, ping, lifetime, labels, value) - VALUES - (?1, ?2, ?3, ?4, ?5) - ON CONFLICT(id, ping, labels) DO UPDATE SET - lifetime = excluded.lifetime, - value = excluded.value - "#; - - for ping in send_in_pings - .iter() - .chain(need_metrics.then_some(&ping_name)) - { - let new_value = { - let mut stmt = tx.prepare_cached(value_sql).unwrap(); - let mut rows = stmt - .query(params![ - full_id, - ping, - lifetime.as_str().to_string(), - metric_name - ]) - .unwrap(); - - if let Ok(Some(row)) = rows.next() { - let blob: Vec = row.get(0).unwrap(); - let old_value = rmp_serde::from_slice(&blob).ok(); - transform(old_value) - } else { - transform(None) - } - }; - - { - let mut stmt = tx.prepare_cached(insert_sql).unwrap(); - let encoded = - rmp_serde::to_vec(&new_value).expect("IMPOSSIBLE: Serializing metric failed"); - stmt.execute(params![ - full_id, - ping, - lifetime.as_str(), - metric_name, - encoded - ]) - .unwrap(); - } - } + let inner = CommonMetricData { + category: String::from("glean.error"), + name: String::from(error.as_str()), + send_in_pings, + lifetime, + label: Some(MetricLabel::Static(String::from(metric_name))), + ..Default::default() + }; + let cmd = CommonMetricDataInternal { + inner, + disabled: AtomicU8::new(0), + }; + _ = glean + .storage() + .record_with_transaction(glean, tx, &cmd, transform); } /// Gets the number of recorded errors for the given metric and error type. From 8e5fe3abe8305749902996a07c1c11cd17311158 Mon Sep 17 00:00:00 2001 From: Jan-Erik Rediger Date: Mon, 16 Feb 2026 14:30:21 +0100 Subject: [PATCH 25/30] Test: ensure we don't record errors if we aren't enabled --- glean-core/tests/collection_enabled.rs | 37 ++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/glean-core/tests/collection_enabled.rs b/glean-core/tests/collection_enabled.rs index 78ae96dcfe..de36f415a7 100644 --- a/glean-core/tests/collection_enabled.rs +++ b/glean-core/tests/collection_enabled.rs @@ -9,6 +9,7 @@ use glean_core::metrics::*; use glean_core::CommonMetricData; use glean_core::Glean; use glean_core::Lifetime; +use glean_core::{test_get_num_recorded_errors, ErrorType}; fn nofollows_ping(glean: &mut Glean) -> PingType { // When `follows_collection_enabled=false` then by default `enabled=false` @@ -198,3 +199,39 @@ fn queued_nofollows_pings_are_not_removed() { let (glean, _t) = new_glean(Some(t)); assert_eq!(1, get_queued_pings(glean.get_data_path()).unwrap().len()); } + +#[test] +fn label_errors_when_collection_disabled() { + let (mut glean, _t) = new_glean(None); + + let manual_ping = manual_ping(&mut glean); + glean.set_ping_enabled(&manual_ping, false); + + let labeled = LabeledCounter::new( + LabeledMetricData::Common { + cmd: CommonMetricData { + name: "labeled_metric".into(), + category: "telemetry".into(), + send_in_pings: vec!["manual".into()], + disabled: false, + lifetime: Lifetime::Ping, + ..Default::default() + }, + }, + None, + ); + + let long_label = "a".repeat(112); + let metric = labeled.get(&long_label); + + // Attempt to increment the counter with zero + metric.add_sync(&glean, 1); + // Check that nothing was recorded. + assert!(metric.get_value(&glean, Some("manual")).is_none()); + + // Make sure that the error has been recorded + assert_eq!( + Ok(1), + test_get_num_recorded_errors(&glean, metric.meta(), ErrorType::InvalidLabel) + ); +} From 624cb287286f5530368aaecfc511e874354c1bf7 Mon Sep 17 00:00:00 2001 From: Jan-Erik Rediger Date: Mon, 16 Feb 2026 16:52:14 +0100 Subject: [PATCH 26/30] Handle sqlite load errors --- glean-core/metrics.yaml | 8 ++-- glean-core/src/core/mod.rs | 8 ++-- glean-core/src/database/sqlite.rs | 69 +++++++++++++++++++++++++++--- glean-core/src/error.rs | 19 ++++++++ glean-core/src/internal_metrics.rs | 8 ++-- glean-core/src/lib_unit_tests.rs | 6 +-- 6 files changed, 97 insertions(+), 21 deletions(-) diff --git a/glean-core/metrics.yaml b/glean-core/metrics.yaml index 0f4b8aac5a..eb61cca5ef 100644 --- a/glean-core/metrics.yaml +++ b/glean-core/metrics.yaml @@ -924,17 +924,17 @@ glean.database: - glean-team@mozilla.com expires: never - rkv_load_error: + load_error: type: string description: | - If there was an error loading the RKV database, record it. + If there was an error loading the sqlite database, record it. send_in_pings: - metrics - health bugs: - - https://bugzilla.mozilla.org/show_bug.cgi?id=1815253 + - https://bugzilla.mozilla.org/show_bug.cgi?id=TODO data_reviews: - - https://bugzilla.mozilla.org/show_bug.cgi?id=1815253 + - https://bugzilla.mozilla.org/show_bug.cgi?id=TODO data_sensitivity: - technical notification_emails: diff --git a/glean-core/src/core/mod.rs b/glean-core/src/core/mod.rs index bc2f8d7e2e..f145af411d 100644 --- a/glean-core/src/core/mod.rs +++ b/glean-core/src/core/mod.rs @@ -680,14 +680,12 @@ impl Glean { .accumulate_sync(self, size.get() as i64) } - if let Some(rkv_load_state) = self + if let Some(load_state) = self .data_store .as_ref() - .and_then(|database| database.rkv_load_state()) + .and_then(|database| database.load_state()) { - self.database_metrics - .rkv_load_error - .set_sync(self, rkv_load_state) + self.database_metrics.load_error.set_sync(self, load_state) } } diff --git a/glean-core/src/database/sqlite.rs b/glean-core/src/database/sqlite.rs index 2b64ade12e..9b8eaf30ce 100644 --- a/glean-core/src/database/sqlite.rs +++ b/glean-core/src/database/sqlite.rs @@ -13,14 +13,17 @@ use rusqlite::params; use rusqlite::types::FromSqlError; use rusqlite::OptionalExtension; use rusqlite::Transaction; +use rusqlite::{Error as SqlError, ErrorCode}; use connection::Connection; use schema::Schema; +pub use schema::SchemaError; use crate::common_metric_data::CommonMetricDataInternal; use crate::database::migration; use crate::metrics::dual_labeled_counter::RECORD_SEPARATOR; use crate::metrics::Metric; +use crate::Error; use crate::Glean; use crate::Lifetime; use crate::Result; @@ -28,12 +31,21 @@ use crate::Result; mod connection; mod schema; +#[derive(Debug)] +pub enum LoadState { + Ok, + Err(Error), +} + pub struct Database { /// The database connection. pub(crate) conn: connection::Connection, /// Initial file size when opening the database. pub(crate) file_size: Option, + + /// Load state + load_state: LoadState, } impl MallocSizeOf for Database { @@ -83,6 +95,45 @@ fn database_size(dir: &Path) -> Option { NonZeroU64::new(total_size) } +pub fn sqlite_open(path: &Path) -> std::result::Result<(Connection, LoadState), Error> { + // TODO: Make this more robust, use the correct errors and see how we can test all the branches + // properly. + match Connection::new::(path) { + Err(e @ SchemaError::UnsupportedSchemaVersion(_)) => Err(e.into()), + Err(e @ SchemaError::Sqlite(SqlError::SqliteFailure(err, _))) => { + match err.code { + ErrorCode::PermissionDenied => Err(e.into()), + ErrorCode::NotADatabase => { + log::debug!("sqlite failed: not a database. starting from scratch."); + fs::remove_file(path).map_err(|_| rkv::StoreError::FileInvalid)?; + // Now try again, we only handle that error once. + let conn = Connection::new::(path)?; + Ok((conn, LoadState::Err(e.into()))) + } + ErrorCode::CannotOpen => { + log::debug!("sqlite failed: cannot open. starting from scratch."); + fs::remove_file(path).map_err(|_| rkv::StoreError::FileInvalid)?; + // Now try again, we only handle that error once. + let conn = Connection::new::(path)?; + Ok((conn, LoadState::Err(e.into()))) + } + _ => Err(e.into()), + } + } + Err(err @ SchemaError::Sqlite(SqlError::SqlInputError { .. })) => { + log::debug!("sqlite failed: schema migration failed. starting from scratch."); + fs::remove_file(path).map_err(|_| rkv::StoreError::FileInvalid)?; + // Now try again, we only handle that error once. + let conn = Connection::new::(path)?; + Ok((conn, LoadState::Err(err.into()))) + } + other => { + let conn = other?; + Ok((conn, LoadState::Ok)) + } + } +} + impl Database { /// Initializes the data store. /// @@ -101,9 +152,13 @@ impl Database { fs::create_dir_all(&path)?; let store_path = path.join(DEFAULT_DATABASE_FILE_NAME); let sqlite_exists = store_path.exists(); - let conn = Connection::new::(&store_path).unwrap(); + let (conn, load_state) = sqlite_open(&store_path)?; - let db = Self { conn, file_size }; + let db = Self { + conn, + file_size, + load_state, + }; if sqlite_exists { log::debug!("SQLite database already exists. Not trying to migrate Rkv"); @@ -119,9 +174,13 @@ impl Database { self.file_size } - /// Get the rkv load state. - pub fn rkv_load_state(&self) -> Option { - None + /// Get the load state. + pub fn load_state(&self) -> Option { + if let LoadState::Err(e) = &self.load_state { + Some(e.to_string()) + } else { + None + } } /// Iterates with the provided transaction function diff --git a/glean-core/src/error.rs b/glean-core/src/error.rs index 51e9fa7a17..58823a93d9 100644 --- a/glean-core/src/error.rs +++ b/glean-core/src/error.rs @@ -9,6 +9,8 @@ use std::result; use rkv::StoreError; +use crate::database::sqlite::SchemaError; + /// A specialized [`Result`] type for this crate's operations. /// /// This is generally used to avoid writing out [`Error`] directly and @@ -68,6 +70,9 @@ pub enum ErrorKind { /// Database/SQLite error SQLite(rusqlite::Error), + + /// Schema error + Schema(SchemaError), } /// A specialized [`Error`] type for this crate's operations. @@ -125,6 +130,7 @@ impl Display for Error { ), UuidError(e) => write!(f, "Failed to parse UUID: {}", e), SQLite(e) => write!(f, "SQLite error: {}", e), + Schema(e) => write!(f, "Schema error: {}", e), } } } @@ -167,6 +173,19 @@ impl From for Error { } } +impl From for Error { + fn from(error: SchemaError) -> Error { + match error { + SchemaError::Sqlite(err) => Error { + kind: ErrorKind::SQLite(err), + }, + err => Error { + kind: ErrorKind::Schema(err), + }, + } + } +} + impl From for Error { fn from(error: OsString) -> Error { Error { diff --git a/glean-core/src/internal_metrics.rs b/glean-core/src/internal_metrics.rs index 00704f4b6f..b418b93316 100644 --- a/glean-core/src/internal_metrics.rs +++ b/glean-core/src/internal_metrics.rs @@ -355,8 +355,8 @@ impl UploadMetrics { pub struct DatabaseMetrics { pub size: MemoryDistributionMetric, - /// RKV's load result, indicating success or relaying the detected error. - pub rkv_load_error: StringMetric, + /// sqlite's load result, indicating success or relaying the detected error. + pub load_error: StringMetric, /// The time it takes for a write-commit for the Glean database. pub write_time: TimingDistributionMetric, @@ -377,8 +377,8 @@ impl DatabaseMetrics { MemoryUnit::Byte, ), - rkv_load_error: StringMetric::new(CommonMetricData { - name: "rkv_load_error".into(), + load_error: StringMetric::new(CommonMetricData { + name: "load_error".into(), category: "glean.database".into(), send_in_pings: vec!["metrics".into(), "health".into()], lifetime: Lifetime::Ping, diff --git a/glean-core/src/lib_unit_tests.rs b/glean-core/src/lib_unit_tests.rs index 97df93f2b7..aa454de082 100644 --- a/glean-core/src/lib_unit_tests.rs +++ b/glean-core/src/lib_unit_tests.rs @@ -1190,10 +1190,10 @@ fn records_database_file_size() { // We should see the database containing some data. assert!(data.sum > 0); - let rkv_load_state = &glean.database_metrics.rkv_load_error; - let rkv_load_error = rkv_load_state.get_value(&glean, "metrics"); + let load_state = &glean.database_metrics.load_error; + let load_error = load_state.get_value(&glean, "metrics"); - assert_eq!(rkv_load_error, None); + assert_eq!(load_error, None); } #[cfg(not(target_os = "windows"))] From e6b6df17f30618610a2299b59494e50d9b51da93 Mon Sep 17 00:00:00 2001 From: Jan-Erik Rediger Date: Tue, 17 Feb 2026 11:40:28 +0100 Subject: [PATCH 27/30] Test several sqlite edge cases What individual tests do should be clear from their name or further comments inline. --- glean-core/src/database/sqlite.rs | 2 +- glean-core/tests/sqlite.rs | 185 ++++++++++++++++++++++++++++++ 2 files changed, 186 insertions(+), 1 deletion(-) create mode 100644 glean-core/tests/sqlite.rs diff --git a/glean-core/src/database/sqlite.rs b/glean-core/src/database/sqlite.rs index 9b8eaf30ce..d82040fc0d 100644 --- a/glean-core/src/database/sqlite.rs +++ b/glean-core/src/database/sqlite.rs @@ -276,7 +276,7 @@ impl Database { }) .optional() }) - .unwrap_or(None) + .unwrap_or(None) // TODO: Should we handle the error here properly? } /// Determines if the storage has the given metric. diff --git a/glean-core/tests/sqlite.rs b/glean-core/tests/sqlite.rs new file mode 100644 index 0000000000..28c6ba64e5 --- /dev/null +++ b/glean-core/tests/sqlite.rs @@ -0,0 +1,185 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +mod common; +use std::fs; + +use crate::common::*; + +use glean_core::metrics::*; +use glean_core::CommonMetricData; +use glean_core::Lifetime; +use rusqlite::params; +use uuid::uuid; + +fn clientid_metric() -> UuidMetric { + UuidMetric::new(CommonMetricData { + name: "client_id".into(), + category: "".into(), + send_in_pings: vec!["glean_client_info".into()], + lifetime: Lifetime::User, + disabled: false, + label: None, + }) +} + +#[test] +fn database_file_is_not_sqlite() { + let temp = { + let (glean, temp) = new_glean(None); + drop(glean); + temp + }; + + { + let path = temp.path().join("db").join("glean.sqlite"); + fs::remove_file(&path).unwrap(); + fs::write(&path, "not sqlite").unwrap(); + } + + let (glean, _temp) = new_glean(Some(temp)); + + let client_id = clientid_metric().get_value(&glean, None); + assert!(client_id.is_some()); +} + +#[test] +fn database_contains_wrong_table() { + let temp = { + let (glean, temp) = new_glean(None); + drop(glean); + temp + }; + + { + let path = temp.path().join("db").join("glean.sqlite"); + fs::remove_file(&path).unwrap(); + + let conn = rusqlite::Connection::open(path).unwrap(); + conn.execute("CREATE TABLE telemetry (a TEXT)", ()).unwrap(); + } + + let (glean, _temp) = new_glean(Some(temp)); + + let client_id = clientid_metric().get_value(&glean, None); + assert!(client_id.is_some()); +} + +#[test] +#[ignore] +fn database_contains_correct_user_version_but_wrong_table() { + let temp = { + let (glean, temp) = new_glean(None); + drop(glean); + temp + }; + + { + let path = temp.path().join("db").join("glean.sqlite"); + let conn = rusqlite::Connection::open(path).unwrap(); + conn.execute("DROP TABLE telemetry", ()).unwrap(); + conn.execute("CREATE TABLE telemetry (a TEXT)", ()).unwrap(); + } + + let (glean, _temp) = new_glean(Some(temp)); + + let client_id = clientid_metric().get_value(&glean, None); + assert!(client_id.is_some()); +} + +#[test] +fn invalid_msgpack_value() { + let (first_client_id, temp) = { + let (glean, temp) = new_glean(None); + let client_id = clientid_metric().get_value(&glean, None).unwrap(); + drop(glean); + (client_id, temp) + }; + + { + let path = temp.path().join("db").join("glean.sqlite"); + let conn = rusqlite::Connection::open(path).unwrap(); + conn.execute( + "UPDATE telemetry SET value = ?1 WHERE id = 'client_id'", + params![b"c0ffeec0-ffee-c0ff-eec0-ffeec0ffeec0"], + ) + .unwrap(); + + // Also remove the client_id.txt so the client_id is not re-set from it. + fs::remove_file(temp.path().join("client_id.txt")).unwrap(); + } + + let (glean, _temp) = new_glean(Some(temp)); + + let client_id = clientid_metric().get_value(&glean, None).unwrap(); + let known_id = uuid!("c0ffeec0-ffee-c0ff-eec0-ffeec0ffeec0"); + assert_ne!(known_id, client_id); + assert_ne!(first_client_id, client_id); +} + +#[test] +fn higher_user_version_upgrade_does_not_crash() { + let (first_client_id, temp) = { + let (glean, temp) = new_glean(None); + let client_id = clientid_metric().get_value(&glean, None).unwrap(); + drop(glean); + (client_id, temp) + }; + + { + let path = temp.path().join("db").join("glean.sqlite"); + let conn = rusqlite::Connection::open(path).unwrap(); + conn.execute_batch("PRAGMA user_version = 2").unwrap(); + } + + let (glean, _temp) = new_glean(Some(temp)); + + let client_id = clientid_metric().get_value(&glean, None).unwrap(); + assert_eq!(first_client_id, client_id); +} + +// Permissions only really work on Unix systems, definitely not on Windows +#[cfg(unix)] +mod unix { + use glean_core::Glean; + + use super::*; + + #[test] + fn database_permission_error() { + let temp = tempfile::tempdir().unwrap(); + + let db_path = temp.path().join("db"); + fs::create_dir_all(&db_path).unwrap(); + let path = db_path.join("glean.sqlite"); + fs::write(&path, "").unwrap(); + let attr = fs::metadata(&path).unwrap(); + let original_permissions = attr.permissions(); + let mut permissions = original_permissions.clone(); + permissions.set_readonly(true); + fs::set_permissions(&path, permissions).unwrap(); + + let cfg = glean_core::InternalConfiguration { + data_path: path.display().to_string(), + application_id: GLOBAL_APPLICATION_ID.into(), + language_binding_name: "Rust".into(), + upload_enabled: true, + max_events: None, + delay_ping_lifetime_io: false, + app_build: "Unknown".into(), + use_core_mps: false, + trim_data_to_registered_pings: false, + log_level: None, + rate_limit: None, + enable_event_timestamps: false, + experimentation_id: None, + enable_internal_pings: true, + ping_schedule: Default::default(), + ping_lifetime_threshold: 0, + ping_lifetime_max_time: 0, + }; + let glean = Glean::new(cfg); + assert!(glean.is_err()); + } +} From 98425c13df1ea07241fe2eaabfe887ba3ca6cff8 Mon Sep 17 00:00:00 2001 From: Jan-Erik Rediger Date: Tue, 17 Feb 2026 15:13:49 +0100 Subject: [PATCH 28/30] Test what happens if database is otherwise locked This currently fails. The database is locked, so Glean can't access it. It's unclear how we should handle that. It's not a particular likely case to happen in practice. --- glean-core/tests/sqlite.rs | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/glean-core/tests/sqlite.rs b/glean-core/tests/sqlite.rs index 28c6ba64e5..3f678f6661 100644 --- a/glean-core/tests/sqlite.rs +++ b/glean-core/tests/sqlite.rs @@ -11,6 +11,7 @@ use glean_core::metrics::*; use glean_core::CommonMetricData; use glean_core::Lifetime; use rusqlite::params; +use rusqlite::TransactionBehavior; use uuid::uuid; fn clientid_metric() -> UuidMetric { @@ -183,3 +184,29 @@ mod unix { assert!(glean.is_err()); } } + +// TODO: +// This currently fails. +// The database is locked, so Glean can't access it. +// It's unclear how we should handle that. +// It's not a particular likely case to happen in practice. +#[test] +#[ignore] +fn database_externally_locked() { + let temp = { + let (glean, temp) = new_glean(None); + drop(glean); + temp + }; + + let path = temp.path().join("db").join("glean.sqlite"); + let mut conn = rusqlite::Connection::open(path).unwrap(); + let _tx = conn + .transaction_with_behavior(TransactionBehavior::Immediate) + .unwrap(); + + let (glean, _temp) = new_glean(Some(temp)); + + let client_id = clientid_metric().get_value(&glean, None); + assert!(client_id.is_some()); +} From 14f464281e56fc5fc269d0c0770f3be2e09cc1ef Mon Sep 17 00:00:00 2001 From: Jan-Erik Rediger Date: Thu, 19 Feb 2026 10:21:00 +0100 Subject: [PATCH 29/30] Handle possible SQLite query errors and bubble them up instead of blindly panicking --- glean-core/src/database/sqlite.rs | 57 ++++++++++++++++--------------- glean-core/src/storage/mod.rs | 25 ++++++++++---- 2 files changed, 47 insertions(+), 35 deletions(-) diff --git a/glean-core/src/database/sqlite.rs b/glean-core/src/database/sqlite.rs index d82040fc0d..9b954fe666 100644 --- a/glean-core/src/database/sqlite.rs +++ b/glean-core/src/database/sqlite.rs @@ -199,7 +199,12 @@ impl Database { /// # Panics /// /// This function will **not** panic on database errors. - pub fn iter_store(&self, lifetime: Lifetime, storage_name: &str, mut transaction_fn: F) + pub fn iter_store( + &self, + lifetime: Lifetime, + storage_name: &str, + mut transaction_fn: F, + ) -> Result<()> where F: FnMut(&[u8], &[&str], &Metric), { @@ -214,34 +219,30 @@ impl Database { AND ping = ?2 "#; - self.conn - .read(|conn| { - let mut stmt = conn.prepare_cached(iter_sql).unwrap(); - let rows = stmt - .query_map( - params![lifetime.as_str().to_string(), storage_name], - |row| { - let id: String = row.get(0)?; - let blob: Vec = row.get(1)?; - let labels: String = row.get(2)?; - let blob: Metric = rmp_serde::from_slice(&blob) - .map_err(|_| FromSqlError::InvalidType)?; - Ok((id, labels, blob)) - }, - ) - .unwrap(); - - for row in rows { - let Ok((metric_id, labels, metric)) = row else { - continue; - }; - let labels = labels.split(RECORD_SEPARATOR).collect::>(); - transaction_fn(metric_id.as_bytes(), &labels, &metric); - } + self.conn.read(|conn| { + let mut stmt = conn.prepare_cached(iter_sql)?; + let rows = stmt.query_map( + params![lifetime.as_str().to_string(), storage_name], + |row| { + let id: String = row.get(0)?; + let blob: Vec = row.get(1)?; + let labels: String = row.get(2)?; + let blob: Metric = + rmp_serde::from_slice(&blob).map_err(|_| FromSqlError::InvalidType)?; + Ok((id, labels, blob)) + }, + )?; - Result::<(), ()>::Ok(()) - }) - .unwrap() + for row in rows { + let Ok((metric_id, labels, metric)) = row else { + continue; + }; + let labels = labels.split(RECORD_SEPARATOR).collect::>(); + transaction_fn(metric_id.as_bytes(), &labels, &metric); + } + + Ok(()) + }) } /// TODO diff --git a/glean-core/src/storage/mod.rs b/glean-core/src/storage/mod.rs index c602a5a4e4..c47eb901df 100644 --- a/glean-core/src/storage/mod.rs +++ b/glean-core/src/storage/mod.rs @@ -141,13 +141,21 @@ impl StorageManager { } }; - storage.iter_store(Lifetime::Ping, store_name, &mut snapshotter); - storage.iter_store(Lifetime::Application, store_name, &mut snapshotter); - storage.iter_store(Lifetime::User, store_name, &mut snapshotter); + if let Err(e) = storage.iter_store(Lifetime::Ping, store_name, &mut snapshotter) { + log::debug!("could not snapshot ping lifetime store: {e:?}"); + } + if let Err(e) = storage.iter_store(Lifetime::Application, store_name, &mut snapshotter) { + log::debug!("could not snapshot ping lifetime store: {e:?}"); + } + if let Err(e) = storage.iter_store(Lifetime::User, store_name, &mut snapshotter) { + log::debug!("could not snapshot ping lifetime store: {e:?}"); + } // Add send in all pings client.annotations if store_name != "glean_client_info" { - storage.iter_store(Lifetime::Application, "all-pings", snapshotter); + if let Err(e) = storage.iter_store(Lifetime::Application, "all-pings", snapshotter) { + log::debug!("could not snapshot metrics for 'all-pings': {e:?}"); + } } if clear_store { @@ -190,8 +198,9 @@ impl StorageManager { } }; - storage.iter_store(metric_lifetime, store_name, &mut snapshotter); - + storage + .iter_store(metric_lifetime, store_name, &mut snapshotter) + .ok()?; snapshot } @@ -267,7 +276,9 @@ impl StorageManager { } }; - storage.iter_store(Lifetime::Application, store_name, &mut snapshotter); + storage + .iter_store(Lifetime::Application, store_name, &mut snapshotter) + .ok()?; if snapshot.is_empty() { None From 7872360c347dc65ed22ce6645a903af3173b3089 Mon Sep 17 00:00:00 2001 From: Jan-Erik Rediger Date: Thu, 19 Feb 2026 16:55:47 +0100 Subject: [PATCH 30/30] A test to verify that all metrics are properly migrated. The data was generated with cargo run -p glean-tests --bin verify-data -- tmp on an Rkv-powered Glean checkout. The database (`tmp/db/data.safe.bin`) was then copied into glean-core/rlb/tests/rkv-database.safe.bin --- Cargo.lock | 1 + glean-core/rlb-tests/Cargo.toml | 3 + glean-core/rlb-tests/build.rs | 9 + glean-core/rlb-tests/metrics.yaml | 257 ++++++++++++++++++ glean-core/rlb-tests/pings.yaml | 22 ++ glean-core/rlb-tests/src/bin/verify-data.rs | 254 +++++++++++++++++ glean-core/rlb-tests/tests/it.rs | 16 ++ .../rlb-tests/tests/rkv-database.safe.bin | Bin 0 -> 4065 bytes 8 files changed, 562 insertions(+) create mode 100644 glean-core/rlb-tests/build.rs create mode 100644 glean-core/rlb-tests/metrics.yaml create mode 100644 glean-core/rlb-tests/pings.yaml create mode 100644 glean-core/rlb-tests/src/bin/verify-data.rs create mode 100644 glean-core/rlb-tests/tests/rkv-database.safe.bin diff --git a/Cargo.lock b/Cargo.lock index eddf0ceea9..a9c3fafce7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -640,6 +640,7 @@ dependencies = [ "env_logger", "flate2", "glean", + "glean-build", "libc", "log", "once_cell", diff --git a/glean-core/rlb-tests/Cargo.toml b/glean-core/rlb-tests/Cargo.toml index 4a879203af..77be9081fb 100644 --- a/glean-core/rlb-tests/Cargo.toml +++ b/glean-core/rlb-tests/Cargo.toml @@ -17,3 +17,6 @@ serde_json = "1.0.44" [dev-dependencies] assert_cmd = "2.1.2" tempfile = "3.1.0" + +[build-dependencies] +glean-build = { path = "../build" } diff --git a/glean-core/rlb-tests/build.rs b/glean-core/rlb-tests/build.rs new file mode 100644 index 0000000000..8a17552ca4 --- /dev/null +++ b/glean-core/rlb-tests/build.rs @@ -0,0 +1,9 @@ +use glean_build::Builder; + +fn main() { + Builder::default() + .file("metrics.yaml") + .file("pings.yaml") + .generate() + .expect("Error generating Glean Rust bindings"); +} diff --git a/glean-core/rlb-tests/metrics.yaml b/glean-core/rlb-tests/metrics.yaml new file mode 100644 index 0000000000..f8c0146131 --- /dev/null +++ b/glean-core/rlb-tests/metrics.yaml @@ -0,0 +1,257 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# This file defines the metrics that are recorded by the Glean SDK. They are +# automatically converted to Rust code at build time using the `glean_parser` +# PyPI package. + +--- + +$schema: moz://mozilla.org/schemas/glean/metrics/2-0-0 + +test.metrics: + sample_counter: + type: counter + description: | + Just testing booleans + bugs: + - https://bugzilla.mozilla.org/123456789 + data_reviews: + - N/A + notification_emails: + - CHANGE-ME@example.com + expires: never + send_in_pings: + - prototype + - usage-reporting + + sample_url: + type: url + description: | + Just testing booleans + bugs: + - https://bugzilla.mozilla.org/123456789 + data_reviews: + - N/A + notification_emails: + - CHANGE-ME@example.com + expires: never + send_in_pings: + - prototype + - usage-reporting + + sample_boolean: + type: boolean + description: | + Just testing booleans + bugs: + - https://bugzilla.mozilla.org/123456789 + data_reviews: + - N/A + notification_emails: + - CHANGE-ME@example.com + expires: never + send_in_pings: + - prototype + - usage-reporting + + sample_labeled_counter: &defaults + type: labeled_counter + description: | + Just testing labeled_counter. + bugs: + - https://bugzilla.mozilla.org/1907991 + data_reviews: + - N/A + notification_emails: + - nobody@example.com + expires: never + send_in_pings: + - prototype + no_lint: + - COMMON_PREFIX + + sample_labeled_string: + <<: *defaults + type: labeled_string + description: | + Just testing labeled_string + bugs: + - https://bugzilla.mozilla.org/1907991 + data_reviews: + - N/A + notification_emails: + - nobody@example.com + expires: never + send_in_pings: + - prototype + no_lint: + - COMMON_PREFIX + + timings: + <<: *defaults + type: timing_distribution + time_unit: millisecond + +party: + balloons: + type: object + description: | + Just testing objects + bugs: + - https://bugzilla.mozilla.org/1839640 + data_reviews: + - N/A + notification_emails: + - CHANGE-ME@example.com + expires: never + send_in_pings: + - prototype + structure: + type: array + items: + type: object + properties: + colour: + type: string + diameter: + type: number + + drinks: + type: object + description: | + Just testing objects + bugs: + - https://bugzilla.mozilla.org/1910809 + data_reviews: + - N/A + notification_emails: + - CHANGE-ME@example.com + expires: never + send_in_pings: + - prototype + structure: + type: array + items: + type: object + properties: + name: + type: string + ingredients: + type: array + items: + type: string + + chooser: + type: object + description: | + Array of key-value elements + bugs: + - https://bugzilla.mozilla.org/123456789 + data_reviews: + - http://example.com/reviews + notification_emails: + - CHANGE-ME@example.com + expires: never + structure: + type: array + items: + type: object + properties: + key: + type: string + value: + oneOf: + - type: string + - type: number + - type: boolean + +test.dual_labeled: + static_static: + type: dual_labeled_counter + description: > + A dual labeled counter with static keys and categories + dual_labels: + key: + description: > + The key for the dual labeled counter + labels: + - key1 + - key2 + category: + description: > + The category for the dual labeled counter + labels: + - category1 + - category2 + bugs: + - https://bugzilla.mozilla.org/11137353 + data_reviews: + - http://example.com/reviews + notification_emails: + - CHANGE-ME@example.com + expires: never + + dynamic_static: + type: dual_labeled_counter + description: > + A dual labeled counter with static keys and dynamic categories + dual_labels: + key: + description: > + The key for the dual labeled counter + labels: + - key1 + - key2 + category: + description: > + The category for the dual labeled counter + bugs: + - https://bugzilla.mozilla.org/11137353 + data_reviews: + - http://example.com/reviews + notification_emails: + - CHANGE-ME@example.com + expires: never + + static_dynamic: + type: dual_labeled_counter + description: > + A dual labeled counter with dynamic keys and static categories + dual_labels: + key: + description: > + The key for the dual labeled counter + category: + description: > + The category for the dual labeled counter + labels: + - category1 + - category2 + bugs: + - https://bugzilla.mozilla.org/11137353 + data_reviews: + - http://example.com/reviews + notification_emails: + - CHANGE-ME@example.com + expires: never + + dynamic_dynamic: + type: dual_labeled_counter + description: > + A dual labeled counter with dynamic keys and dynamic categories + dual_labels: + key: + description: > + The key for the dual labeled counter + category: + description: > + The category for the dual labeled counter + bugs: + - https://bugzilla.mozilla.org/11137353 + data_reviews: + - http://example.com/reviews + notification_emails: + - CHANGE-ME@example.com + expires: never diff --git a/glean-core/rlb-tests/pings.yaml b/glean-core/rlb-tests/pings.yaml new file mode 100644 index 0000000000..5abc28a675 --- /dev/null +++ b/glean-core/rlb-tests/pings.yaml @@ -0,0 +1,22 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# This file defines the built-in pings that are recorded by the Glean SDK. They +# are automatically converted to Rust code at build time using the +# `glean_parser` PyPI package. + +--- + +$schema: moz://mozilla.org/schemas/glean/pings/2-0-0 + +prototype: + description: | + A sample custom ping. + include_client_id: true + bugs: + - https://bugzilla.mozilla.org/123456789 + data_reviews: + - N/A + notification_emails: + - CHANGE-ME@example.com diff --git a/glean-core/rlb-tests/src/bin/verify-data.rs b/glean-core/rlb-tests/src/bin/verify-data.rs new file mode 100644 index 0000000000..1ddd5933b8 --- /dev/null +++ b/glean-core/rlb-tests/src/bin/verify-data.rs @@ -0,0 +1,254 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::time::Duration; +use std::{env, fs}; + +use glean::{ClientInfoMetrics, ConfigurationBuilder, ErrorType, TestGetValue, net}; + +#[allow(clippy::all)] // Don't lint generated code. +pub mod glean_metrics { + include!(concat!(env!("OUT_DIR"), "/glean_metrics.rs")); +} + +#[derive(Debug)] +struct Uploader; + +impl net::PingUploader for Uploader { + fn upload(&self, _upload_request: net::CapablePingUploadRequest) -> net::UploadResult { + net::UploadResult::recoverable_failure() + } +} + +fn main() { + env_logger::init(); + + let mut args = env::args().skip(1); + + let data_path = args.next().expect("data path required"); + let verify = args.next().map(|arg| arg == "verify").unwrap_or(false); + if !verify { + println!("Removing {data_path}"); + _ = fs::remove_dir_all(&data_path); + } + + let cfg = ConfigurationBuilder::new(true, data_path, "org.mozilla.glean_core.example") + .with_server_endpoint("invalid-test-host") + .with_use_core_mps(false) + .with_uploader(Uploader) + .build(); + + let client_info = ClientInfoMetrics { + app_build: env!("CARGO_PKG_VERSION").to_string(), + app_display_version: env!("CARGO_PKG_VERSION").to_string(), + channel: None, + locale: None, + }; + + _ = &*glean_metrics::prototype; + glean::initialize(cfg, client_info); + + if verify { + println!("Verifying metric values..."); + assert!( + glean_metrics::test_metrics::sample_boolean + .test_get_value(None) + .unwrap() + ); + assert_eq!( + Some(1), + glean_metrics::test_metrics::sample_counter.test_get_value(None) + ); + assert_eq!( + "https://example.com", + glean_metrics::test_metrics::sample_url + .test_get_value(None) + .unwrap() + ); + assert_eq!( + 1, + glean_metrics::test_metrics::sample_url + .test_get_num_recorded_errors(ErrorType::InvalidValue) + ); + + assert_eq!( + 1, + glean_metrics::test_metrics::sample_labeled_counter + .get("test") + .test_get_value(None) + .unwrap() + ); + assert_eq!( + "foo", + glean_metrics::test_metrics::sample_labeled_string + .get("test") + .test_get_value(None) + .unwrap() + ); + + let exp_balloons = serde_json::json!([ + { "colour": "red", "diameter": 5 }, + { "colour": "blue" }, + ]); + assert_eq!( + Some(exp_balloons), + glean_metrics::party::balloons.test_get_value(None) + ); + + let exp_chooser = serde_json::json!([ + { "key": "fortytwo", "value": 42 }, + { "key": "to-be", "value": false }, + ]); + assert_eq!( + Some(exp_chooser), + glean_metrics::party::chooser.test_get_value(None) + ); + assert_eq!( + 2, + glean_metrics::test_dual_labeled::static_static + .get("key1", "category1") + .test_get_value(None) + .unwrap() + ); + assert_eq!( + 0, + glean_metrics::test_dual_labeled::static_static + .test_get_num_recorded_errors(ErrorType::InvalidLabel) + ); + assert_eq!( + 0, + glean_metrics::test_dual_labeled::dynamic_static + .test_get_num_recorded_errors(ErrorType::InvalidLabel) + ); + assert_eq!( + Some(3), + glean_metrics::test_dual_labeled::dynamic_static + .get("party", "category1") + .test_get_value(None) + ); + assert_eq!( + 0, + glean_metrics::test_dual_labeled::static_dynamic + .test_get_num_recorded_errors(ErrorType::InvalidLabel) + ); + assert_eq!( + Some(4), + glean_metrics::test_dual_labeled::static_dynamic + .get("key1", "balloons") + .test_get_value(None) + ); + assert_eq!( + 0, + glean_metrics::test_dual_labeled::dynamic_dynamic + .test_get_num_recorded_errors(ErrorType::InvalidLabel) + ); + assert_eq!( + Some(5), + glean_metrics::test_dual_labeled::dynamic_dynamic + .get("party", "balloons") + .test_get_value(None) + ); + + assert_eq!( + Some(6), + glean_metrics::test_dual_labeled::static_static + .get("__other__", "__other__") + .test_get_value(None) + ); + + assert_eq!( + 0, + glean_metrics::party::drinks.test_get_num_recorded_errors(ErrorType::InvalidValue) + ); + + let timings = glean_metrics::test_metrics::timings + .test_get_value(None) + .unwrap(); + assert_eq!(100, timings.count); + assert_eq!(1_000_000_000, timings.sum); + assert_eq!(100, timings.values[&9975792]); + + println!("OK."); + } else { + println!("Setting metric values..."); + glean_metrics::test_metrics::sample_boolean.set(true); + glean_metrics::test_metrics::sample_counter.add(1); + glean_metrics::test_metrics::sample_url.set("https://example.com"); + glean_metrics::test_metrics::sample_url.set("data:application/json"); + glean_metrics::test_metrics::sample_labeled_counter + .get("test") + .add(1); + glean_metrics::test_metrics::sample_labeled_string + .get("test") + .set(String::from("foo")); + + use glean_metrics::party::{BalloonsObject, BalloonsObjectItem}; + let balloons = BalloonsObject::from([ + BalloonsObjectItem { + colour: Some("red".to_string()), + diameter: Some(5), + }, + BalloonsObjectItem { + colour: Some("blue".to_string()), + diameter: None, + }, + ]); + glean_metrics::party::balloons.set(balloons); + + use glean_metrics::party::{ChooserObject, ChooserObjectItem, ChooserObjectItemValueEnum}; + let mut ch = ChooserObject::new(); + let it = ChooserObjectItem { + key: Some("fortytwo".to_string()), + value: Some(ChooserObjectItemValueEnum::Number(42)), + }; + ch.push(it); + let it = ChooserObjectItem { + key: Some("to-be".to_string()), + value: Some(ChooserObjectItemValueEnum::Boolean(false)), + }; + ch.push(it); + glean_metrics::party::chooser.set(ch); + + glean_metrics::test_dual_labeled::static_static + .get("key1", "category1") + .add(2); + glean_metrics::test_dual_labeled::dynamic_static + .get("party", "category1") + .add(3); + glean_metrics::test_dual_labeled::static_dynamic + .get("key1", "balloons") + .add(4); + glean_metrics::test_dual_labeled::dynamic_dynamic + .get("party", "balloons") + .add(5); + + // Testing the `__other__` label. + glean_metrics::test_dual_labeled::static_static + .get("party", "balloons") + .add(6); + + // Testing with empty and null values. + let drinks = serde_json::json!([ + { "name": "lemonade", "ingredients": ["lemon", "water", "sugar"] }, + { "name": "sparkling-water", "ingredients": [] }, + { "name": "still-water", "ingredients": null }, + ]); + glean_metrics::party::drinks.set_string(drinks.to_string()); + + { + let mut buffer = glean_metrics::test_metrics::timings.start_buffer(); + + let mock_duration = Duration::from_millis(10); + for _ in 0..100 { + buffer.accumulate(mock_duration.as_millis() as u64); + } + } + + // Ensure Glean actually catches up. + _ = glean_metrics::party::drinks.test_get_value(None); + println!("OK."); + } + + glean::shutdown(); +} diff --git a/glean-core/rlb-tests/tests/it.rs b/glean-core/rlb-tests/tests/it.rs index ddf712b995..71bc65a68a 100644 --- a/glean-core/rlb-tests/tests/it.rs +++ b/glean-core/rlb-tests/tests/it.rs @@ -295,3 +295,19 @@ fn enabled_pings() { let payload = fs::read_to_string(&entries[0]).unwrap(); assert!(payload.contains("/two/"), "Payload: {payload}"); } + +#[test] +fn rkv_sqlite_migration() { + let tempdir = tempfile::tempdir().unwrap(); + + let db_dir = tempdir.path().join("db"); + fs::create_dir_all(&db_dir).unwrap(); + let rkv_db = db_dir.join("data.safe.bin"); + fs::write(&rkv_db, include_bytes!("rkv-database.safe.bin")).unwrap(); + + cargo_bin_cmd!("verify-data") + .arg(tempdir.path()) + .arg("verify") + .assert() + .success(); +} diff --git a/glean-core/rlb-tests/tests/rkv-database.safe.bin b/glean-core/rlb-tests/tests/rkv-database.safe.bin new file mode 100644 index 0000000000000000000000000000000000000000..8858f9339dfdabcad0055201f8d50bdadbab4023 GIT binary patch literal 4065 zcmcInTW{k;6i(UYVu3(hg;-RIyIdq1$9A*1u?U0^5|5yL+ErzZ?MX7~#A9a0?M79_ z3vaygfcOpk03LbbfnNjvfHUJc>85egY$;aS8GDYubD8hVIlk4Zubta&QYk9U^aXc+ z$Bh$CNs(ln(LyJz7>j+J=R+srg*bunuABFV>5#h5s=#wxSh2e7` z`|gq=R%j{a=ivc*9|lnnez-ND zX0vN-Q*U_Kk8AtQmhL)rE+{r$L46ajg6Sj^RiV*Z?{b_MTrOELF;KYe$j_80%2`48 zYvdEc8PJP~A~0{LL{;gGX-)G^t#qxX!zM$kk_(bssHC1JW5RePX-CpboKQ(p-prC) zfVZ~+6ZR}eu+N;F*L9N@VKg)|*Gs?EG&X?d!tswk+xGtK+w=Rs%YQx(>|Lw16zS{7 z0|UGcPE9MFi&MyErb?o-DV517VR|cH&^H{v`0|w{Up~0Pmxb~h(};;Wqgt{|IXY1( zg#@13FedQ7q8&|@4xCt^$gJcvsU%-ZtdE^9%g8PTfU5)|2xQVi&h6>-vy=CJaub96 zgPT7(^FcbHkD1cSAB_oD^uRtU$c*|UKTrMhgR6w-2^<(|3&9dd3Tc$sT=LJ4ke|!G z{KqVL-}%-EdgE!*Y1DT8Jc+KtTX6ONNEAHHmt*&YHw*1ImML#ua=3&tHxII@5YTRX z=N7$h+{BOOSYRNLYvbr+0CGlnLyq~jo{{> z?q3OBgD#AQ5JVRm8gWfo{d8O06`L2$jd<(u{L_e1vuMA}&qUD;2_g5j1woJKI6nE{@7ZPMIPfJO4Ku;xf7J z38}QDb}b}s-*%7+%6c)eh)5Q%?m1d5(<5V7H5LNv(Pm&M`B>S#_ndbI@O3wzYF(;P zr$Zmx79q&Q413`4dO`Rvv>t+S literal 0 HcmV?d00001