From 56555e726ccf567fd91a344cb49d710d155862b9 Mon Sep 17 00:00:00 2001 From: Timothy DeHerrera Date: Wed, 12 Nov 2025 14:50:14 -0700 Subject: [PATCH 01/33] feat: enable full-scale Nix evaluation in sandboxed environment - Refactor nixec crate from monolithic main function to more modular code - Implement complete sandboxed nix-instantiate execution using birdcage - Add public run_nixec() function for testing and external usage - Update birdcage to git version with fixing nix call `unshare(CLONE_FS)` - Add anyhow dependency and improve error handling - Extract hardcoded paths to constants for maintainability - Update build system and development environment dependencies --- Cargo.lock | 29 ++---- Cargo.toml | 3 +- build/Cargo.nix | 37 +++++--- crates/nixec/Cargo.toml | 1 + crates/nixec/src/main.rs | 200 ++++++++++++++++++++++++++++----------- dev/env/shell.nix | 1 + 6 files changed, 187 insertions(+), 84 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 245173c..ec5b67a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -380,8 +380,7 @@ dependencies = [ [[package]] name = "birdcage" version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "848df95320021558dd6bb4c26de3fe66724cdcbdbbf3fa720150b52b086ae568" +source = "git+https://github.com/nrdxp/birdcage?branch=clone_fs#7d294c333b1d515dd6d175c1477bc3ee4d44bdfe" dependencies = [ "bitflags 2.10.0", "libc", @@ -1197,7 +1196,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.1", + "windows-sys 0.60.2", ] [[package]] @@ -4044,7 +4043,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.61.1", + "windows-sys 0.60.2", ] [[package]] @@ -4110,7 +4109,7 @@ dependencies = [ "portable-atomic", "portable-atomic-util", "serde_core", - "windows-sys 0.61.1", + "windows-sys 0.60.2", ] [[package]] @@ -4642,6 +4641,7 @@ dependencies = [ name = "nixec" version = "0.1.0" dependencies = [ + "anyhow", "birdcage", "thiserror 2.0.17", ] @@ -4680,7 +4680,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.1", + "windows-sys 0.60.2", ] [[package]] @@ -5644,7 +5644,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.4.15", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -5657,7 +5657,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.11.0", - "windows-sys 0.61.1", + "windows-sys 0.60.2", ] [[package]] @@ -6567,7 +6567,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix 1.1.2", - "windows-sys 0.61.1", + "windows-sys 0.60.2", ] [[package]] @@ -7549,7 +7549,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.1", + "windows-sys 0.48.0", ] [[package]] @@ -7712,15 +7712,6 @@ dependencies = [ "windows-targets 0.52.6", ] -[[package]] -name = "windows-sys" -version = "0.59.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" -dependencies = [ - "windows-targets 0.52.6", -] - [[package]] name = "windows-sys" version = "0.60.2" diff --git a/Cargo.toml b/Cargo.toml index e187c9a..2dcaf7c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,7 +13,7 @@ version = "0.3.0" [profile.release] codegen-units = 1 lto = true -opt-level = 3 +opt-level = "z" strip = true [dependencies] @@ -83,6 +83,7 @@ prodash = { version = "30.0.1", features = [ ] } [patch.crates-io] +birdcage = { git = "https://github.com/nrdxp/birdcage", branch = "clone_fs" } tracing = { git = "https://github.com/nrdxp/tracing", branch = "hierarchical" } tracing-appender = { git = "https://github.com/nrdxp/tracing", branch = "hierarchical" } tracing-core = { git = "https://github.com/nrdxp/tracing", branch = "hierarchical" } diff --git a/build/Cargo.nix b/build/Cargo.nix index bd20900..a0c61a8 100644 --- a/build/Cargo.nix +++ b/build/Cargo.nix @@ -15213,6 +15213,10 @@ rec { ]; src = lib.cleanSourceWith { filter = sourceFilter; src = ../crates/nixec; }; dependencies = [ + { + name = "anyhow"; + packageId = "anyhow"; + } { name = "birdcage"; packageId = "birdcage"; @@ -18761,9 +18765,9 @@ rec { }; "rustls" = rec { crateName = "rustls"; - version = "0.23.14"; + version = "0.23.35"; edition = "2021"; - sha256 = "1a0b2sdvq69vqrz08wvjmlqafzh7pfgzhn9j0n107f9wd529jpa1"; + sha256 = "13xxk2qqchibd7pr0laqq6pzayx9xm4rb45d8rz68kvxday58gsk"; dependencies = [ { name = "log"; @@ -18812,13 +18816,14 @@ rec { ]; features = { "aws-lc-rs" = [ "aws_lc_rs" ]; - "aws_lc_rs" = [ "dep:aws-lc-rs" "webpki/aws_lc_rs" ]; + "aws_lc_rs" = [ "dep:aws-lc-rs" "webpki/aws-lc-rs" "aws-lc-rs/aws-lc-sys" "aws-lc-rs/prebuilt-nasm" ]; "brotli" = [ "dep:brotli" "dep:brotli-decompressor" "std" ]; - "default" = [ "aws_lc_rs" "logging" "std" "tls12" ]; - "fips" = [ "aws_lc_rs" "aws-lc-rs?/fips" ]; + "default" = [ "aws_lc_rs" "logging" "prefer-post-quantum" "std" "tls12" ]; + "fips" = [ "aws_lc_rs" "aws-lc-rs?/fips" "webpki/aws-lc-rs-fips" ]; "hashbrown" = [ "dep:hashbrown" ]; "log" = [ "dep:log" ]; "logging" = [ "log" ]; + "prefer-post-quantum" = [ "aws_lc_rs" ]; "read_buf" = [ "rustversion" "std" ]; "ring" = [ "dep:ring" "webpki/ring" ]; "rustversion" = [ "dep:rustversion" ]; @@ -18882,11 +18887,19 @@ rec { }; "rustls-pki-types" = rec { crateName = "rustls-pki-types"; - version = "1.9.0"; + version = "1.13.0"; edition = "2021"; - sha256 = "0mcc901b4hm2ql2qwpf2gzqhqn6d7iag92hr872wjr8c6wsnws8f"; + sha256 = "0yjzsnpv1sjbnfxbbmrnyimd23jip48nav6l9hr1rjd06vcjl64l"; libName = "rustls_pki_types"; + dependencies = [ + { + name = "zeroize"; + packageId = "zeroize"; + optional = true; + } + ]; features = { + "alloc" = [ "dep:zeroize" ]; "default" = [ "alloc" ]; "std" = [ "alloc" ]; "web" = [ "web-time" ]; @@ -18896,9 +18909,9 @@ rec { }; "rustls-webpki" = rec { crateName = "rustls-webpki"; - version = "0.102.8"; + version = "0.103.8"; edition = "2021"; - sha256 = "1sdy8ks86b7jpabpnb2px2s7f1sq8v0nqf6fnlvwzm4vfk41pjk4"; + sha256 = "0lpymb84bi5d2pm017n39nbiaa5cd046hgz06ir29ql6a8pzmz9g"; libName = "webpki"; dependencies = [ { @@ -18920,8 +18933,10 @@ rec { ]; features = { "alloc" = [ "ring?/alloc" "pki-types/alloc" ]; - "aws_lc_rs" = [ "dep:aws-lc-rs" ]; - "default" = [ "std" "ring" ]; + "aws-lc-rs" = [ "dep:aws-lc-rs" "aws-lc-rs/aws-lc-sys" "aws-lc-rs/prebuilt-nasm" ]; + "aws-lc-rs-fips" = [ "dep:aws-lc-rs" "aws-lc-rs/fips" ]; + "aws-lc-rs-unstable" = [ "aws-lc-rs" "aws-lc-rs/unstable" ]; + "default" = [ "std" ]; "ring" = [ "dep:ring" ]; "std" = [ "alloc" "pki-types/std" ]; }; diff --git a/crates/nixec/Cargo.toml b/crates/nixec/Cargo.toml index c480e58..cc8ce44 100644 --- a/crates/nixec/Cargo.toml +++ b/crates/nixec/Cargo.toml @@ -6,4 +6,5 @@ version = "0.1.0" [dependencies] birdcage = "^0.8" +anyhow.workspace = true thiserror.workspace = true diff --git a/crates/nixec/src/main.rs b/crates/nixec/src/main.rs index 9974924..111e64d 100644 --- a/crates/nixec/src/main.rs +++ b/crates/nixec/src/main.rs @@ -12,39 +12,178 @@ use birdcage::process::Command; use birdcage::{Birdcage, Exception, Sandbox}; use thiserror::Error; +//================================================================================================ +// Constants +//================================================================================================ + +const SSL_CERT_PATH: &str = "/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"; +const RESOLV_CONF_PATH: &str = "/etc/resolv.conf"; +const DEV_NULL_PATH: &str = "/dev/null"; + //================================================================================================ // Types //================================================================================================ +/// Configuration for nixec execution. +#[derive(Debug)] +struct NixecConfig { + nix_dir: PathBuf, + git_dir: PathBuf, + utils_dir: PathBuf, + nix_store: PathBuf, + nix_config: String, +} + /// Represents the possible errors that can occur during `nixec` execution. #[derive(Error, Debug)] -enum NixecError { +pub enum NixecError { /// Error indicating that the `nix` executable could not be found in the system's PATH. - #[error("No `nix` executable in PATH")] - NoNix, + #[error("No `{0}` executable in PATH")] + NoBin(String), /// Error originating from the `birdcage` sandboxing library. #[error(transparent)] ExceptionFailed(#[from] birdcage::error::Error), /// Error from I/O operations, typically when executing a command. #[error(transparent)] CommandFailed(#[from] std::io::Error), + /// Error for UTF-8 conversion failures. + #[error(transparent)] + Utf8Error(#[from] std::string::FromUtf8Error), /// Error for when the Nix store path could not be determined. #[error("Failed to determine nix store path")] StorePath, } -/// A specialized `Result` type for `nixec` operations. -type Result = std::result::Result; - //================================================================================================ // Functions //================================================================================================ +/// Sets up the necessary paths and configuration for nixec. +fn setup_paths() -> Result { + let nix_dir = bin_dir("nix")?; + let git_dir = bin_dir("git")?; + let utils_dir = bin_dir("coreutils")?; + let nix_instantiate = nix_dir.join("nix-instantiate"); + let nix = nix_dir.join("nix"); + + let nix_store: PathBuf = String::from_utf8( + UnsafeCommand::new(&nix_instantiate) + .args(["--eval", "--raw", "--expr", "builtins.storeDir"]) + .output()? + .stdout, + ) + .map_err(|_| NixecError::StorePath)? + .trim() + .into(); + + let nix_config: String = String::from_utf8( + UnsafeCommand::new(&nix) + .args(["config", "show"]) + .output()? + .stdout, + )? + .trim() + .lines() + .chain(std::iter::once( + format!("ssl-cert-file = {}", SSL_CERT_PATH).as_str(), + )) + .collect::>() + .join("\n"); + + Ok(NixecConfig { + nix_dir, + git_dir, + utils_dir, + nix_store, + nix_config, + }) +} + +/// Configures the sandbox with necessary exceptions and environment variables. +fn configure_sandbox(config: &NixecConfig) -> Result { + let cwd = env::current_dir()?; + let mut sandbox = Birdcage::new(); + + sandbox.add_exception(Exception::Read(cwd))?; + + if let Ok(home) = std::env::var("HOME") { + let cache = std::env::var("XDG_CACHE_HOME") + .map(PathBuf::from) + .unwrap_or(PathBuf::from(home).join(".cache")); + unsafe { env::set_var("XDG_CACHE_HOME", &cache) }; + sandbox.add_exception(Exception::Environment("XDG_CACHE_HOME".into()))?; + sandbox.add_exception(Exception::WriteAndRead(cache.join("nix")))?; + }; + + let nix_root = config + .nix_store + .parent() + .map(Path::to_path_buf) + .ok_or(NixecError::StorePath)?; + + sandbox.add_exception(Exception::ExecuteAndRead(nix_root))?; + unsafe { env::set_var("NIX_CONFIG", &config.nix_config) }; + unsafe { + env::set_var( + "PATH", + format!( + "{}:{}:{}", + config.nix_dir.display(), + config.git_dir.display(), + config.utils_dir.display() + ), + ) + }; + unsafe { env::set_var("GIT_SSL_CAINFO", SSL_CERT_PATH) }; + sandbox.add_exception(Exception::Environment("HOME".into()))?; + sandbox.add_exception(Exception::Environment("NIX_CONFIG".into()))?; + sandbox.add_exception(Exception::Environment("PATH".into()))?; + sandbox.add_exception(Exception::Environment("GIT_SSL_CAINFO".into()))?; + sandbox.add_exception(Exception::ExecuteAndRead(config.nix_dir.clone()))?; + sandbox.add_exception(Exception::ExecuteAndRead(config.git_dir.clone()))?; + sandbox.add_exception(Exception::Read(RESOLV_CONF_PATH.into()))?; + sandbox.add_exception(Exception::WriteAndRead(DEV_NULL_PATH.into()))?; + sandbox.add_exception(Exception::Networking)?; + + Ok(sandbox) +} + +/// Runs the nix-instantiate command within the configured sandbox. +fn run_command_in_sandbox( + config: &NixecConfig, + sandbox: Birdcage, + args: &[String], +) -> Result { + let nix_instantiate = config.nix_dir.join("nix-instantiate"); + let mut command = Command::new(nix_instantiate); + command.args(args); + + let output = sandbox.spawn(command)?.wait_with_output()?; + println!("{}", String::from_utf8(output.stdout)?); + println!("{}", String::from_utf8(output.stderr)?); + + Ok(ExitCode::from(output.status.code().unwrap_or(1) as u8)) +} + +/// Runs nixec with the given arguments. This is the main logic that can be called for testing. +pub fn run_nixec(args: Vec) -> Result { + let config = setup_paths()?; + let sandbox = configure_sandbox(&config)?; + let sandbox_args = &args[1..]; // Assuming args[0] is program name + run_command_in_sandbox(&config, sandbox, sandbox_args) +} + +/// The main entry point for the `nixec` executable. +fn main() -> anyhow::Result { + let args: Vec = env::args().collect(); + run_nixec(args).map_err(Into::into) +} + /// Locates the directory containing a given executable. /// /// This function searches the directories listed in the `PATH` environment variable /// to find the specified executable and returns its parent directory. -fn bin_dir(exec_name: &str) -> Result { +fn bin_dir(exec_name: &str) -> Result { env::var_os("PATH") .and_then(|paths| { env::split_paths(&paths) @@ -60,50 +199,5 @@ fn bin_dir(exec_name: &str) -> Result { }) .next() }) - .ok_or(NixecError::NoNix) -} - -/// The main entry point for the `nixec` executable. -/// -/// This function sets up the sandbox, determines the Nix store path, and executes -/// `nix-instantiate` with the provided arguments within the sandbox. -fn main() -> Result { - let nix_dir = bin_dir("nix")?; - let nix_instantiate = nix_dir.join("nix-instantiate"); - let cwd = env::current_dir()?; - - let args: Vec = env::args().collect(); - let sandbox_args = &args[1..]; - - let mut sandbox = Birdcage::new(); - - sandbox.add_exception(Exception::Read(cwd))?; - - let nix_store: PathBuf = String::from_utf8( - UnsafeCommand::new(nix_instantiate.clone()) - .args(["--eval", "--expr", "builtins.storeDir"]) - .output()? - .stdout, - ) - .map_err(|_| NixecError::StorePath)? - .trim() - .trim_matches('"') - .into(); - - sandbox.add_exception(Exception::ExecuteAndRead( - nix_store - .parent() - .map(Path::to_path_buf) - .ok_or(NixecError::StorePath)?, - ))?; - unsafe { env::set_var("HOME", "/homeless-shelter") }; - sandbox.add_exception(Exception::Environment("HOME".into()))?; - - sandbox.add_exception(Exception::ExecuteAndRead(nix_dir))?; - let mut command = Command::new(nix_instantiate); - command.args(sandbox_args); - - let output = sandbox.spawn(command)?.wait_with_output()?; - - Ok(ExitCode::from(output.status.code().unwrap_or(1) as u8)) + .ok_or(NixecError::NoBin(exec_name.into())) } diff --git a/dev/env/shell.nix b/dev/env/shell.nix index 10543e4..475082b 100644 --- a/dev/env/shell.nix +++ b/dev/env/shell.nix @@ -19,6 +19,7 @@ pkgs.mkShell.override { stdenv = pkgs.clangStdenv; } { pkg-config nodePackages.prettier mod.fenix.default.rustfmt + crate2nix nil toolchain mold From e9019260ccd8b733733ec5c5ca6d5512934f820d Mon Sep 17 00:00:00 2001 From: Timothy DeHerrera Date: Thu, 13 Nov 2025 15:35:47 -0700 Subject: [PATCH 02/33] feat(nixec): setup restricted evaluation Inside our sandbox env this works very similarly to pure evaluation without the "copy the world" requirement. Basically, any path outside of the current directory is invisible to the evaluation and eval will abort with an error if you attempt to access them. The only remaining "holes" are impure builtins of nix expressions, such as `builtins.getEnv` which are plugged by atom-nix creating a totally pure evaluation context for much less cost. Currently this change hardcodes allowed uris, since they are required to specify when setting restricted eval. In the future we should allow setting these externally, either via a flag or perhaps an environmental variable. --- crates/nixec/src/main.rs | 40 ++++++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/crates/nixec/src/main.rs b/crates/nixec/src/main.rs index 111e64d..e04bc76 100644 --- a/crates/nixec/src/main.rs +++ b/crates/nixec/src/main.rs @@ -84,9 +84,13 @@ fn setup_paths() -> Result { )? .trim() .lines() - .chain(std::iter::once( + .chain([ format!("ssl-cert-file = {}", SSL_CERT_PATH).as_str(), - )) + "pure-eval = false", + "restrict-eval = true", + // FIXME: hardcoded for now, should allow setting externally + "allowed-uris = git+https: https: ssh: git+ssh:", + ]) .collect::>() .join("\n"); @@ -104,15 +108,14 @@ fn configure_sandbox(config: &NixecConfig) -> Result { let cwd = env::current_dir()?; let mut sandbox = Birdcage::new(); - sandbox.add_exception(Exception::Read(cwd))?; - if let Ok(home) = std::env::var("HOME") { let cache = std::env::var("XDG_CACHE_HOME") .map(PathBuf::from) .unwrap_or(PathBuf::from(home).join(".cache")); unsafe { env::set_var("XDG_CACHE_HOME", &cache) }; - sandbox.add_exception(Exception::Environment("XDG_CACHE_HOME".into()))?; - sandbox.add_exception(Exception::WriteAndRead(cache.join("nix")))?; + sandbox + .add_exception(Exception::Environment("XDG_CACHE_HOME".into()))? + .add_exception(Exception::WriteAndRead(cache.join("nix")))?; }; let nix_root = config @@ -121,7 +124,6 @@ fn configure_sandbox(config: &NixecConfig) -> Result { .map(Path::to_path_buf) .ok_or(NixecError::StorePath)?; - sandbox.add_exception(Exception::ExecuteAndRead(nix_root))?; unsafe { env::set_var("NIX_CONFIG", &config.nix_config) }; unsafe { env::set_var( @@ -135,15 +137,21 @@ fn configure_sandbox(config: &NixecConfig) -> Result { ) }; unsafe { env::set_var("GIT_SSL_CAINFO", SSL_CERT_PATH) }; - sandbox.add_exception(Exception::Environment("HOME".into()))?; - sandbox.add_exception(Exception::Environment("NIX_CONFIG".into()))?; - sandbox.add_exception(Exception::Environment("PATH".into()))?; - sandbox.add_exception(Exception::Environment("GIT_SSL_CAINFO".into()))?; - sandbox.add_exception(Exception::ExecuteAndRead(config.nix_dir.clone()))?; - sandbox.add_exception(Exception::ExecuteAndRead(config.git_dir.clone()))?; - sandbox.add_exception(Exception::Read(RESOLV_CONF_PATH.into()))?; - sandbox.add_exception(Exception::WriteAndRead(DEV_NULL_PATH.into()))?; - sandbox.add_exception(Exception::Networking)?; + unsafe { env::set_var("NIX_PATH", format!("eval={}", cwd.display())) }; + sandbox + .add_exception(Exception::Environment("HOME".into()))? + .add_exception(Exception::Environment("NIX_CONFIG".into()))? + .add_exception(Exception::Environment("NIX_PATH".into()))? + .add_exception(Exception::Environment("PATH".into()))? + .add_exception(Exception::Environment("GIT_SSL_CAINFO".into()))? + .add_exception(Exception::Read(cwd))? + .add_exception(Exception::Read(RESOLV_CONF_PATH.into()))? + .add_exception(Exception::ExecuteAndRead(nix_root))? + .add_exception(Exception::ExecuteAndRead(config.nix_dir.clone()))? + .add_exception(Exception::ExecuteAndRead(config.git_dir.clone()))? + .add_exception(Exception::ExecuteAndRead(config.utils_dir.clone()))? + .add_exception(Exception::WriteAndRead(DEV_NULL_PATH.into()))? + .add_exception(Exception::Networking)?; Ok(sandbox) } From 9df162f4d158ecb579b5bb4e85d2de39332f89ee Mon Sep 17 00:00:00 2001 From: Timothy DeHerrera Date: Thu, 13 Nov 2025 17:41:45 -0700 Subject: [PATCH 03/33] release(nix-lock): 0.1.6 -> 0.2.0 Adds a new "import.nix" expression used to import this specific version of the larger lock expressions. The import.nix will be read at eka compile time and stored in the binary itself, making it convenient to call into atoms without any implicit nix code, and without making the baked in expression too rigid, since it is versions and well specified in the lock. The fact that we bake this particular expression into the lock can be considered an "implementation detail" to make calling it more straight forward. --- nix-lock/atom.toml | 2 +- nix-lock/default.nix | 35 ++++++++++++++++++++++++++++------- nix-lock/import.nix | 18 ++++++++++++++++++ 3 files changed, 47 insertions(+), 8 deletions(-) create mode 100644 nix-lock/import.nix diff --git a/nix-lock/atom.toml b/nix-lock/atom.toml index 8b01128..999690c 100644 --- a/nix-lock/atom.toml +++ b/nix-lock/atom.toml @@ -1,6 +1,6 @@ [package] label = "nix-lock" -version = "0.1.6" +version = "0.2.0" [locker.eka] root_hash = "4abbd2644bc3585e9be95deb277ccf48f6ed26ac" diff --git a/nix-lock/default.nix b/nix-lock/default.nix index 4167259..b7a6b13 100644 --- a/nix-lock/default.nix +++ b/nix-lock/default.nix @@ -7,6 +7,24 @@ root: lockstr: let unknownErr = "unknown atom type encountered"; lock_toml = builtins.fromTOML lockstr; + pureScope = { + __getEnv = ""; + __nixPath = [ ]; + __currentTime = 0; + __currentSystem = + extraConfig.platform or abort + "Accessing the current system is impure. Set the platform in the config instead"; + __storePath = abort "Making explicit dependencies on store paths is illegal."; + builtins = builtins.removeAttrs builtins [ + "nixPath" + "storePath" + "currentSystem" + "currentTime" + "getEnv" + ]; + }; + Import = scopedImport pureScope; + f = root: lock: let @@ -52,10 +70,13 @@ let if builtins.pathExists tomlPath then builtins.fromTOML (builtins.readFile tomlPath) else { }; trvialComposer = root: args: - scopedImport { - atoms = args.extern or { }; - cfg = args.config or { }; - } root; + scopedImport ( + pureScope + // { + atoms = args.extern or { }; + cfg = args.config or { }; + } + ) root; in let composeKind = lock.compose.use or null; @@ -96,7 +117,7 @@ let }; in { - import = path: import (fetch + "/${path}"); + import = path: Import (fetch + "/${path}"); src = fetch; }; "nix+tar" = @@ -108,7 +129,7 @@ let }; in { - import = path: import (fetch + "/${path}"); + import = path: Import (fetch + "/${path}"); src = fetch; }; "nix" = @@ -120,7 +141,7 @@ let }; in { - import = path: import (fetch + "/${path}"); + import = path: Import (fetch + "/${path}"); src = fetch; }; "nix+src" = diff --git a/nix-lock/import.nix b/nix-lock/import.nix new file mode 100644 index 0000000..aacb0d1 --- /dev/null +++ b/nix-lock/import.nix @@ -0,0 +1,18 @@ +args@{ root, ... }: +let + lockstr = builtins.readFile (root + "/atom.lock"); + lock = builtins.fromTOML lockstr; + inherit (lock) locker; + lockexpr = import ( + builtins.fetchGit { + inherit (locker) rev; + name = locker.label; + url = locker.mirror; + ref = "refs/eka/atoms/${locker.label}/${locker.version}"; + } + ); +in +lockexpr root lockstr { + extraExtern = args.extraExtern or { }; + extraConfig = args.extraConfig or { }; +} From 4788980f9341d394e0ec22d505bfcb42d5a0451d Mon Sep 17 00:00:00 2001 From: Timothy DeHerrera Date: Fri, 14 Nov 2025 11:07:54 -0700 Subject: [PATCH 04/33] feat: store nix-lock import in binary Eventually we should fetch this expression at runtime, but this will serve as a good stopgap for boilerplate free execution into atoms for the time being while the API is still unstable. --- build/atom.lock | 6 +++--- build/default.nix | 16 ++++++++-------- build/eka/mod.nix | 7 ++++--- crates/eka-root-macro/src/lib.rs | 28 +++++++++++++++++++++++----- 4 files changed, 38 insertions(+), 19 deletions(-) diff --git a/build/atom.lock b/build/atom.lock index 69c70b5..91f2d97 100644 --- a/build/atom.lock +++ b/build/atom.lock @@ -15,9 +15,9 @@ mirrors = ["https://github.com/ekala-project/atom"] [locker] label = "nix-lock" -version = "0.1.6" +version = "0.2.0" set = "4abbd2644bc3585e9be95deb277ccf48f6ed26ac" -rev = "e711aa1f48d877652dd2ba724d4af752be7b5371" +rev = "c526c47a5d51feb2b6f86b3c2b9c90752903f51c" mirror = "https://github.com/ekala-project/eka" id = "r3rlam6bd31t9i04ggug7avs8vf8981q7a3b6gobemk1gfs50qm0" @@ -44,4 +44,4 @@ id = "3ot6f917im3d5qhs1jdo2samkthlahg5qcr4ql5e2neomv2v6dt0" type = "nix+git" name = "eka" url = "https://github.com/ekala-project/eka.git" -rev = "aa7399c89d6afae67e2eccd63e242ccd029dccd3" +rev = "e699f085b1ce4c37d9706bd445223f5cb430c626" diff --git a/build/default.nix b/build/default.nix index ef41a21..4c218ca 100644 --- a/build/default.nix +++ b/build/default.nix @@ -3,19 +3,19 @@ let lock = builtins.fromTOML lockstr; inherit (lock) locker; ekaSrc = builtins.head (builtins.filter (x: x.name == "eka") lock.deps); - lockexpr = - import - <| builtins.fetchGit { - inherit (locker) rev; - name = locker.label; - url = locker.mirror; - ref = "refs/eka/atoms/${locker.label}/${locker.version}"; - }; + fetch = builtins.fetchGit { + inherit (locker) rev; + name = locker.label; + url = locker.mirror; + ref = "refs/eka/atoms/${locker.label}/${locker.version}"; + }; + lockexpr = import fetch; in lockexpr ./. lockstr { # FIXME: this is the only way to get the repository source into the atom for now # The long-term solution to this problem is specified in ../adrs/0010-electron-sources.md extraExtern.ext = rec { + inherit locker fetch; cargo-lock = import (src + "/build/Cargo.nix"); src = if builtins.pathExists ../. then diff --git a/build/eka/mod.nix b/build/eka/mod.nix index d8e5af0..262640f 100644 --- a/build/eka/mod.nix +++ b/build/eka/mod.nix @@ -20,9 +20,10 @@ in # split up the atom crate and use some of its functionality in the proc macro # to compute these from the local repository without having to contact the # network. - EKA_ROOT_COMMIT_HASH = "4abbd2644bc3585e9be95deb277ccf48f6ed26ac"; - EKA_ORIGIN_URL = "https://github.com/ekala-project/eka"; - EKA_LOCK_REV = "e711aa1f48d877652dd2ba724d4af752be7b5371"; + EKA_ROOT_COMMIT_HASH = ext.locker.set; + EKA_ORIGIN_URL = ext.locker.mirror; + EKA_LOCK_REV = ext.locker.rev; + EKA_LOCK_IMPORT = ext.fetch + "/import.nix"; }; } // builtins.listToAttrs ( diff --git a/crates/eka-root-macro/src/lib.rs b/crates/eka-root-macro/src/lib.rs index e81f667..143807e 100644 --- a/crates/eka-root-macro/src/lib.rs +++ b/crates/eka-root-macro/src/lib.rs @@ -8,8 +8,8 @@ use quote::quote; const LOCK_LABEL: &str = "nix-lock"; const LOCK_MAJOR: u64 = 0; -const LOCK_MINOR: u64 = 1; -const LOCK_PATCH: u64 = 6; +const LOCK_MINOR: u64 = 2; +const LOCK_PATCH: u64 = 0; /// Computes Eka's repository root commit hash at compile time #[proc_macro] @@ -43,16 +43,34 @@ pub fn eka_origin_info(_input: TokenStream) -> TokenStream { } else { lock_rev() }; + let lock_import = if let Ok(file) = std::env::var("EKA_LOCK_IMPORT") { + std::fs::read_to_string(file).expect("could not read lock import from evironment") + } else { + get_repo() + .to_thread_local() + .find_commit(rev) + .ok() + .and_then(|c| c.tree().ok()) + .and_then(|t| t.find_entry("import.nix").and_then(|e| e.object().ok())) + .and_then(|o| String::from_utf8(o.data.clone()).ok()) + .expect("could not read lock import expression") + }; + let rev_tokens = rev.iter().map(|&byte| quote! { #byte }); quote! { - const LOCK_MAJOR: u64 = #LOCK_MAJOR; - const LOCK_MINOR: u64 = #LOCK_MINOR; - const LOCK_PATCH: u64 = #LOCK_PATCH; + /// the expression used to import the lockfile parser, pulled directly from the locker atom itself + /// TODO: in the future, we will want to pull this at runtime + pub const LOCK_IMPORT: &str = #lock_import; + pub(crate) const LOCK_LABEL: &str = #LOCK_LABEL; pub(crate) const LOCK_REV: [u8; 20] = [#(#rev_tokens),*]; pub(crate) const EKA_ORIGIN_URL: &str = #url; pub(crate) const EKA_ROOT_COMMIT_HASH: [u8; 20] = [#(#root_tokens),*]; + + const LOCK_MAJOR: u64 = #LOCK_MAJOR; + const LOCK_MINOR: u64 = #LOCK_MINOR; + const LOCK_PATCH: u64 = #LOCK_PATCH; } .into() } From ce8e3c4f386280a85155d963ea6f1565f1a1c530 Mon Sep 17 00:00:00 2001 From: Timothy DeHerrera Date: Sat, 15 Nov 2025 02:24:30 -0700 Subject: [PATCH 05/33] fix(git): retry if transports can't persist Some transports cannot persist multiple connections. We want to be able to maximally reuse transports whenever we can while still degrading gracefully when we have a transport implement which cannot. So during fetching, we call the network as usual, but if it fails, we check if the original transport we were handed is not able to persist via the `connection_persists_across_multiple_requests` method, and if not, we try again with a fresh transport. This avoids pointless retries if the connection failed for some other reason while allowing us to handle those transports in gix which do not currently support more than one request. Since we cannot know if our passed transport has already been used beforehand, we attempt the network call to find out first, then check. --- crates/atom/src/storage/git.rs | 114 ++++++++++++++++++++------------- 1 file changed, 71 insertions(+), 43 deletions(-) diff --git a/crates/atom/src/storage/git.rs b/crates/atom/src/storage/git.rs index 47dd4a8..8572386 100644 --- a/crates/atom/src/storage/git.rs +++ b/crates/atom/src/storage/git.rs @@ -690,35 +690,48 @@ impl super::QueryStore for gix::Url { }; let config = gix::config::File::from_globals()?; - let (mut cascade, _, prompt_opts) = gix::config::credential_helpers( - self.to_owned(), - &config, - true, - gix::config::section::is_trusted, - Environment { - xdg_config_home: Permission::Allow, - home: Permission::Allow, - http_transport: Permission::Allow, - identity: Permission::Allow, - objects: Permission::Allow, - git_prefix: Permission::Allow, - ssh_prefix: Permission::Allow, - }, - false, - )?; - let authenticate = Box::new(move |action| cascade.invoke(action, prompt_opts.clone())); - - let mut handshake = gix::protocol::fetch::handshake( - &mut *transport, - authenticate, - Vec::new(), - &mut prodash::progress::Discard, - ) - .map_err(|e| { - tracing::error!(url = %self, "couldn't establish a handshake with the remote"); - Box::new(e) - })?; + let shake_hands = |transport: &mut Box| { + let (mut cascade, _, prompt_opts) = gix::config::credential_helpers( + self.to_owned(), + &config, + true, + gix::config::section::is_trusted, + Environment { + xdg_config_home: Permission::Allow, + home: Permission::Allow, + http_transport: Permission::Allow, + identity: Permission::Allow, + objects: Permission::Allow, + git_prefix: Permission::Allow, + ssh_prefix: Permission::Allow, + }, + false, + )?; + let authenticate = Box::new(move |action| cascade.invoke(action, prompt_opts.clone())); + + gix::protocol::fetch::handshake( + &mut *transport, + authenticate, + Vec::new(), + &mut prodash::progress::Discard, + ) + .map_err(Box::new) + .map_err(Error::Handshake) + }; + + let mut handshake = shake_hands(transport) + .or_else(|e| { + if !transport.connection_persists_across_multiple_requests() { + let mut transport = self.get_transport()?; + shake_hands(&mut transport) + } else { + Err(e) + } + }) + .inspect_err(|_| { + tracing::error!(url = %self, "couldn't establish a handshake with the remote"); + })?; tracing::debug!(?targets, url = %self, "checking remote for refs"); use gix::refspec::parse::Operation; @@ -820,8 +833,6 @@ impl<'repo> super::QueryStore for gix::Remote<'repo> { use tracing::level_filters::LevelFilter; let tree = Root::new(); - let sync_progress = tree.add_child("sync"); - let init_progress = tree.add_child("init"); let _ = if LevelFilter::current() > LevelFilter::WARN { Some(setup_line_renderer(&tree)) } else { @@ -834,22 +845,39 @@ impl<'repo> super::QueryStore for gix::Remote<'repo> { .replace_refspecs(references, Direction::Fetch) .map_err(Box::new)?; - let transport = if let Some(transport) = transport { - transport - } else { - &mut remote.get_transport()? - }; + let fetch = |transport: Option<&mut Self::Transport>| { + let sync_progress = tree.add_child("sync"); + let init_progress = tree.add_child("init"); - let client = remote.to_connection_with_transport(transport); + let transport = if let Some(transport) = transport { + transport + } else { + &mut remote.get_transport()? + }; - let query = client - .prepare_fetch(sync_progress, Options::default()) - .map_err(Box::new)?; + remote + .to_connection_with_transport(transport) + .prepare_fetch(sync_progress, Options::default()) + .map_err(Box::new) + .map_err(Error::Refs) + .and_then(|p| { + Ok(p.with_write_packed_refs_only(true) + .receive(init_progress, &AtomicBool::new(false)) + .map_err(Box::new)?) + }) + }; - let outcome = query - .with_write_packed_refs_only(true) - .receive(init_progress, &AtomicBool::new(false)) - .map_err(Box::new)?; + let supports_multiple = transport + .as_ref() + .is_some_and(|t| t.connection_persists_across_multiple_requests()); + + let outcome = fetch(transport).or_else(|e| { + if !supports_multiple { + fetch(None) + } else { + Err(e) + } + })?; Ok(outcome.ref_map.remote_refs) } From 25298f1acde3b66899c8194186afe53c0f52d2b4 Mon Sep 17 00:00:00 2001 From: Timothy DeHerrera Date: Sat, 15 Nov 2025 02:57:25 -0700 Subject: [PATCH 06/33] fix(git): get_ref needs to be filtered Even though git only fetches the objects specified in the target, the server can actually return more refs than requested as an optimization. For this reason, the call to `next()` would return the wrong ref. Instead we use `find` to ensure the ref we return matches the one we actually requested from the server. --- crates/atom/src/storage/git.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/crates/atom/src/storage/git.rs b/crates/atom/src/storage/git.rs index 8572386..228847b 100644 --- a/crates/atom/src/storage/git.rs +++ b/crates/atom/src/storage/git.rs @@ -774,7 +774,10 @@ impl super::QueryStore for gix::Url { let name = target.as_ref().to_string(); self.get_refs(Some(target), transport).and_then(|r| { r.into_iter() - .next() + .find(|r| { + let (n, ..) = r.unpack(); + name.contains(n.to_string().as_str()) + }) .ok_or(Error::NoRef(name, self.to_string())) }) } @@ -893,7 +896,10 @@ impl<'repo> super::QueryStore for gix::Remote<'repo> { let name = target.as_ref().to_string(); self.get_refs(Some(target), transport).and_then(|r| { r.into_iter() - .next() + .find(|r| { + let (n, ..) = r.unpack(); + name.contains(n.to_string().as_str()) + }) .ok_or(Error::NoRef(name, self.symbol().to_owned())) }) } From 3fb694316232b39932dd5dfb036452fa18c6e3a0 Mon Sep 17 00:00:00 2001 From: Timothy DeHerrera Date: Sat, 15 Nov 2025 04:25:14 -0700 Subject: [PATCH 07/33] feat(storage::git): add caching for remote atoms Implement bare monorepo cache for git remote atoms, using base58-encoded genesis hashes as remote names. Store atoms under refs//