From 69cfd0e7ca36ba272bda06f3ea58570944d61d51 Mon Sep 17 00:00:00 2001 From: Ivan Nikolic Date: Tue, 30 Dec 2025 17:20:40 +0800 Subject: [PATCH] jeff flake without py env stuff --- flake.lock | 101 ++++++++++++++++++- flake.nix | 288 ++++++++++++++++++++++++++++++++++++++++++++--------- 2 files changed, 339 insertions(+), 50 deletions(-) diff --git a/flake.lock b/flake.lock index e6d920a293..5231ed9039 100644 --- a/flake.lock +++ b/flake.lock @@ -18,6 +18,78 @@ "type": "github" } }, + "home-manager": { + "inputs": { + "nixpkgs": [ + "xome", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1753983724, + "narHash": "sha256-2vlAOJv4lBrE+P1uOGhZ1symyjXTRdn/mz0tZ6faQcg=", + "owner": "nix-community", + "repo": "home-manager", + "rev": "7035020a507ed616e2b20c61491ae3eaa8e5462c", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "home-manager", + "type": "github" + } + }, + "lib": { + "inputs": { + "flakeUtils": [ + "flake-utils" + ], + "libSource": "libSource" + }, + "locked": { + "lastModified": 1764022662, + "narHash": "sha256-vS3EeyELqCskh88JkUW/ce8A8b3m+iRPLPd4kDRTqPY=", + "owner": "jeff-hykin", + "repo": "quick-nix-toolkits", + "rev": "de1cc174579ecc7b655de5ba9618548d1b72306c", + "type": "github" + }, + "original": { + "owner": "jeff-hykin", + "repo": "quick-nix-toolkits", + "type": "github" + } + }, + "libSource": { + "locked": { + "lastModified": 1766884708, + "narHash": "sha256-x8nyRwtD0HMeYtX60xuIuZJbwwoI7/UKAdCiATnQNz0=", + "owner": "nix-community", + "repo": "nixpkgs.lib", + "rev": "15177f81ad356040b4460a676838154cbf7f6213", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "nixpkgs.lib", + "type": "github" + } + }, + "libSource_2": { + "locked": { + "lastModified": 1753579242, + "narHash": "sha256-zvaMGVn14/Zz8hnp4VWT9xVnhc8vuL3TStRqwk22biA=", + "owner": "divnix", + "repo": "nixpkgs.lib", + "rev": "0f36c44e01a6129be94e3ade315a5883f0228a6e", + "type": "github" + }, + "original": { + "owner": "divnix", + "repo": "nixpkgs.lib", + "type": "github" + } + }, "nixpkgs": { "locked": { "lastModified": 1748929857, @@ -37,7 +109,9 @@ "root": { "inputs": { "flake-utils": "flake-utils", - "nixpkgs": "nixpkgs" + "lib": "lib", + "nixpkgs": "nixpkgs", + "xome": "xome" } }, "systems": { @@ -54,6 +128,31 @@ "repo": "default", "type": "github" } + }, + "xome": { + "inputs": { + "flake-utils": [ + "flake-utils" + ], + "home-manager": "home-manager", + "libSource": "libSource_2", + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1765466883, + "narHash": "sha256-c4YxXoS6U9BFcxP4TWZirwycaxT2oFyPMeyVp5vrME8=", + "owner": "jeff-hykin", + "repo": "xome", + "rev": "1f3507c4985e05177bd1a5b57d2862e30bb5da9b", + "type": "github" + }, + "original": { + "owner": "jeff-hykin", + "repo": "xome", + "type": "github" + } } }, "root": "root", diff --git a/flake.nix b/flake.nix index 6491c5d88a..5b3c7ad1dc 100644 --- a/flake.nix +++ b/flake.nix @@ -4,9 +4,14 @@ inputs = { nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; flake-utils.url = "github:numtide/flake-utils"; + lib.url = "github:jeff-hykin/quick-nix-toolkits"; + lib.inputs.flakeUtils.follows = "flake-utils"; + xome.url = "github:jeff-hykin/xome"; + xome.inputs.nixpkgs.follows = "nixpkgs"; + xome.inputs.flake-utils.follows = "flake-utils"; }; - outputs = { self, nixpkgs, flake-utils, ... }: + outputs = { self, nixpkgs, flake-utils, lib, xome, ... }: flake-utils.lib.eachDefaultSystem (system: let pkgs = import nixpkgs { inherit system; }; @@ -14,80 +19,265 @@ # ------------------------------------------------------------ # 1. Shared package list (tool-chain + project deps) # ------------------------------------------------------------ - devPackages = with pkgs; [ + # we "flag" each package with what we need it for (e.g. LD_LIBRARY_PATH, nativeBuildInputs vs buildInputs, etc) + aggregation = lib.aggregator [ ### Core shell & utils - bashInteractive coreutils gh - stdenv.cc.cc.lib pcre2 + { vals.pkg=pkgs.bashInteractive; flags={}; } + { vals.pkg=pkgs.coreutils; flags={}; } + { vals.pkg=pkgs.gh; flags={}; } + { vals.pkg=pkgs.stdenv.cc.cc.lib; flags.ldLibraryGroup=true; } + { vals.pkg=pkgs.stdenv.cc; flags.ldLibraryGroup=true; } + { vals.pkg=pkgs.cctools; flags={}; onlyIf=pkgs.stdenv.isDarwin; } # for pip install opencv-python + { vals.pkg=pkgs.pcre2; flags={ ldLibraryGroup=pkgs.stdenv.isDarwin; packageConfGroup=pkgs.stdenv.isDarwin; }; } + { vals.pkg=pkgs.libsysprof-capture; flags.packageConfGroup=true; onlyIf=pkgs.stdenv.isDarwin; } + { vals.pkg=pkgs.xcbuild; flags={}; } + { vals.pkg=pkgs.git-lfs; flags={}; } + { vals.pkg=pkgs.gnugrep; flags={}; } + { vals.pkg=pkgs.gnused; flags={}; } + { vals.pkg=pkgs.iproute2; flags={}; onlyIf=pkgs.stdenv.isLinux; } + { vals.pkg=pkgs.pkg-config; flags={}; } + { vals.pkg=pkgs.git; flags={}; } + { vals.pkg=pkgs.opensshWithKerberos;flags={}; } + { vals.pkg=pkgs.unixtools.ifconfig; flags={}; } + { vals.pkg=pkgs.unixtools.netstat; flags={}; } + + # when pip packages call cc with -I/usr/include, that causes problems on some machines, this swaps that out for the nix cc headers + # this is only necessary for pip packages from venv, pip packages from nixpkgs.python312Packages.* already have "-I/usr/include" patched with the nix equivalent + { + vals.pkg=(pkgs.writeShellScriptBin + "cc-no-usr-include" + '' + #!${pkgs.bash}/bin/bash + set -euo pipefail + + real_cc="${pkgs.stdenv.cc}/bin/gcc" + + args=() + for a in "$@"; do + case "$a" in + -I/usr/include|-I/usr/local/include) + # drop these + ;; + *) + args+=("$a") + ;; + esac + done + + exec "$real_cc" "''${args[@]}" + '' + ); + flags={}; + } ### Python + static analysis - python312 python312Packages.pip python312Packages.setuptools - python312Packages.virtualenv pre-commit + { vals.pkg=pkgs.python312; flags={}; vals.pythonMinorVersion="12";} + { vals.pkg=pkgs.python312Packages.pip; flags={}; } + { vals.pkg=pkgs.python312Packages.setuptools; flags={}; } + { vals.pkg=pkgs.python312Packages.virtualenv; flags={}; } + { vals.pkg=pkgs.pre-commit; flags={}; } ### Runtime deps - python312Packages.pyaudio portaudio ffmpeg_6 ffmpeg_6.dev + { vals.pkg=pkgs.portaudio; flags={ldLibraryGroup=true; packageConfGroup=true;}; } + { vals.pkg=pkgs.ffmpeg_6; flags={}; } + { vals.pkg=pkgs.ffmpeg_6.dev; flags={}; } ### Graphics / X11 stack - libGL libGLU mesa glfw - xorg.libX11 xorg.libXi xorg.libXext xorg.libXrandr xorg.libXinerama - xorg.libXcursor xorg.libXfixes xorg.libXrender xorg.libXdamage - xorg.libXcomposite xorg.libxcb xorg.libXScrnSaver xorg.libXxf86vm - - udev SDL2 SDL2.dev zlib + { vals.pkg=pkgs.libGL; flags.ldLibraryGroup=true; onlyIf=pkgs.stdenv.isLinux; } + { vals.pkg=pkgs.libGLU; flags.ldLibraryGroup=true; onlyIf=pkgs.stdenv.isLinux; } + { vals.pkg=pkgs.mesa; flags.ldLibraryGroup=true; onlyIf=pkgs.stdenv.isLinux; } + { vals.pkg=pkgs.glfw; flags.ldLibraryGroup=true; onlyIf=pkgs.stdenv.isLinux; } + { vals.pkg=pkgs.xorg.libX11; flags.ldLibraryGroup=true; onlyIf=pkgs.stdenv.isLinux; } + { vals.pkg=pkgs.xorg.libXi; flags.ldLibraryGroup=true; onlyIf=pkgs.stdenv.isLinux; } + { vals.pkg=pkgs.xorg.libXext; flags.ldLibraryGroup=true; onlyIf=pkgs.stdenv.isLinux; } + { vals.pkg=pkgs.xorg.libXrandr; flags.ldLibraryGroup=true; onlyIf=pkgs.stdenv.isLinux; } + { vals.pkg=pkgs.xorg.libXinerama; flags.ldLibraryGroup=true; onlyIf=pkgs.stdenv.isLinux; } + { vals.pkg=pkgs.xorg.libXcursor; flags.ldLibraryGroup=true; onlyIf=pkgs.stdenv.isLinux; } + { vals.pkg=pkgs.xorg.libXfixes; flags.ldLibraryGroup=true; onlyIf=pkgs.stdenv.isLinux; } + { vals.pkg=pkgs.xorg.libXrender; flags.ldLibraryGroup=true; onlyIf=pkgs.stdenv.isLinux; } + { vals.pkg=pkgs.xorg.libXdamage; flags.ldLibraryGroup=true; onlyIf=pkgs.stdenv.isLinux; } + { vals.pkg=pkgs.xorg.libXcomposite; flags.ldLibraryGroup=true; onlyIf=pkgs.stdenv.isLinux; } + { vals.pkg=pkgs.xorg.libxcb; flags.ldLibraryGroup=true; onlyIf=pkgs.stdenv.isLinux; } + { vals.pkg=pkgs.xorg.libXScrnSaver; flags.ldLibraryGroup=true; onlyIf=pkgs.stdenv.isLinux; } + { vals.pkg=pkgs.xorg.libXxf86vm; flags.ldLibraryGroup=true; onlyIf=pkgs.stdenv.isLinux; } + { vals.pkg=pkgs.udev; flags.ldLibraryGroup=true; onlyIf=pkgs.stdenv.isLinux; } + { vals.pkg=pkgs.SDL2; flags.ldLibraryGroup=true; onlyIf=pkgs.stdenv.isLinux; } + { vals.pkg=pkgs.SDL2.dev; flags.ldLibraryGroup=true; onlyIf=pkgs.stdenv.isLinux; } + { vals.pkg=pkgs.zlib; flags.ldLibraryGroup=true; onlyIf=pkgs.stdenv.isLinux; } ### GTK / OpenCV helpers - glib gtk3 gdk-pixbuf gobject-introspection + { vals.pkg=pkgs.glib; flags.ldLibraryGroup=true; onlyIf=pkgs.stdenv.isLinux; } + { vals.pkg=pkgs.gtk3; flags.ldLibraryGroup=true; onlyIf=pkgs.stdenv.isLinux; } + { vals.pkg=pkgs.gdk-pixbuf; flags.ldLibraryGroup=true; onlyIf=pkgs.stdenv.isLinux; } + { vals.pkg=pkgs.gobject-introspection; flags.ldLibraryGroup=true; onlyIf=pkgs.stdenv.isLinux; } ### GStreamer - gst_all_1.gstreamer gst_all_1.gst-plugins-base gst_all_1.gst-plugins-good - gst_all_1.gst-plugins-bad gst_all_1.gst-plugins-ugly - python312Packages.gst-python + { vals.pkg=pkgs.gst_all_1.gstreamer; flags.ldLibraryGroup=true; flags.giTypelibGroup=true; } + { vals.pkg=pkgs.gst_all_1.gst-plugins-base; flags.ldLibraryGroup=true; flags.giTypelibGroup=true; } + { vals.pkg=pkgs.gst_all_1.gst-plugins-good; flags={}; onlyIf=pkgs.stdenv.isLinux; } + { vals.pkg=pkgs.gst_all_1.gst-plugins-bad; flags={}; onlyIf=pkgs.stdenv.isLinux; } + { vals.pkg=pkgs.gst_all_1.gst-plugins-ugly; flags={}; onlyIf=pkgs.stdenv.isLinux; } + { vals.pkg=pkgs.python312Packages.gst-python; flags={}; onlyIf=pkgs.stdenv.isLinux; } ### Open3D & build-time - eigen cmake ninja jsoncpp libjpeg libjpeg_turbo libpng + { vals.pkg=pkgs.eigen; flags={}; } + { vals.pkg=pkgs.cmake; flags={}; } + { vals.pkg=pkgs.ninja; flags={}; } + { vals.pkg=pkgs.jsoncpp; flags={}; } + { vals.pkg=pkgs.libjpeg; flags.ldLibraryGroup=true; } + { vals.pkg=pkgs.libjpeg_turbo; flags.ldLibraryGroup=true; } + { vals.pkg=pkgs.libpng; flags={}; } ### LCM (Lightweight Communications and Marshalling) - lcm + { vals.pkg=pkgs.lcm; flags.ldLibraryGroup=true; onlyIf=pkgs.stdenv.isLinux; } + # lcm works on darwin, but only after two fixes (1. pkg-config, 2. fsync) + { + onlyIf=pkgs.stdenv.isDarwin; + flags.ldLibraryGroup=true; + flags.manualPythonPackages=true; + vals.pkg=pkgs.lcm.overrideAttrs (old: + let + # 1. fix pkg-config on darwin + pkgConfPackages = aggregation.getAll { hasAllFlags=[ "packageConfGroup" ]; attrPath=[ "pkg" ]; }; + packageConfPackagesString = (aggregation.getAll { + hasAllFlags=[ "packageConfGroup" ]; + attrPath=[ "pkg" ]; + strAppend="/lib/pkgconfig"; + strJoin=":"; + }); + in + { + buildInputs = (old.buildInputs or []) ++ pkgConfPackages; + nativeBuildInputs = (old.nativeBuildInputs or []) ++ [ pkgs.pkg-config pkgs.python312 ]; + # 1. fix pkg-config on darwin + env.PKG_CONFIG_PATH = packageConfPackagesString; + # 2. Fix fsync on darwin + patches = [ + (pkgs.writeText "lcm-darwin-fsync.patch" "--- ./lcm-logger/lcm_logger.c 2025-11-14 09:46:01.000000000 -0600\n+++ ./lcm-logger/lcm_logger.c 2025-11-14 09:47:05.000000000 -0600\n@@ -428,9 +428,13 @@\n if (needs_flushed) {\n fflush(logger->log->f);\n #ifndef WIN32\n+#ifdef __APPLE__\n+ fsync(fileno(logger->log->f));\n+#else\n // Perform a full fsync operation after flush\n fdatasync(fileno(logger->log->f));\n #endif\n+#endif\n logger->last_fflush_time = log_event->timestamp;\n }\n") + ]; + } + ); + } ]; # ------------------------------------------------------------ - # 2. Host interactive shell → `nix develop` + # 2. group / aggregate our packages # ------------------------------------------------------------ - devShell = pkgs.mkShell { - packages = devPackages; - shellHook = '' + devPackages = aggregation.getAll { attrPath=[ "pkg" ]; }; + ldLibraryPackages = aggregation.getAll { hasAllFlags=[ "ldLibraryGroup" ]; attrPath=[ "pkg" ]; }; + giTypelibPackagesString = aggregation.getAll { + hasAllFlags=[ "giTypelibGroup" ]; + attrPath=[ "pkg" ]; + strAppend="/lib/girepository-1.0"; + strJoin=":"; + }; + packageConfPackagesString = (aggregation.getAll { + hasAllFlags=[ "packageConfGroup" ]; + attrPath=[ "pkg" ]; + strAppend="/lib/pkgconfig"; + strJoin=":"; + }); + manualPythonPackages = (aggregation.getAll { + hasAllFlags=[ "manualPythonPackages" ]; + attrPath=[ "pkg" ]; + strAppend="/lib/python3.${aggregation.mergedVals.pythonMinorVersion}/site-packages"; + strJoin=":"; + }); + + # ------------------------------------------------------------ + # 3. Host interactive shell → `nix develop` + # ------------------------------------------------------------ + shellHook = '' + shopt -s nullglob 2>/dev/null || setopt +o nomatch 2>/dev/null || true # allow globs to be empty without throwing an error + if [ "$OSTYPE" = "linux-gnu" ]; then + export CC="cc-no-usr-include" # basically patching for nix # Create nvidia-only lib symlinks to avoid glibc conflicts - NVIDIA_LIBS_DIR="/tmp/nix-nvidia-libs-$$" + NVIDIA_LIBS_DIR="/tmp/nix-nvidia-libs" mkdir -p "$NVIDIA_LIBS_DIR" - for lib in /usr/lib/libcuda.so* /usr/lib/libnvidia*.so*; do + for lib in /usr/lib/libcuda.so* /usr/lib/libnvidia*.so* /usr/lib/x86_64-linux-gnu/libnvidia*.so*; do [ -e "$lib" ] && ln -sf "$lib" "$NVIDIA_LIBS_DIR/" 2>/dev/null done + fi + export LD_LIBRARY_PATH="$NVIDIA_LIBS_DIR:${pkgs.lib.makeLibraryPath ldLibraryPackages}:$LD_LIBRARY_PATH" + export LIBRARY_PATH="$LD_LIBRARY_PATH" # fixes python find_library for pyaudio + export DISPLAY=:0 + export GI_TYPELIB_PATH="${giTypelibPackagesString}:$GI_TYPELIB_PATH" + export PKG_CONFIG_PATH=${lib.escapeShellArg packageConfPackagesString} + export PYTHONPATH="$PYTHONPATH:"${lib.escapeShellArg manualPythonPackages} + # CC, CFLAGS, and LDFLAGS are bascially all for `pip install pyaudio` + export CFLAGS="$(pkg-config --cflags portaudio-2.0) $CFLAGS" + export LDFLAGS="-L$(pkg-config --variable=libdir portaudio-2.0) $LDFLAGS" + + # without this alias, the pytest uses the non-venv python and fails + alias pytest="python -m pytest" + + PROJECT_ROOT=$(git rev-parse --show-toplevel 2>/dev/null || echo "$PWD") + if [ -f "$PROJECT_ROOT/env/bin/activate" ]; then + . "$PROJECT_ROOT/env/bin/activate" + fi + + [ -f "$PROJECT_ROOT/motd" ] && cat "$PROJECT_ROOT/motd" + [ -f "$PROJECT_ROOT/.pre-commit-config.yaml" ] && pre-commit install --install-hooks + ''; + devShells = { + # basic shell (blends with your current environment) + default = pkgs.mkShell { + buildInputs = devPackages; + shellHook = shellHook; + }; + # strict shell (creates a fake home, only select exteral commands (e.g. sudo) from your system are available) + isolated = (xome.simpleMakeHomeFor { + inherit pkgs; + pure = true; + commandPassthrough = [ "sudo" "nvim" "code" "sysctl" "sw_vers" "git" "vim" "emacs" "openssl" "ssh" "osascript" "otool" "hidutil" "logger" "codesign" ]; # e.g. use external nvim instead of nix's + # commonly needed for MacOS: [ "osascript" "otool" "hidutil" "logger" "codesign" ] + homeSubpathPassthrough = [ "cache/nix/" ]; # share nix cache between projects + homeModule = { + # for home-manager examples, see: + # https://deepwiki.com/nix-community/home-manager/5-configuration-examples + # all home-manager options: + # https://nix-community.github.io/home-manager/options.xhtml + home.homeDirectory = "/tmp/virtual_homes/dimos"; + home.stateVersion = "25.11"; + home.packages = devPackages; - export LD_LIBRARY_PATH="$NVIDIA_LIBS_DIR:${pkgs.lib.makeLibraryPath [ - pkgs.stdenv.cc.cc.lib pkgs.libGL pkgs.libGLU pkgs.mesa pkgs.glfw - pkgs.xorg.libX11 pkgs.xorg.libXi pkgs.xorg.libXext pkgs.xorg.libXrandr - pkgs.xorg.libXinerama pkgs.xorg.libXcursor pkgs.xorg.libXfixes - pkgs.xorg.libXrender pkgs.xorg.libXdamage pkgs.xorg.libXcomposite - pkgs.xorg.libxcb pkgs.xorg.libXScrnSaver pkgs.xorg.libXxf86vm - pkgs.udev pkgs.portaudio pkgs.SDL2.dev pkgs.zlib pkgs.glib pkgs.gtk3 - pkgs.gdk-pixbuf pkgs.gobject-introspection pkgs.lcm - pkgs.gst_all_1.gstreamer pkgs.gst_all_1.gst-plugins-base - pkgs.libjpeg_turbo]}:$LD_LIBRARY_PATH" - - export DISPLAY=:0 - export GI_TYPELIB_PATH="${pkgs.gst_all_1.gstreamer}/lib/girepository-1.0:${pkgs.gst_all_1.gst-plugins-base}/lib/girepository-1.0:$GI_TYPELIB_PATH" - - PROJECT_ROOT=$(git rev-parse --show-toplevel 2>/dev/null || echo "$PWD") - if [ -f "$PROJECT_ROOT/env/bin/activate" ]; then - . "$PROJECT_ROOT/env/bin/activate" - fi - - [ -f "$PROJECT_ROOT/motd" ] && cat "$PROJECT_ROOT/motd" - [ -f "$PROJECT_ROOT/.pre-commit-config.yaml" ] && pre-commit install --install-hooks - ''; + programs = { + home-manager = { + enable = true; + }; + zsh = { + enable = true; + enableCompletion = true; + autosuggestion.enable = true; + syntaxHighlighting.enable = true; + shellAliases.ll = "ls -la"; + history.size = 100000; + # this is kinda like .zshrc + initContent = '' + # most people expect comments in their shell to to work + setopt interactivecomments + # fix emoji prompt offset issues (this shouldn't lock people into English b/c LANG can be non-english) + export LC_CTYPE=en_US.UTF-8 + ${shellHook} + ''; + }; + starship = { + enable = true; + enableZshIntegration = true; + settings = { + character = { + success_symbol = "[▣](bold green)"; + error_symbol = "[▣](bold red)"; + }; + }; + }; + }; + }; + }).default; }; # ------------------------------------------------------------ - # 3. Closure copied into the OCI image rootfs + # 4. Closure copied into the OCI image rootfs # ------------------------------------------------------------ imageRoot = pkgs.buildEnv { name = "dimos-image-root"; @@ -97,7 +287,7 @@ in { ## Local dev shell - devShells.default = devShell; + devShells = devShells; ## Layered docker image with DockerTools packages.devcontainer = pkgs.dockerTools.buildLayeredImage {