From 36a874d0e86140f6fc1ca9268a70544a4a50de8e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 17 Oct 2025 13:56:17 +0200 Subject: [PATCH 1/8] Source: Add skip() method This allows FdSource to efficiently skip data we don't care about. --- src/libutil/include/nix/util/serialise.hh | 5 +++ src/libutil/serialise.cc | 48 +++++++++++++++++++++-- 2 files changed, 50 insertions(+), 3 deletions(-) diff --git a/src/libutil/include/nix/util/serialise.hh b/src/libutil/include/nix/util/serialise.hh index 16e0d0fa568..8799e128fc4 100644 --- a/src/libutil/include/nix/util/serialise.hh +++ b/src/libutil/include/nix/util/serialise.hh @@ -97,6 +97,8 @@ struct Source void drainInto(Sink & sink); std::string drain(); + + virtual void skip(size_t len); }; /** @@ -177,6 +179,7 @@ struct FdSource : BufferedSource Descriptor fd; size_t read = 0; BackedStringView endOfFileError{"unexpected end-of-file"}; + bool isSeekable = true; FdSource() : fd(INVALID_DESCRIPTOR) @@ -200,6 +203,8 @@ struct FdSource : BufferedSource */ bool hasData(); + void skip(size_t len) override; + protected: size_t readUnbuffered(char * data, size_t len) override; private: diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 15629935e12..bdce956f311 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -94,9 +94,8 @@ void Source::drainInto(Sink & sink) { std::array buf; while (true) { - size_t n; try { - n = read(buf.data(), buf.size()); + auto n = read(buf.data(), buf.size()); sink({buf.data(), n}); } catch (EndOfFile &) { break; @@ -111,6 +110,16 @@ std::string Source::drain() return std::move(s.s); } +void Source::skip(size_t len) +{ + std::array buf; + while (len) { + auto n = read(buf.data(), std::min(len, buf.size())); + assert(n <= len); + len -= n; + } +} + size_t BufferedSource::read(char * data, size_t len) { if (!buffer) @@ -120,7 +129,7 @@ size_t BufferedSource::read(char * data, size_t len) bufPosIn = readUnbuffered(buffer.get(), bufSize); /* Copy out the data in the buffer. */ - size_t n = len > bufPosIn - bufPosOut ? bufPosIn - bufPosOut : len; + auto n = std::min(len, bufPosIn - bufPosOut); memcpy(data, buffer.get() + bufPosOut, n); bufPosOut += n; if (bufPosIn == bufPosOut) @@ -191,6 +200,39 @@ bool FdSource::hasData() } } +void FdSource::skip(size_t len) +{ + /* Discard data in the buffer. */ + if (len && buffer && bufPosIn - bufPosOut) { + if (len >= bufPosIn - bufPosOut) { + len -= bufPosIn - bufPosOut; + bufPosIn = bufPosOut = 0; + } else { + bufPosOut += len; + len = 0; + } + } + +#ifndef _WIN32 + /* If we can, seek forward in the file to skip the rest. */ + if (isSeekable && len) { + if (lseek(fd, len, SEEK_CUR) == -1) { + if (errno == ESPIPE) + isSeekable = false; + else + throw SysError("seeking forward in file"); + } else { + read += len; + return; + } + } +#endif + + /* Otherwise, skip by reading. */ + if (len) + BufferedSource::skip(len); +} + size_t StringSource::read(char * data, size_t len) { if (pos == s.size()) From cc3a1ac1430216546de60463907cf7f4076d550e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 17 Oct 2025 18:32:47 +0200 Subject: [PATCH 2/8] NullFileSystemObjectSink: Skip over file contents --- src/libutil/archive.cc | 7 ++++++- src/libutil/fs-sink.cc | 2 ++ src/libutil/include/nix/util/fs-sink.hh | 8 ++++++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index b978ac4dbff..560757e1e45 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -132,6 +132,11 @@ static void parseContents(CreateRegularFileSink & sink, Source & source) sink.preallocateContents(size); + if (sink.skipContents) { + source.skip(size + (size % 8 ? 8 - (size % 8) : 0)); + return; + } + uint64_t left = size; std::array buf; @@ -166,7 +171,7 @@ static void parse(FileSystemObjectSink & sink, Source & source, const CanonPath auto expectTag = [&](std::string_view expected) { auto tag = getString(); if (tag != expected) - throw badArchive("expected tag '%s', got '%s'", expected, tag); + throw badArchive("expected tag '%s', got '%s'", expected, tag.substr(0, 1024)); }; expectTag("("); diff --git a/src/libutil/fs-sink.cc b/src/libutil/fs-sink.cc index 6efd5e0c7e2..45ef57a9f5b 100644 --- a/src/libutil/fs-sink.cc +++ b/src/libutil/fs-sink.cc @@ -196,6 +196,8 @@ void NullFileSystemObjectSink::createRegularFile( void isExecutable() override {} } crf; + crf.skipContents = true; + // Even though `NullFileSystemObjectSink` doesn't do anything, it's important // that we call the function, to e.g. advance the parser using this // sink. diff --git a/src/libutil/include/nix/util/fs-sink.hh b/src/libutil/include/nix/util/fs-sink.hh index f96fe3ef954..bd2db7f53e6 100644 --- a/src/libutil/include/nix/util/fs-sink.hh +++ b/src/libutil/include/nix/util/fs-sink.hh @@ -14,6 +14,14 @@ namespace nix { */ struct CreateRegularFileSink : Sink { + /** + * If set to true, the sink will not be called with the contents + * of the file. `preallocateContents()` will still be called to + * convey the file size. Useful for sinks that want to efficiently + * discard the contents of the file. + */ + bool skipContents = false; + virtual void isExecutable() = 0; /** From aa1569017e76bcf2324bad6d13413b3f0ada5840 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 17 Oct 2025 20:23:20 +0200 Subject: [PATCH 3/8] nix store dump-path: Refuse to write NARs to the terminal --- src/nix/dump-path.cc | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/nix/dump-path.cc b/src/nix/dump-path.cc index 8475655e927..f375b0ac8e4 100644 --- a/src/nix/dump-path.cc +++ b/src/nix/dump-path.cc @@ -4,6 +4,14 @@ using namespace nix; +static FdSink getNarSink() +{ + auto fd = getStandardOutput(); + if (isatty(fd)) + throw UsageError("refusing to write NAR to a terminal"); + return FdSink(std::move(fd)); +} + struct CmdDumpPath : StorePathCommand { std::string description() override @@ -20,7 +28,7 @@ struct CmdDumpPath : StorePathCommand void run(ref store, const StorePath & storePath) override { - FdSink sink(getStandardOutput()); + auto sink = getNarSink(); store->narFromPath(storePath, sink); sink.flush(); } @@ -51,7 +59,7 @@ struct CmdDumpPath2 : Command void run() override { - FdSink sink(getStandardOutput()); + auto sink = getNarSink(); dumpPath(path, sink); sink.flush(); } From e8ab7e4018c05b1a051107e75cc2af3ccda465c8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 17 Oct 2025 13:57:46 +0200 Subject: [PATCH 4/8] nix nario list: Efficiently skip NARs This makes it way faster: on a 15 GB system closure, from 7.42s to 0.42s on a cold page cache, and from 5.64s to 0.13s on a hot cache. --- src/nix/nario.cc | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/nix/nario.cc b/src/nix/nario.cc index 0439c5c6809..8b093470390 100644 --- a/src/nix/nario.cc +++ b/src/nix/nario.cc @@ -154,9 +154,7 @@ struct CmdNarioList : Command addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs) override { logger->cout(fmt("%s: %d bytes", printStorePath(info.path), info.narSize)); - // Discard the NAR. - NullFileSystemObjectSink parseSink; - parseDump(parseSink, source); + source.skip(info.narSize); } StorePath addToStoreFromDump( From 7e31bdc64ac1c99285cf9d6ff2c31fe374c0a330 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 17 Oct 2025 14:19:56 +0200 Subject: [PATCH 5/8] nix nario list: Add --json flag --- src/nix/nario-list.md | 27 ++++++++++++++++++++++++++- src/nix/nario.cc | 21 +++++++++++++++++++-- tests/functional/export.sh | 3 +++ 3 files changed, 48 insertions(+), 3 deletions(-) diff --git a/src/nix/nario-list.md b/src/nix/nario-list.md index 80c1f10d7a0..ea833586074 100644 --- a/src/nix/nario-list.md +++ b/src/nix/nario-list.md @@ -5,12 +5,37 @@ R""( * List the contents of a nario file: ```console - # nix nario list < dump + # nix nario list < dump.nario /nix/store/4y1jj6cwvslmfh1bzkhbvhx77az6yf00-xgcc-14.2.1.20250322-libgcc: 201856 bytes /nix/store/d8hnbm5hvbg2vza50garppb63y724i94-libunistring-1.3: 2070240 bytes … ``` +* Use `--json` to get detailed information in JSON format: + + ```console + # nix nario list --json < dump.nario + { + "paths": { + "/nix/store/m1r53pnn…-hello-2.12.1": { + "ca": null, + "deriver": "/nix/store/qa8is0vm…-hello-2.12.1.drv", + "narHash": "sha256-KSCYs4J7tFa+oX7W5M4D7ZYNvrWtdcWTdTL5fQk+za8=", + "narSize": 234672, + "references": [ + "/nix/store/g8zyryr9…-glibc-2.40-66", + "/nix/store/m1r53pnn…-hello-2.12.1" + ], + "registrationTime": 1756900709, + "signatures": [ "cache.nixos.org-1:QbG7A…" ], + "ultimate": false + }, + … + }, + "version": 1 + } + ``` + # Description This command lists the contents of a nario file read from standard input. diff --git a/src/nix/nario.cc b/src/nix/nario.cc index 8b093470390..27b598c2c51 100644 --- a/src/nix/nario.cc +++ b/src/nix/nario.cc @@ -6,6 +6,8 @@ #include "nix/util/fs-sink.hh" #include "nix/util/archive.hh" +#include + using namespace nix; struct CmdNario : NixMultiCommand @@ -98,7 +100,7 @@ struct CmdNarioImport : StoreCommand, MixNoCheckSigs static auto rCmdNarioImport = registerCommand2({"nario", "import"}); -struct CmdNarioList : Command +struct CmdNarioList : Command, MixJSON { std::string description() override { @@ -129,6 +131,8 @@ struct CmdNarioList : Command struct ListingStore : Store { + std::optional json; + ListingStore(ref config) : Store{*config} { @@ -153,7 +157,12 @@ struct CmdNarioList : Command void addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs) override { - logger->cout(fmt("%s: %d bytes", printStorePath(info.path), info.narSize)); + if (json) { + auto obj = info.toJSON(*this, true, HashFormat::SRI); + ; + json->emplace(printStorePath(info.path), std::move(obj)); + } else + logger->cout(fmt("%s: %d bytes", printStorePath(info.path), info.narSize)); source.skip(info.narSize); } @@ -189,7 +198,15 @@ struct CmdNarioList : Command auto source{getNarioSource()}; auto config = make_ref(StoreConfig::Params()); ListingStore lister(config); + if (json) + lister.json = nlohmann::json::object(); importPaths(lister, source, NoCheckSigs); + if (json) { + auto j = nlohmann::json::object(); + j["version"] = 1; + j["paths"] = std::move(*lister.json); + printJSON(j); + } } }; diff --git a/tests/functional/export.sh b/tests/functional/export.sh index 83797a2a25a..d052d0fe69a 100755 --- a/tests/functional/export.sh +++ b/tests/functional/export.sh @@ -7,6 +7,7 @@ TODO_NixOS clearStore outPath=$(nix-build dependencies.nix --no-out-link) +drvPath=$(nix path-info --json "$outPath" | jq -r .\""$outPath"\".deriver) nix-store --export $outPath > $TEST_ROOT/exp expectStderr 1 nix nario export "$outPath" | grepQuiet "required argument.*missing" @@ -60,3 +61,5 @@ clearStore expectStderr 1 nix nario import < $TEST_ROOT/exp_all | grepQuiet "lacks a signature" nix nario import --trusted-public-keys "$public_key" < $TEST_ROOT/exp_all [[ $(nix path-info --json "$outPath" | jq -r .[].signatures[]) =~ my-key: ]] + +[[ $(nix nario list --json < "$TEST_ROOT/exp_all" | jq -r ".paths.\"$outPath\".deriver") = $drvPath ]] From a4af7d4f79d52d2b4e1c1b66bb73f8308540fb89 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 17 Oct 2025 20:19:46 +0200 Subject: [PATCH 6/8] nix nario list: Show NAR contents With `--json`, it shows information in the same format as `nix store ls --json` (i.e. the NAR listing format). --- src/nix/nario.cc | 118 +++++++++++++++++++++++++++++++++++-- tests/functional/export.sh | 14 ++++- 2 files changed, 124 insertions(+), 8 deletions(-) diff --git a/src/nix/nario.cc b/src/nix/nario.cc index 27b598c2c51..4c80510bec5 100644 --- a/src/nix/nario.cc +++ b/src/nix/nario.cc @@ -100,8 +100,103 @@ struct CmdNarioImport : StoreCommand, MixNoCheckSigs static auto rCmdNarioImport = registerCommand2({"nario", "import"}); +nlohmann::json listNar(Source & source) +{ + struct : FileSystemObjectSink + { + nlohmann::json root = nlohmann::json::object(); + + nlohmann::json & makeObject(const CanonPath & path, std::string_view type) + { + auto * cur = &root; + for (auto & c : path) { + assert((*cur)["type"] == "directory"); + auto i = (*cur)["entries"].emplace(c, nlohmann::json::object()).first; + cur = &i.value(); + } + auto inserted = cur->emplace("type", type).second; + assert(inserted); + return *cur; + } + + void createDirectory(const CanonPath & path) override + { + auto & j = makeObject(path, "directory"); + j["entries"] = nlohmann::json::object(); + } + + void createRegularFile(const CanonPath & path, std::function func) override + { + struct : CreateRegularFileSink + { + bool executable = false; + std::optional size; + + void operator()(std::string_view data) override {} + + void preallocateContents(uint64_t s) override + { + size = s; + } + + void isExecutable() override + { + executable = true; + } + } crf; + + crf.skipContents = true; + + func(crf); + + auto & j = makeObject(path, "regular"); + j.emplace("size", crf.size.value()); + if (crf.executable) + j.emplace("executable", true); + } + + void createSymlink(const CanonPath & path, const std::string & target) override + { + auto & j = makeObject(path, "symlink"); + j.emplace("target", target); + } + + } parseSink; + + parseDump(parseSink, source); + + return parseSink.root; +} + +void renderNarListing(std::string_view prefix, const nlohmann::json & root) +{ + std::function recurse; + recurse = [&](const nlohmann::json & json, const CanonPath & path) { + logger->cout(fmt("%s.%s", prefix, path)); + auto type = json["type"]; + if (type == "directory") { + for (auto & entry : json["entries"].items()) { + recurse(entry.value(), path / entry.key()); + } + } + }; + + recurse(root, CanonPath::root); +} + struct CmdNarioList : Command, MixJSON { + bool listContents = true; + + CmdNarioList() + { + addFlag({ + .longName = "no-contents", + .description = "Do not list the contents of store paths.", + .handler = {&listContents, false}, + }); + } + std::string description() override { return "list the contents of a nario file"; @@ -132,9 +227,11 @@ struct CmdNarioList : Command, MixJSON struct ListingStore : Store { std::optional json; + bool listContents; - ListingStore(ref config) + ListingStore(ref config, bool listContents) : Store{*config} + , listContents(listContents) { } @@ -157,13 +254,22 @@ struct CmdNarioList : Command, MixJSON void addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs) override { + std::optional contents; + if (listContents) + contents = listNar(source); + else + source.skip(info.narSize); + if (json) { auto obj = info.toJSON(*this, true, HashFormat::SRI); - ; + if (contents) + obj.emplace("contents", *contents); json->emplace(printStorePath(info.path), std::move(obj)); - } else - logger->cout(fmt("%s: %d bytes", printStorePath(info.path), info.narSize)); - source.skip(info.narSize); + } else { + logger->cout(fmt(ANSI_BOLD "%s:" ANSI_NORMAL " %d bytes", printStorePath(info.path), info.narSize)); + if (contents) + renderNarListing(" ", *contents); + } } StorePath addToStoreFromDump( @@ -197,7 +303,7 @@ struct CmdNarioList : Command, MixJSON auto source{getNarioSource()}; auto config = make_ref(StoreConfig::Params()); - ListingStore lister(config); + ListingStore lister(config, listContents); if (json) lister.json = nlohmann::json::object(); importPaths(lister, source, NoCheckSigs); diff --git a/tests/functional/export.sh b/tests/functional/export.sh index d052d0fe69a..f6aa565e46c 100755 --- a/tests/functional/export.sh +++ b/tests/functional/export.sh @@ -50,7 +50,8 @@ nix nario import --no-check-sigs < $TEST_ROOT/exp_all nix path-info "$outPath" # Test `nix nario list`. -nix nario list < $TEST_ROOT/exp_all | grepQuiet "dependencies-input-0: .* bytes" +nix nario list < $TEST_ROOT/exp_all +nix nario list < $TEST_ROOT/exp_all | grepQuiet ".*dependencies-input-0.*bytes" # Test format 2 (including signatures). nix key generate-secret --key-name my-key > $TEST_ROOT/secret @@ -62,4 +63,13 @@ expectStderr 1 nix nario import < $TEST_ROOT/exp_all | grepQuiet "lacks a signat nix nario import --trusted-public-keys "$public_key" < $TEST_ROOT/exp_all [[ $(nix path-info --json "$outPath" | jq -r .[].signatures[]) =~ my-key: ]] -[[ $(nix nario list --json < "$TEST_ROOT/exp_all" | jq -r ".paths.\"$outPath\".deriver") = $drvPath ]] +# Test json listing. +json=$(nix nario list --json < "$TEST_ROOT/exp_all") +[[ $(printf "%s" "$json" | jq -r ".paths.\"$outPath\".deriver") = "$drvPath" ]] +[[ $(printf "%s" "$json" | jq -r ".paths.\"$outPath\".contents.type") = directory ]] +[[ $(printf "%s" "$json" | jq -r ".paths.\"$outPath\".contents.entries.foobar.type") = regular ]] +[[ $(printf "%s" "$json" | jq ".paths.\"$outPath\".contents.entries.foobar.size") = 7 ]] + +json=$(nix nario list --json --no-contents < "$TEST_ROOT/exp_all") +[[ $(printf "%s" "$json" | jq -r ".paths.\"$outPath\".deriver") = "$drvPath" ]] +[[ $(printf "%s" "$json" | jq -r ".paths.\"$outPath\".contents.type") = null ]] From a582f17b1fb25368df29e4ff4e259c66733157b7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 17 Oct 2025 20:30:15 +0200 Subject: [PATCH 7/8] Fix error message --- src/nix/nario.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nix/nario.cc b/src/nix/nario.cc index 4c80510bec5..0168aa0ed63 100644 --- a/src/nix/nario.cc +++ b/src/nix/nario.cc @@ -61,7 +61,7 @@ struct CmdNarioExport : StorePathsCommand { auto fd = getStandardOutput(); if (isatty(fd)) - throw UsageError("refusing to write nario to standard output"); + throw UsageError("refusing to write nario to a terminal"); FdSink sink(std::move(fd)); exportPaths(*store, StorePathSet(storePaths.begin(), storePaths.end()), sink, version); } @@ -73,7 +73,7 @@ static FdSource getNarioSource() { auto fd = getStandardInput(); if (isatty(fd)) - throw UsageError("refusing to read nario from standard input"); + throw UsageError("refusing to read nario from a terminal"); return FdSource(std::move(fd)); } From 107a93e889030ed4eca8b543854d3bec5f35b806 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 20 Oct 2025 13:40:19 +0200 Subject: [PATCH 8/8] Use a smaller buffer --- src/libutil/serialise.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index bdce956f311..47a00c8d660 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -112,7 +112,7 @@ std::string Source::drain() void Source::skip(size_t len) { - std::array buf; + std::array buf; while (len) { auto n = read(buf.data(), std::min(len, buf.size())); assert(n <= len);