Skip to content

Cache getLegacyGitAccessor() #396

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
edolstra merged 3 commits into from
Mar 24, 2026
Merged

Cache getLegacyGitAccessor() #396

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
8f22e91
Make getLegacyGitAccessor() less spammy
edolstra Mar 24, 2026
84f79c0
Cache getLegacyGitAccessor()
edolstra Mar 24, 2026
39a204a
Remove unused gitInitialBranch constant
edolstra Mar 24, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 37 additions & 18 deletions src/libfetchers/git.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,12 +32,6 @@ namespace nix::fetchers {

namespace {

// Explicit initial branch of our bare repo to suppress warnings from new version of git.
// The value itself does not matter, since we always fetch a specific revision or branch.
// It is set with `-c init.defaultBranch=` instead of `--initial-branch=` to stay compatible with
// old version of git, which will ignore unrecognized `-c` options.
const std::string gitInitialBranch = "__nix_dummy_branch";

bool isCacheFileWithinTtl(time_t now, const struct stat & st)
{
return st.st_mtime + static_cast<time_t>(settings.tarballTtl) > now;
Expand Down Expand Up @@ -118,7 +112,7 @@ std::optional<std::string> readHeadCached(const std::string & actualUrl, bool sh
std::optional<std::string> cachedRef;
if (stat(headRefFile.string().c_str(), &st) == 0) {
cachedRef = readHead(cacheDir);
if (cachedRef != std::nullopt && *cachedRef != gitInitialBranch && isCacheFileWithinTtl(now, st)) {
if (cachedRef != std::nullopt && isCacheFileWithinTtl(now, st)) {
debug("using cached HEAD ref '%s' for repo '%s'", *cachedRef, actualUrl);
return cachedRef;
}
Expand Down Expand Up @@ -800,23 +794,51 @@ struct GitInputScheme : InputScheme
* checkout`.
*/
ref<SourceAccessor> getLegacyGitAccessor(
const Settings & settings,
Store & store,
RepoInfo & repoInfo,
const std::filesystem::path & repoDir,
const Hash & rev,
GitAccessorOptions & options) const
{
if (!options.submodules)
options.exportIgnore = true;

auto fingerprint = options.makeFingerprint(rev) + ";legacy";

auto cacheKey = makeSourcePathToHashCacheKey(fingerprint, ContentAddressMethod::Raw::NixArchive, "/");

auto makeAccessor = [&](const auto & storePath) -> ref<SourceAccessor> {
auto accessor = store.getFSAccessor(storePath);
accessor->fingerprint = fingerprint;
return ref{accessor};
};

if (auto res = settings.getCache()->lookup(cacheKey)) {
auto hash = Hash::parseSRI(fetchers::getStrAttr(*res, "hash"));
auto storePath = store.makeFixedOutputPathFromCA(
"source", ContentAddressWithReferences::fromParts(ContentAddressMethod::Raw::NixArchive, hash, {}));
store.addTempRoot(storePath);
if (store.maybeQueryPathInfo(storePath)) {
debug("using cached legacy export of revision '%s'", rev.gitRev());
return makeAccessor(storePath);
}
}

debug("doing legacy export of revision '%s'", rev.gitRev());

auto tmpDir = createTempDir();
AutoDelete delTmpDir(tmpDir, true);

auto storePath =
options.submodules
? [&]() {
// Nix < 2.20 used `git checkout` for repos with submodules.
runProgram({.program = "git", .args = {"init", tmpDir}});
runProgram({.program = "git", .args = {"init", tmpDir, "-b", "master"}});
Copy link
Copy Markdown
Member

@cole-h cole-h Mar 24, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why is the -b master required now? Just to make it "reproducible" (more easily correctly cache-able)?

Copy link
Copy Markdown
Member

@cole-h cole-h Mar 24, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Also related (I think): the LLM comment above.

Copy link
Copy Markdown
Collaborator Author

@edolstra edolstra Mar 24, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's to shut up a harmless warning about not having a default branch.

runProgram({.program = "git", .args = {"-C", tmpDir, "remote", "add", "origin", repoDir}});
runProgram({.program = "git", .args = {"-C", tmpDir, "fetch", "origin", rev.gitRev()}});
runProgram({.program = "git", .args = {"-C", tmpDir, "checkout", rev.gitRev()}});
runProgram(
{.program = "git", .args = {"-C", tmpDir, "fetch", "--quiet", "origin", rev.gitRev()}});
runProgram({.program = "git", .args = {"-C", tmpDir, "checkout", "--quiet", rev.gitRev()}});
Comment thread
edolstra marked this conversation as resolved.
PathFilter filter = [&](const Path & path) { return baseNameOf(path) != ".git"; };
return store.addToStore(
"source",
Expand All @@ -828,8 +850,6 @@ struct GitInputScheme : InputScheme
}()
: [&]() {
// Nix < 2.20 used `git archive` for repos without submodules.
options.exportIgnore = true;

auto source = sinkToSource([&](Sink & sink) {
runProgram2(
{.program = "git",
Expand All @@ -842,11 +862,10 @@ struct GitInputScheme : InputScheme
return store.addToStore("source", {getFSSourceAccessor(), CanonPath(tmpDir.string())});
}();

auto accessor = store.getFSAccessor(storePath);

accessor->fingerprint = options.makeFingerprint(rev) + ";legacy";
settings.getCache()->upsert(
cacheKey, {{"hash", store.queryPathInfo(storePath)->narHash.to_string(HashFormat::SRI, true)}});

return ref{accessor};
return makeAccessor(storePath);
}

std::optional<std::pair<ref<SourceAccessor>, Input>> getAccessorFromCommit(
Expand Down Expand Up @@ -1005,7 +1024,7 @@ struct GitInputScheme : InputScheme
* as well. */
warn("Using Nix 2.19 semantics to export Git repository '%s'.", input.to_string());
auto accessorModern = accessor;
accessor = getLegacyGitAccessor(store, repoInfo, repoDir, rev, options);
accessor = getLegacyGitAccessor(settings, store, repoInfo, repoDir, rev, options);
if (expectedNarHash) {
auto narHashLegacy =
fetchToStore2(settings, store, {accessor}, FetchMode::DryRun, input.getName()).second;
Expand All @@ -1023,7 +1042,7 @@ struct GitInputScheme : InputScheme
if (expectedNarHash) {
auto narHashNew = fetchToStore2(settings, store, {accessor}, FetchMode::DryRun, input.getName()).second;
if (expectedNarHash != narHashNew) {
auto accessorLegacy = getLegacyGitAccessor(store, repoInfo, repoDir, rev, options);
auto accessorLegacy = getLegacyGitAccessor(settings, store, repoInfo, repoDir, rev, options);
auto narHashLegacy =
fetchToStore2(settings, store, {accessorLegacy}, FetchMode::DryRun, input.getName()).second;
if (expectedNarHash == narHashLegacy) {
Expand Down
Loading