From 12c05839da8a9b8e1e306682a9f8e488c6fec139 Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Thu, 29 Mar 2012 18:20:31 +0200 Subject: [PATCH 1/2] Add the '--show-progress' flag to nix-copy-closure --- doc/manual/nix-copy-closure.xml | 8 ++++++++ perl/lib/Nix/CopyClosure.pm | 7 ++++--- scripts/nix-copy-closure.in | 10 ++++++++-- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/doc/manual/nix-copy-closure.xml b/doc/manual/nix-copy-closure.xml index 45cfc0f3424..4b5ce755c67 100644 --- a/doc/manual/nix-copy-closure.xml +++ b/doc/manual/nix-copy-closure.xml @@ -26,6 +26,7 @@ + user@machine @@ -110,6 +111,13 @@ those paths. If this bothers you, use + + + Show the progress of each path's transfer as it's made. + This requires the pv utility to be in PATH. + + + Also copy the outputs of store derivations included diff --git a/perl/lib/Nix/CopyClosure.pm b/perl/lib/Nix/CopyClosure.pm index 08573b2ab1f..4794fed9d00 100644 --- a/perl/lib/Nix/CopyClosure.pm +++ b/perl/lib/Nix/CopyClosure.pm @@ -6,11 +6,12 @@ use Nix::Store; sub copyTo { - my ($sshHost, $sshOpts, $storePaths, $compressor, $decompressor, $includeOutputs, $dryRun, $sign) = @_; + my ($sshHost, $sshOpts, $storePaths, $compressor, $decompressor, $includeOutputs, $dryRun, $sign, $progressViewer) = @_; $compressor = "$compressor |" if $compressor ne ""; $decompressor = "$decompressor |" if $decompressor ne ""; - + $progressViewer = "$progressViewer |" if $progressViewer ne ""; + # Get the closure of this path. my @closure = reverse(topoSortPaths(computeFSClosure(0, $includeOutputs, map { followLinksToStorePath $_ } @{$storePaths}))); @@ -34,7 +35,7 @@ sub copyTo { if (scalar @missing > 0) { print STDERR "copying ", scalar @missing, " missing paths to ‘$sshHost’...\n"; unless ($dryRun) { - open SSH, "| $compressor ssh $sshHost @{$sshOpts} '$decompressor nix-store --import' > /dev/null" or die; + open SSH, "| $progressViewer $compressor ssh $sshHost @{$sshOpts} '$decompressor nix-store --import' > /dev/null" or die; exportPaths(fileno(SSH), $sign, @missing); close SSH or die "copying store paths to remote machine `$sshHost' failed: $?"; } diff --git a/scripts/nix-copy-closure.in b/scripts/nix-copy-closure.in index 1ed24d28520..a38a31a8b5f 100755 --- a/scripts/nix-copy-closure.in +++ b/scripts/nix-copy-closure.in @@ -23,6 +23,8 @@ my $sign = 0; my $compressor = ""; my $decompressor = ""; +my $progressViewer = ""; + my $toMode = 1; my $includeOutputs = 0; @@ -60,6 +62,9 @@ while (@ARGV) { elsif ($arg eq "--include-outputs") { $includeOutputs = 1; } + elsif ($arg eq "--show-progress") { + $progressViewer = "pv"; + } elsif ($arg eq "--dry-run") { $dryRun = 1; } @@ -76,7 +81,7 @@ openSSHConnection $sshHost or die "$0: unable to start SSH\n"; if ($toMode) { # Copy TO the remote machine. - Nix::CopyClosure::copyTo($sshHost, [ @sshOpts ], [ @storePaths ], $compressor, $decompressor, $includeOutputs, $dryRun, $sign); + Nix::CopyClosure::copyTo($sshHost, [ @sshOpts ], [ @storePaths ], $compressor, $decompressor, $includeOutputs, $dryRun, $sign, $progressViewer); } else { # Copy FROM the remote machine. @@ -101,9 +106,10 @@ else { # Copy FROM the remote machine. print STDERR "copying ", scalar @missing, " missing paths from ‘$sshHost’...\n"; $compressor = "| $compressor" if $compressor ne ""; $decompressor = "$decompressor |" if $decompressor ne ""; + $progressViewer = "$progressViewer |" if $progressViewer ne ""; unless ($dryRun) { my $extraOpts = $sign ? "--sign" : ""; - system("set -f; ssh $sshHost @sshOpts 'nix-store --export $extraOpts @missing $compressor' | $decompressor $Nix::Config::binDir/nix-store --import > /dev/null") == 0 + system("set -f; ssh $sshHost @sshOpts 'nix-store --export $extraOpts @missing $compressor' | $decompressor $progressViewer $Nix::Config::binDir/nix-store --import > /dev/null") == 0 or die "copying store paths from remote machine `$sshHost' failed: $?"; } } From e0ec68147024e2fc54f30681e05a86517d4f3bb0 Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Thu, 29 Mar 2012 18:58:14 +0200 Subject: [PATCH 2/2] nix-copy-closure: Move the progressViewer directly adjacent to the ssh call so that network progress is what's measured --- perl/lib/Nix/CopyClosure.pm | 2 +- scripts/nix-copy-closure.in | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/perl/lib/Nix/CopyClosure.pm b/perl/lib/Nix/CopyClosure.pm index 4794fed9d00..79c6dfcccbf 100644 --- a/perl/lib/Nix/CopyClosure.pm +++ b/perl/lib/Nix/CopyClosure.pm @@ -35,7 +35,7 @@ sub copyTo { if (scalar @missing > 0) { print STDERR "copying ", scalar @missing, " missing paths to ‘$sshHost’...\n"; unless ($dryRun) { - open SSH, "| $progressViewer $compressor ssh $sshHost @{$sshOpts} '$decompressor nix-store --import' > /dev/null" or die; + open SSH, "| $compressor $progressViewer ssh $sshHost @{$sshOpts} '$decompressor nix-store --import' > /dev/null" or die; exportPaths(fileno(SSH), $sign, @missing); close SSH or die "copying store paths to remote machine `$sshHost' failed: $?"; } diff --git a/scripts/nix-copy-closure.in b/scripts/nix-copy-closure.in index a38a31a8b5f..ca240d3ee5f 100755 --- a/scripts/nix-copy-closure.in +++ b/scripts/nix-copy-closure.in @@ -109,7 +109,7 @@ else { # Copy FROM the remote machine. $progressViewer = "$progressViewer |" if $progressViewer ne ""; unless ($dryRun) { my $extraOpts = $sign ? "--sign" : ""; - system("set -f; ssh $sshHost @sshOpts 'nix-store --export $extraOpts @missing $compressor' | $decompressor $progressViewer $Nix::Config::binDir/nix-store --import > /dev/null") == 0 + system("set -f; ssh $sshHost @sshOpts 'nix-store --export $extraOpts @missing $compressor' | $progressViewer $decompressor $Nix::Config::binDir/nix-store --import > /dev/null") == 0 or die "copying store paths from remote machine `$sshHost' failed: $?"; } }