diff --git a/Makefile b/Makefile index 0e570fe..48404b3 100644 --- a/Makefile +++ b/Makefile @@ -36,6 +36,7 @@ test: go test ./... -v tidy: + gofmt -s -w . go mod tidy build: diff --git a/README.md b/README.md index fa65c77..fcf9307 100644 --- a/README.md +++ b/README.md @@ -1,19 +1,21 @@ # backup-rsync -Local backup using rsync as a data copier and ZFS datasets as a destination. +Backup using `rsync` as an engine. + +NOTE: Using rsync in remote mode is not a use case considered for this tool. +Both the source and destination are local mounted drives, ensuring efficient and direct data transfer. Go tool used for my own private purposes. **Use at your own risk!** -## Goals +## Features -- Have the coverage of paths in the source devices checked for completeness -- Document the data copy jobs -- Easily run single jobs on the command line -- Extensive logging of the operations performed -- Dry run for checking what would be performed -- Provide checks for ZFS (snapshot management, size limits reached, ...) +- The tool checks that all specified source paths are covered, ensuring completeness of backups. +- Each data copy job is defined and documented in the configuration file. +- Individual jobs can be executed directly from the command line. +- All backup operations are extensively logged, including detailed rsync output and job summaries. +- A dry run mode is available to preview actions without making changes. ## Configuration File Format (`sync.yaml`) diff --git a/backup/cmd/config.go b/backup/cmd/config.go index 1e297c3..c3ecdd3 100644 --- a/backup/cmd/config.go +++ b/backup/cmd/config.go @@ -19,7 +19,6 @@ func buildConfigCommand() *cobra.Command { Run: func(cmd *cobra.Command, args []string) { configPath, _ := cmd.Flags().GetString("config") cfg := internal.LoadResolvedConfig(configPath) - fmt.Printf("Resolved Configuration:\n%s\n", cfg) }, } diff --git a/backup/cmd/list.go b/backup/cmd/list.go index 6ad9bb2..f387b41 100644 --- a/backup/cmd/list.go +++ b/backup/cmd/list.go @@ -1,29 +1,23 @@ package cmd import ( - "fmt" - "backup-rsync/backup/internal" "github.com/spf13/cobra" ) -func listCommands(cfg internal.Config) { - logPath := internal.GetLogPath(false) - for _, job := range cfg.Jobs { - jobLogPath := fmt.Sprintf("%s/job-%s.log", logPath, job.Name) - internal.ExecuteJob(job, false, true, jobLogPath) - } -} - func buildListCommand() *cobra.Command { return &cobra.Command{ Use: "list", Short: "List the commands that will be executed", Run: func(cmd *cobra.Command, args []string) { configPath, _ := cmd.Flags().GetString("config") + rsyncPath, _ := cmd.Flags().GetString("rsync-path") cfg := internal.LoadResolvedConfig(configPath) - listCommands(cfg) + command := internal.NewRSyncCommand(rsyncPath) + command.ListOnly = true + + cfg.Apply(command) }, } } diff --git a/backup/cmd/run.go b/backup/cmd/run.go index 9eb5810..4c5f9fe 100644 --- a/backup/cmd/run.go +++ b/backup/cmd/run.go @@ -15,7 +15,9 @@ func buildRunCommand() *cobra.Command { rsyncPath, _ := cmd.Flags().GetString("rsync-path") cfg := internal.LoadResolvedConfig(configPath) - internal.ExecuteSyncJobs(cfg, false, rsyncPath) + command := internal.NewRSyncCommand(rsyncPath) + + cfg.Apply(command) }, } } diff --git a/backup/cmd/simulate.go b/backup/cmd/simulate.go index 4c2e798..b6d815f 100644 --- a/backup/cmd/simulate.go +++ b/backup/cmd/simulate.go @@ -15,7 +15,10 @@ func buildSimulateCommand() *cobra.Command { rsyncPath, _ := cmd.Flags().GetString("rsync-path") cfg := internal.LoadResolvedConfig(configPath) - internal.ExecuteSyncJobs(cfg, true, rsyncPath) + command := internal.NewRSyncCommand(rsyncPath) + command.Simulate = true + + cfg.Apply(command) }, } } diff --git a/backup/cmd/version.go b/backup/cmd/version.go index b3638c1..f5948fa 100644 --- a/backup/cmd/version.go +++ b/backup/cmd/version.go @@ -13,11 +13,10 @@ func buildVersionCommand() *cobra.Command { Use: "version", Short: "Prints the rsync version, protocol version, and full path to the rsync binary.", Run: func(cmd *cobra.Command, args []string) { - var executor internal.CommandExecutor = &internal.RealCommandExecutor{} - rsyncPath, _ := cmd.Flags().GetString("rsync-path") + rsync := internal.NewRSyncCommand(rsyncPath) - output, err := internal.FetchRsyncVersion(executor, rsyncPath) + output, err := rsync.GetVersionInfo() if err != nil { fmt.Printf("%v\n", err) diff --git a/backup/internal/helper.go b/backup/internal/helper.go index 869c29c..887a53f 100644 --- a/backup/internal/helper.go +++ b/backup/internal/helper.go @@ -3,6 +3,7 @@ package internal import ( "fmt" + "io" "log" "os" "strings" @@ -16,20 +17,19 @@ func NormalizePath(path string) string { const FilePermission = 0644 const LogDirPermission = 0755 -func GetLogPath(create bool) string { +func GetLogPath() string { logPath := "logs/sync-" + time.Now().Format("2006-01-02T15-04-05") - if create { - err := os.MkdirAll(logPath, LogDirPermission) - if err != nil { - log.Fatalf("Failed to create log directory: %v", err) - } + + err := os.MkdirAll(logPath, LogDirPermission) + if err != nil { + log.Fatalf("Failed to create log directory: %v", err) } return logPath } -func ExecuteSyncJobs(cfg Config, simulate bool, rsyncPath string) { - logPath := GetLogPath(true) +func createFileLogger() (*log.Logger, string) { + logPath := GetLogPath() overallLogPath := logPath + "/summary.log" @@ -45,21 +45,33 @@ func ExecuteSyncJobs(cfg Config, simulate bool, rsyncPath string) { } }() - overallLogger := log.New(overallLogFile, "", log.LstdFlags) + logger := log.New(overallLogFile, "", log.LstdFlags) + + return logger, logPath +} + +func createLogger(rsync RSyncCommand) (*log.Logger, string) { + if rsync.ListOnly { + return log.New(io.Discard, "", 0), "" + } + + return createFileLogger() +} - var executor CommandExecutor = &RealCommandExecutor{} +func (cfg Config) Apply(rsync RSyncCommand) { + overallLogger, logPath := createLogger(rsync) - versionInfo, err := FetchRsyncVersion(executor, rsyncPath) + versionInfo, err := rsync.GetVersionInfo() if err != nil { overallLogger.Printf("Failed to fetch rsync version: %v", err) } else { - overallLogger.Printf("Rsync Binary Path: %s", rsyncPath) + overallLogger.Printf("Rsync Binary Path: %s", rsync.BinPath) overallLogger.Printf("Rsync Version Info: %s", versionInfo) } for _, job := range cfg.Jobs { jobLogPath := fmt.Sprintf("%s/job-%s.log", logPath, job.Name) - status := ExecuteJob(job, simulate, false, jobLogPath) + status := job.Apply(rsync, jobLogPath) overallLogger.Printf("STATUS [%s]: %s", job.Name, status) fmt.Printf("Status [%s]: %s\n", job.Name, status) } diff --git a/backup/internal/job.go b/backup/internal/job.go index 46e4808..d475578 100644 --- a/backup/internal/job.go +++ b/backup/internal/job.go @@ -5,50 +5,20 @@ import ( "strings" ) -func BuildRsyncVersionCmd() []string { - return []string{"--version"} -} - -func BuildRsyncCmd(job Job, simulate bool, logPath string) []string { - args := []string{"-aiv", "--stats"} - if job.Delete { - args = append(args, "--delete") - } - - if logPath != "" { - args = append(args, "--log-file="+logPath) - } - - for _, excl := range job.Exclusions { - args = append(args, "--exclude="+excl) - } - - args = append(args, job.Source, job.Target) - if simulate { - args = append([]string{"--dry-run"}, args...) - } - - return args -} - -func ExecuteJob(job Job, simulate bool, show bool, logPath string) string { - return ExecuteJobWithExecutor(job, simulate, show, logPath, &RealCommandExecutor{}) -} - -func ExecuteJobWithExecutor(job Job, simulate bool, show bool, logPath string, executor CommandExecutor) string { +func (job Job) Apply(rsync RSyncCommand, logPath string) string { if !job.Enabled { return "SKIPPED" } - args := BuildRsyncCmd(job, simulate, logPath) + args := rsync.ArgumentsForJob(job, logPath) fmt.Printf("Job: %s\n", job.Name) - fmt.Printf("Command: rsync %s\n", strings.Join(args, " ")) + fmt.Printf("Command: rsync %s %s\n", rsync.BinPath, strings.Join(args, " ")) - if show { + if rsync.ListOnly { return "SUCCESS" } - out, err := executor.Execute("rsync", args...) + out, err := rsync.Executor.Execute(rsync.BinPath, args...) fmt.Printf("Output:\n%s\n", string(out)) if err != nil { diff --git a/backup/internal/command_executor.go b/backup/internal/job_runner.go similarity index 60% rename from backup/internal/command_executor.go rename to backup/internal/job_runner.go index 453466f..33a9c53 100644 --- a/backup/internal/command_executor.go +++ b/backup/internal/job_runner.go @@ -7,16 +7,16 @@ import ( "strings" ) -// CommandExecutor interface for executing commands. -type CommandExecutor interface { +// JobRunner interface for executing commands. +type JobRunner interface { Execute(name string, args ...string) ([]byte, error) } -// RealCommandExecutor implements CommandExecutor using actual os/exec. -type RealCommandExecutor struct{} +// RealSync implements JobRunner using actual os/exec. +type RealSync struct{} // Execute runs the actual command. -func (r *RealCommandExecutor) Execute(name string, args ...string) ([]byte, error) { +func (r *RealSync) Execute(name string, args ...string) ([]byte, error) { ctx := context.Background() cmd := exec.CommandContext(ctx, name, args...) diff --git a/backup/internal/rsync.go b/backup/internal/rsync.go index a20d53b..e52ccf6 100644 --- a/backup/internal/rsync.go +++ b/backup/internal/rsync.go @@ -10,14 +10,28 @@ import ( var ErrInvalidRsyncVersion = errors.New("invalid rsync version output") var ErrInvalidRsyncPath = errors.New("rsync path must be an absolute path") -func FetchRsyncVersion(executor CommandExecutor, rsyncPath string) (string, error) { +type RSyncCommand struct { + BinPath string + Simulate bool + ListOnly bool + Executor JobRunner +} + +func NewRSyncCommand(binPath string) RSyncCommand { + return RSyncCommand{ + BinPath: binPath, + Executor: &RealSync{}, + } +} + +func (command RSyncCommand) GetVersionInfo() (string, error) { + rsyncPath := command.BinPath + if !filepath.IsAbs(rsyncPath) { return "", fmt.Errorf("%w: \"%s\"", ErrInvalidRsyncPath, rsyncPath) } - cmdArgs := BuildRsyncVersionCmd() - - output, err := executor.Execute(rsyncPath, cmdArgs...) + output, err := command.Executor.Execute(rsyncPath, "--version") if err != nil { return "", fmt.Errorf("error fetching rsync version: %w", err) } @@ -29,3 +43,25 @@ func FetchRsyncVersion(executor CommandExecutor, rsyncPath string) (string, erro return string(output), nil } + +func (command RSyncCommand) ArgumentsForJob(job Job, logPath string) []string { + args := []string{"-aiv", "--stats"} + if job.Delete { + args = append(args, "--delete") + } + + if logPath != "" { + args = append(args, "--log-file="+logPath) + } + + for _, excl := range job.Exclusions { + args = append(args, "--exclude="+excl) + } + + args = append(args, job.Source, job.Target) + if command.Simulate { + args = append([]string{"--dry-run"}, args...) + } + + return args +} diff --git a/backup/internal/test/job_test.go b/backup/internal/test/job_test.go index 78aaf9d..6f167b6 100644 --- a/backup/internal/test/job_test.go +++ b/backup/internal/test/job_test.go @@ -2,7 +2,6 @@ package internal_test import ( "backup-rsync/backup/internal" - "strings" "testing" "github.com/stretchr/testify/assert" @@ -61,26 +60,16 @@ func WithExclusions(exclusions []string) Option { } } -func TestBuildRsyncCmd(t *testing.T) { - job := *NewJob( - WithSource("/home/user/Music/"), - WithTarget("/target/user/music/home"), - WithExclusions([]string{"*.tmp", "node_modules/"}), - ) - args := internal.BuildRsyncCmd(job, true, "") - - expectedArgs := []string{ - "--dry-run", "-aiv", "--stats", "--delete", - "--exclude=*.tmp", "--exclude=node_modules/", - "/home/user/Music/", "/target/user/music/home", +func newMockRSyncCommand(simulate bool) internal.RSyncCommand { + return internal.RSyncCommand{ + BinPath: "/usr/bin/rsync", + Simulate: simulate, + Executor: &MockCommandExecutor{}, } - - assert.Equal(t, strings.Join(expectedArgs, " "), strings.Join(args, " ")) } -func TestExecuteJob(t *testing.T) { - // Create mock executor - mockExecutor := &MockCommandExecutor{} +func TestApply(t *testing.T) { + rsync := newMockRSyncCommand(true) job := *NewJob( WithName("test_job"), @@ -88,10 +77,15 @@ func TestExecuteJob(t *testing.T) { WithTarget("/mnt/backup1/test/"), WithExclusions([]string{"*.tmp"}), ) - simulate := true - status := internal.ExecuteJobWithExecutor(job, simulate, false, "", mockExecutor) + status := job.Apply(rsync, "") assert.Equal(t, statusSuccess, status) +} +func TestApply_Disabled(t *testing.T) { + command := internal.RSyncCommand{ + BinPath: "/usr/bin/rsync", + Executor: &MockCommandExecutor{}, + } disabledJob := *NewJob( WithName("disabled_job"), @@ -100,8 +94,12 @@ func TestExecuteJob(t *testing.T) { WithEnabled(false), ) - status = internal.ExecuteJobWithExecutor(disabledJob, simulate, false, "", mockExecutor) + status := disabledJob.Apply(command, "") assert.Equal(t, "SKIPPED", status) +} + +func TestApply_Invalid(t *testing.T) { + rsync := newMockRSyncCommand(false) // Test case for failure (simulate by providing invalid source path) invalidJob := *NewJob( @@ -110,14 +108,12 @@ func TestExecuteJob(t *testing.T) { WithTarget("/mnt/backup1/invalid/"), ) - status = internal.ExecuteJobWithExecutor(invalidJob, false, false, "", mockExecutor) + status := invalidJob.Apply(rsync, "") assert.Equal(t, "FAILURE", status) } -// Ensure all references to ExecuteJob are prefixed with internal. func TestJobSkippedEnabledTrue(t *testing.T) { - // Create mock executor - mockExecutor := &MockCommandExecutor{} + rsync := newMockRSyncCommand(false) job := *NewJob( WithName("test_job"), @@ -125,13 +121,12 @@ func TestJobSkippedEnabledTrue(t *testing.T) { WithTarget("/mnt/backup1/test/"), ) - status := internal.ExecuteJobWithExecutor(job, true, false, "", mockExecutor) + status := job.Apply(rsync, "") assert.Equal(t, statusSuccess, status) } func TestJobSkippedEnabledFalse(t *testing.T) { - // Create mock executor (won't be used since job is disabled) - mockExecutor := &MockCommandExecutor{} + rsync := newMockRSyncCommand(false) disabledJob := *NewJob( WithName("disabled_job"), @@ -140,13 +135,12 @@ func TestJobSkippedEnabledFalse(t *testing.T) { WithEnabled(false), ) - status := internal.ExecuteJobWithExecutor(disabledJob, true, false, "", mockExecutor) + status := disabledJob.Apply(rsync, "") assert.Equal(t, "SKIPPED", status) } func TestJobSkippedEnabledOmitted(t *testing.T) { - // Create mock executor - mockExecutor := &MockCommandExecutor{} + rsync := newMockRSyncCommand(false) job := *NewJob( WithName("omitted_enabled_job"), @@ -154,13 +148,14 @@ func TestJobSkippedEnabledOmitted(t *testing.T) { WithTarget("/mnt/backup1/omitted/"), ) - status := internal.ExecuteJobWithExecutor(job, true, false, "", mockExecutor) + status := job.Apply(rsync, "") assert.Equal(t, statusSuccess, status) } -func TestExecuteJobWithMockedRsync(t *testing.T) { - // Create mock executor +func TestApplyWithMockedRsync(t *testing.T) { mockExecutor := &MockCommandExecutor{} + rsync := newMockRSyncCommand(true) + rsync.Executor = mockExecutor job := *NewJob( WithName("test_job"), @@ -168,13 +163,13 @@ func TestExecuteJobWithMockedRsync(t *testing.T) { WithTarget("/mnt/backup1/test/"), WithExclusions([]string{"*.tmp"}), ) - status := internal.ExecuteJobWithExecutor(job, true, false, "", mockExecutor) + status := job.Apply(rsync, "") assert.Equal(t, statusSuccess, status) assert.NotEmpty(t, mockExecutor.CapturedCommands) cmd := mockExecutor.CapturedCommands[0] - assert.Equal(t, "rsync", cmd.Name, "Command name mismatch") + assert.Equal(t, "/usr/bin/rsync", cmd.Name, "Command name mismatch") assert.Contains(t, cmd.Args, "--dry-run", "Expected --dry-run flag in command arguments") } diff --git a/backup/internal/test/mock_executor_test.go b/backup/internal/test/mock_executor_test.go index 388f96b..54ee553 100644 --- a/backup/internal/test/mock_executor_test.go +++ b/backup/internal/test/mock_executor_test.go @@ -8,7 +8,7 @@ import ( // Static error for testing. var ErrExitStatus23 = errors.New("exit status 23") -// MockCommandExecutor implements CommandExecutor for testing. +// MockCommandExecutor implements JobRunner for testing. type MockCommandExecutor struct { CapturedCommands []MockCommand Output string @@ -39,7 +39,7 @@ func (m *MockCommandExecutor) Execute(name string, args ...string) ([]byte, erro } // Simulate specific scenarios for rsync. - if name == "rsync" { + if name == "/usr/bin/rsync" { argsStr := strings.Join(args, " ") if strings.Contains(argsStr, "/invalid/source/path") { diff --git a/backup/internal/test/rsync_test.go b/backup/internal/test/rsync_test.go index 773a7ae..707ecd7 100644 --- a/backup/internal/test/rsync_test.go +++ b/backup/internal/test/rsync_test.go @@ -3,6 +3,7 @@ package internal_test import ( "backup-rsync/backup/internal" "errors" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -13,49 +14,83 @@ var errCommandNotFound = errors.New("command not found") const rsyncPath = "/usr/bin/rsync" +func TestBuildRsyncCmd(t *testing.T) { + job := *NewJob( + WithSource("/home/user/Music/"), + WithTarget("/target/user/music/home"), + WithExclusions([]string{"*.tmp", "node_modules/"}), + ) + command := internal.RSyncCommand{ + BinPath: rsyncPath, + Simulate: true, + Executor: nil, + } + args := command.ArgumentsForJob(job, "") + + expectedArgs := []string{ + "--dry-run", "-aiv", "--stats", "--delete", + "--exclude=*.tmp", "--exclude=node_modules/", + "/home/user/Music/", "/target/user/music/home", + } + + assert.Equal(t, strings.Join(expectedArgs, " "), strings.Join(args, " ")) +} + func TestFetchRsyncVersion_Success(t *testing.T) { - executor := &MockCommandExecutor{ - Output: "rsync version 3.2.3 protocol version 31\n", - Error: nil, + rsync := internal.RSyncCommand{ + BinPath: rsyncPath, + Executor: &MockCommandExecutor{ + Output: "rsync version 3.2.3 protocol version 31\n", + Error: nil, + }, } - versionInfo, err := internal.FetchRsyncVersion(executor, rsyncPath) + versionInfo, err := rsync.GetVersionInfo() require.NoError(t, err) assert.Equal(t, "rsync version 3.2.3 protocol version 31\n", versionInfo) } func TestFetchRsyncVersion_CommandError(t *testing.T) { - executor := &MockCommandExecutor{ - Output: "", - Error: errCommandNotFound, + rsync := internal.RSyncCommand{ + BinPath: rsyncPath, + Executor: &MockCommandExecutor{ + Output: "", + Error: errCommandNotFound, + }, } - versionInfo, err := internal.FetchRsyncVersion(executor, rsyncPath) + versionInfo, err := rsync.GetVersionInfo() require.Error(t, err) assert.Empty(t, versionInfo) } func TestFetchRsyncVersion_InvalidOutput(t *testing.T) { - executor := &MockCommandExecutor{ - Output: "invalid output", - Error: nil, + rsync := internal.RSyncCommand{ + BinPath: rsyncPath, + Executor: &MockCommandExecutor{ + Output: "invalid output", + Error: nil, + }, } - versionInfo, err := internal.FetchRsyncVersion(executor, rsyncPath) + versionInfo, err := rsync.GetVersionInfo() require.Error(t, err) assert.Empty(t, versionInfo) } func TestFetchRsyncVersion_EmptyPath(t *testing.T) { - executor := &MockCommandExecutor{ - Output: "", - Error: nil, + rsync := internal.RSyncCommand{ + BinPath: "", + Executor: &MockCommandExecutor{ + Output: "", + Error: nil, + }, } - versionInfo, err := internal.FetchRsyncVersion(executor, "") + versionInfo, err := rsync.GetVersionInfo() require.Error(t, err) require.EqualError(t, err, "rsync path must be an absolute path: \"\"") @@ -63,12 +98,15 @@ func TestFetchRsyncVersion_EmptyPath(t *testing.T) { } func TestFetchRsyncVersion_IncompletePath(t *testing.T) { - executor := &MockCommandExecutor{ - Output: "", - Error: nil, + rsync := internal.RSyncCommand{ + BinPath: "bin/rsync", + Executor: &MockCommandExecutor{ + Output: "", + Error: nil, + }, } - versionInfo, err := internal.FetchRsyncVersion(executor, "bin/rsync") + versionInfo, err := rsync.GetVersionInfo() require.Error(t, err) require.EqualError(t, err, "rsync path must be an absolute path: \"bin/rsync\"") diff --git a/backup/internal/types.go b/backup/internal/types.go index 9b553b3..1326999 100644 --- a/backup/internal/types.go +++ b/backup/internal/types.go @@ -23,6 +23,8 @@ type Config struct { } // Job represents a backup job configuration for a source/target pair. +// +//nolint:recvcheck // UnmarshalYAML requires pointer receiver while Apply uses value receiver type Job struct { Name string `yaml:"name"` Source string `yaml:"source"`