Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ test:
go test ./... -v

tidy:
gofmt -s -w .
go mod tidy

build:
Expand Down
18 changes: 10 additions & 8 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,19 +1,21 @@
# backup-rsync

Local backup using rsync as a data copier and ZFS datasets as a destination.
Backup using `rsync` as an engine.

NOTE: Using rsync in remote mode is not a use case considered for this tool.
Both the source and destination are local mounted drives, ensuring efficient and direct data transfer.

Go tool used for my own private purposes.

**Use at your own risk!**

## Goals
## Features

- Have the coverage of paths in the source devices checked for completeness
- Document the data copy jobs
- Easily run single jobs on the command line
- Extensive logging of the operations performed
- Dry run for checking what would be performed
- Provide checks for ZFS (snapshot management, size limits reached, ...)
- The tool checks that all specified source paths are covered, ensuring completeness of backups.
- Each data copy job is defined and documented in the configuration file.
- Individual jobs can be executed directly from the command line.
- All backup operations are extensively logged, including detailed rsync output and job summaries.
- A dry run mode is available to preview actions without making changes.

## Configuration File Format (`sync.yaml`)

Expand Down
1 change: 0 additions & 1 deletion backup/cmd/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ func buildConfigCommand() *cobra.Command {
Run: func(cmd *cobra.Command, args []string) {
configPath, _ := cmd.Flags().GetString("config")
cfg := internal.LoadResolvedConfig(configPath)

fmt.Printf("Resolved Configuration:\n%s\n", cfg)
},
}
Expand Down
16 changes: 5 additions & 11 deletions backup/cmd/list.go
Original file line number Diff line number Diff line change
@@ -1,29 +1,23 @@
package cmd

import (
"fmt"

"backup-rsync/backup/internal"

"github.com/spf13/cobra"
)

func listCommands(cfg internal.Config) {
logPath := internal.GetLogPath(false)
for _, job := range cfg.Jobs {
jobLogPath := fmt.Sprintf("%s/job-%s.log", logPath, job.Name)
internal.ExecuteJob(job, false, true, jobLogPath)
}
}

func buildListCommand() *cobra.Command {
return &cobra.Command{
Use: "list",
Short: "List the commands that will be executed",
Run: func(cmd *cobra.Command, args []string) {
configPath, _ := cmd.Flags().GetString("config")
rsyncPath, _ := cmd.Flags().GetString("rsync-path")
cfg := internal.LoadResolvedConfig(configPath)
listCommands(cfg)
command := internal.NewRSyncCommand(rsyncPath)
command.ListOnly = true

cfg.Apply(command)
},
}
}
4 changes: 3 additions & 1 deletion backup/cmd/run.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,9 @@ func buildRunCommand() *cobra.Command {
rsyncPath, _ := cmd.Flags().GetString("rsync-path")

cfg := internal.LoadResolvedConfig(configPath)
internal.ExecuteSyncJobs(cfg, false, rsyncPath)
command := internal.NewRSyncCommand(rsyncPath)

cfg.Apply(command)
},
}
}
5 changes: 4 additions & 1 deletion backup/cmd/simulate.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,10 @@ func buildSimulateCommand() *cobra.Command {
rsyncPath, _ := cmd.Flags().GetString("rsync-path")

cfg := internal.LoadResolvedConfig(configPath)
internal.ExecuteSyncJobs(cfg, true, rsyncPath)
command := internal.NewRSyncCommand(rsyncPath)
command.Simulate = true

cfg.Apply(command)
},
}
}
5 changes: 2 additions & 3 deletions backup/cmd/version.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,10 @@ func buildVersionCommand() *cobra.Command {
Use: "version",
Short: "Prints the rsync version, protocol version, and full path to the rsync binary.",
Run: func(cmd *cobra.Command, args []string) {
var executor internal.CommandExecutor = &internal.RealCommandExecutor{}

rsyncPath, _ := cmd.Flags().GetString("rsync-path")
rsync := internal.NewRSyncCommand(rsyncPath)

output, err := internal.FetchRsyncVersion(executor, rsyncPath)
output, err := rsync.GetVersionInfo()
if err != nil {
fmt.Printf("%v\n", err)

Expand Down
38 changes: 25 additions & 13 deletions backup/internal/helper.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package internal

import (
"fmt"
"io"
"log"
"os"
"strings"
Expand All @@ -16,20 +17,19 @@ func NormalizePath(path string) string {
const FilePermission = 0644
const LogDirPermission = 0755

func GetLogPath(create bool) string {
func GetLogPath() string {
logPath := "logs/sync-" + time.Now().Format("2006-01-02T15-04-05")
if create {
err := os.MkdirAll(logPath, LogDirPermission)
if err != nil {
log.Fatalf("Failed to create log directory: %v", err)
}

err := os.MkdirAll(logPath, LogDirPermission)
if err != nil {
log.Fatalf("Failed to create log directory: %v", err)
}

return logPath
}

func ExecuteSyncJobs(cfg Config, simulate bool, rsyncPath string) {
logPath := GetLogPath(true)
func createFileLogger() (*log.Logger, string) {
logPath := GetLogPath()

overallLogPath := logPath + "/summary.log"

Expand All @@ -45,21 +45,33 @@ func ExecuteSyncJobs(cfg Config, simulate bool, rsyncPath string) {
}
}()

overallLogger := log.New(overallLogFile, "", log.LstdFlags)
logger := log.New(overallLogFile, "", log.LstdFlags)

return logger, logPath
}

func createLogger(rsync RSyncCommand) (*log.Logger, string) {
if rsync.ListOnly {
return log.New(io.Discard, "", 0), ""
}

return createFileLogger()
}

var executor CommandExecutor = &RealCommandExecutor{}
func (cfg Config) Apply(rsync RSyncCommand) {
overallLogger, logPath := createLogger(rsync)

versionInfo, err := FetchRsyncVersion(executor, rsyncPath)
versionInfo, err := rsync.GetVersionInfo()
if err != nil {
overallLogger.Printf("Failed to fetch rsync version: %v", err)
} else {
overallLogger.Printf("Rsync Binary Path: %s", rsyncPath)
overallLogger.Printf("Rsync Binary Path: %s", rsync.BinPath)
overallLogger.Printf("Rsync Version Info: %s", versionInfo)
}

for _, job := range cfg.Jobs {
jobLogPath := fmt.Sprintf("%s/job-%s.log", logPath, job.Name)
status := ExecuteJob(job, simulate, false, jobLogPath)
status := job.Apply(rsync, jobLogPath)
overallLogger.Printf("STATUS [%s]: %s", job.Name, status)
fmt.Printf("Status [%s]: %s\n", job.Name, status)
}
Expand Down
40 changes: 5 additions & 35 deletions backup/internal/job.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,50 +5,20 @@ import (
"strings"
)

func BuildRsyncVersionCmd() []string {
return []string{"--version"}
}

func BuildRsyncCmd(job Job, simulate bool, logPath string) []string {
args := []string{"-aiv", "--stats"}
if job.Delete {
args = append(args, "--delete")
}

if logPath != "" {
args = append(args, "--log-file="+logPath)
}

for _, excl := range job.Exclusions {
args = append(args, "--exclude="+excl)
}

args = append(args, job.Source, job.Target)
if simulate {
args = append([]string{"--dry-run"}, args...)
}

return args
}

func ExecuteJob(job Job, simulate bool, show bool, logPath string) string {
return ExecuteJobWithExecutor(job, simulate, show, logPath, &RealCommandExecutor{})
}

func ExecuteJobWithExecutor(job Job, simulate bool, show bool, logPath string, executor CommandExecutor) string {
func (job Job) Apply(rsync RSyncCommand, logPath string) string {
if !job.Enabled {
return "SKIPPED"
}

args := BuildRsyncCmd(job, simulate, logPath)
args := rsync.ArgumentsForJob(job, logPath)
fmt.Printf("Job: %s\n", job.Name)
fmt.Printf("Command: rsync %s\n", strings.Join(args, " "))
fmt.Printf("Command: rsync %s %s\n", rsync.BinPath, strings.Join(args, " "))

if show {
if rsync.ListOnly {
return "SUCCESS"
}

out, err := executor.Execute("rsync", args...)
out, err := rsync.Executor.Execute(rsync.BinPath, args...)
fmt.Printf("Output:\n%s\n", string(out))

if err != nil {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,16 +7,16 @@ import (
"strings"
)

// CommandExecutor interface for executing commands.
type CommandExecutor interface {
// JobRunner interface for executing commands.
type JobRunner interface {
Execute(name string, args ...string) ([]byte, error)
}

// RealCommandExecutor implements CommandExecutor using actual os/exec.
type RealCommandExecutor struct{}
// RealSync implements JobRunner using actual os/exec.
type RealSync struct{}

// Execute runs the actual command.
func (r *RealCommandExecutor) Execute(name string, args ...string) ([]byte, error) {
func (r *RealSync) Execute(name string, args ...string) ([]byte, error) {
ctx := context.Background()
cmd := exec.CommandContext(ctx, name, args...)

Expand Down
44 changes: 40 additions & 4 deletions backup/internal/rsync.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,14 +10,28 @@ import (
var ErrInvalidRsyncVersion = errors.New("invalid rsync version output")
var ErrInvalidRsyncPath = errors.New("rsync path must be an absolute path")

func FetchRsyncVersion(executor CommandExecutor, rsyncPath string) (string, error) {
type RSyncCommand struct {
BinPath string
Simulate bool
ListOnly bool
Executor JobRunner
}

func NewRSyncCommand(binPath string) RSyncCommand {
return RSyncCommand{
BinPath: binPath,
Executor: &RealSync{},
}
}

func (command RSyncCommand) GetVersionInfo() (string, error) {
rsyncPath := command.BinPath

if !filepath.IsAbs(rsyncPath) {
return "", fmt.Errorf("%w: \"%s\"", ErrInvalidRsyncPath, rsyncPath)
}

cmdArgs := BuildRsyncVersionCmd()

output, err := executor.Execute(rsyncPath, cmdArgs...)
output, err := command.Executor.Execute(rsyncPath, "--version")
if err != nil {
return "", fmt.Errorf("error fetching rsync version: %w", err)
}
Expand All @@ -29,3 +43,25 @@ func FetchRsyncVersion(executor CommandExecutor, rsyncPath string) (string, erro

return string(output), nil
}

func (command RSyncCommand) ArgumentsForJob(job Job, logPath string) []string {
args := []string{"-aiv", "--stats"}
if job.Delete {
args = append(args, "--delete")
}

if logPath != "" {
args = append(args, "--log-file="+logPath)
}

for _, excl := range job.Exclusions {
args = append(args, "--exclude="+excl)
}

args = append(args, job.Source, job.Target)
if command.Simulate {
args = append([]string{"--dry-run"}, args...)
}

return args
}
Loading