feat: add logging package

.gitignore

@@ -1 +1,2 @@
-/build/
+/build/
+/logs/

cmd/app.go

@@ -1,6 +1,7 @@
 package cmd
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -9,6 +10,7 @@ import (
 	"github.com/spf13/cobra"
 
 	"github.com/apecloud/datasafed/pkg/config"
+	"github.com/apecloud/datasafed/pkg/logging"
 	"github.com/apecloud/datasafed/pkg/storage"
 	"github.com/apecloud/datasafed/pkg/storage/rclone"
 )
@@ -20,24 +22,39 @@ const (
 
 var (
 	rootCmd = &cobra.Command{
-		Use:   "datasafed",
-		Short: "`datasafed` is a command line tool for managing remote storages.",
+		Use:           "datasafed",
+		Short:         "`datasafed` is a command line tool for managing remote storages.",
+		SilenceErrors: true,
+		SilenceUsage:  true,
 	}
 
 	configFile       string
 	doNotInitStorage bool
 	globalStorage    storage.Storage
+	appCtx           context.Context = context.Background()
+	onFinishFuncs    []func()
 )
 
 func init() {
 	rootCmd.PersistentPreRunE = func(cmd *cobra.Command, args []string) error {
-		if doNotInitStorage {
-			return nil
+		appCtx = logging.WithLogger(appCtx, logging.DefaultLoggerFactory)
+		if !doNotInitStorage {
+			if err := initStorage(); err != nil {
+				return err
+			}
 		}
-		return initStorage()
+		return nil
+	}
+	rootCmd.PersistentPostRunE = func(cmd *cobra.Command, args []string) error {
+		for _, fn := range onFinishFuncs {
+			fn()
+		}
+		return nil
 	}
 	rootCmd.PersistentFlags().StringVarP(&configFile, "conf", "c",
 		"/etc/datasafed/datasafed.conf", "config file")
+
+	logging.Attach(rootCmd)
 }
 
 // RootCommand returns the root command.

cmd/list.go

@@ -2,7 +2,6 @@ package cmd
 
 import (
 	"cmp"
-	"context"
 	"encoding/json"
 	"fmt"
 	"os"
@@ -46,14 +45,14 @@ datasafed list /
 # List one file and extract its size
 datasafed list somefile.txt -o long | awk '{print $2}'
 
-# List all files under the directory
-datasafed list -r -f /some/dir
+# List all files under the directory (ends with '/')
+datasafed list -r -f /some/dir/
 
 # List files modified within 1 hour and sort the result by size
-datasafed list -r -f -s size --newer-than $(( $(date +%s) - 3600 )) /some/dir
+datasafed list -r -f -s size --newer-than $(( $(date +%s) - 3600 )) /some/dir/
 
 # List files with the name pattern
-datasafed list --name "*.txt" /some/dir
+datasafed list --name "*.txt" /some/dir/
 `),
 		Args: cobra.ExactArgs(1),
 		Run: func(cmd *cobra.Command, args []string) {
@@ -96,7 +95,7 @@ func doList(opts *listOptions, cmd *cobra.Command, args []string) {
 		Recursive: opts.recursive,
 		MaxDepth:  opts.maxDepth,
 	}
-	entries, err := globalStorage.List(context.Background(), rpath, lopts)
+	entries, err := globalStorage.List(appCtx, rpath, lopts)
 	exitIfError(err)
 
 	entries = filterEntries(entries, opts)

cmd/mkdir.go

@@ -1,16 +1,17 @@
 package cmd
 
 import (
-	"context"
 	"strings"
 
 	"github.com/spf13/cobra"
 )
 
 func init() {
 	cmd := &cobra.Command{
-		Use:   "mkdir rpath",
-		Short: "Create an empty remote directory.",
+		Use: "mkdir rpath",
+		Short: "Create an empty remote directory." +
+			"Some storage backends, such as S3, do not have the concept of a directory, " +
+			"in which case the command will directly return success with no effect.",
 		Example: strings.TrimSpace(`
 # Create an empty directory
 datasafed mkdir some/dir
@@ -22,6 +23,6 @@ datasafed mkdir some/dir
 }
 
 func doMkdir(cmd *cobra.Command, args []string) {
-	err := globalStorage.Mkdir(context.Background(), args[0])
+	err := globalStorage.Mkdir(appCtx, args[0])
 	exitIfError(err)
 }

cmd/pull.go

@@ -1,7 +1,6 @@
 package cmd
 
 import (
-	"context"
 	"fmt"
 	"io"
 	"os"
@@ -51,7 +50,7 @@ func doPull(cmd *cobra.Command, args []string) {
 		defer f.Close()
 		out = f
 	}
-	err := globalStorage.Pull(context.Background(), rpath, out)
+	err := globalStorage.Pull(appCtx, rpath, out)
 	if err != nil {
 		err = fmt.Errorf("pull %q: %w", rpath, err)
 	}

cmd/push.go

@@ -1,7 +1,6 @@
 package cmd
 
 import (
-	"context"
 	"fmt"
 	"io"
 	"os"
@@ -40,7 +39,7 @@ func doPush(cmd *cobra.Command, args []string) {
 		defer f.Close()
 		in = f
 	}
-	err := globalStorage.Push(context.Background(), in, rpath)
+	err := globalStorage.Push(appCtx, in, rpath)
 	if err != nil {
 		err = fmt.Errorf("push to %q: %w", rpath, err)
 	}

cmd/rm.go

@@ -1,7 +1,6 @@
 package cmd
 
 import (
-	"context"
 	"strings"
 
 	"github.com/spf13/cobra"
@@ -33,6 +32,6 @@ datasafed rm -r some/path/to/dir
 }
 
 func doRm(opts *rmOptions, cmd *cobra.Command, args []string) {
-	err := globalStorage.Remove(context.Background(), args[0], opts.recursive)
+	err := globalStorage.Remove(appCtx, args[0], opts.recursive)
 	exitIfError(err)
 }

cmd/rmdir.go

@@ -1,7 +1,6 @@
 package cmd
 
 import (
-	"context"
 	"strings"
 
 	"github.com/spf13/cobra"
@@ -11,6 +10,9 @@ func init() {
 	cmd := &cobra.Command{
 		Use:   "rmdir rpath",
 		Short: "Remove an empty remote directory.",
+		Long: "Remove an empty remote directory.\n" +
+			"Some storage backends, such as S3, do not have the concept of a directory, " +
+			"in which case the command will directly return success with no effect.",
 		Example: strings.TrimSpace(`
 # Remove an empty directory
 datasafed rmdir some/dir
@@ -22,6 +24,6 @@ datasafed rmdir some/dir
 }
 
 func doRmdir(cmd *cobra.Command, args []string) {
-	err := globalStorage.Rmdir(context.Background(), args[0])
+	err := globalStorage.Rmdir(appCtx, args[0])
 	exitIfError(err)
 }

cmd/stat.go

@@ -1,7 +1,6 @@
 package cmd
 
 import (
-	"context"
 	"encoding/json"
 	"fmt"
 	"strings"
@@ -23,8 +22,8 @@ func init() {
 # Stat a file
 datasafed stat path/to/file.txt
 
-# Stat a directory with json output
-datasafed stat -json path/to/dir
+# Stat a directory (ends with '/') with json output
+datasafed stat -json path/to/dir/
 `),
 		Args: cobra.ExactArgs(1),
 		Run: func(cmd *cobra.Command, args []string) {
@@ -37,7 +36,7 @@ datasafed stat -json path/to/dir
 
 func doStat(opts *statOptions, cmd *cobra.Command, args []string) {
 	rpath := args[0]
-	result, err := globalStorage.Stat(context.Background(), rpath)
+	result, err := globalStorage.Stat(appCtx, rpath)
 	exitIfError(err)
 	if !opts.json {
 		fmt.Printf("TotalSize: %d\n", result.TotalSize)

go.mod

@@ -3,8 +3,11 @@ module github.com/apecloud/datasafed
 go 1.21
 
 require (
+	github.com/fatih/color v1.16.0
 	github.com/rclone/rclone v1.63.1
 	github.com/spf13/cobra v1.7.0
+	github.com/stretchr/testify v1.8.3
+	go.uber.org/zap v1.26.0
 	gopkg.in/ini.v1 v1.67.0
 )
 
@@ -28,6 +31,7 @@ require (
 	github.com/colinmarc/hdfs/v2 v2.3.0 // indirect
 	github.com/coreos/go-semver v0.3.1 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.2 // indirect
 	github.com/geoffgarside/ber v1.1.0 // indirect
@@ -66,7 +70,7 @@ require (
 	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
-	github.com/mattn/go-isatty v0.0.16 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/ncw/go-acd v0.0.0-20201019170801-fe55f33415b1 // indirect
@@ -77,6 +81,7 @@ require (
 	github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
 	github.com/pkg/sftp v1.13.6-0.20230213180117-971c283182b6 // indirect
 	github.com/pkg/xattr v0.4.9 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
 	github.com/prometheus/client_golang v1.14.0 // indirect
 	github.com/prometheus/client_model v0.3.0 // indirect
@@ -106,11 +111,12 @@ require (
 	github.com/zeebo/errs v1.3.0 // indirect
 	go.etcd.io/bbolt v1.3.7 // indirect
 	go.opencensus.io v0.24.0 // indirect
+	go.uber.org/multierr v1.10.0 // indirect
 	golang.org/x/crypto v0.7.0 // indirect
 	golang.org/x/net v0.8.0 // indirect
 	golang.org/x/oauth2 v0.6.0 // indirect
 	golang.org/x/sync v0.1.0 // indirect
-	golang.org/x/sys v0.8.0 // indirect
+	golang.org/x/sys v0.14.0 // indirect
 	golang.org/x/term v0.7.0 // indirect
 	golang.org/x/text v0.8.0 // indirect
 	golang.org/x/time v0.3.0 // indirect

go.sum

@@ -105,6 +105,8 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
+github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
 github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
 github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
 github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU=
@@ -283,8 +285,9 @@ github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ
 github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
-github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ=
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/miekg/dns v1.1.42/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4=
@@ -444,6 +447,12 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
 go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
+go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk=
+go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo=
+go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ=
+go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
+go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo=
+go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
@@ -610,8 +619,9 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q=
+golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=

pkg/logging/broadcast.go

@@ -0,0 +1,29 @@
+package logging
+
+import (
+	"go.uber.org/zap"
+	"go.uber.org/zap/zapcore"
+)
+
+// Broadcast is a logger that broadcasts each log message to multiple loggers.
+func Broadcast(logger ...Logger) Logger {
+	var cores []zapcore.Core
+
+	var singleName string
+
+	for _, l := range logger {
+		dl := l.Desugar()
+
+		if singleName == "" {
+			singleName = dl.Name()
+		}
+
+		if dl.Name() != singleName {
+			singleName = "-"
+		}
+
+		cores = append(cores, dl.Core())
+	}
+
+	return zap.New(zapcore.NewTee(cores...)).Sugar().Named(singleName)
+}