refactor: make List() base on a callback

cmd/list.go

@@ -1,9 +1,11 @@
 package cmd
 
 import (
+	"bufio"
 	"cmp"
 	"encoding/json"
 	"fmt"
+	"io"
 	"os"
 	"path/filepath"
 	"slices"
@@ -64,8 +66,8 @@ datasafed list --name "*.txt" /some/dir/
 	pflags.BoolVarP(&opts.filesOnly, "files-only", "f", false, "list files only")
 	pflags.BoolVarP(&opts.recursive, "recursive", "r", false, "list recursively")
 	pflags.IntVar(&opts.maxDepth, "max-depth", 0, "max depth when listing recursively")
-	pflags.StringVarP(&opts.sort, "sort", "s", "path",
-		fmt.Sprintf("sort by which field, choices: %q", validSorts))
+	pflags.StringVarP(&opts.sort, "sort", "s", "",
+		fmt.Sprintf("sort by which field, choices: %q, this option conflicts with --recursive", validSorts))
 	pflags.BoolVar(&opts.reverse, "reverse", false, "reverse order")
 	pflags.Int64Var(&opts.newer, "newer-than", 0,
 		"list only entries whose last modification time is newer than the specified unix timestamp (exclusive)")
@@ -77,57 +79,67 @@ datasafed list --name "*.txt" /some/dir/
 		fmt.Sprintf("output format, choices: %q", validOutputFormats))
 
 	cmd.MarkFlagsMutuallyExclusive("dirs-only", "files-only")
+	cmd.MarkFlagsMutuallyExclusive("recursive", "sort")
 
 	rootCmd.AddCommand(cmd)
 }
 
 func doList(opts *listOptions, cmd *cobra.Command, args []string) {
-	if !slices.Contains(validSorts, opts.sort) {
+	if opts.sort != "" && !slices.Contains(validSorts, opts.sort) {
 		exitIfError(fmt.Errorf("invalid sort: %q", opts.sort))
 	}
 	if !slices.Contains(validOutputFormats, opts.format) {
 		exitIfError(fmt.Errorf("invalid output format: %q", opts.format))
 	}
+	bufStdout := bufio.NewWriterSize(os.Stdout, 8*1024)
+	filter := getFilterFn(opts)
+	printer := getPrinter(opts, bufStdout)
+	var cb func(storage.DirEntry) error
+	var entries []storage.DirEntry
+	if opts.recursive {
+		cb = func(entry storage.DirEntry) error {
+			if filter(entry) {
+				printer.printItem(entry)
+			}
+			return nil
+		}
+	} else {
+		cb = func(entry storage.DirEntry) error {
+			if filter(entry) {
+				entries = append(entries, entry)
+			}
+			return nil
+		}
+	}
+
+	if opts.recursive {
+		printer.printHeader()
+	}
 	rpath := args[0]
 	lopts := &storage.ListOptions{
 		DirsOnly:  opts.dirsOnly,
 		FilesOnly: opts.filesOnly,
 		Recursive: opts.recursive,
 		MaxDepth:  opts.maxDepth,
 	}
-	entries, err := globalStorage.List(appCtx, rpath, lopts)
+	err := globalStorage.List(appCtx, rpath, lopts, cb)
 	exitIfError(err)
+	if opts.recursive {
+		printer.printFooter()
+	}
 
-	entries = filterEntries(entries, opts)
-	sortEntries(entries, opts)
-	switch opts.format {
-	case "short":
-		for _, entry := range entries {
-			fmt.Println(toPath(entry))
-		}
-	case "long":
+	if !opts.recursive {
+		sortEntries(entries, opts)
+		printer.printHeader()
 		for _, entry := range entries {
-			fmt.Printf("%s\t%d\t%s\n", entry.MTime().Format(time.RFC3339), entry.Size(), toPath(entry))
-		}
-	case "json":
-		if len(entries) == 0 {
-			fmt.Print("[]\n")
-		} else {
-			fmt.Print("[\n")
-			enc := json.NewEncoder(os.Stdout)
-			enc.SetIndent("  ", "  ")
-			first := true
-			for _, entry := range entries {
-				if !first {
-					fmt.Print("  ,\n")
-				}
-				first = false
-				fmt.Print("  ")
-				printJson(entry, enc)
+			if filter(entry) {
+				printer.printItem(entry)
 			}
-			fmt.Print("]\n")
 		}
+		printer.printFooter()
 	}
+
+	bufStdout.Flush()
 }
 
 func toPath(entry storage.DirEntry) string {
@@ -138,6 +150,30 @@ func toPath(entry storage.DirEntry) string {
 	return path
 }
 
+func getFilterFn(opts *listOptions) func(entry storage.DirEntry) bool {
+	return func(entry storage.DirEntry) bool {
+		matchPattern := func(entry storage.DirEntry) bool { return true }
+		if opts.namePattern != "" {
+			matchPattern = func(entry storage.DirEntry) bool {
+				// ignore errors
+				matched, _ := filepath.Match(opts.namePattern, entry.Name())
+				return matched
+			}
+		}
+		mt := entry.MTime().Unix()
+		if opts.newer > 0 && opts.newer >= mt {
+			return false
+		}
+		if opts.older > 0 && opts.older <= mt {
+			return false
+		}
+		if !matchPattern(entry) {
+			return false
+		}
+		return true
+	}
+}
+
 func printJson(entry storage.DirEntry, enc *json.Encoder) {
 	type jsonEntry struct {
 		Path     string `json:"path"`
@@ -155,33 +191,54 @@ func printJson(entry storage.DirEntry, enc *json.Encoder) {
 	})
 }
 
-func filterEntries(entries []storage.DirEntry, opts *listOptions) []storage.DirEntry {
-	matchPattern := func(entry storage.DirEntry) bool { return true }
-	if opts.namePattern != "" {
-		matchPattern = func(entry storage.DirEntry) bool {
-			// ignore errors
-			matched, _ := filepath.Match(opts.namePattern, entry.Name())
-			return matched
-		}
-	}
-	var filtered []storage.DirEntry
-	for _, entry := range entries {
-		mt := entry.MTime().Unix()
-		if opts.newer > 0 && opts.newer >= mt {
-			continue
+func getPrinter(opts *listOptions, out io.Writer) *printer {
+	switch opts.format {
+	case "short":
+		return &printer{
+			printHeader: func() {},
+			printItem: func(entry storage.DirEntry) {
+				fmt.Fprintln(out, toPath(entry))
+			},
+			printFooter: func() {},
 		}
-		if opts.older > 0 && opts.older <= mt {
-			continue
+	case "long":
+		return &printer{
+			printHeader: func() {},
+			printItem: func(entry storage.DirEntry) {
+				fmt.Fprintf(out, "%s\t%d\t%s\n", entry.MTime().Format(time.RFC3339), entry.Size(), toPath(entry))
+			},
+			printFooter: func() {},
 		}
-		if !matchPattern(entry) {
-			continue
+	case "json":
+		enc := json.NewEncoder(out)
+		enc.SetIndent("  ", "  ")
+		first := true
+		return &printer{
+			printHeader: func() {
+				fmt.Fprintf(out, "[")
+			},
+			printItem: func(entry storage.DirEntry) {
+				if first {
+					fmt.Fprintf(out, "\n")
+				} else if !first {
+					fmt.Fprintf(out, "  ,\n")
+				}
+				first = false
+				fmt.Fprintf(out, "  ")
+				printJson(entry, enc)
+			},
+			printFooter: func() {
+				fmt.Fprintf(out, "]\n")
+			},
 		}
-		filtered = append(filtered, entry)
 	}
-	return filtered
+	panic(fmt.Sprintf("unsupported format %s", opts.format))
 }
 
 func sortEntries(entries []storage.DirEntry, opts *listOptions) {
+	if opts.sort == "" {
+		opts.sort = "path"
+	}
 	var getter func(storage.DirEntry) any
 	var compare func(any, any) int
 	switch opts.sort {
@@ -215,3 +272,9 @@ func sortEntries(entries []storage.DirEntry, opts *listOptions) {
 		return cmpVal
 	})
 }
+
+type printer struct {
+	printHeader func()
+	printItem   func(entry storage.DirEntry)
+	printFooter func()
+}

cmd/mkdir.go

@@ -8,8 +8,9 @@ import (
 
 func init() {
 	cmd := &cobra.Command{
-		Use: "mkdir rpath",
-		Short: "Create an empty remote directory." +
+		Use:   "mkdir rpath",
+		Short: "Create an empty remote directory.",
+		Long: "Remove an empty remote directory.\n" +
 			"Some storage backends, such as S3, do not have the concept of a directory, " +
 			"in which case the command will directly return success with no effect.",
 		Example: strings.TrimSpace(`

pkg/config/config.go

@@ -23,7 +23,9 @@ func newConfig(cfg *ini.File) *Config {
 			if strings.HasSuffix(key, ProcessorObscure) {
 				// replace the KV with the obscured version
 				newKey := strings.TrimSuffix(key, ProcessorObscure)
-				sec.NewKey(newKey, obscure.MustObscure(k.Value()))
+				if _, err := sec.NewKey(newKey, obscure.MustObscure(k.Value())); err != nil {
+					panic(err)
+				}
 				sec.DeleteKey(key)
 			}
 		}

pkg/storage/kopia/blobstorageimpl.go

@@ -65,17 +65,20 @@ func (b *blobStorageImpl) GetBlobFromPath(ctx context.Context, dirPath, filePath
 
 func (b *blobStorageImpl) GetMetadataFromPath(ctx context.Context, dirPath, filePath string) (blob.Metadata, error) {
 	log(ctx).Debugf("GetMetadata dir %s file %s", dirPath, filePath)
-	entries, err := b.s.List(ctx, filePath, &storage.ListOptions{PathIsFile: true})
+	var entry storage.DirEntry
+	err := b.s.List(ctx, filePath, &storage.ListOptions{PathIsFile: true}, func(en storage.DirEntry) error {
+		entry = en
+		return nil
+	})
 	if err != nil {
 		if errors.Is(err, storage.ErrObjectNotFound) {
 			return blob.Metadata{}, blob.ErrBlobNotFound
 		}
 		return blob.Metadata{}, fmt.Errorf("fail to OpenFile, err: %w", err)
 	}
-	if len(entries) != 1 {
-		return blob.Metadata{}, fmt.Errorf("expect List() to return single entry for path %s, got %d", filePath, len(entries))
+	if entry == nil {
+		return blob.Metadata{}, blob.ErrBlobNotFound
 	}
-	entry := entries[0]
 	return blob.Metadata{
 		Length:    entry.Size(),
 		Timestamp: entry.MTime(),
@@ -116,18 +119,15 @@ func (b *blobStorageImpl) DeleteBlobInPath(ctx context.Context, dirPath, filePat
 func (b *blobStorageImpl) ReadDir(ctx context.Context, path string) ([]os.FileInfo, error) {
 	log(ctx).Debugf("ReadDir path %s", path)
 	var fileInfos []os.FileInfo
-	_, err := b.s.List(ctx, path, &storage.ListOptions{
-		Recursive: false,
-		Callback: func(en storage.DirEntry) error {
-			info := fileInfo{
-				name:    en.Name(),
-				size:    en.Size(),
-				modTime: en.MTime(),
-				isDir:   en.IsDir(),
-			}
-			fileInfos = append(fileInfos, &info)
-			return nil
-		},
+	err := b.s.List(ctx, path, &storage.ListOptions{Recursive: false}, func(en storage.DirEntry) error {
+		info := fileInfo{
+			name:    en.Name(),
+			size:    en.Size(),
+			modTime: en.MTime(),
+			isDir:   en.IsDir(),
+		}
+		fileInfos = append(fileInfos, &info)
+		return nil
 	})
 	return fileInfos, err
 }