Skip to content

feat: integrate with kopia #6

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
zjx20 merged 1 commit into from
Dec 25, 2023
Merged

feat: integrate with kopia #6

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
80 changes: 52 additions & 28 deletions cmd/app.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,20 +4,25 @@ import (
"context"
"fmt"
"os"
"path/filepath"
"strconv"
"strings"

"github.com/spf13/cobra"

"github.com/apecloud/datasafed/pkg/config"
"github.com/apecloud/datasafed/pkg/logging"
"github.com/apecloud/datasafed/pkg/storage"
"github.com/apecloud/datasafed/pkg/storage/kopia"
"github.com/apecloud/datasafed/pkg/storage/rclone"
)

const (
rootKey = "root"
backendBasePathEnv = "DATASAFED_BACKEND_BASE_PATH"
backendBasePathEnv = "DATASAFED_BACKEND_BASE_PATH"
kopiaRepoRootEnv = "DATASAFED_KOPIA_REPO_ROOT"
kopiaPasswordEnv = "DATASAFED_KOPIA_PASSWORD"
kopiaDisableCacheEnv = "DATASAFED_KOPIA_DISABLE_CACHE"
kopiaMaintenanceEnv = "DATASAFED_KOPIA_MAINTENANCE"
kopiaSafetyEnv = "DATASAFED_KOPIA_SAFETY"
)

var (
Expand Down Expand Up @@ -75,39 +80,58 @@ func initStorage() error {
return err
}

basePath := strings.TrimSpace(os.Getenv(backendBasePathEnv))
storageConf := config.GetGlobal().GetAll(config.StorageSection)
adjustRoot(storageConf)
var err error
globalStorage, err = rclone.New(storageConf)
if err != nil {
return err

if kopiaRoot := strings.TrimSpace(os.Getenv(kopiaRepoRootEnv)); kopiaRoot != "" {
return initKopiaStorage(storageConf, basePath, kopiaRoot)
} else {
st, err := createStorage(storageConf, basePath)
if err != nil {
return err
}
globalStorage = st
return nil
}
return nil
}

func adjustRoot(storageConf map[string]string) {
basePath := os.Getenv(backendBasePathEnv)
if basePath == "" {
return
func initKopiaStorage(storageConf map[string]string, basePath, kopiaRoot string) error {
underlying, err := createStorage(storageConf, "")
if err != nil {
return err
}
basePath = filepath.Clean(basePath)
if strings.HasPrefix(basePath, "..") {
exitIfError(fmt.Errorf("invalid base path %q from env %s",
os.Getenv(backendBasePathEnv), backendBasePathEnv))
kopia.SetUnderlyingStorage(underlying)
storageConf[kopia.RepoRootKey] = kopiaRoot
storageConf[kopia.PasswordKey] = strings.TrimSpace(os.Getenv(kopiaPasswordEnv))
storageConf[kopia.DisableCacheKey] = strings.TrimSpace(os.Getenv(kopiaDisableCacheEnv))
st, err := kopia.New(appCtx, storageConf, basePath)
if err != nil {
return err
}
if basePath == "." {
basePath = ""
} else {
basePath = strings.TrimPrefix(basePath, "/")
basePath = strings.TrimPrefix(basePath, "./")
globalStorage = st

maintenance := os.Getenv(kopiaMaintenanceEnv)
if ok, _ := strconv.ParseBool(maintenance); ok {
onFinish(func() {
err := kopia.RunMaintenance(appCtx, globalStorage, os.Getenv(kopiaSafetyEnv))
if err != nil {
fmt.Fprintf(os.Stderr, "RunMaintenance() failed, err: %v\n", err)
}
})
}
root := storageConf[rootKey]
if strings.HasSuffix(root, "/") {
root = root + basePath
} else {
root = root + "/" + basePath
return nil
}

func createStorage(conf map[string]string, basePath string) (storage.Storage, error) {
cloneConf := make(map[string]string, len(conf))
for k, v := range conf {
cloneConf[k] = v
}
storageConf[rootKey] = root
return rclone.New(appCtx, cloneConf, basePath)
}

func onFinish(fn func()) {
onFinishFuncs = append(onFinishFuncs, fn)
}

func Execute() {
Expand Down
77 changes: 46 additions & 31 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,22 +4,23 @@ go 1.21

require (
github.com/fatih/color v1.16.0
github.com/kopia/kopia v0.15.0
github.com/rclone/rclone v1.63.1
github.com/spf13/cobra v1.7.0
github.com/stretchr/testify v1.8.3
github.com/stretchr/testify v1.8.4
go.uber.org/zap v1.26.0
gopkg.in/ini.v1 v1.67.0
)

require (
cloud.google.com/go/compute v1.19.0 // indirect
cloud.google.com/go/compute v1.23.0 // indirect
cloud.google.com/go/compute/metadata v0.2.3 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.8.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.1 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0 // indirect
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 // indirect
github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1 // indirect
github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd // indirect
github.com/Microsoft/go-winio v0.5.2 // indirect
github.com/abbot/go-http-auth v0.4.0 // indirect
Expand All @@ -28,24 +29,31 @@ require (
github.com/buengese/sgzip v0.1.1 // indirect
github.com/calebcase/tmpfile v1.0.3 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/chmduquesne/rollinghash v4.0.0+incompatible // indirect
github.com/colinmarc/hdfs/v2 v2.3.0 // indirect
github.com/coreos/go-semver v0.3.1 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5 // indirect
github.com/edsrzf/mmap-go v1.1.0 // indirect
github.com/gabriel-vasile/mimetype v1.4.2 // indirect
github.com/geoffgarside/ber v1.1.0 // indirect
github.com/go-chi/chi/v5 v5.0.8 // indirect
github.com/go-logr/logr v1.2.4 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-ole/go-ole v1.2.6 // indirect
github.com/gofrs/flock v0.8.1 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
github.com/golang-jwt/jwt/v5 v5.0.0 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/google/go-querystring v1.1.0 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect
github.com/googleapis/gax-go/v2 v2.8.0 // indirect
github.com/google/s2a-go v0.1.7 // indirect
github.com/google/uuid v1.3.1 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.1 // indirect
github.com/googleapis/gax-go/v2 v2.12.0 // indirect
github.com/hashicorp/cronexpr v1.1.2 // indirect
github.com/hashicorp/errwrap v1.0.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/go-uuid v1.0.3 // indirect
Expand All @@ -61,40 +69,43 @@ require (
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/jtolio/eventkit v0.0.0-20221004135224-074cf276595b // indirect
github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 // indirect
github.com/klauspost/compress v1.16.5 // indirect
github.com/klauspost/cpuid/v2 v2.0.12 // indirect
github.com/klauspost/compress v1.17.0 // indirect
github.com/klauspost/cpuid/v2 v2.2.5 // indirect
github.com/klauspost/pgzip v1.2.6 // indirect
github.com/klauspost/reedsolomon v1.11.8 // indirect
github.com/koofr/go-httpclient v0.0.0-20230225102643-5d51a2e9dea6 // indirect
github.com/koofr/go-koofrclient v0.0.0-20221207135200-cbd7fc9ad6a6 // indirect
github.com/kr/fs v0.1.0 // indirect
github.com/kr/pretty v0.3.0 // indirect
github.com/kylelemons/godebug v1.1.0 // indirect
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/natefinch/atomic v1.0.1 // indirect
github.com/ncw/go-acd v0.0.0-20201019170801-fe55f33415b1 // indirect
github.com/ncw/swift/v2 v2.0.1 // indirect
github.com/oracle/oci-go-sdk/v65 v65.34.0 // indirect
github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 // indirect
github.com/pierrec/lz4 v2.6.1+incompatible // indirect
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
github.com/pkg/sftp v1.13.6-0.20230213180117-971c283182b6 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pkg/sftp v1.13.6 // indirect
github.com/pkg/xattr v0.4.9 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
github.com/prometheus/client_golang v1.14.0 // indirect
github.com/prometheus/client_model v0.3.0 // indirect
github.com/prometheus/common v0.37.0 // indirect
github.com/prometheus/procfs v0.8.0 // indirect
github.com/prometheus/client_golang v1.17.0 // indirect
github.com/prometheus/client_model v0.5.0 // indirect
github.com/prometheus/common v0.44.0 // indirect
github.com/prometheus/procfs v0.11.1 // indirect
github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 // indirect
github.com/rclone/ftp v0.0.0-20230327202000-dadc1f64e87d // indirect
github.com/rfjakob/eme v1.1.2 // indirect
github.com/rogpeppe/go-internal v1.8.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/shirou/gopsutil/v3 v3.23.5 // indirect
github.com/shoenig/go-m1cpu v0.1.6 // indirect
github.com/sirupsen/logrus v1.9.0 // indirect
github.com/sirupsen/logrus v1.9.3 // indirect
github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 // indirect
github.com/sony/gobreaker v0.5.0 // indirect
github.com/spacemonkeygo/monkit/v3 v3.0.19 // indirect
Expand All @@ -111,20 +122,24 @@ require (
github.com/zeebo/errs v1.3.0 // indirect
go.etcd.io/bbolt v1.3.7 // indirect
go.opencensus.io v0.24.0 // indirect
go.uber.org/multierr v1.10.0 // indirect
golang.org/x/crypto v0.7.0 // indirect
golang.org/x/net v0.8.0 // indirect
golang.org/x/oauth2 v0.6.0 // indirect
golang.org/x/sync v0.1.0 // indirect
go.opentelemetry.io/otel v1.19.0 // indirect
go.opentelemetry.io/otel/metric v1.19.0 // indirect
go.opentelemetry.io/otel/trace v1.19.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/crypto v0.14.0 // indirect
golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 // indirect
golang.org/x/net v0.17.0 // indirect
golang.org/x/oauth2 v0.13.0 // indirect
golang.org/x/sync v0.4.0 // indirect
golang.org/x/sys v0.14.0 // indirect
golang.org/x/term v0.7.0 // indirect
golang.org/x/text v0.8.0 // indirect
golang.org/x/term v0.13.0 // indirect
golang.org/x/text v0.13.0 // indirect
golang.org/x/time v0.3.0 // indirect
google.golang.org/api v0.115.0 // indirect
google.golang.org/api v0.146.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20230331144136-dcfb400f0633 // indirect
google.golang.org/grpc v1.54.0 // indirect
google.golang.org/protobuf v1.30.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20230920204549-e6e6cdab5c13 // indirect
google.golang.org/grpc v1.58.2 // indirect
google.golang.org/protobuf v1.31.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
storj.io/common v0.0.0-20221123115229-fed3e6651b63 // indirect
Expand Down
Loading