kvdb: add sqlite

docs/release-notes/release-notes-0.16.0.md

@@ -330,6 +330,12 @@ Keysend](https://github.com/lightningnetwork/lnd/pull/7334).
 * [Store AckedUpdates in a more compact
   way](https://github.com/lightningnetwork/lnd/pull/7055)
 
+## DB
+
+* [Add a sqlite backend 
+  option](https://github.com/lightningnetwork/lnd/pull/7251) to the kvdb 
+  package.
+
 ## Pathfinding
 
 * [Pathfinding takes capacity of edges into account to improve success
@@ -389,6 +395,7 @@ refactor the itest for code health and maintenance.
 * Antoni Spaanderman
 * Carla Kirk-Cohen
 * Carsten Otto
+* Chris Geihsler
 * Conner Babinchak
 * cutiful
 * Daniel McNally

kvdb/backend.go

@@ -247,20 +247,49 @@ func updateLastCompactionDate(dbFile string) error {
 func GetTestBackend(path, name string) (Backend, func(), error) {
 	empty := func() {}
 
+	// Note that for tests, we expect only one db backend build flag
+	// (or none) to be set at a time and thus one of the following switch
+	// cases should ever be true
 	switch {
 	case PostgresBackend:
 		key := filepath.Join(path, name)
 		keyHash := sha256.Sum256([]byte(key))
 
-		f, err := NewPostgresFixture("test_" + hex.EncodeToString(keyHash[:]))
+		f, err := NewPostgresFixture("test_" + hex.EncodeToString(
+			keyHash[:]),
+		)
 		if err != nil {
 			return nil, func() {}, err
 		}
 		return f.DB(), func() {
 			_ = f.DB().Close()
 		}, nil
 
-	case TestBackend == BoltBackendName:
+	case EtcdBackend:
+		etcdConfig, cancel, err := StartEtcdTestBackend(path, 0, 0, "")
+		if err != nil {
+			return nil, empty, err
+		}
+		backend, err := Open(
+			EtcdBackendName, context.TODO(), etcdConfig,
+		)
+		return backend, cancel, err
+
+	case SqliteBackend:
+		dbPath := filepath.Join(path, name)
+		keyHash := sha256.Sum256([]byte(dbPath))
+		sqliteDb, err := StartSqliteTestBackend(
+			path, name, "test_"+hex.EncodeToString(keyHash[:]),
+		)
+		if err != nil {
+			return nil, empty, err
+		}
+
+		return sqliteDb, func() {
+			_ = sqliteDb.Close()
+		}, nil
+
+	default:
 		db, err := GetBoltBackend(&BoltBackendConfig{
 			DBPath:         path,
 			DBFileName:     name,
@@ -271,17 +300,6 @@ func GetTestBackend(path, name string) (Backend, func(), error) {
 			return nil, nil, err
 		}
 		return db, empty, nil
-
-	case TestBackend == EtcdBackendName:
-		etcdConfig, cancel, err := StartEtcdTestBackend(path, 0, 0, "")
-		if err != nil {
-			return nil, empty, err
-		}
-		backend, err := Open(
-			EtcdBackendName, context.TODO(), etcdConfig,
-		)
-		return backend, cancel, err
-
 	}
 
 	return nil, nil, fmt.Errorf("unknown backend")

kvdb/config.go

@@ -18,6 +18,11 @@ const (
 	// by a live instance of postgres.
 	PostgresBackendName = "postgres"
 
+	// SqliteBackendName is the name of the backend that should be passed
+	// into kvdb.Create to initialize a new instance of kvdb.Backend backed
+	// by a live instance of sqlite.
+	SqliteBackendName = "sqlite"
+
 	// DefaultBoltAutoCompactMinAge is the default minimum time that must
 	// have passed since a bolt database file was last compacted for the
 	// compaction to be considered again.

kvdb/go.mod

@@ -1,28 +1,113 @@
 module github.com/lightningnetwork/lnd/kvdb
 
 require (
-	github.com/andybalholm/brotli v1.0.3 // indirect
 	github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f
 	github.com/btcsuite/btcwallet/walletdb v1.3.6-0.20210803004036-eebed51155ec
 	github.com/davecgh/go-spew v1.1.1
 	github.com/fergusstrange/embedded-postgres v1.10.0
-	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/btree v1.0.1
 	github.com/jackc/pgx/v4 v4.13.0
-	github.com/klauspost/compress v1.13.6 // indirect
-	github.com/klauspost/pgzip v1.2.5 // indirect
-	github.com/lib/pq v1.10.3 // indirect
 	github.com/lightningnetwork/lnd/healthcheck v1.0.0
-	github.com/nwaples/rardecode v1.1.2 // indirect
-	github.com/pierrec/lz4/v4 v4.1.8 // indirect
 	github.com/stretchr/testify v1.7.0
-	github.com/ulikunitz/xz v0.5.10 // indirect
 	go.etcd.io/bbolt v1.3.6
 	go.etcd.io/etcd/api/v3 v3.5.0
 	go.etcd.io/etcd/client/pkg/v3 v3.5.0
 	go.etcd.io/etcd/client/v3 v3.5.0
 	go.etcd.io/etcd/server/v3 v3.5.0
 	golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4
+	modernc.org/sqlite v1.20.3
+)
+
+require (
+	github.com/andybalholm/brotli v1.0.3 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.1.1 // indirect
+	github.com/coreos/go-semver v0.3.0 // indirect
+	github.com/coreos/go-systemd/v22 v22.3.2 // indirect
+	github.com/dsnet/compress v0.0.1 // indirect
+	github.com/dustin/go-humanize v1.0.0 // indirect
+	github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/golang/snappy v0.0.4 // indirect
+	github.com/google/uuid v1.3.0 // indirect
+	github.com/gorilla/websocket v1.4.2 // indirect
+	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
+	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
+	github.com/jackc/chunkreader/v2 v2.0.1 // indirect
+	github.com/jackc/pgconn v1.10.0 // indirect
+	github.com/jackc/pgio v1.0.0 // indirect
+	github.com/jackc/pgpassfile v1.0.0 // indirect
+	github.com/jackc/pgproto3/v2 v2.1.1 // indirect
+	github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect
+	github.com/jackc/pgtype v1.8.1 // indirect
+	github.com/jonboulle/clockwork v0.2.2 // indirect
+	github.com/json-iterator/go v1.1.11 // indirect
+	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
+	github.com/klauspost/compress v1.13.6 // indirect
+	github.com/klauspost/pgzip v1.2.5 // indirect
+	github.com/lib/pq v1.10.3 // indirect
+	github.com/lightningnetwork/lnd/ticker v1.0.0 // indirect
+	github.com/mattn/go-isatty v0.0.16 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
+	github.com/mholt/archiver/v3 v3.5.0 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.1 // indirect
+	github.com/nwaples/rardecode v1.1.2 // indirect
+	github.com/pierrec/lz4/v4 v4.1.8 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/prometheus/client_golang v1.11.0 // indirect
+	github.com/prometheus/client_model v0.2.0 // indirect
+	github.com/prometheus/common v0.26.0 // indirect
+	github.com/prometheus/procfs v0.6.0 // indirect
+	github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
+	github.com/sirupsen/logrus v1.7.0 // indirect
+	github.com/soheilhy/cmux v0.1.5 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect
+	github.com/ulikunitz/xz v0.5.10 // indirect
+	github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
+	github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect
+	go.etcd.io/etcd/client/v2 v2.305.0 // indirect
+	go.etcd.io/etcd/pkg/v3 v3.5.0 // indirect
+	go.etcd.io/etcd/raft/v3 v3.5.0 // indirect
+	go.opentelemetry.io/contrib v0.20.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 // indirect
+	go.opentelemetry.io/otel v0.20.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp v0.20.0 // indirect
+	go.opentelemetry.io/otel/metric v0.20.0 // indirect
+	go.opentelemetry.io/otel/sdk v0.20.0 // indirect
+	go.opentelemetry.io/otel/sdk/export/metric v0.20.0 // indirect
+	go.opentelemetry.io/otel/sdk/metric v0.20.0 // indirect
+	go.opentelemetry.io/otel/trace v0.20.0 // indirect
+	go.opentelemetry.io/proto/otlp v0.7.0 // indirect
+	go.uber.org/atomic v1.7.0 // indirect
+	go.uber.org/multierr v1.6.0 // indirect
+	go.uber.org/zap v1.17.0 // indirect
+	golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 // indirect
+	golang.org/x/mod v0.4.2 // indirect
+	golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab // indirect
+	golang.org/x/text v0.3.6 // indirect
+	golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect
+	golang.org/x/tools v0.1.2 // indirect
+	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
+	google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c // indirect
+	google.golang.org/grpc v1.38.0 // indirect
+	google.golang.org/protobuf v1.26.0 // indirect
+	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+	lukechampine.com/uint128 v1.2.0 // indirect
+	modernc.org/cc/v3 v3.40.0 // indirect
+	modernc.org/ccgo/v3 v3.16.13 // indirect
+	modernc.org/libc v1.22.2 // indirect
+	modernc.org/mathutil v1.5.0 // indirect
+	modernc.org/memory v1.4.0 // indirect
+	modernc.org/opt v0.1.3 // indirect
+	modernc.org/strutil v1.1.3 // indirect
+	modernc.org/token v1.0.1 // indirect
+	sigs.k8s.io/yaml v1.2.0 // indirect
 )
 
 // This replace is for https://github.com/advisories/GHSA-w73w-5m7g-f7qc
@@ -35,4 +120,4 @@ replace github.com/ulikunitz/xz => github.com/ulikunitz/xz v0.5.8
 // https://deps.dev/advisory/OSV/GO-2021-0053?from=%2Fgo%2Fgithub.com%252Fgogo%252Fprotobuf%2Fv1.3.1
 replace github.com/gogo/protobuf => github.com/gogo/protobuf v1.3.2
 
-go 1.16
+go 1.18