From 680394518f573b967830f713feb13efbcec391a3 Mon Sep 17 00:00:00 2001 From: Andras Banki-Horvath Date: Wed, 14 Aug 2024 11:38:03 +0200 Subject: [PATCH 01/21] mod: temporarily replace sqldb with local version --- go.mod | 4 ++++ go.sum | 2 -- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 7680509fd39..ba53873e13e 100644 --- a/go.mod +++ b/go.mod @@ -207,6 +207,10 @@ replace github.com/gogo/protobuf => github.com/gogo/protobuf v1.3.2 // allows us to specify that as an option. replace google.golang.org/protobuf => github.com/lightninglabs/protobuf-go-hex-display v1.30.0-hex-display +// Temporary replace until https://github.com/lightningnetwork/lnd/pull/8831 is +// merged. +replace github.com/lightningnetwork/lnd/sqldb => ./sqldb + // If you change this please also update docs/INSTALL.md and GO_VERSION in // Makefile (then run `make lint` to see where else it needs to be updated as // well). diff --git a/go.sum b/go.sum index 5ed9cd20462..72bf21ab6ed 100644 --- a/go.sum +++ b/go.sum @@ -464,8 +464,6 @@ github.com/lightningnetwork/lnd/kvdb v1.4.12 h1:Y0WY5Tbjyjn6eCYh068qkWur5oFtioJl github.com/lightningnetwork/lnd/kvdb v1.4.12/go.mod h1:hx9buNcxsZpZwh8m1sjTQwy2SOeBoWWOZ3RnOQkMsxI= github.com/lightningnetwork/lnd/queue v1.1.1 h1:99ovBlpM9B0FRCGYJo6RSFDlt8/vOkQQZznVb18iNMI= github.com/lightningnetwork/lnd/queue v1.1.1/go.mod h1:7A6nC1Qrm32FHuhx/mi1cieAiBZo5O6l8IBIoQxvkz4= -github.com/lightningnetwork/lnd/sqldb v1.0.6 h1:LJdDSVdN33bVBIefsaJlPW9PDAm6GrXlyFucmzSJ3Ts= -github.com/lightningnetwork/lnd/sqldb v1.0.6/go.mod h1:OG09zL/PHPaBJefp4HsPz2YLUJ+zIQHbpgCtLnOx8I4= github.com/lightningnetwork/lnd/ticker v1.1.1 h1:J/b6N2hibFtC7JLV77ULQp++QLtCwT6ijJlbdiZFbSM= github.com/lightningnetwork/lnd/ticker v1.1.1/go.mod h1:waPTRAAcwtu7Ji3+3k+u/xH5GHovTsCoSVpho0KDvdA= github.com/lightningnetwork/lnd/tlv v1.3.0 h1:exS/KCPEgpOgviIttfiXAPaUqw2rHQrnUOpP7HPBPiY= From 9acd06d29682ab62f378a642636e58f224dde080 Mon Sep 17 00:00:00 2001 From: Andras Banki-Horvath Date: Tue, 12 Nov 2024 16:25:29 +0100 Subject: [PATCH 02/21] sqldb: add table to track custom SQL migrations This commit adds the migration_tracker table which we'll use to track if a custom migration has already been done. --- sqldb/sqlc/migration.sql.go | 60 +++++++++++++++++++ .../000005_migration_tracker.down.sql | 1 + .../000005_migration_tracker.up.sql | 17 ++++++ sqldb/sqlc/models.go | 5 ++ sqldb/sqlc/querier.go | 4 ++ sqldb/sqlc/queries/migration.sql | 21 +++++++ 6 files changed, 108 insertions(+) create mode 100644 sqldb/sqlc/migration.sql.go create mode 100644 sqldb/sqlc/migrations/000005_migration_tracker.down.sql create mode 100644 sqldb/sqlc/migrations/000005_migration_tracker.up.sql create mode 100644 sqldb/sqlc/queries/migration.sql diff --git a/sqldb/sqlc/migration.sql.go b/sqldb/sqlc/migration.sql.go new file mode 100644 index 00000000000..d65ff74f5d6 --- /dev/null +++ b/sqldb/sqlc/migration.sql.go @@ -0,0 +1,60 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.25.0 +// source: migration.sql + +package sqlc + +import ( + "context" + "time" +) + +const getDatabaseVersion = `-- name: GetDatabaseVersion :one +SELECT + version +FROM + migration_tracker +ORDER BY + version DESC +LIMIT 1 +` + +func (q *Queries) GetDatabaseVersion(ctx context.Context) (int32, error) { + row := q.db.QueryRowContext(ctx, getDatabaseVersion) + var version int32 + err := row.Scan(&version) + return version, err +} + +const getMigration = `-- name: GetMigration :one +SELECT + migration_time +FROM + migration_tracker +WHERE + version = $1 +` + +func (q *Queries) GetMigration(ctx context.Context, version int32) (time.Time, error) { + row := q.db.QueryRowContext(ctx, getMigration, version) + var migration_time time.Time + err := row.Scan(&migration_time) + return migration_time, err +} + +const setMigration = `-- name: SetMigration :exec +INSERT INTO + migration_tracker (version, migration_time) +VALUES ($1, $2) +` + +type SetMigrationParams struct { + Version int32 + MigrationTime time.Time +} + +func (q *Queries) SetMigration(ctx context.Context, arg SetMigrationParams) error { + _, err := q.db.ExecContext(ctx, setMigration, arg.Version, arg.MigrationTime) + return err +} diff --git a/sqldb/sqlc/migrations/000005_migration_tracker.down.sql b/sqldb/sqlc/migrations/000005_migration_tracker.down.sql new file mode 100644 index 00000000000..5f86e385c2b --- /dev/null +++ b/sqldb/sqlc/migrations/000005_migration_tracker.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS migration_tracker; diff --git a/sqldb/sqlc/migrations/000005_migration_tracker.up.sql b/sqldb/sqlc/migrations/000005_migration_tracker.up.sql new file mode 100644 index 00000000000..fa55812b528 --- /dev/null +++ b/sqldb/sqlc/migrations/000005_migration_tracker.up.sql @@ -0,0 +1,17 @@ +-- The migration_tracker table keeps track of migrations that have been applied +-- to the database. This table ensures that migrations are idempotent and are +-- only run once. It tracks a global database version that encompasses both +-- schema migrations handled by golang-migrate and custom in-code migrations +-- for more complex data conversions that cannot be expressed in pure SQL. +CREATE TABLE IF NOT EXISTS migration_tracker ( + -- version is the global version of the migration. Note that we + -- intentionally don't set it as PRIMARY KEY as it'd auto increment on + -- SQLite and our sqlc workflow will replace it with an auto + -- incrementing SERIAL on Postgres too. UNIQUE achieves the same effect + -- without the auto increment. + version INTEGER UNIQUE NOT NULL, + + -- migration_time is the timestamp at which the migration was run. + migration_time TIMESTAMP NOT NULL +); + diff --git a/sqldb/sqlc/models.go b/sqldb/sqlc/models.go index 83be5a708fa..1476d0470d6 100644 --- a/sqldb/sqlc/models.go +++ b/sqldb/sqlc/models.go @@ -91,3 +91,8 @@ type InvoiceSequence struct { Name string CurrentValue int64 } + +type MigrationTracker struct { + Version int32 + MigrationTime time.Time +} diff --git a/sqldb/sqlc/querier.go b/sqldb/sqlc/querier.go index 04b61c7007f..268ca027c1f 100644 --- a/sqldb/sqlc/querier.go +++ b/sqldb/sqlc/querier.go @@ -7,6 +7,7 @@ package sqlc import ( "context" "database/sql" + "time" ) type Querier interface { @@ -17,6 +18,7 @@ type Querier interface { FetchSettledAMPSubInvoices(ctx context.Context, arg FetchSettledAMPSubInvoicesParams) ([]FetchSettledAMPSubInvoicesRow, error) FilterInvoices(ctx context.Context, arg FilterInvoicesParams) ([]Invoice, error) GetAMPInvoiceID(ctx context.Context, setID []byte) (int64, error) + GetDatabaseVersion(ctx context.Context) (int32, error) // This method may return more than one invoice if filter using multiple fields // from different invoices. It is the caller's responsibility to ensure that // we bubble up an error in those cases. @@ -25,6 +27,7 @@ type Querier interface { GetInvoiceFeatures(ctx context.Context, invoiceID int64) ([]InvoiceFeature, error) GetInvoiceHTLCCustomRecords(ctx context.Context, invoiceID int64) ([]GetInvoiceHTLCCustomRecordsRow, error) GetInvoiceHTLCs(ctx context.Context, invoiceID int64) ([]InvoiceHtlc, error) + GetMigration(ctx context.Context, version int32) (time.Time, error) InsertAMPSubInvoiceHTLC(ctx context.Context, arg InsertAMPSubInvoiceHTLCParams) error InsertInvoice(ctx context.Context, arg InsertInvoiceParams) (int64, error) InsertInvoiceFeature(ctx context.Context, arg InsertInvoiceFeatureParams) error @@ -37,6 +40,7 @@ type Querier interface { OnInvoiceCanceled(ctx context.Context, arg OnInvoiceCanceledParams) error OnInvoiceCreated(ctx context.Context, arg OnInvoiceCreatedParams) error OnInvoiceSettled(ctx context.Context, arg OnInvoiceSettledParams) error + SetMigration(ctx context.Context, arg SetMigrationParams) error UpdateAMPSubInvoiceHTLCPreimage(ctx context.Context, arg UpdateAMPSubInvoiceHTLCPreimageParams) (sql.Result, error) UpdateAMPSubInvoiceState(ctx context.Context, arg UpdateAMPSubInvoiceStateParams) error UpdateInvoiceAmountPaid(ctx context.Context, arg UpdateInvoiceAmountPaidParams) (sql.Result, error) diff --git a/sqldb/sqlc/queries/migration.sql b/sqldb/sqlc/queries/migration.sql new file mode 100644 index 00000000000..aed90d1938d --- /dev/null +++ b/sqldb/sqlc/queries/migration.sql @@ -0,0 +1,21 @@ +-- name: SetMigration :exec +INSERT INTO + migration_tracker (version, migration_time) +VALUES ($1, $2); + +-- name: GetMigration :one +SELECT + migration_time +FROM + migration_tracker +WHERE + version = $1; + +-- name: GetDatabaseVersion :one +SELECT + version +FROM + migration_tracker +ORDER BY + version DESC +LIMIT 1; From b789fb2db3a5df30f2e9682c4b5358564b7d7491 Mon Sep 17 00:00:00 2001 From: Andras Banki-Horvath Date: Fri, 22 Nov 2024 18:29:54 +0100 Subject: [PATCH 03/21] sqldb: add support for custom in-code migrations This commit introduces support for custom, in-code migrations, allowing a specific Go function to be executed at a designated database version during sqlc migrations. If the current database version surpasses the specified version, the migration will be skipped. --- lncfg/db.go | 3 +- sqldb/migrations.go | 204 ++++++++++++++++++++++++++ sqldb/migrations_test.go | 293 ++++++++++++++++++++++++++++++++++++++ sqldb/postgres.go | 43 ++++-- sqldb/postgres_fixture.go | 4 +- sqldb/sqlite.go | 38 ++++- 6 files changed, 565 insertions(+), 20 deletions(-) diff --git a/lncfg/db.go b/lncfg/db.go index 3d45bb78b1b..a6598e66dab 100644 --- a/lncfg/db.go +++ b/lncfg/db.go @@ -452,7 +452,7 @@ func (db *DB) GetBackends(ctx context.Context, chanDBPath, var nativeSQLStore *sqldb.BaseDB if db.UseNativeSQL { nativePostgresStore, err := sqldb.NewPostgresStore( - db.Postgres, + db.Postgres, sqldb.GetMigrations(), ) if err != nil { return nil, fmt.Errorf("error opening "+ @@ -576,6 +576,7 @@ func (db *DB) GetBackends(ctx context.Context, chanDBPath, nativeSQLiteStore, err := sqldb.NewSqliteStore( db.Sqlite, path.Join(chanDBPath, SqliteNativeDBName), + sqldb.GetMigrations(), ) if err != nil { return nil, fmt.Errorf("error opening "+ diff --git a/sqldb/migrations.go b/sqldb/migrations.go index 9d394ceed19..83634d0e511 100644 --- a/sqldb/migrations.go +++ b/sqldb/migrations.go @@ -2,22 +2,104 @@ package sqldb import ( "bytes" + "context" + "database/sql" "errors" + "fmt" "io" "io/fs" "net/http" "strings" + "time" "github.com/btcsuite/btclog/v2" "github.com/golang-migrate/migrate/v4" "github.com/golang-migrate/migrate/v4/database" "github.com/golang-migrate/migrate/v4/source/httpfs" + "github.com/lightningnetwork/lnd/sqldb/sqlc" ) +var ( + // migrationConfig defines a list of migrations to be applied to the + // database. Each migration is assigned a version number, determining + // its execution order. + // The schema version, tracked by golang-migrate, ensures migrations are + // applied to the correct schema. For migrations involving only schema + // changes, the migration function can be left nil. For custom + // migrations an implemented migration function is required. + // + // NOTE: The migration function may have runtime dependencies, which + // must be injected during runtime. + migrationConfig = []MigrationConfig{ + { + Name: "000001_invoices", + Version: 1, + SchemaVersion: 1, + }, + { + Name: "000002_amp_invoices", + Version: 2, + SchemaVersion: 2, + }, + { + Name: "000003_invoice_events", + Version: 3, + SchemaVersion: 3, + }, + { + Name: "000004_invoice_expiry_fix", + Version: 4, + SchemaVersion: 4, + }, + { + Name: "000005_migration_tracker", + Version: 5, + SchemaVersion: 5, + }, + } +) + +// MigrationConfig is a configuration struct that describes SQL migrations. Each +// migration is associated with a specific schema version and a global database +// version. Migrations are applied in the order of their global database +// version. If a migration includes a non-nil MigrationFn, it is executed after +// the SQL schema has been migrated to the corresponding schema version. +type MigrationConfig struct { + // Name is the name of the migration. + Name string + + // Version represents the "global" database version for this migration. + // Unlike the schema version tracked by golang-migrate, it encompasses + // all migrations, including those managed by golang-migrate as well + // as custom in-code migrations. + Version int + + // SchemaVersion represents the schema version tracked by golang-migrate + // at which the migration is applied. + SchemaVersion int + + // MigrationFn is the function executed for custom migrations at the + // specified version. It is used to handle migrations that cannot be + // performed through SQL alone. If set to nil, no custom migration is + // applied. + MigrationFn func(tx *sqlc.Queries) error +} + // MigrationTarget is a functional option that can be passed to applyMigrations // to specify a target version to migrate to. type MigrationTarget func(mig *migrate.Migrate) error +// MigrationExecutor is an interface that abstracts the migration functionality. +type MigrationExecutor interface { + // ExecuteMigrations runs database migrations up to the specified target + // version or all migrations if no target is specified. A migration may + // include a schema change, a custom migration function, or both. + // Developers must ensure that migrations are defined in the correct + // order. Migration details are stored in the global variable + // migrationConfig. + ExecuteMigrations(target MigrationTarget) error +} + var ( // TargetLatest is a MigrationTarget that migrates to the latest // version available. @@ -34,6 +116,14 @@ var ( } ) +// GetMigrations returns a copy of the migration configuration. +func GetMigrations() []MigrationConfig { + migrations := make([]MigrationConfig, len(migrationConfig)) + copy(migrations, migrationConfig) + + return migrations +} + // migrationLogger is a logger that wraps the passed btclog.Logger so it can be // used to log migrations. type migrationLogger struct { @@ -216,3 +306,117 @@ func (t *replacerFile) Close() error { // instance, so there's nothing to do for us here. return nil } + +// MigrationTxOptions is the implementation of the TxOptions interface for +// migration transactions. +type MigrationTxOptions struct { +} + +// ReadOnly returns false to indicate that migration transactions are not read +// only. +func (m *MigrationTxOptions) ReadOnly() bool { + return false +} + +// ApplyMigrations applies the provided migrations to the database in sequence. +// It ensures migrations are executed in the correct order, applying both custom +// migration functions and SQL migrations as needed. +func ApplyMigrations(ctx context.Context, db *BaseDB, + migrator MigrationExecutor, migrations []MigrationConfig) error { + + // Ensure that the migrations are sorted by version. + for i := 0; i < len(migrations); i++ { + if migrations[i].Version != i+1 { + return fmt.Errorf("migration version %d is out of "+ + "order. Expected %d", migrations[i].Version, + i+1) + } + } + // Construct a transaction executor to apply custom migrations. + executor := NewTransactionExecutor(db, func(tx *sql.Tx) *sqlc.Queries { + return db.WithTx(tx) + }) + + currentVersion := 0 + version, err := db.GetDatabaseVersion(ctx) + if !errors.Is(err, sql.ErrNoRows) { + if err != nil { + return fmt.Errorf("error getting current database "+ + "version: %w", err) + } + + currentVersion = int(version) + } + + for _, migration := range migrations { + if migration.Version <= currentVersion { + log.Infof("Skipping migration '%s' (version %d) as it "+ + "has already been applied", migration.Name, + migration.Version) + + continue + } + + log.Infof("Migrating SQL schema to version %d", + migration.SchemaVersion) + + // Execute SQL schema migrations up to the target version. + err = migrator.ExecuteMigrations( + TargetVersion(uint(migration.SchemaVersion)), + ) + if err != nil { + return fmt.Errorf("error executing schema migrations "+ + "to target version %d: %w", + migration.SchemaVersion, err) + } + + var opts MigrationTxOptions + + // Run the custom migration as a transaction to ensure + // atomicity. If successful, mark the migration as complete in + // the migration tracker table. + err = executor.ExecTx(ctx, &opts, func(tx *sqlc.Queries) error { + // Apply the migration function if one is provided. + if migration.MigrationFn != nil { + log.Infof("Applying custom migration '%v' "+ + "(version %d) to schema version %d", + migration.Name, migration.Version, + migration.SchemaVersion) + + err = migration.MigrationFn(tx) + if err != nil { + return fmt.Errorf("error applying "+ + "migration '%v' (version %d) "+ + "to schema version %d: %w", + migration.Name, + migration.Version, + migration.SchemaVersion, err) + } + + log.Infof("Migration '%v' (version %d) "+ + "applied ", migration.Name, + migration.Version) + } + + // Mark the migration as complete by adding the version + // to the migration tracker table along with the current + // timestamp. + err = tx.SetMigration(ctx, sqlc.SetMigrationParams{ + Version: int32(migration.Version), + MigrationTime: time.Now(), + }) + if err != nil { + return fmt.Errorf("error setting migration "+ + "version %d: %w", migration.Version, + err) + } + + return nil + }, func() {}) + if err != nil { + return err + } + } + + return nil +} diff --git a/sqldb/migrations_test.go b/sqldb/migrations_test.go index cd55e92cb86..284ba8e991b 100644 --- a/sqldb/migrations_test.go +++ b/sqldb/migrations_test.go @@ -2,8 +2,15 @@ package sqldb import ( "context" + "database/sql" + "fmt" + "path/filepath" "testing" + "github.com/golang-migrate/migrate/v4" + "github.com/golang-migrate/migrate/v4/database" + pgx_migrate "github.com/golang-migrate/migrate/v4/database/pgx/v5" + sqlite_migrate "github.com/golang-migrate/migrate/v4/database/sqlite" "github.com/lightningnetwork/lnd/sqldb/sqlc" "github.com/stretchr/testify/require" ) @@ -152,3 +159,289 @@ func testInvoiceExpiryMigration(t *testing.T, makeDB makeMigrationTestDB) { require.NoError(t, err) require.Equal(t, expected, invoices) } + +// TestCustomMigration tests that a custom in-code migrations are correctly +// executed during the migration process. +func TestCustomMigration(t *testing.T) { + var customMigrationLog []string + + logMigration := func(name string) { + customMigrationLog = append(customMigrationLog, name) + } + + // Some migrations to use for both the failure and success tests. Note + // that the migrations are not in order to test that they are executed + // in the correct order. + migrations := []MigrationConfig{ + { + Name: "1", + Version: 1, + SchemaVersion: 1, + MigrationFn: func(*sqlc.Queries) error { + logMigration("1") + + return nil + }, + }, + { + Name: "2", + Version: 2, + SchemaVersion: 1, + MigrationFn: func(*sqlc.Queries) error { + logMigration("2") + + return nil + }, + }, + { + Name: "3", + Version: 3, + SchemaVersion: 2, + MigrationFn: func(*sqlc.Queries) error { + logMigration("3") + + return nil + }, + }, + } + + tests := []struct { + name string + migrations []MigrationConfig + expectedSuccess bool + expectedMigrationLog []string + expectedSchemaVersion int + expectedVersion int + }{ + { + name: "success", + migrations: migrations, + expectedSuccess: true, + expectedMigrationLog: []string{"1", "2", "3"}, + expectedSchemaVersion: 2, + expectedVersion: 3, + }, + { + name: "unordered migrations", + migrations: append([]MigrationConfig{ + { + Name: "4", + Version: 4, + SchemaVersion: 3, + MigrationFn: func(*sqlc.Queries) error { + logMigration("4") + + return nil + }, + }, + }, migrations...), + expectedSuccess: false, + expectedMigrationLog: nil, + expectedSchemaVersion: 0, + }, + { + name: "failure of migration 4", + migrations: append(migrations, MigrationConfig{ + Name: "4", + Version: 4, + SchemaVersion: 3, + MigrationFn: func(*sqlc.Queries) error { + return fmt.Errorf("migration 4 failed") + }, + }), + expectedSuccess: false, + expectedMigrationLog: []string{"1", "2", "3"}, + // Since schema migration is a separate step we expect + // that migrating up to 3 succeeded. + expectedSchemaVersion: 3, + // We still remain on version 3 though. + expectedVersion: 3, + }, + { + name: "success of migration 4", + migrations: append(migrations, MigrationConfig{ + Name: "4", + Version: 4, + SchemaVersion: 3, + MigrationFn: func(*sqlc.Queries) error { + logMigration("4") + + return nil + }, + }), + expectedSuccess: true, + expectedMigrationLog: []string{"1", "2", "3", "4"}, + expectedSchemaVersion: 3, + expectedVersion: 4, + }, + } + + ctxb := context.Background() + for _, test := range tests { + // checkSchemaVersion checks the database schema version against + // the expected version. + getSchemaVersion := func(t *testing.T, + driver database.Driver, dbName string) int { + + sqlMigrate, err := migrate.NewWithInstance( + "migrations", nil, dbName, driver, + ) + require.NoError(t, err) + + version, _, err := sqlMigrate.Version() + if err != migrate.ErrNilVersion { + require.NoError(t, err) + } + + return int(version) + } + + t.Run("SQLite "+test.name, func(t *testing.T) { + customMigrationLog = nil + + // First instantiate the database and run the migrations + // including the custom migrations. + t.Logf("Creating new SQLite DB for testing migrations") + + dbFileName := filepath.Join(t.TempDir(), "tmp.db") + var ( + db *SqliteStore + err error + ) + + // Run the migration 3 times to test that the migrations + // are idempotent. + for i := 0; i < 3; i++ { + db, err = NewSqliteStore(&SqliteConfig{ + SkipMigrations: false, + }, dbFileName, test.migrations) + if db != nil { + dbToCleanup := db.DB + t.Cleanup(func() { + require.NoError( + t, dbToCleanup.Close(), + ) + }) + } + + if test.expectedSuccess { + require.NoError(t, err) + } else { + require.Error(t, err) + + // Also repoen the DB without migrations + // so we can read versions. + db, err = NewSqliteStore(&SqliteConfig{ + SkipMigrations: true, + }, dbFileName, nil) + require.NoError(t, err) + } + + require.Equal(t, + test.expectedMigrationLog, + customMigrationLog, + ) + + // Create the migration executor to be able to + // query the current schema version. + driver, err := sqlite_migrate.WithInstance( + db.DB, &sqlite_migrate.Config{}, + ) + require.NoError(t, err) + + require.Equal( + t, test.expectedSchemaVersion, + getSchemaVersion(t, driver, ""), + ) + + // Check the migraton version in the database. + version, err := db.GetDatabaseVersion(ctxb) + if test.expectedSchemaVersion != 0 { + require.NoError(t, err) + } else { + require.Equal(t, sql.ErrNoRows, err) + } + + require.Equal( + t, test.expectedVersion, int(version), + ) + } + }) + + t.Run("Postgres "+test.name, func(t *testing.T) { + customMigrationLog = nil + + // First create a temporary Postgres database to run + // the migrations on. + fixture := NewTestPgFixture( + t, DefaultPostgresFixtureLifetime, + ) + t.Cleanup(func() { + fixture.TearDown(t) + }) + + dbName := randomDBName(t) + + // Next instantiate the database and run the migrations + // including the custom migrations. + t.Logf("Creating new Postgres DB '%s' for testing "+ + "migrations", dbName) + + _, err := fixture.db.ExecContext( + context.Background(), "CREATE DATABASE "+dbName, + ) + require.NoError(t, err) + + cfg := fixture.GetConfig(dbName) + var db *PostgresStore + + // Run the migration 3 times to test that the migrations + // are idempotent. + for i := 0; i < 3; i++ { + cfg.SkipMigrations = false + db, err = NewPostgresStore(cfg, test.migrations) + + if test.expectedSuccess { + require.NoError(t, err) + } else { + require.Error(t, err) + + // Also repoen the DB without migrations + // so we can read versions. + cfg.SkipMigrations = true + db, err = NewPostgresStore(cfg, nil) + require.NoError(t, err) + } + + require.Equal(t, + test.expectedMigrationLog, + customMigrationLog, + ) + + // Create the migration executor to be able to + // query the current version. + driver, err := pgx_migrate.WithInstance( + db.DB, &pgx_migrate.Config{}, + ) + require.NoError(t, err) + + require.Equal( + t, test.expectedSchemaVersion, + getSchemaVersion(t, driver, ""), + ) + + // Check the migraton version in the database. + version, err := db.GetDatabaseVersion(ctxb) + if test.expectedSchemaVersion != 0 { + require.NoError(t, err) + } else { + require.Equal(t, sql.ErrNoRows, err) + } + + require.Equal( + t, test.expectedVersion, int(version), + ) + } + }) + } +} diff --git a/sqldb/postgres.go b/sqldb/postgres.go index c8553915741..4884943f06c 100644 --- a/sqldb/postgres.go +++ b/sqldb/postgres.go @@ -1,6 +1,7 @@ package sqldb import ( + "context" "database/sql" "fmt" "net/url" @@ -32,6 +33,9 @@ var ( "BIGINT PRIMARY KEY": "BIGSERIAL PRIMARY KEY", "TIMESTAMP": "TIMESTAMP WITHOUT TIME ZONE", } + + // Make sure PostgresStore implements the MigrationExecutor interface. + _ MigrationExecutor = (*PostgresStore)(nil) ) // replacePasswordInDSN takes a DSN string and returns it with the password @@ -85,43 +89,62 @@ type PostgresStore struct { // NewPostgresStore creates a new store that is backed by a Postgres database // backend. -func NewPostgresStore(cfg *PostgresConfig) (*PostgresStore, error) { +func NewPostgresStore(cfg *PostgresConfig, migrations []MigrationConfig) ( + *PostgresStore, error) { + sanitizedDSN, err := replacePasswordInDSN(cfg.Dsn) if err != nil { return nil, err } log.Infof("Using SQL database '%s'", sanitizedDSN) - rawDB, err := sql.Open("pgx", cfg.Dsn) + db, err := sql.Open("pgx", cfg.Dsn) if err != nil { return nil, err } + // Ensure the migration tracker table exists before running migrations. + // This table tracks migration progress and ensures compatibility with + // SQLC query generation. If the table is already created by an SQLC + // migration, this operation becomes a no-op. + migrationTrackerSQL := ` + CREATE TABLE IF NOT EXISTS migration_tracker ( + version INTEGER UNIQUE NOT NULL, + migration_time TIMESTAMP NOT NULL + );` + + _, err = db.Exec(migrationTrackerSQL) + if err != nil { + return nil, fmt.Errorf("error creating migration tracker: %w", + err) + } + maxConns := defaultMaxConns if cfg.MaxConnections > 0 { maxConns = cfg.MaxConnections } - rawDB.SetMaxOpenConns(maxConns) - rawDB.SetMaxIdleConns(maxConns) - rawDB.SetConnMaxLifetime(connIdleLifetime) + db.SetMaxOpenConns(maxConns) + db.SetMaxIdleConns(maxConns) + db.SetConnMaxLifetime(connIdleLifetime) - queries := sqlc.New(rawDB) + queries := sqlc.New(db) s := &PostgresStore{ cfg: cfg, BaseDB: &BaseDB{ - DB: rawDB, + DB: db, Queries: queries, }, } // Execute migrations unless configured to skip them. if !cfg.SkipMigrations { - err := s.ExecuteMigrations(TargetLatest) + err := ApplyMigrations( + context.Background(), s.BaseDB, s, migrations, + ) if err != nil { - return nil, fmt.Errorf("error executing migrations: %w", - err) + return nil, err } } diff --git a/sqldb/postgres_fixture.go b/sqldb/postgres_fixture.go index da5769c429e..284cd0c8c7d 100644 --- a/sqldb/postgres_fixture.go +++ b/sqldb/postgres_fixture.go @@ -148,7 +148,7 @@ func NewTestPostgresDB(t *testing.T, fixture *TestPgFixture) *PostgresStore { require.NoError(t, err) cfg := fixture.GetConfig(dbName) - store, err := NewPostgresStore(cfg) + store, err := NewPostgresStore(cfg, GetMigrations()) require.NoError(t, err) return store @@ -172,7 +172,7 @@ func NewTestPostgresDBWithVersion(t *testing.T, fixture *TestPgFixture, storeCfg := fixture.GetConfig(dbName) storeCfg.SkipMigrations = true - store, err := NewPostgresStore(storeCfg) + store, err := NewPostgresStore(storeCfg, GetMigrations()) require.NoError(t, err) err = store.ExecuteMigrations(TargetVersion(version)) diff --git a/sqldb/sqlite.go b/sqldb/sqlite.go index 99e55d6eafb..bf192eb0f89 100644 --- a/sqldb/sqlite.go +++ b/sqldb/sqlite.go @@ -3,6 +3,7 @@ package sqldb import ( + "context" "database/sql" "fmt" "net/url" @@ -34,6 +35,9 @@ var ( sqliteSchemaReplacements = map[string]string{ "BIGINT PRIMARY KEY": "INTEGER PRIMARY KEY", } + + // Make sure SqliteStore implements the MigrationExecutor interface. + _ MigrationExecutor = (*SqliteStore)(nil) ) // SqliteStore is a database store implementation that uses a sqlite backend. @@ -45,7 +49,9 @@ type SqliteStore struct { // NewSqliteStore attempts to open a new sqlite database based on the passed // config. -func NewSqliteStore(cfg *SqliteConfig, dbPath string) (*SqliteStore, error) { +func NewSqliteStore(cfg *SqliteConfig, dbPath string, + migrations []MigrationConfig) (*SqliteStore, error) { + // The set of pragma options are accepted using query options. For now // we only want to ensure that foreign key constraints are properly // enforced. @@ -102,6 +108,23 @@ func NewSqliteStore(cfg *SqliteConfig, dbPath string) (*SqliteStore, error) { return nil, err } + // Create the migration tracker table before starting migrations to + // ensure it can be used to track migration progress. Note that a + // corresponding SQLC migration also creates this table, making this + // operation a no-op in that context. Its purpose is to ensure + // compatibility with SQLC query generation. + migrationTrackerSQL := ` + CREATE TABLE IF NOT EXISTS migration_tracker ( + version INTEGER UNIQUE NOT NULL, + migration_time TIMESTAMP NOT NULL + );` + + _, err = db.Exec(migrationTrackerSQL) + if err != nil { + return nil, fmt.Errorf("error creating migration tracker: %w", + err) + } + db.SetMaxOpenConns(defaultMaxConns) db.SetMaxIdleConns(defaultMaxConns) db.SetConnMaxLifetime(connIdleLifetime) @@ -117,10 +140,11 @@ func NewSqliteStore(cfg *SqliteConfig, dbPath string) (*SqliteStore, error) { // Execute migrations unless configured to skip them. if !cfg.SkipMigrations { - if err := s.ExecuteMigrations(TargetLatest); err != nil { - return nil, fmt.Errorf("error executing migrations: "+ - "%w", err) - + err := ApplyMigrations( + context.Background(), s.BaseDB, s, migrations, + ) + if err != nil { + return nil, err } } @@ -157,7 +181,7 @@ func NewTestSqliteDB(t *testing.T) *SqliteStore { dbFileName := filepath.Join(t.TempDir(), "tmp.db") sqlDB, err := NewSqliteStore(&SqliteConfig{ SkipMigrations: false, - }, dbFileName) + }, dbFileName, GetMigrations()) require.NoError(t, err) t.Cleanup(func() { @@ -180,7 +204,7 @@ func NewTestSqliteDBWithVersion(t *testing.T, version uint) *SqliteStore { dbFileName := filepath.Join(t.TempDir(), "tmp.db") sqlDB, err := NewSqliteStore(&SqliteConfig{ SkipMigrations: true, - }, dbFileName) + }, dbFileName, nil) require.NoError(t, err) err = sqlDB.ExecuteMigrations(TargetVersion(version)) From 91c3e1496f4d3e29ca84ac6ed7c5e2eabc5bf630 Mon Sep 17 00:00:00 2001 From: Andras Banki-Horvath Date: Mon, 25 Nov 2024 20:29:08 +0100 Subject: [PATCH 04/21] sqldb: separate migration execution from construction This commit separates the execution of SQL and in-code migrations from their construction. This change is necessary because, currently, the SQL schema is migrated during the construction phase in the lncfg package. However, migrations are typically executed when individual stores are constructed within the configuration builder. --- config_builder.go | 26 +++++++++++++++++++------ lncfg/db.go | 19 +++++++++---------- sqldb/interfaces.go | 12 ++++++++++++ sqldb/migrations_test.go | 31 ++++++++++++++++++------------ sqldb/no_sqlite.go | 24 ++++++++++++++++++++++- sqldb/postgres.go | 34 ++++++++++++++++++++------------- sqldb/postgres_fixture.go | 8 ++++++-- sqldb/sqlite.go | 40 ++++++++++++++++++++++++++------------- 8 files changed, 137 insertions(+), 57 deletions(-) diff --git a/config_builder.go b/config_builder.go index afafbed7543..b5e19e5dc7c 100644 --- a/config_builder.go +++ b/config_builder.go @@ -932,10 +932,10 @@ type DatabaseInstances struct { // the btcwallet's loader. WalletDB btcwallet.LoaderOption - // NativeSQLStore is a pointer to a native SQL store that can be used - // for native SQL queries for tables that already support it. This may - // be nil if the use-native-sql flag was not set. - NativeSQLStore *sqldb.BaseDB + // NativeSQLStore holds a reference to the native SQL store that can + // be used for native SQL queries for tables that already support it. + // This may be nil if the use-native-sql flag was not set. + NativeSQLStore sqldb.DB } // DefaultDatabaseBuilder is a type that builds the default database backends @@ -1079,6 +1079,19 @@ func (d *DefaultDatabaseBuilder) BuildDatabase( // Instantiate a native SQL invoice store if the flag is set. if d.cfg.DB.UseNativeSQL { + // We need to apply all migrations to the native SQL store + // before we can use it. + err := dbs.NativeSQLStore.ApplyAllMigrations( + ctx, sqldb.GetMigrations(), + ) + if err != nil { + cleanUp() + err := fmt.Errorf("unable to apply migrations: %w", err) + d.logger.Error(err) + + return nil, nil, err + } + // KV invoice db resides in the same database as the channel // state DB. Let's query the database to see if we have any // invoices there. If we do, we won't allow the user to start @@ -1107,10 +1120,11 @@ func (d *DefaultDatabaseBuilder) BuildDatabase( return nil, nil, err } + baseDB := dbs.NativeSQLStore.GetBaseDB() executor := sqldb.NewTransactionExecutor( - dbs.NativeSQLStore, + baseDB, func(tx *sql.Tx) invoices.SQLInvoiceQueries { - return dbs.NativeSQLStore.WithTx(tx) + return baseDB.WithTx(tx) }, ) diff --git a/lncfg/db.go b/lncfg/db.go index a6598e66dab..040b3e8d1b4 100644 --- a/lncfg/db.go +++ b/lncfg/db.go @@ -231,10 +231,10 @@ type DatabaseBackends struct { // the underlying wallet database from. WalletDB btcwallet.LoaderOption - // NativeSQLStore is a pointer to a native SQL store that can be used - // for native SQL queries for tables that already support it. This may - // be nil if the use-native-sql flag was not set. - NativeSQLStore *sqldb.BaseDB + // NativeSQLStore holds a reference to the native SQL store that can + // be used for native SQL queries for tables that already support it. + // This may be nil if the use-native-sql flag was not set. + NativeSQLStore sqldb.DB // Remote indicates whether the database backends are remote, possibly // replicated instances or local bbolt or sqlite backed databases. @@ -449,17 +449,17 @@ func (db *DB) GetBackends(ctx context.Context, chanDBPath, } closeFuncs[NSWalletDB] = postgresWalletBackend.Close - var nativeSQLStore *sqldb.BaseDB + var nativeSQLStore sqldb.DB if db.UseNativeSQL { nativePostgresStore, err := sqldb.NewPostgresStore( - db.Postgres, sqldb.GetMigrations(), + db.Postgres, ) if err != nil { return nil, fmt.Errorf("error opening "+ "native postgres store: %v", err) } - nativeSQLStore = nativePostgresStore.BaseDB + nativeSQLStore = nativePostgresStore closeFuncs[PostgresBackend] = nativePostgresStore.Close } @@ -571,19 +571,18 @@ func (db *DB) GetBackends(ctx context.Context, chanDBPath, } closeFuncs[NSWalletDB] = sqliteWalletBackend.Close - var nativeSQLStore *sqldb.BaseDB + var nativeSQLStore sqldb.DB if db.UseNativeSQL { nativeSQLiteStore, err := sqldb.NewSqliteStore( db.Sqlite, path.Join(chanDBPath, SqliteNativeDBName), - sqldb.GetMigrations(), ) if err != nil { return nil, fmt.Errorf("error opening "+ "native SQLite store: %v", err) } - nativeSQLStore = nativeSQLiteStore.BaseDB + nativeSQLStore = nativeSQLiteStore closeFuncs[SqliteBackend] = nativeSQLiteStore.Close } diff --git a/sqldb/interfaces.go b/sqldb/interfaces.go index 3c042aa5a75..1c5b4878fbd 100644 --- a/sqldb/interfaces.go +++ b/sqldb/interfaces.go @@ -355,6 +355,18 @@ func (t *TransactionExecutor[Q]) ExecTx(ctx context.Context, ) } +// DB is an interface that represents a generic SQL database. It provides +// methods to apply migrations and access the underlying database connection. +type DB interface { + // GetBaseDB returns the underlying BaseDB instance. + GetBaseDB() *BaseDB + + // ApplyAllMigrations applies all migrations to the database including + // both sqlc and custom in-code migrations. + ApplyAllMigrations(ctx context.Context, + customMigrations []MigrationConfig) error +} + // BaseDB is the base database struct that each implementation can embed to // gain some common functionality. type BaseDB struct { diff --git a/sqldb/migrations_test.go b/sqldb/migrations_test.go index 284ba8e991b..385840364cd 100644 --- a/sqldb/migrations_test.go +++ b/sqldb/migrations_test.go @@ -314,16 +314,19 @@ func TestCustomMigration(t *testing.T) { for i := 0; i < 3; i++ { db, err = NewSqliteStore(&SqliteConfig{ SkipMigrations: false, - }, dbFileName, test.migrations) - if db != nil { - dbToCleanup := db.DB - t.Cleanup(func() { - require.NoError( - t, dbToCleanup.Close(), - ) - }) - } + }, dbFileName) + require.NoError(t, err) + + dbToCleanup := db.DB + t.Cleanup(func() { + require.NoError( + t, dbToCleanup.Close(), + ) + }) + err = db.ApplyAllMigrations( + ctxb, test.migrations, + ) if test.expectedSuccess { require.NoError(t, err) } else { @@ -333,7 +336,7 @@ func TestCustomMigration(t *testing.T) { // so we can read versions. db, err = NewSqliteStore(&SqliteConfig{ SkipMigrations: true, - }, dbFileName, nil) + }, dbFileName) require.NoError(t, err) } @@ -399,8 +402,12 @@ func TestCustomMigration(t *testing.T) { // are idempotent. for i := 0; i < 3; i++ { cfg.SkipMigrations = false - db, err = NewPostgresStore(cfg, test.migrations) + db, err = NewPostgresStore(cfg) + require.NoError(t, err) + err = db.ApplyAllMigrations( + ctxb, test.migrations, + ) if test.expectedSuccess { require.NoError(t, err) } else { @@ -409,7 +416,7 @@ func TestCustomMigration(t *testing.T) { // Also repoen the DB without migrations // so we can read versions. cfg.SkipMigrations = true - db, err = NewPostgresStore(cfg, nil) + db, err = NewPostgresStore(cfg) require.NoError(t, err) } diff --git a/sqldb/no_sqlite.go b/sqldb/no_sqlite.go index 9ea35c43c64..ad0cae6e4f8 100644 --- a/sqldb/no_sqlite.go +++ b/sqldb/no_sqlite.go @@ -2,7 +2,15 @@ package sqldb -import "fmt" +import ( + "context" + "fmt" +) + +var ( + // Make sure SqliteStore implements the DB interface. + _ DB = (*SqliteStore)(nil) +) // SqliteStore is a database store implementation that uses a sqlite backend. type SqliteStore struct { @@ -16,3 +24,17 @@ type SqliteStore struct { func NewSqliteStore(cfg *SqliteConfig, dbPath string) (*SqliteStore, error) { return nil, fmt.Errorf("SQLite backend not supported in WebAssembly") } + +// GetBaseDB returns the underlying BaseDB instance for the SQLite store. +// It is a trivial helper method to comply with the sqldb.DB interface. +func (s *SqliteStore) GetBaseDB() *BaseDB { + return s.BaseDB +} + +// ApplyAllMigrations applies both the SQLC and custom in-code migrations to +// the SQLite database. +func (s *SqliteStore) ApplyAllMigrations(context.Context, + []MigrationConfig) error { + + return fmt.Errorf("SQLite backend not supported in WebAssembly") +} diff --git a/sqldb/postgres.go b/sqldb/postgres.go index 4884943f06c..455ecb40572 100644 --- a/sqldb/postgres.go +++ b/sqldb/postgres.go @@ -36,6 +36,9 @@ var ( // Make sure PostgresStore implements the MigrationExecutor interface. _ MigrationExecutor = (*PostgresStore)(nil) + + // Make sure PostgresStore implements the DB interface. + _ DB = (*PostgresStore)(nil) ) // replacePasswordInDSN takes a DSN string and returns it with the password @@ -89,9 +92,7 @@ type PostgresStore struct { // NewPostgresStore creates a new store that is backed by a Postgres database // backend. -func NewPostgresStore(cfg *PostgresConfig, migrations []MigrationConfig) ( - *PostgresStore, error) { - +func NewPostgresStore(cfg *PostgresConfig) (*PostgresStore, error) { sanitizedDSN, err := replacePasswordInDSN(cfg.Dsn) if err != nil { return nil, err @@ -130,25 +131,32 @@ func NewPostgresStore(cfg *PostgresConfig, migrations []MigrationConfig) ( queries := sqlc.New(db) - s := &PostgresStore{ + return &PostgresStore{ cfg: cfg, BaseDB: &BaseDB{ DB: db, Queries: queries, }, - } + }, nil +} + +// GetBaseDB returns the underlying BaseDB instance for the Postgres store. +// It is a trivial helper method to comply with the sqldb.DB interface. +func (s *PostgresStore) GetBaseDB() *BaseDB { + return s.BaseDB +} + +// ApplyAllMigrations applies both the SQLC and custom in-code migrations to the +// Postgres database. +func (s *PostgresStore) ApplyAllMigrations(ctx context.Context, + migrations []MigrationConfig) error { // Execute migrations unless configured to skip them. - if !cfg.SkipMigrations { - err := ApplyMigrations( - context.Background(), s.BaseDB, s, migrations, - ) - if err != nil { - return nil, err - } + if s.cfg.SkipMigrations { + return nil } - return s, nil + return ApplyMigrations(ctx, s.BaseDB, s, migrations) } // ExecuteMigrations runs migrations for the Postgres database, depending on the diff --git a/sqldb/postgres_fixture.go b/sqldb/postgres_fixture.go index 284cd0c8c7d..ce21aab7d43 100644 --- a/sqldb/postgres_fixture.go +++ b/sqldb/postgres_fixture.go @@ -148,9 +148,13 @@ func NewTestPostgresDB(t *testing.T, fixture *TestPgFixture) *PostgresStore { require.NoError(t, err) cfg := fixture.GetConfig(dbName) - store, err := NewPostgresStore(cfg, GetMigrations()) + store, err := NewPostgresStore(cfg) require.NoError(t, err) + require.NoError(t, store.ApplyAllMigrations( + context.Background(), GetMigrations()), + ) + return store } @@ -172,7 +176,7 @@ func NewTestPostgresDBWithVersion(t *testing.T, fixture *TestPgFixture, storeCfg := fixture.GetConfig(dbName) storeCfg.SkipMigrations = true - store, err := NewPostgresStore(storeCfg, GetMigrations()) + store, err := NewPostgresStore(storeCfg) require.NoError(t, err) err = store.ExecuteMigrations(TargetVersion(version)) diff --git a/sqldb/sqlite.go b/sqldb/sqlite.go index bf192eb0f89..59cb0356929 100644 --- a/sqldb/sqlite.go +++ b/sqldb/sqlite.go @@ -38,6 +38,9 @@ var ( // Make sure SqliteStore implements the MigrationExecutor interface. _ MigrationExecutor = (*SqliteStore)(nil) + + // Make sure SqliteStore implements the DB interface. + _ DB = (*SqliteStore)(nil) ) // SqliteStore is a database store implementation that uses a sqlite backend. @@ -49,9 +52,7 @@ type SqliteStore struct { // NewSqliteStore attempts to open a new sqlite database based on the passed // config. -func NewSqliteStore(cfg *SqliteConfig, dbPath string, - migrations []MigrationConfig) (*SqliteStore, error) { - +func NewSqliteStore(cfg *SqliteConfig, dbPath string) (*SqliteStore, error) { // The set of pragma options are accepted using query options. For now // we only want to ensure that foreign key constraints are properly // enforced. @@ -138,17 +139,26 @@ func NewSqliteStore(cfg *SqliteConfig, dbPath string, }, } + return s, nil +} + +// GetBaseDB returns the underlying BaseDB instance for the SQLite store. +// It is a trivial helper method to comply with the sqldb.DB interface. +func (s *SqliteStore) GetBaseDB() *BaseDB { + return s.BaseDB +} + +// ApplyAllMigrations applies both the SQLC and custom in-code migrations to the +// SQLite database. +func (s *SqliteStore) ApplyAllMigrations(ctx context.Context, + migrations []MigrationConfig) error { + // Execute migrations unless configured to skip them. - if !cfg.SkipMigrations { - err := ApplyMigrations( - context.Background(), s.BaseDB, s, migrations, - ) - if err != nil { - return nil, err - } + if s.cfg.SkipMigrations { + return nil } - return s, nil + return ApplyMigrations(ctx, s.BaseDB, s, migrations) } // ExecuteMigrations runs migrations for the sqlite database, depending on the @@ -181,9 +191,13 @@ func NewTestSqliteDB(t *testing.T) *SqliteStore { dbFileName := filepath.Join(t.TempDir(), "tmp.db") sqlDB, err := NewSqliteStore(&SqliteConfig{ SkipMigrations: false, - }, dbFileName, GetMigrations()) + }, dbFileName) require.NoError(t, err) + require.NoError(t, sqlDB.ApplyAllMigrations( + context.Background(), GetMigrations()), + ) + t.Cleanup(func() { require.NoError(t, sqlDB.DB.Close()) }) @@ -204,7 +218,7 @@ func NewTestSqliteDBWithVersion(t *testing.T, version uint) *SqliteStore { dbFileName := filepath.Join(t.TempDir(), "tmp.db") sqlDB, err := NewSqliteStore(&SqliteConfig{ SkipMigrations: true, - }, dbFileName, nil) + }, dbFileName) require.NoError(t, err) err = sqlDB.ExecuteMigrations(TargetVersion(version)) From 115f96c29aef9664cb8bee731d965756fd75c09d Mon Sep 17 00:00:00 2001 From: Andras Banki-Horvath Date: Tue, 11 Jun 2024 22:38:38 +0200 Subject: [PATCH 05/21] multi: add call to directly insert an AMP sub-invoice --- invoices/sql_store.go | 4 ++++ sqldb/sqlc/amp_invoices.sql.go | 29 +++++++++++++++++++++++++++++ sqldb/sqlc/querier.go | 1 + sqldb/sqlc/queries/amp_invoices.sql | 8 ++++++++ 4 files changed, 42 insertions(+) diff --git a/invoices/sql_store.go b/invoices/sql_store.go index 5459ec26c73..24aa63da052 100644 --- a/invoices/sql_store.go +++ b/invoices/sql_store.go @@ -79,6 +79,10 @@ type SQLInvoiceQueries interface { //nolint:interfacebloat UpsertAMPSubInvoice(ctx context.Context, arg sqlc.UpsertAMPSubInvoiceParams) (sql.Result, error) + // TODO(bhandras): remove this once migrations have been separated out. + InsertAMPSubInvoice(ctx context.Context, + arg sqlc.InsertAMPSubInvoiceParams) error + UpdateAMPSubInvoiceState(ctx context.Context, arg sqlc.UpdateAMPSubInvoiceStateParams) error diff --git a/sqldb/sqlc/amp_invoices.sql.go b/sqldb/sqlc/amp_invoices.sql.go index e47b1c803db..182848e1469 100644 --- a/sqldb/sqlc/amp_invoices.sql.go +++ b/sqldb/sqlc/amp_invoices.sql.go @@ -235,6 +235,35 @@ func (q *Queries) GetAMPInvoiceID(ctx context.Context, setID []byte) (int64, err return invoice_id, err } +const insertAMPSubInvoice = `-- name: InsertAMPSubInvoice :exec +INSERT INTO amp_sub_invoices ( + set_id, state, created_at, settled_at, settle_index, invoice_id +) VALUES ( + $1, $2, $3, $4, $5, $6 +) +` + +type InsertAMPSubInvoiceParams struct { + SetID []byte + State int16 + CreatedAt time.Time + SettledAt sql.NullTime + SettleIndex sql.NullInt64 + InvoiceID int64 +} + +func (q *Queries) InsertAMPSubInvoice(ctx context.Context, arg InsertAMPSubInvoiceParams) error { + _, err := q.db.ExecContext(ctx, insertAMPSubInvoice, + arg.SetID, + arg.State, + arg.CreatedAt, + arg.SettledAt, + arg.SettleIndex, + arg.InvoiceID, + ) + return err +} + const insertAMPSubInvoiceHTLC = `-- name: InsertAMPSubInvoiceHTLC :exec INSERT INTO amp_sub_invoice_htlcs ( invoice_id, set_id, htlc_id, root_share, child_index, hash, preimage diff --git a/sqldb/sqlc/querier.go b/sqldb/sqlc/querier.go index 268ca027c1f..4ef50362e3f 100644 --- a/sqldb/sqlc/querier.go +++ b/sqldb/sqlc/querier.go @@ -28,6 +28,7 @@ type Querier interface { GetInvoiceHTLCCustomRecords(ctx context.Context, invoiceID int64) ([]GetInvoiceHTLCCustomRecordsRow, error) GetInvoiceHTLCs(ctx context.Context, invoiceID int64) ([]InvoiceHtlc, error) GetMigration(ctx context.Context, version int32) (time.Time, error) + InsertAMPSubInvoice(ctx context.Context, arg InsertAMPSubInvoiceParams) error InsertAMPSubInvoiceHTLC(ctx context.Context, arg InsertAMPSubInvoiceHTLCParams) error InsertInvoice(ctx context.Context, arg InsertInvoiceParams) (int64, error) InsertInvoiceFeature(ctx context.Context, arg InsertInvoiceFeatureParams) error diff --git a/sqldb/sqlc/queries/amp_invoices.sql b/sqldb/sqlc/queries/amp_invoices.sql index 1fad75e0da9..1184fd2a418 100644 --- a/sqldb/sqlc/queries/amp_invoices.sql +++ b/sqldb/sqlc/queries/amp_invoices.sql @@ -65,3 +65,11 @@ SET preimage = $5 WHERE a.invoice_id = $1 AND a.set_id = $2 AND a.htlc_id = ( SELECT id FROM invoice_htlcs AS i WHERE i.chan_id = $3 AND i.htlc_id = $4 ); + +-- name: InsertAMPSubInvoice :exec +INSERT INTO amp_sub_invoices ( + set_id, state, created_at, settled_at, settle_index, invoice_id +) VALUES ( + $1, $2, $3, $4, $5, $6 +); + From 3820497d7ff84cf0dd733ba0982598dd224b69f5 Mon Sep 17 00:00:00 2001 From: Andras Banki-Horvath Date: Tue, 11 Jun 2024 22:27:43 +0200 Subject: [PATCH 06/21] sqldb: set settled_at and settle_index on invocie insertion is set Previously we intentially did not set settled_at and settle_index when inserting a new invoice as those fields are set when we settle an invoice through the usual invoice update. As migration requires that we set these nullable fields, we can safely add them. --- sqldb/sqlc/invoices.sql.go | 55 +++++++++++++++++++++++++++++++++ sqldb/sqlc/querier.go | 1 + sqldb/sqlc/queries/invoices.sql | 10 ++++++ 3 files changed, 66 insertions(+) diff --git a/sqldb/sqlc/invoices.sql.go b/sqldb/sqlc/invoices.sql.go index 9e31380abb8..4356b74c9cb 100644 --- a/sqldb/sqlc/invoices.sql.go +++ b/sqldb/sqlc/invoices.sql.go @@ -533,6 +533,61 @@ func (q *Queries) InsertInvoiceHTLCCustomRecord(ctx context.Context, arg InsertI return err } +const insertMigratedInvoice = `-- name: InsertMigratedInvoice :one +INSERT INTO invoices ( + hash, preimage, settle_index, settled_at, memo, amount_msat, cltv_delta, + expiry, payment_addr, payment_request, payment_request_hash, state, + amount_paid_msat, is_amp, is_hodl, is_keysend, created_at +) VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17 +) RETURNING id +` + +type InsertMigratedInvoiceParams struct { + Hash []byte + Preimage []byte + SettleIndex sql.NullInt64 + SettledAt sql.NullTime + Memo sql.NullString + AmountMsat int64 + CltvDelta sql.NullInt32 + Expiry int32 + PaymentAddr []byte + PaymentRequest sql.NullString + PaymentRequestHash []byte + State int16 + AmountPaidMsat int64 + IsAmp bool + IsHodl bool + IsKeysend bool + CreatedAt time.Time +} + +func (q *Queries) InsertMigratedInvoice(ctx context.Context, arg InsertMigratedInvoiceParams) (int64, error) { + row := q.db.QueryRowContext(ctx, insertMigratedInvoice, + arg.Hash, + arg.Preimage, + arg.SettleIndex, + arg.SettledAt, + arg.Memo, + arg.AmountMsat, + arg.CltvDelta, + arg.Expiry, + arg.PaymentAddr, + arg.PaymentRequest, + arg.PaymentRequestHash, + arg.State, + arg.AmountPaidMsat, + arg.IsAmp, + arg.IsHodl, + arg.IsKeysend, + arg.CreatedAt, + ) + var id int64 + err := row.Scan(&id) + return id, err +} + const nextInvoiceSettleIndex = `-- name: NextInvoiceSettleIndex :one UPDATE invoice_sequences SET current_value = current_value + 1 WHERE name = 'settle_index' diff --git a/sqldb/sqlc/querier.go b/sqldb/sqlc/querier.go index 4ef50362e3f..3471dc01574 100644 --- a/sqldb/sqlc/querier.go +++ b/sqldb/sqlc/querier.go @@ -34,6 +34,7 @@ type Querier interface { InsertInvoiceFeature(ctx context.Context, arg InsertInvoiceFeatureParams) error InsertInvoiceHTLC(ctx context.Context, arg InsertInvoiceHTLCParams) (int64, error) InsertInvoiceHTLCCustomRecord(ctx context.Context, arg InsertInvoiceHTLCCustomRecordParams) error + InsertMigratedInvoice(ctx context.Context, arg InsertMigratedInvoiceParams) (int64, error) NextInvoiceSettleIndex(ctx context.Context) (int64, error) OnAMPSubInvoiceCanceled(ctx context.Context, arg OnAMPSubInvoiceCanceledParams) error OnAMPSubInvoiceCreated(ctx context.Context, arg OnAMPSubInvoiceCreatedParams) error diff --git a/sqldb/sqlc/queries/invoices.sql b/sqldb/sqlc/queries/invoices.sql index 2a49553e658..52abd9fc77c 100644 --- a/sqldb/sqlc/queries/invoices.sql +++ b/sqldb/sqlc/queries/invoices.sql @@ -7,6 +7,16 @@ INSERT INTO invoices ( $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15 ) RETURNING id; +-- name: InsertMigratedInvoice :one +INSERT INTO invoices ( + hash, preimage, settle_index, settled_at, memo, amount_msat, cltv_delta, + expiry, payment_addr, payment_request, payment_request_hash, state, + amount_paid_msat, is_amp, is_hodl, is_keysend, created_at +) VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17 +) RETURNING id; + + -- name: InsertInvoiceFeature :exec INSERT INTO invoice_features ( invoice_id, feature From b7d743929dde51b31d3a63073e6498dcb0231109 Mon Sep 17 00:00:00 2001 From: Andras Banki-Horvath Date: Wed, 11 Sep 2024 14:14:25 +0200 Subject: [PATCH 07/21] sqldb: add a temporary index to store KV invoice hash to ID mapping --- sqldb/migrations.go | 5 ++ sqldb/sqlc/invoices.sql.go | 56 +++++++++++++++++++ .../000006_invoice_migration.down.sql | 1 + .../000006_invoice_migration.up.sql | 17 ++++++ sqldb/sqlc/models.go | 6 ++ sqldb/sqlc/querier.go | 4 ++ sqldb/sqlc/queries/invoices.sql | 20 +++++++ 7 files changed, 109 insertions(+) create mode 100644 sqldb/sqlc/migrations/000006_invoice_migration.down.sql create mode 100644 sqldb/sqlc/migrations/000006_invoice_migration.up.sql diff --git a/sqldb/migrations.go b/sqldb/migrations.go index 83634d0e511..7722fda4f46 100644 --- a/sqldb/migrations.go +++ b/sqldb/migrations.go @@ -56,6 +56,11 @@ var ( Version: 5, SchemaVersion: 5, }, + { + Name: "000006_invoice_migration", + Version: 6, + SchemaVersion: 6, + }, } ) diff --git a/sqldb/sqlc/invoices.sql.go b/sqldb/sqlc/invoices.sql.go index 4356b74c9cb..0fea7541e02 100644 --- a/sqldb/sqlc/invoices.sql.go +++ b/sqldb/sqlc/invoices.sql.go @@ -11,6 +11,15 @@ import ( "time" ) +const clearKVInvoiceHashIndex = `-- name: ClearKVInvoiceHashIndex :exec +DELETE FROM invoice_payment_hashes +` + +func (q *Queries) ClearKVInvoiceHashIndex(ctx context.Context) error { + _, err := q.db.ExecContext(ctx, clearKVInvoiceHashIndex) + return err +} + const deleteCanceledInvoices = `-- name: DeleteCanceledInvoices :execresult DELETE FROM invoices @@ -405,6 +414,19 @@ func (q *Queries) GetInvoiceHTLCs(ctx context.Context, invoiceID int64) ([]Invoi return items, nil } +const getKVInvoicePaymentHashByAddIndex = `-- name: GetKVInvoicePaymentHashByAddIndex :one +SELECT hash +FROM invoice_payment_hashes +WHERE add_index = $1 +` + +func (q *Queries) GetKVInvoicePaymentHashByAddIndex(ctx context.Context, addIndex int64) ([]byte, error) { + row := q.db.QueryRowContext(ctx, getKVInvoicePaymentHashByAddIndex, addIndex) + var hash []byte + err := row.Scan(&hash) + return hash, err +} + const insertInvoice = `-- name: InsertInvoice :one INSERT INTO invoices ( hash, preimage, memo, amount_msat, cltv_delta, expiry, payment_addr, @@ -533,6 +555,24 @@ func (q *Queries) InsertInvoiceHTLCCustomRecord(ctx context.Context, arg InsertI return err } +const insertKVInvoiceKeyAndAddIndex = `-- name: InsertKVInvoiceKeyAndAddIndex :exec +INSERT INTO invoice_payment_hashes ( + id, add_index +) VALUES ( + $1, $2 +) +` + +type InsertKVInvoiceKeyAndAddIndexParams struct { + ID int32 + AddIndex int64 +} + +func (q *Queries) InsertKVInvoiceKeyAndAddIndex(ctx context.Context, arg InsertKVInvoiceKeyAndAddIndexParams) error { + _, err := q.db.ExecContext(ctx, insertKVInvoiceKeyAndAddIndex, arg.ID, arg.AddIndex) + return err +} + const insertMigratedInvoice = `-- name: InsertMigratedInvoice :one INSERT INTO invoices ( hash, preimage, settle_index, settled_at, memo, amount_msat, cltv_delta, @@ -601,6 +641,22 @@ func (q *Queries) NextInvoiceSettleIndex(ctx context.Context) (int64, error) { return current_value, err } +const setKVInvoicePaymentHash = `-- name: SetKVInvoicePaymentHash :exec +UPDATE invoice_payment_hashes +SET hash = $2 +WHERE id = $1 +` + +type SetKVInvoicePaymentHashParams struct { + ID int32 + Hash []byte +} + +func (q *Queries) SetKVInvoicePaymentHash(ctx context.Context, arg SetKVInvoicePaymentHashParams) error { + _, err := q.db.ExecContext(ctx, setKVInvoicePaymentHash, arg.ID, arg.Hash) + return err +} + const updateInvoiceAmountPaid = `-- name: UpdateInvoiceAmountPaid :execresult UPDATE invoices SET amount_paid_msat = $2 diff --git a/sqldb/sqlc/migrations/000006_invoice_migration.down.sql b/sqldb/sqlc/migrations/000006_invoice_migration.down.sql new file mode 100644 index 00000000000..a95d34f3a67 --- /dev/null +++ b/sqldb/sqlc/migrations/000006_invoice_migration.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS invoice_payment_hashes; diff --git a/sqldb/sqlc/migrations/000006_invoice_migration.up.sql b/sqldb/sqlc/migrations/000006_invoice_migration.up.sql new file mode 100644 index 00000000000..2e78b03724a --- /dev/null +++ b/sqldb/sqlc/migrations/000006_invoice_migration.up.sql @@ -0,0 +1,17 @@ +-- invoice_payment_hashes table contains the hash of the invoices. This table +-- is used during KV to SQL invoice migration as in our KV representation we +-- don't have a mapping from hash to add index. +CREATE TABLE IF NOT EXISTS invoice_payment_hashes ( + -- id represents is the key of the invoice in the KV store. + id INTEGER PRIMARY KEY, + + -- add_index is the KV add index of the invoice. + add_index BIGINT NOT NULL, + + -- hash is the payment hash for this invoice. + hash BLOB +); + +-- Create an indexes on the add_index and hash columns to speed up lookups. +CREATE INDEX IF NOT EXISTS invoice_payment_hashes_add_index_idx ON invoice_payment_hashes(add_index); +CREATE INDEX IF NOT EXISTS invoice_payment_hashes_hash_idx ON invoice_payment_hashes(hash); diff --git a/sqldb/sqlc/models.go b/sqldb/sqlc/models.go index 1476d0470d6..f69d0352bb2 100644 --- a/sqldb/sqlc/models.go +++ b/sqldb/sqlc/models.go @@ -87,6 +87,12 @@ type InvoiceHtlcCustomRecord struct { HtlcID int64 } +type InvoicePaymentHash struct { + ID int32 + AddIndex int64 + Hash []byte +} + type InvoiceSequence struct { Name string CurrentValue int64 diff --git a/sqldb/sqlc/querier.go b/sqldb/sqlc/querier.go index 3471dc01574..6f05b32c2cd 100644 --- a/sqldb/sqlc/querier.go +++ b/sqldb/sqlc/querier.go @@ -11,6 +11,7 @@ import ( ) type Querier interface { + ClearKVInvoiceHashIndex(ctx context.Context) error DeleteCanceledInvoices(ctx context.Context) (sql.Result, error) DeleteInvoice(ctx context.Context, arg DeleteInvoiceParams) (sql.Result, error) FetchAMPSubInvoiceHTLCs(ctx context.Context, arg FetchAMPSubInvoiceHTLCsParams) ([]FetchAMPSubInvoiceHTLCsRow, error) @@ -27,6 +28,7 @@ type Querier interface { GetInvoiceFeatures(ctx context.Context, invoiceID int64) ([]InvoiceFeature, error) GetInvoiceHTLCCustomRecords(ctx context.Context, invoiceID int64) ([]GetInvoiceHTLCCustomRecordsRow, error) GetInvoiceHTLCs(ctx context.Context, invoiceID int64) ([]InvoiceHtlc, error) + GetKVInvoicePaymentHashByAddIndex(ctx context.Context, addIndex int64) ([]byte, error) GetMigration(ctx context.Context, version int32) (time.Time, error) InsertAMPSubInvoice(ctx context.Context, arg InsertAMPSubInvoiceParams) error InsertAMPSubInvoiceHTLC(ctx context.Context, arg InsertAMPSubInvoiceHTLCParams) error @@ -34,6 +36,7 @@ type Querier interface { InsertInvoiceFeature(ctx context.Context, arg InsertInvoiceFeatureParams) error InsertInvoiceHTLC(ctx context.Context, arg InsertInvoiceHTLCParams) (int64, error) InsertInvoiceHTLCCustomRecord(ctx context.Context, arg InsertInvoiceHTLCCustomRecordParams) error + InsertKVInvoiceKeyAndAddIndex(ctx context.Context, arg InsertKVInvoiceKeyAndAddIndexParams) error InsertMigratedInvoice(ctx context.Context, arg InsertMigratedInvoiceParams) (int64, error) NextInvoiceSettleIndex(ctx context.Context) (int64, error) OnAMPSubInvoiceCanceled(ctx context.Context, arg OnAMPSubInvoiceCanceledParams) error @@ -42,6 +45,7 @@ type Querier interface { OnInvoiceCanceled(ctx context.Context, arg OnInvoiceCanceledParams) error OnInvoiceCreated(ctx context.Context, arg OnInvoiceCreatedParams) error OnInvoiceSettled(ctx context.Context, arg OnInvoiceSettledParams) error + SetKVInvoicePaymentHash(ctx context.Context, arg SetKVInvoicePaymentHashParams) error SetMigration(ctx context.Context, arg SetMigrationParams) error UpdateAMPSubInvoiceHTLCPreimage(ctx context.Context, arg UpdateAMPSubInvoiceHTLCPreimageParams) (sql.Result, error) UpdateAMPSubInvoiceState(ctx context.Context, arg UpdateAMPSubInvoiceStateParams) error diff --git a/sqldb/sqlc/queries/invoices.sql b/sqldb/sqlc/queries/invoices.sql index 52abd9fc77c..025189a65ec 100644 --- a/sqldb/sqlc/queries/invoices.sql +++ b/sqldb/sqlc/queries/invoices.sql @@ -179,3 +179,23 @@ INSERT INTO invoice_htlc_custom_records ( SELECT ihcr.htlc_id, key, value FROM invoice_htlcs ih JOIN invoice_htlc_custom_records ihcr ON ih.id=ihcr.htlc_id WHERE ih.invoice_id = $1; + +-- name: InsertKVInvoiceKeyAndAddIndex :exec +INSERT INTO invoice_payment_hashes ( + id, add_index +) VALUES ( + $1, $2 +); + +-- name: SetKVInvoicePaymentHash :exec +UPDATE invoice_payment_hashes +SET hash = $2 +WHERE id = $1; + +-- name: GetKVInvoicePaymentHashByAddIndex :one +SELECT hash +FROM invoice_payment_hashes +WHERE add_index = $1; + +-- name: ClearKVInvoiceHashIndex :exec +DELETE FROM invoice_payment_hashes; From d65b63056887a56341c4823780635115e2911159 Mon Sep 17 00:00:00 2001 From: Andras Banki-Horvath Date: Mon, 2 Dec 2024 19:13:19 +0100 Subject: [PATCH 08/21] sqldb: remove unused preimage query parameter --- sqldb/sqlc/invoices.sql.go | 7 +------ sqldb/sqlc/queries/invoices.sql | 3 --- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/sqldb/sqlc/invoices.sql.go b/sqldb/sqlc/invoices.sql.go index 0fea7541e02..b98a78c69ee 100644 --- a/sqldb/sqlc/invoices.sql.go +++ b/sqldb/sqlc/invoices.sql.go @@ -191,11 +191,8 @@ WHERE ( i.hash = $3 OR $3 IS NULL ) AND ( - i.preimage = $4 OR + i.payment_addr = $4 OR $4 IS NULL -) AND ( - i.payment_addr = $5 OR - $5 IS NULL ) GROUP BY i.id LIMIT 2 @@ -205,7 +202,6 @@ type GetInvoiceParams struct { SetID []byte AddIndex sql.NullInt64 Hash []byte - Preimage []byte PaymentAddr []byte } @@ -217,7 +213,6 @@ func (q *Queries) GetInvoice(ctx context.Context, arg GetInvoiceParams) ([]Invoi arg.SetID, arg.AddIndex, arg.Hash, - arg.Preimage, arg.PaymentAddr, ) if err != nil { diff --git a/sqldb/sqlc/queries/invoices.sql b/sqldb/sqlc/queries/invoices.sql index 025189a65ec..f57c9ab7659 100644 --- a/sqldb/sqlc/queries/invoices.sql +++ b/sqldb/sqlc/queries/invoices.sql @@ -47,9 +47,6 @@ WHERE ( ) AND ( i.hash = sqlc.narg('hash') OR sqlc.narg('hash') IS NULL -) AND ( - i.preimage = sqlc.narg('preimage') OR - sqlc.narg('preimage') IS NULL ) AND ( i.payment_addr = sqlc.narg('payment_addr') OR sqlc.narg('payment_addr') IS NULL From be18f55ca19ebc7e9b8cbc7ad75d212d27d90765 Mon Sep 17 00:00:00 2001 From: Andras Banki-Horvath Date: Tue, 11 Jun 2024 22:39:18 +0200 Subject: [PATCH 09/21] invoices: extract method to create invoice insertion params --- invoices/sql_store.go | 113 +++++++++++++++++++++++++----------------- 1 file changed, 67 insertions(+), 46 deletions(-) diff --git a/invoices/sql_store.go b/invoices/sql_store.go index 24aa63da052..6cd2176d0c2 100644 --- a/invoices/sql_store.go +++ b/invoices/sql_store.go @@ -204,6 +204,66 @@ func NewSQLStore(db BatchedSQLInvoiceQueries, } } +func makeInsertInvoiceParams(invoice *Invoice, paymentHash lntypes.Hash) ( + sqlc.InsertInvoiceParams, error) { + + // Precompute the payment request hash so we can use it in the query. + var paymentRequestHash []byte + if len(invoice.PaymentRequest) > 0 { + h := sha256.New() + h.Write(invoice.PaymentRequest) + paymentRequestHash = h.Sum(nil) + } + + params := sqlc.InsertInvoiceParams{ + Hash: paymentHash[:], + AmountMsat: int64(invoice.Terms.Value), + CltvDelta: sqldb.SQLInt32( + invoice.Terms.FinalCltvDelta, + ), + Expiry: int32(invoice.Terms.Expiry.Seconds()), + // Note: keysend invoices don't have a payment request. + PaymentRequest: sqldb.SQLStr(string( + invoice.PaymentRequest), + ), + PaymentRequestHash: paymentRequestHash, + State: int16(invoice.State), + AmountPaidMsat: int64(invoice.AmtPaid), + IsAmp: invoice.IsAMP(), + IsHodl: invoice.HodlInvoice, + IsKeysend: invoice.IsKeysend(), + CreatedAt: invoice.CreationDate.UTC(), + } + + if invoice.Memo != nil { + // Store the memo as a nullable string in the database. Note + // that for compatibility reasons, we store the value as a valid + // string even if it's empty. + params.Memo = sql.NullString{ + String: string(invoice.Memo), + Valid: true, + } + } + + // Some invoices may not have a preimage, like in the case of HODL + // invoices. + if invoice.Terms.PaymentPreimage != nil { + preimage := *invoice.Terms.PaymentPreimage + if preimage == UnknownPreimage { + return sqlc.InsertInvoiceParams{}, + errors.New("cannot use all-zeroes preimage") + } + params.Preimage = preimage[:] + } + + // Some non MPP payments may have the default (invalid) value. + if invoice.Terms.PaymentAddr != BlankPayAddr { + params.PaymentAddr = invoice.Terms.PaymentAddr[:] + } + + return params, nil +} + // AddInvoice inserts the targeted invoice into the database. If the invoice has // *any* payment hashes which already exists within the database, then the // insertion will be aborted and rejected due to the strict policy banning any @@ -224,55 +284,16 @@ func (i *SQLStore) AddInvoice(ctx context.Context, invoiceID int64 ) - // Precompute the payment request hash so we can use it in the query. - var paymentRequestHash []byte - if len(newInvoice.PaymentRequest) > 0 { - h := sha256.New() - h.Write(newInvoice.PaymentRequest) - paymentRequestHash = h.Sum(nil) + insertInvoiceParams, err := makeInsertInvoiceParams( + newInvoice, paymentHash, + ) + if err != nil { + return 0, err } - err := i.db.ExecTx(ctx, &writeTxOpts, func(db SQLInvoiceQueries) error { - params := sqlc.InsertInvoiceParams{ - Hash: paymentHash[:], - Memo: sqldb.SQLStr(string(newInvoice.Memo)), - AmountMsat: int64(newInvoice.Terms.Value), - // Note: BOLT12 invoices don't have a final cltv delta. - CltvDelta: sqldb.SQLInt32( - newInvoice.Terms.FinalCltvDelta, - ), - Expiry: int32(newInvoice.Terms.Expiry.Seconds()), - // Note: keysend invoices don't have a payment request. - PaymentRequest: sqldb.SQLStr(string( - newInvoice.PaymentRequest), - ), - PaymentRequestHash: paymentRequestHash, - State: int16(newInvoice.State), - AmountPaidMsat: int64(newInvoice.AmtPaid), - IsAmp: newInvoice.IsAMP(), - IsHodl: newInvoice.HodlInvoice, - IsKeysend: newInvoice.IsKeysend(), - CreatedAt: newInvoice.CreationDate.UTC(), - } - - // Some invoices may not have a preimage, like in the case of - // HODL invoices. - if newInvoice.Terms.PaymentPreimage != nil { - preimage := *newInvoice.Terms.PaymentPreimage - if preimage == UnknownPreimage { - return errors.New("cannot use all-zeroes " + - "preimage") - } - params.Preimage = preimage[:] - } - - // Some non MPP payments may have the default (invalid) value. - if newInvoice.Terms.PaymentAddr != BlankPayAddr { - params.PaymentAddr = newInvoice.Terms.PaymentAddr[:] - } - + err = i.db.ExecTx(ctx, &writeTxOpts, func(db SQLInvoiceQueries) error { var err error - invoiceID, err = db.InsertInvoice(ctx, params) + invoiceID, err = db.InsertInvoice(ctx, insertInvoiceParams) if err != nil { return fmt.Errorf("unable to insert invoice: %w", err) } From 43797d6be7ee3a17ee078da62a14a5f1889531f7 Mon Sep 17 00:00:00 2001 From: Andras Banki-Horvath Date: Wed, 11 Sep 2024 14:20:25 +0200 Subject: [PATCH 10/21] invoices: add method to create payment hash index Certain invoices may not have a deterministic payment hash. For such invoices we still store the payment hashes in our KV database, but we do not have a sufficient index to retrieve them. This PR adds such index to the SQL database that will be used during migration to retrieve payment hashes. --- invoices/sql_migration.go | 128 ++++++++++++++++++++++++++++++++++++++ invoices/sql_store.go | 11 ++++ 2 files changed, 139 insertions(+) create mode 100644 invoices/sql_migration.go diff --git a/invoices/sql_migration.go b/invoices/sql_migration.go new file mode 100644 index 00000000000..2bdb14048cf --- /dev/null +++ b/invoices/sql_migration.go @@ -0,0 +1,128 @@ +package invoices + +import ( + "bytes" + "context" + "encoding/binary" + "fmt" + + "github.com/lightningnetwork/lnd/kvdb" + "github.com/lightningnetwork/lnd/lntypes" + "github.com/lightningnetwork/lnd/sqldb/sqlc" +) + +var ( + // invoiceBucket is the name of the bucket within the database that + // stores all data related to invoices no matter their final state. + // Within the invoice bucket, each invoice is keyed by its invoice ID + // which is a monotonically increasing uint32. + invoiceBucket = []byte("invoices") + + // invoiceIndexBucket is the name of the sub-bucket within the + // invoiceBucket which indexes all invoices by their payment hash. The + // payment hash is the sha256 of the invoice's payment preimage. This + // index is used to detect duplicates, and also to provide a fast path + // for looking up incoming HTLCs to determine if we're able to settle + // them fully. + // + // maps: payHash => invoiceKey + invoiceIndexBucket = []byte("paymenthashes") + + // numInvoicesKey is the name of key which houses the auto-incrementing + // invoice ID which is essentially used as a primary key. With each + // invoice inserted, the primary key is incremented by one. This key is + // stored within the invoiceIndexBucket. Within the invoiceBucket + // invoices are uniquely identified by the invoice ID. + numInvoicesKey = []byte("nik") + + // addIndexBucket is an index bucket that we'll use to create a + // monotonically increasing set of add indexes. Each time we add a new + // invoice, this sequence number will be incremented and then populated + // within the new invoice. + // + // In addition to this sequence number, we map: + // + // addIndexNo => invoiceKey + addIndexBucket = []byte("invoice-add-index") +) + +// createInvoiceHashIndex generates a hash index that contains payment hashes +// for each invoice in the database. Retrieving the payment hash for certain +// invoices, such as those created for spontaneous AMP payments, can be +// challenging because the hash is not directly derivable from the invoice's +// parameters and is stored separately in the `paymenthashes` bucket. This +// bucket maps payment hashes to invoice keys, but for migration purposes, we +// need the ability to query in the reverse direction. This function establishes +// a new index in the SQL database that maps each invoice key to its +// corresponding payment hash. +func createInvoiceHashIndex(ctx context.Context, db kvdb.Backend, + tx SQLInvoiceQueries) error { + + return db.View(func(kvTx kvdb.RTx) error { + invoices := kvTx.ReadBucket(invoiceBucket) + if invoices == nil { + return ErrNoInvoicesCreated + } + + invoiceIndex := invoices.NestedReadBucket( + invoiceIndexBucket, + ) + if invoiceIndex == nil { + return ErrNoInvoicesCreated + } + + addIndex := invoices.NestedReadBucket(addIndexBucket) + if addIndex == nil { + return ErrNoInvoicesCreated + } + + // First, iterate over all elements in the add index bucket and + // insert the add index value for the corresponding invoice key + // in the payment_hashes table. + err := addIndex.ForEach(func(k, v []byte) error { + // The key is the add index, and the value is + // the invoice key. + addIndexNo := binary.BigEndian.Uint64(k) + invoiceKey := binary.BigEndian.Uint32(v) + + return tx.InsertKVInvoiceKeyAndAddIndex(ctx, + sqlc.InsertKVInvoiceKeyAndAddIndexParams{ + ID: int32(invoiceKey), + AddIndex: int64(addIndexNo), + }, + ) + }) + if err != nil { + return err + } + + // Next, iterate over all hashes in the invoice index bucket and + // set the hash to the corresponding the invoice key in the + // payment_hashes table. + return invoiceIndex.ForEach(func(k, v []byte) error { + // Skip the special numInvoicesKey as that does + // not point to a valid invoice. + if bytes.Equal(k, numInvoicesKey) { + return nil + } + + // The key is the payment hash, and the value + // is the invoice key. + if len(k) != lntypes.HashSize { + return fmt.Errorf("invalid payment "+ + "hash length: expected %v, "+ + "got %v", lntypes.HashSize, + len(k)) + } + + invoiceKey := binary.BigEndian.Uint32(v) + + return tx.SetKVInvoicePaymentHash(ctx, + sqlc.SetKVInvoicePaymentHashParams{ + ID: int32(invoiceKey), + Hash: k, + }, + ) + }) + }, func() {}) +} diff --git a/invoices/sql_store.go b/invoices/sql_store.go index 6cd2176d0c2..839b19a54c3 100644 --- a/invoices/sql_store.go +++ b/invoices/sql_store.go @@ -123,6 +123,17 @@ type SQLInvoiceQueries interface { //nolint:interfacebloat OnAMPSubInvoiceSettled(ctx context.Context, arg sqlc.OnAMPSubInvoiceSettledParams) error + + // Migration specific methods. + // TODO(bhandras): remove this once migrations have been separated out. + InsertKVInvoiceKeyAndAddIndex(ctx context.Context, + arg sqlc.InsertKVInvoiceKeyAndAddIndexParams) error + + SetKVInvoicePaymentHash(ctx context.Context, + arg sqlc.SetKVInvoicePaymentHashParams) error + + GetKVInvoicePaymentHashByAddIndex(ctx context.Context, addIndex int64) ( + []byte, error) } var _ InvoiceDB = (*SQLStore)(nil) From 708bed517d3704d5c8e9b72edef4a531ab44a638 Mon Sep 17 00:00:00 2001 From: Andras Banki-Horvath Date: Tue, 11 Jun 2024 22:33:15 +0200 Subject: [PATCH 11/21] invoices: add migration code for a single invoice --- invoices/sql_migration.go | 273 +++++++++++++++++++++ invoices/sql_migration_test.go | 421 +++++++++++++++++++++++++++++++++ invoices/sql_store.go | 4 + 3 files changed, 698 insertions(+) create mode 100644 invoices/sql_migration_test.go diff --git a/invoices/sql_migration.go b/invoices/sql_migration.go index 2bdb14048cf..47ee3e3299d 100644 --- a/invoices/sql_migration.go +++ b/invoices/sql_migration.go @@ -5,9 +5,13 @@ import ( "context" "encoding/binary" "fmt" + "strconv" + "time" + "github.com/lightningnetwork/lnd/graph/db/models" "github.com/lightningnetwork/lnd/kvdb" "github.com/lightningnetwork/lnd/lntypes" + "github.com/lightningnetwork/lnd/sqldb" "github.com/lightningnetwork/lnd/sqldb/sqlc" ) @@ -126,3 +130,272 @@ func createInvoiceHashIndex(ctx context.Context, db kvdb.Backend, }) }, func() {}) } + +// toInsertMigratedInvoiceParams creates the parameters for inserting a migrated +// invoice into the SQL database. The parameters are derived from the original +// invoice insert parameters. +func toInsertMigratedInvoiceParams( + params sqlc.InsertInvoiceParams) sqlc.InsertMigratedInvoiceParams { + + return sqlc.InsertMigratedInvoiceParams{ + Hash: params.Hash, + Preimage: params.Preimage, + Memo: params.Memo, + AmountMsat: params.AmountMsat, + CltvDelta: params.CltvDelta, + Expiry: params.Expiry, + PaymentAddr: params.PaymentAddr, + PaymentRequest: params.PaymentRequest, + PaymentRequestHash: params.PaymentRequestHash, + State: params.State, + AmountPaidMsat: params.AmountPaidMsat, + IsAmp: params.IsAmp, + IsHodl: params.IsHodl, + IsKeysend: params.IsKeysend, + CreatedAt: params.CreatedAt, + } +} + +// MigrateSingleInvoice migrates a single invoice to the new SQL schema. Note +// that perfect equality between the old and new schemas is not achievable, as +// the invoice's add index cannot be mapped directly to its ID due to SQL’s +// auto-incrementing primary key. The ID returned from the insert will instead +// serve as the add index in the new schema. +func MigrateSingleInvoice(ctx context.Context, tx SQLInvoiceQueries, + invoice *Invoice, paymentHash lntypes.Hash) error { + + insertInvoiceParams, err := makeInsertInvoiceParams( + invoice, paymentHash, + ) + if err != nil { + return err + } + + // Convert the insert invoice parameters to the migrated invoice insert + // parameters. + insertMigratedInvoiceParams := toInsertMigratedInvoiceParams( + insertInvoiceParams, + ) + + // If the invoice is settled, we'll also set the timestamp and the index + // at which it was settled. + if invoice.State == ContractSettled { + if invoice.SettleIndex == 0 { + return fmt.Errorf("settled invoice %s missing settle "+ + "index", paymentHash) + } + + if invoice.SettleDate.IsZero() { + return fmt.Errorf("settled invoice %s missing settle "+ + "date", paymentHash) + } + + insertMigratedInvoiceParams.SettleIndex = sqldb.SQLInt64( + invoice.SettleIndex, + ) + insertMigratedInvoiceParams.SettledAt = sqldb.SQLTime( + invoice.SettleDate.UTC(), + ) + } + + // First we need to insert the invoice itself so we can use the "add + // index" which in this case is the auto incrementing primary key that + // is returned from the insert. + invoiceID, err := tx.InsertMigratedInvoice( + ctx, insertMigratedInvoiceParams, + ) + if err != nil { + return fmt.Errorf("unable to insert invoice: %w", err) + } + + // Insert the invoice's features. + for feature := range invoice.Terms.Features.Features() { + params := sqlc.InsertInvoiceFeatureParams{ + InvoiceID: invoiceID, + Feature: int32(feature), + } + + err := tx.InsertInvoiceFeature(ctx, params) + if err != nil { + return fmt.Errorf("unable to insert invoice "+ + "feature(%v): %w", feature, err) + } + } + + sqlHtlcIDs := make(map[models.CircuitKey]int64) + + // Now insert the HTLCs of the invoice. We'll also keep track of the SQL + // ID of each HTLC so we can use it when inserting the AMP sub invoices. + for circuitKey, htlc := range invoice.Htlcs { + htlcParams := sqlc.InsertInvoiceHTLCParams{ + HtlcID: int64(circuitKey.HtlcID), + ChanID: strconv.FormatUint( + circuitKey.ChanID.ToUint64(), 10, + ), + AmountMsat: int64(htlc.Amt), + AcceptHeight: int32(htlc.AcceptHeight), + AcceptTime: htlc.AcceptTime.UTC(), + ExpiryHeight: int32(htlc.Expiry), + State: int16(htlc.State), + InvoiceID: invoiceID, + } + + // Leave the MPP amount as NULL if the MPP total amount is zero. + if htlc.MppTotalAmt != 0 { + htlcParams.TotalMppMsat = sqldb.SQLInt64( + int64(htlc.MppTotalAmt), + ) + } + + // Leave the resolve time as NULL if the HTLC is not resolved. + if !htlc.ResolveTime.IsZero() { + htlcParams.ResolveTime = sqldb.SQLTime( + htlc.ResolveTime.UTC(), + ) + } + + sqlID, err := tx.InsertInvoiceHTLC(ctx, htlcParams) + if err != nil { + return fmt.Errorf("unable to insert invoice htlc: %w", + err) + } + + sqlHtlcIDs[circuitKey] = sqlID + + // Store custom records. + for key, value := range htlc.CustomRecords { + err = tx.InsertInvoiceHTLCCustomRecord( + ctx, sqlc.InsertInvoiceHTLCCustomRecordParams{ + Key: int64(key), + Value: value, + HtlcID: sqlID, + }, + ) + if err != nil { + return err + } + } + } + + if !invoice.IsAMP() { + return nil + } + + for setID, ampState := range invoice.AMPState { + // Find the earliest HTLC of the AMP invoice, which will + // be used as the creation date of this sub invoice. + var createdAt time.Time + for circuitKey := range ampState.InvoiceKeys { + htlc := invoice.Htlcs[circuitKey] + if createdAt.IsZero() { + createdAt = htlc.AcceptTime.UTC() + continue + } + + if createdAt.After(htlc.AcceptTime) { + createdAt = htlc.AcceptTime.UTC() + } + } + + params := sqlc.InsertAMPSubInvoiceParams{ + SetID: setID[:], + State: int16(ampState.State), + CreatedAt: createdAt, + InvoiceID: invoiceID, + } + + if ampState.SettleIndex != 0 { + if ampState.SettleDate.IsZero() { + return fmt.Errorf("settled AMP sub invoice %x "+ + "missing settle date", setID) + } + + params.SettledAt = sqldb.SQLTime( + ampState.SettleDate.UTC(), + ) + + params.SettleIndex = sqldb.SQLInt64( + ampState.SettleIndex, + ) + } + + err := tx.InsertAMPSubInvoice(ctx, params) + if err != nil { + return fmt.Errorf("unable to insert AMP sub invoice: "+ + "%w", err) + } + + // Now we can add the AMP HTLCs to the database. + for circuitKey := range ampState.InvoiceKeys { + htlc := invoice.Htlcs[circuitKey] + rootShare := htlc.AMP.Record.RootShare() + + sqlHtlcID, ok := sqlHtlcIDs[circuitKey] + if !ok { + return fmt.Errorf("missing htlc for AMP htlc: "+ + "%v", circuitKey) + } + + params := sqlc.InsertAMPSubInvoiceHTLCParams{ + InvoiceID: invoiceID, + SetID: setID[:], + HtlcID: sqlHtlcID, + RootShare: rootShare[:], + ChildIndex: int64(htlc.AMP.Record.ChildIndex()), + Hash: htlc.AMP.Hash[:], + } + + if htlc.AMP.Preimage != nil { + params.Preimage = htlc.AMP.Preimage[:] + } + + err = tx.InsertAMPSubInvoiceHTLC(ctx, params) + if err != nil { + return fmt.Errorf("unable to insert AMP sub "+ + "invoice: %w", err) + } + } + } + + return nil +} + +// OverrideInvoiceTimeZone overrides the time zone of the invoice to the local +// time zone and chops off the nanosecond part for comparison. This is needed +// because KV database stores times as-is which as an unwanted side effect would +// fail migration due to time comparison expecting both the original and +// migrated invoices to be in the same local time zone and in microsecond +// precision. Note that PostgreSQL stores times in microsecond precision while +// SQLite can store times in nanosecond precision if using TEXT storage class. +func OverrideInvoiceTimeZone(invoice *Invoice) { + fixTime := func(t time.Time) time.Time { + return t.In(time.Local).Truncate(time.Microsecond) + } + + invoice.CreationDate = fixTime(invoice.CreationDate) + + if !invoice.SettleDate.IsZero() { + invoice.SettleDate = fixTime(invoice.SettleDate) + } + + if invoice.IsAMP() { + for setID, ampState := range invoice.AMPState { + if ampState.SettleDate.IsZero() { + continue + } + + ampState.SettleDate = fixTime(ampState.SettleDate) + invoice.AMPState[setID] = ampState + } + } + + for _, htlc := range invoice.Htlcs { + if !htlc.AcceptTime.IsZero() { + htlc.AcceptTime = fixTime(htlc.AcceptTime) + } + + if !htlc.ResolveTime.IsZero() { + htlc.ResolveTime = fixTime(htlc.ResolveTime) + } + } +} diff --git a/invoices/sql_migration_test.go b/invoices/sql_migration_test.go new file mode 100644 index 00000000000..179097f4891 --- /dev/null +++ b/invoices/sql_migration_test.go @@ -0,0 +1,421 @@ +package invoices + +import ( + "context" + crand "crypto/rand" + "database/sql" + "math/rand" + "sync/atomic" + "testing" + "time" + + "github.com/lightningnetwork/lnd/clock" + "github.com/lightningnetwork/lnd/graph/db/models" + "github.com/lightningnetwork/lnd/lntypes" + "github.com/lightningnetwork/lnd/lnwire" + "github.com/lightningnetwork/lnd/record" + "github.com/lightningnetwork/lnd/sqldb" + "github.com/stretchr/testify/require" + "pgregory.net/rapid" +) + +var ( + // testHtlcIDSequence is a global counter for generating unique HTLC + // IDs. + testHtlcIDSequence uint64 +) + +// randomString generates a random string of a given length using rapid. +func randomStringRapid(t *rapid.T, length int) string { + // Define the character set for the string. + const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" //nolint:ll + + // Generate a string by selecting random characters from the charset. + runes := make([]rune, length) + for i := range runes { + // Draw a random index and use it to select a character from the + // charset. + index := rapid.IntRange(0, len(charset)-1).Draw(t, "charIndex") + runes[i] = rune(charset[index]) + } + + return string(runes) +} + +// randTimeBetween generates a random time between min and max. +func randTimeBetween(min, max time.Time) time.Time { + var timeZones = []*time.Location{ + time.UTC, + time.FixedZone("EST", -5*3600), + time.FixedZone("MST", -7*3600), + time.FixedZone("PST", -8*3600), + time.FixedZone("CEST", 2*3600), + } + + // Ensure max is after min + if max.Before(min) { + min, max = max, min + } + + // Calculate the range in nanoseconds + duration := max.Sub(min) + randDuration := time.Duration(rand.Int63n(duration.Nanoseconds())) + + // Generate the random time + randomTime := min.Add(randDuration) + + // Assign a random time zone + randomTimeZone := timeZones[rand.Intn(len(timeZones))] + + // Return the time in the random time zone + return randomTime.In(randomTimeZone) +} + +// randTime generates a random time between 2009 and 2140. +func randTime() time.Time { + min := time.Date(2009, 1, 3, 0, 0, 0, 0, time.UTC) + max := time.Date(2140, 1, 1, 0, 0, 0, 1000, time.UTC) + + return randTimeBetween(min, max) +} + +func randInvoiceTime(invoice *Invoice) time.Time { + return randTimeBetween( + invoice.CreationDate, + invoice.CreationDate.Add(invoice.Terms.Expiry), + ) +} + +// randHTLCRapid generates a random HTLC for an invoice using rapid to randomize +// its parameters. +func randHTLCRapid(t *rapid.T, invoice *Invoice, amt lnwire.MilliSatoshi) ( + models.CircuitKey, *InvoiceHTLC) { + + htlc := &InvoiceHTLC{ + Amt: amt, + AcceptHeight: rapid.Uint32Range(1, 999).Draw(t, "AcceptHeight"), + AcceptTime: randInvoiceTime(invoice), + Expiry: rapid.Uint32Range(1, 999).Draw(t, "Expiry"), + } + + // Set MPP total amount if MPP feature is enabled in the invoice. + if invoice.Terms.Features.HasFeature(lnwire.MPPRequired) { + htlc.MppTotalAmt = invoice.Terms.Value + } + + // Set the HTLC state and resolve time based on the invoice state. + switch invoice.State { + case ContractSettled: + htlc.State = HtlcStateSettled + htlc.ResolveTime = randInvoiceTime(invoice) + + case ContractCanceled: + htlc.State = HtlcStateCanceled + htlc.ResolveTime = randInvoiceTime(invoice) + + case ContractAccepted: + htlc.State = HtlcStateAccepted + } + + // Add randomized custom records to the HTLC. + htlc.CustomRecords = make(record.CustomSet) + numRecords := rapid.IntRange(0, 5).Draw(t, "numRecords") + for i := 0; i < numRecords; i++ { + key := rapid.Uint64Range( + record.CustomTypeStart, 1000+record.CustomTypeStart, + ).Draw(t, "customRecordKey") + value := []byte(randomStringRapid(t, 10)) + htlc.CustomRecords[key] = value + } + + // Generate a unique HTLC ID and assign it to a channel ID. + htlcID := atomic.AddUint64(&testHtlcIDSequence, 1) + randChanID := lnwire.NewShortChanIDFromInt(htlcID % 5) + + circuitKey := models.CircuitKey{ + ChanID: randChanID, + HtlcID: htlcID, + } + + return circuitKey, htlc +} + +// generateInvoiceHTLCsRapid generates all HTLCs for an invoice, including AMP +// HTLCs if applicable, using rapid for randomization of HTLC count and +// distribution. +func generateInvoiceHTLCsRapid(t *rapid.T, invoice *Invoice) { + mpp := invoice.Terms.Features.HasFeature(lnwire.MPPRequired) + + // Use rapid to determine the number of HTLCs based on invoice state and + // MPP feature. + numHTLCs := 1 + if invoice.State == ContractOpen { + numHTLCs = 0 + } else if mpp { + numHTLCs = rapid.IntRange(1, 10).Draw(t, "numHTLCs") + } + + total := invoice.Terms.Value + + // Distribute the total amount across the HTLCs, adding any remainder to + // the last HTLC. + if numHTLCs > 0 { + amt := total / lnwire.MilliSatoshi(numHTLCs) + remainder := total - amt*lnwire.MilliSatoshi(numHTLCs) + + for i := 0; i < numHTLCs; i++ { + if i == numHTLCs-1 { + // Add remainder to the last HTLC. + amt += remainder + } + + // Generate an HTLC with a random circuit key and add it + // to the invoice. + circuitKey, htlc := randHTLCRapid(t, invoice, amt) + invoice.Htlcs[circuitKey] = htlc + } + } +} + +// generateAMPHtlcsRapid generates AMP HTLCs for an invoice using rapid to +// randomize various parameters of the HTLCs in the AMP set. +func generateAMPHtlcsRapid(t *rapid.T, invoice *Invoice) { + // Randomly determine the number of AMP sets (1 to 5). + numSetIDs := rapid.IntRange(1, 5).Draw(t, "numSetIDs") + settledIdx := uint64(1) + + for i := 0; i < numSetIDs; i++ { + var setID SetID + _, err := crand.Read(setID[:]) + require.NoError(t, err) + + // Determine the number of HTLCs in this set (1 to 5). + numHTLCs := rapid.IntRange(1, 5).Draw(t, "numHTLCs") + total := invoice.Terms.Value + invoiceKeys := make(map[CircuitKey]struct{}) + + // Calculate the amount per HTLC and account for remainder in + // the final HTLC. + amt := total / lnwire.MilliSatoshi(numHTLCs) + remainder := total - amt*lnwire.MilliSatoshi(numHTLCs) + + var htlcState HtlcState + for j := 0; j < numHTLCs; j++ { + if j == numHTLCs-1 { + amt += remainder + } + + // Generate HTLC with randomized parameters. + circuitKey, htlc := randHTLCRapid(t, invoice, amt) + htlcState = htlc.State + + var ( + rootShare, hash [32]byte + preimage lntypes.Preimage + ) + + // Randomize AMP data fields. + _, err := crand.Read(rootShare[:]) + require.NoError(t, err) + _, err = crand.Read(hash[:]) + require.NoError(t, err) + _, err = crand.Read(preimage[:]) + require.NoError(t, err) + + record := record.NewAMP(rootShare, setID, uint32(j)) + + htlc.AMP = &InvoiceHtlcAMPData{ + Record: *record, + Hash: hash, + Preimage: &preimage, + } + + invoice.Htlcs[circuitKey] = htlc + invoiceKeys[circuitKey] = struct{}{} + } + + ampState := InvoiceStateAMP{ + State: htlcState, + InvoiceKeys: invoiceKeys, + } + if htlcState == HtlcStateSettled { + ampState.SettleIndex = settledIdx + ampState.SettleDate = randInvoiceTime(invoice) + settledIdx++ + } + + // Set the total amount paid if the AMP set is not canceled. + if htlcState != HtlcStateCanceled { + ampState.AmtPaid = invoice.Terms.Value + } + + invoice.AMPState[setID] = ampState + } +} + +// TestMigrateSingleInvoiceRapid tests the migration of single invoices with +// random data variations using rapid. This test generates a random invoice +// configuration and ensures successful migration. +// +// NOTE: This test may need to be changed if the Invoice or any of the related +// types are modified. +func TestMigrateSingleInvoiceRapid(t *testing.T) { + // Create a shared Postgres instance for efficient testing. + pgFixture := sqldb.NewTestPgFixture( + t, sqldb.DefaultPostgresFixtureLifetime, + ) + t.Cleanup(func() { + pgFixture.TearDown(t) + }) + + makeSQLDB := func(t *testing.T, sqlite bool) *SQLStore { + var db *sqldb.BaseDB + if sqlite { + db = sqldb.NewTestSqliteDB(t).BaseDB + } else { + db = sqldb.NewTestPostgresDB(t, pgFixture).BaseDB + } + + executor := sqldb.NewTransactionExecutor( + db, func(tx *sql.Tx) SQLInvoiceQueries { + return db.WithTx(tx) + }, + ) + + testClock := clock.NewTestClock(time.Unix(1, 0)) + + return NewSQLStore(executor, testClock) + } + + // Define property-based test using rapid. + rapid.Check(t, func(rt *rapid.T) { + // Randomized feature flags for MPP and AMP. + mpp := rapid.Bool().Draw(rt, "mpp") + amp := rapid.Bool().Draw(rt, "amp") + + for _, sqlite := range []bool{true, false} { + store := makeSQLDB(t, sqlite) + testMigrateSingleInvoiceRapid(rt, store, mpp, amp) + } + }) +} + +// testMigrateSingleInvoiceRapid is the primary function for the migration of a +// single invoice with random data in a rapid-based test setup. +func testMigrateSingleInvoiceRapid(t *rapid.T, store *SQLStore, mpp bool, + amp bool) { + + ctxb := context.Background() + invoices := make(map[lntypes.Hash]*Invoice) + + for i := 0; i < 100; i++ { + invoice := generateTestInvoiceRapid(t, mpp, amp) + var hash lntypes.Hash + _, err := crand.Read(hash[:]) + require.NoError(t, err) + + invoices[hash] = invoice + } + + var ops SQLInvoiceQueriesTxOptions + err := store.db.ExecTx(ctxb, &ops, func(tx SQLInvoiceQueries) error { + for hash, invoice := range invoices { + err := MigrateSingleInvoice(ctxb, tx, invoice, hash) + require.NoError(t, err) + } + + return nil + }, func() {}) + require.NoError(t, err) + + // Fetch and compare each migrated invoice from the store with the + // original. + for hash, invoice := range invoices { + sqlInvoice, err := store.LookupInvoice( + ctxb, InvoiceRefByHash(hash), + ) + require.NoError(t, err) + + invoice.AddIndex = sqlInvoice.AddIndex + + OverrideInvoiceTimeZone(invoice) + OverrideInvoiceTimeZone(&sqlInvoice) + + require.Equal(t, *invoice, sqlInvoice) + } +} + +// generateTestInvoiceRapid generates a random invoice with variations based on +// mpp and amp flags. +func generateTestInvoiceRapid(t *rapid.T, mpp bool, amp bool) *Invoice { + var preimage lntypes.Preimage + _, err := crand.Read(preimage[:]) + require.NoError(t, err) + + terms := ContractTerm{ + FinalCltvDelta: rapid.Int32Range(1, 1000).Draw( + t, "FinalCltvDelta", + ), + Expiry: time.Duration( + rapid.IntRange(1, 4444).Draw(t, "Expiry"), + ) * time.Minute, + PaymentPreimage: &preimage, + Value: lnwire.MilliSatoshi( + rapid.Int64Range(1, 9999999).Draw(t, "Value"), + ), + PaymentAddr: [32]byte{}, + Features: lnwire.EmptyFeatureVector(), + } + + if amp { + terms.Features.Set(lnwire.AMPRequired) + } else if mpp { + terms.Features.Set(lnwire.MPPRequired) + } + + created := randTime() + + const maxContractState = 3 + state := ContractState( + rapid.IntRange(0, maxContractState).Draw(t, "ContractState"), + ) + var ( + settled time.Time + settleIndex uint64 + ) + if state == ContractSettled { + settled = randTimeBetween(created, created.Add(terms.Expiry)) + settleIndex = rapid.Uint64Range(1, 999).Draw(t, "SettleIndex") + } + + invoice := &Invoice{ + Memo: []byte(randomStringRapid(t, 10)), + PaymentRequest: []byte( + randomStringRapid(t, MaxPaymentRequestSize), + ), + CreationDate: created, + SettleDate: settled, + Terms: terms, + AddIndex: 0, + SettleIndex: settleIndex, + State: state, + AMPState: make(map[SetID]InvoiceStateAMP), + HodlInvoice: rapid.Bool().Draw(t, "HodlInvoice"), + } + + invoice.Htlcs = make(map[models.CircuitKey]*InvoiceHTLC) + + if invoice.IsAMP() { + generateAMPHtlcsRapid(t, invoice) + } else { + generateInvoiceHTLCsRapid(t, invoice) + } + + for _, htlc := range invoice.Htlcs { + if htlc.State == HtlcStateSettled { + invoice.AmtPaid += htlc.Amt + } + } + + return invoice +} diff --git a/invoices/sql_store.go b/invoices/sql_store.go index 839b19a54c3..c9ffcc44c10 100644 --- a/invoices/sql_store.go +++ b/invoices/sql_store.go @@ -32,6 +32,10 @@ type SQLInvoiceQueries interface { //nolint:interfacebloat InsertInvoice(ctx context.Context, arg sqlc.InsertInvoiceParams) (int64, error) + // TODO(bhandras): remove this once migrations have been separated out. + InsertMigratedInvoice(ctx context.Context, + arg sqlc.InsertMigratedInvoiceParams) (int64, error) + InsertInvoiceFeature(ctx context.Context, arg sqlc.InsertInvoiceFeatureParams) error From b92f57e0aea373b525f868c67f265bc4c0114a24 Mon Sep 17 00:00:00 2001 From: Andras Banki-Horvath Date: Wed, 12 Jun 2024 13:32:22 +0200 Subject: [PATCH 12/21] invoices: add migration code that runs a full invoice DB SQL migration --- go.mod | 2 +- invoices/kv_sql_migration_test.go | 146 +++++++++++++++++++++++++++ invoices/sql_migration.go | 159 +++++++++++++++++++++++++++++- invoices/sql_store.go | 10 +- invoices/testdata/channel.db | Bin 0 -> 1048576 bytes 5 files changed, 311 insertions(+), 6 deletions(-) create mode 100644 invoices/kv_sql_migration_test.go create mode 100644 invoices/testdata/channel.db diff --git a/go.mod b/go.mod index ba53873e13e..c660cbb5af7 100644 --- a/go.mod +++ b/go.mod @@ -138,7 +138,7 @@ require ( github.com/opencontainers/image-spec v1.0.2 // indirect github.com/opencontainers/runc v1.1.12 // indirect github.com/ory/dockertest/v3 v3.10.0 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/pmezard/go-difflib v1.0.0 github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/common v0.26.0 // indirect github.com/prometheus/procfs v0.6.0 // indirect diff --git a/invoices/kv_sql_migration_test.go b/invoices/kv_sql_migration_test.go new file mode 100644 index 00000000000..5d36b5c642b --- /dev/null +++ b/invoices/kv_sql_migration_test.go @@ -0,0 +1,146 @@ +package invoices_test + +import ( + "context" + "database/sql" + "testing" + "time" + + "github.com/lightningnetwork/lnd/channeldb" + "github.com/lightningnetwork/lnd/clock" + invpkg "github.com/lightningnetwork/lnd/invoices" + "github.com/lightningnetwork/lnd/sqldb" + "github.com/lightningnetwork/lnd/sqldb/sqlc" + "github.com/stretchr/testify/require" +) + +// TestMigrationWithChannelDB tests the migration of invoices from a bolt backed +// channel.db to a SQL database. Note that this test does not attempt to be a +// complete migration test for all invoice types but rather is added as a tool +// for developers and users to debug invoice migration issues with an actual +// channel.db file. +func TestMigrationWithChannelDB(t *testing.T) { + // First create a shared Postgres instance so we don't spawn a new + // docker container for each test. + pgFixture := sqldb.NewTestPgFixture( + t, sqldb.DefaultPostgresFixtureLifetime, + ) + t.Cleanup(func() { + pgFixture.TearDown(t) + }) + + makeSQLDB := func(t *testing.T, sqlite bool) (*invpkg.SQLStore, + *sqldb.TransactionExecutor[*sqlc.Queries]) { + + var db *sqldb.BaseDB + if sqlite { + db = sqldb.NewTestSqliteDB(t).BaseDB + } else { + db = sqldb.NewTestPostgresDB(t, pgFixture).BaseDB + } + + invoiceExecutor := sqldb.NewTransactionExecutor( + db, func(tx *sql.Tx) invpkg.SQLInvoiceQueries { + return db.WithTx(tx) + }, + ) + + genericExecutor := sqldb.NewTransactionExecutor( + db, func(tx *sql.Tx) *sqlc.Queries { + return db.WithTx(tx) + }, + ) + + testClock := clock.NewTestClock(time.Unix(1, 0)) + + return invpkg.NewSQLStore(invoiceExecutor, testClock), + genericExecutor + } + + migrationTest := func(t *testing.T, kvStore *channeldb.DB, + sqlite bool) { + + sqlInvoiceStore, sqlStore := makeSQLDB(t, sqlite) + ctxb := context.Background() + + const batchSize = 11 + var opts sqldb.MigrationTxOptions + err := sqlStore.ExecTx( + ctxb, &opts, func(tx *sqlc.Queries) error { + return invpkg.MigrateInvoicesToSQL( + ctxb, kvStore.Backend, kvStore, tx, + batchSize, + ) + }, func() {}, + ) + require.NoError(t, err) + + // MigrateInvoices will check if the inserted invoice equals to + // the migrated one, but as a sanity check, we'll also fetch the + // invoices from the store and compare them to the original + // invoices. + query := invpkg.InvoiceQuery{ + IndexOffset: 0, + // As a sanity check, fetch more invoices than we have + // to ensure that we did not add any extra invoices. + // Note that we don't really have a way to know the + // exact number of invoices in the bolt db without first + // iterating over all of them, but for test purposes + // constant should be enough. + NumMaxInvoices: 9999, + } + result1, err := kvStore.QueryInvoices(ctxb, query) + require.NoError(t, err) + numInvoices := len(result1.Invoices) + + result2, err := sqlInvoiceStore.QueryInvoices(ctxb, query) + require.NoError(t, err) + require.Equal(t, numInvoices, len(result2.Invoices)) + + // Simply zero out the add index so we don't fail on that when + // comparing. + for i := 0; i < numInvoices; i++ { + result1.Invoices[i].AddIndex = 0 + result2.Invoices[i].AddIndex = 0 + + // We need to override the timezone of the invoices as + // the provided DB vs the test runners local time zone + // might be different. + invpkg.OverrideInvoiceTimeZone(&result1.Invoices[i]) + invpkg.OverrideInvoiceTimeZone(&result2.Invoices[i]) + + require.Equal( + t, result1.Invoices[i], result2.Invoices[i], + ) + } + } + + tests := []struct { + name string + dbPath string + }{ + { + "empty", + t.TempDir(), + }, + { + "testdata", + "testdata", + }, + } + + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + store := channeldb.OpenForTesting(t, test.dbPath) + + t.Run("Postgres", func(t *testing.T) { + migrationTest(t, store, false) + }) + + t.Run("SQLite", func(t *testing.T) { + migrationTest(t, store, true) + }) + }) + } +} diff --git a/invoices/sql_migration.go b/invoices/sql_migration.go index 47ee3e3299d..af0e4865f47 100644 --- a/invoices/sql_migration.go +++ b/invoices/sql_migration.go @@ -4,15 +4,19 @@ import ( "bytes" "context" "encoding/binary" + "errors" "fmt" + "reflect" "strconv" "time" + "github.com/davecgh/go-spew/spew" "github.com/lightningnetwork/lnd/graph/db/models" "github.com/lightningnetwork/lnd/kvdb" "github.com/lightningnetwork/lnd/lntypes" "github.com/lightningnetwork/lnd/sqldb" "github.com/lightningnetwork/lnd/sqldb/sqlc" + "github.com/pmezard/go-difflib/difflib" ) var ( @@ -48,6 +52,11 @@ var ( // // addIndexNo => invoiceKey addIndexBucket = []byte("invoice-add-index") + + // ErrMigrationMismatch is returned when the migrated invoice does not + // match the original invoice. + ErrMigrationMismatch = fmt.Errorf("migrated invoice does not match " + + "original invoice") ) // createInvoiceHashIndex generates a hash index that contains payment hashes @@ -60,7 +69,7 @@ var ( // a new index in the SQL database that maps each invoice key to its // corresponding payment hash. func createInvoiceHashIndex(ctx context.Context, db kvdb.Backend, - tx SQLInvoiceQueries) error { + tx *sqlc.Queries) error { return db.View(func(kvTx kvdb.RTx) error { invoices := kvTx.ReadBucket(invoiceBucket) @@ -399,3 +408,151 @@ func OverrideInvoiceTimeZone(invoice *Invoice) { } } } + +// MigrateInvoicesToSQL runs the migration of all invoices from the KV database +// to the SQL database. The migration is done in a single transaction to ensure +// that all invoices are migrated or none at all. This function can be run +// multiple times without causing any issues as it will check if the migration +// has already been performed. +func MigrateInvoicesToSQL(ctx context.Context, db kvdb.Backend, + kvStore InvoiceDB, tx *sqlc.Queries, batchSize int) error { + + log.Infof("Starting migration of invoices from KV to SQL") + + offset := uint64(0) + t0 := time.Now() + + // Create the hash index which we will use to look up invoice + // payment hashes by their add index during migration. + err := createInvoiceHashIndex(ctx, db, tx) + if err != nil && !errors.Is(err, ErrNoInvoicesCreated) { + log.Errorf("Unable to create invoice hash index: %v", + err) + + return err + } + log.Debugf("Created SQL invoice hash index in %v", time.Since(t0)) + + total := 0 + // Now we can start migrating the invoices. We'll do this in + // batches to reduce memory usage. + for { + t0 = time.Now() + query := InvoiceQuery{ + IndexOffset: offset, + NumMaxInvoices: uint64(batchSize), + } + + queryResult, err := kvStore.QueryInvoices(ctx, query) + if err != nil && !errors.Is(err, ErrNoInvoicesCreated) { + return fmt.Errorf("unable to query invoices: "+ + "%w", err) + } + + if len(queryResult.Invoices) == 0 { + log.Infof("All invoices migrated") + + break + } + + err = migrateInvoices(ctx, tx, queryResult.Invoices) + if err != nil { + return err + } + + offset = queryResult.LastIndexOffset + total += len(queryResult.Invoices) + log.Debugf("Migrated %d KV invoices to SQL in %v\n", total, + time.Since(t0)) + } + + // Clean up the hash index as it's no longer needed. + err = tx.ClearKVInvoiceHashIndex(ctx) + if err != nil { + return fmt.Errorf("unable to clear invoice hash "+ + "index: %w", err) + } + + log.Infof("Migration of %d invoices from KV to SQL completed", total) + + return nil +} + +func migrateInvoices(ctx context.Context, tx *sqlc.Queries, + invoices []Invoice) error { + + for i, invoice := range invoices { + var paymentHash lntypes.Hash + if invoice.Terms.PaymentPreimage != nil { + paymentHash = invoice.Terms.PaymentPreimage.Hash() + } else { + paymentHashBytes, err := + tx.GetKVInvoicePaymentHashByAddIndex( + ctx, int64(invoice.AddIndex), + ) + if err != nil { + // This would be an unexpected inconsistency + // in the kv database. We can't do much here + // so we'll notify the user and continue. + log.Warnf("Cannot migrate invoice, unable to "+ + "fetch payment hash (add_index=%v): %v", + invoice.AddIndex, err) + + continue + } + + copy(paymentHash[:], paymentHashBytes) + } + + err := MigrateSingleInvoice(ctx, tx, &invoices[i], paymentHash) + if err != nil { + return fmt.Errorf("unable to migrate invoice(%v): %w", + paymentHash, err) + } + + migratedInvoice, err := fetchInvoice( + ctx, tx, InvoiceRefByHash(paymentHash), + ) + if err != nil { + return fmt.Errorf("unable to fetch migrated "+ + "invoice(%v): %w", paymentHash, err) + } + + // Override the time zone for comparison. Note that we need to + // override both invoices as the original invoice is coming from + // KV database, it was stored as a binary serialized Go + // time.Time value which has nanosecond precision but might have + // been created in a different time zone. The migrated invoice + // is stored in SQL in UTC and selected in the local time zone, + // however in PostgreSQL it has microsecond precision while in + // SQLite it has nanosecond precision if using TEXT storage + // class. + OverrideInvoiceTimeZone(&invoice) + OverrideInvoiceTimeZone(migratedInvoice) + + // Override the add index before checking for equality. + migratedInvoice.AddIndex = invoice.AddIndex + + if !reflect.DeepEqual(invoice, *migratedInvoice) { + diff := difflib.UnifiedDiff{ + A: difflib.SplitLines( + spew.Sdump(invoice), + ), + B: difflib.SplitLines( + spew.Sdump(migratedInvoice), + ), + FromFile: "Expected", + FromDate: "", + ToFile: "Actual", + ToDate: "", + Context: 3, + } + diffText, _ := difflib.GetUnifiedDiffString(diff) + + return fmt.Errorf("%w: %v.\n%v", ErrMigrationMismatch, + paymentHash, diffText) + } + } + + return nil +} diff --git a/invoices/sql_store.go b/invoices/sql_store.go index c9ffcc44c10..8a819e5ba9d 100644 --- a/invoices/sql_store.go +++ b/invoices/sql_store.go @@ -138,6 +138,8 @@ type SQLInvoiceQueries interface { //nolint:interfacebloat GetKVInvoicePaymentHashByAddIndex(ctx context.Context, addIndex int64) ( []byte, error) + + ClearKVInvoiceHashIndex(ctx context.Context) error } var _ InvoiceDB = (*SQLStore)(nil) @@ -354,8 +356,8 @@ func (i *SQLStore) AddInvoice(ctx context.Context, // fetchInvoice fetches the common invoice data and the AMP state for the // invoice with the given reference. -func (i *SQLStore) fetchInvoice(ctx context.Context, - db SQLInvoiceQueries, ref InvoiceRef) (*Invoice, error) { +func fetchInvoice(ctx context.Context, db SQLInvoiceQueries, + ref InvoiceRef) (*Invoice, error) { if ref.PayHash() == nil && ref.PayAddr() == nil && ref.SetID() == nil { return nil, ErrInvoiceNotFound @@ -686,7 +688,7 @@ func (i *SQLStore) LookupInvoice(ctx context.Context, readTxOpt := NewSQLInvoiceQueryReadTx() txErr := i.db.ExecTx(ctx, &readTxOpt, func(db SQLInvoiceQueries) error { - invoice, err = i.fetchInvoice(ctx, db, ref) + invoice, err = fetchInvoice(ctx, db, ref) return err }, func() {}) @@ -1387,7 +1389,7 @@ func (i *SQLStore) UpdateInvoice(ctx context.Context, ref InvoiceRef, ref.refModifier = HtlcSetOnlyModifier } - invoice, err := i.fetchInvoice(ctx, db, ref) + invoice, err := fetchInvoice(ctx, db, ref) if err != nil { return err } diff --git a/invoices/testdata/channel.db b/invoices/testdata/channel.db new file mode 100644 index 0000000000000000000000000000000000000000..78741eae9c63a6284b389e9c6b14f71fa4d32aa9 GIT binary patch literal 1048576 zcmeF)2|QG7-#GA@LG~6RR2s6hP?pLbvWALMipXFvmYFd_mQ(iMtzjLm$T;Fq@W%{_zg~PFvFlYSs zq10Of_T7R#fyf>WeE#I5@ti4Yc=nf+?7k%=wTWH>8Q}{EfB*=900@8p2!H?xfB*=9 z00@8p2!H?xfB*=900@8p2!H?xfB*=900@8p2!H?xfB*=900@8p2!H?xfB*=900@8p z2!H?xfB*=900@8p2!H?xfB*=900@A&Y5TjkJ)n#Tfo1C_(5<+y#CIgBX#si2EJr{vcU{LLz3@wlApej4o)J~I*Z${E{%ud>$ucNSCM|;DOBT4R zJgbGTAOHd&00JNY0w4eaAOHd&00JNY0w4eaAOHd&00JNY0w4eaAOHd&00JNY0w4ea zAOHd&00JNY0w4eaAOHd&00JNY0w4eaAOHd&00JNY0w4eaAOHd&00RG$0z#aPaT}PU zru;apaXXPS3{j=pj!=?Za`U8&w^gS{>4b`T#$Ni9##NrVVA_+EHVkF^ZQ{OH;I}hj zA_#y02!H?xfB*=900@8p2!H?xfB*=900@8p2!H?xfB*=900@8p2!H?xfB*=900@8p z2!H?xfB*=900@8p2!H?xfB*=900@8p2!H?xfB*=900@8p2!H?xfWR*T!kmbJQ)hj{ z+W$MF#{1T2JOd3_T*Mcvp_c~m&fkJ!DU^@M$)AT}NuKx`iie{36lXpYZB&+L{&p1e zE`J8a*oy^-oc#AtJc1|g<(of31c#GB^Y4hV_qP?TjCcEAQ7nz-FGhvI%Hxc7hIe}* zD3(CevG*Qe^Oy6*aeVOx6wCAE--}`;6cf>aEq@cmBYEO36pNvlyZ$~DWBYRxWybQC zpc&YF12kaq8x&*fCvsvLRQS<6u^Eba{k&0(hzIgBQH*ViyS)SyW1iR0fUVySVWC~X zC2`_AD3(LtiJZ6>#h4Fw`5&Cvl{0@X+BoKWgAG+|G zZ8h(RR3==Wb=;``Wb@Jjy@0s)n_k)^O%|6N){0i~4-@yD6<7vrUr{w+m5&)t3MUH- z+VY4ZeW^RE)BNRBgl|&N0% z%$^v&FkdW@vtn9J)2^zjkCNRPd)rF4{iCg7)Bk$Qq!0>;sqGWt8%z%WM{D7}nZt*~ zBo8vHZy<@PP4N?$8>@uuq=QC8&Ub8Nfd(QPxXBrPA#wsiRY1^z62yt^J(V0Xk4~mi zr~wPG%E8?Y{xR3am%{LkpoD9a{QQ{wdAKouUjIlx?GRc3cQ*f+KYK7ani)=`k_E3n zgoX|jGa@vU#Gni^yPr1ekO`^?zaB3&w96ix-Q0u5yzx+UZgFEl#|ta@VDjMb#W)*5cGhC%VFh&3mlO1@=kB}PnYpme z&_?;aagwx?g7I4v&A16@#7dLO4sE7`#GK zkNK^D{Nc(EZ<;4q`U(gd-cd6tXqrxw&wxp~6$0Wy= zW1_ot#wVP7M?EHLi88T{5T3B`_`Uh#hnzm=>K$58x-d>lR7gDM&G0k6H@!-m`pWzL zQBKw#;DxqV8a2ONvvk}SlXF*}b}e-8KR7Z;byUHAW0kAhZ{^IogL1MCF`i&O?T+iM zmt9$xNwZ335#};Z?VhSH9AQKI;odxb{wP-(UPwPblBOtPm9x3v?Umk~YVqX5O2a%y zD(1W|ynb$P_Nyp7j)_Ye)RE>C1l#y&1IKLZ<8x+Cjz3={krvvP5=(y(msvAnSKQ4f z^!^_Vp@672veij#SJyj)Jgdu!EXt=%Exu%ySerGp?{VKoUtd9&1GEv`VDhl4-w?so z&$g^)|617%3r09h5uIdg#2T{$p{bvr?NwxC9kj>gevWMR7e5rD zHZLVT!rd$HZB?7@lTdbEcJ&iiUp|~255ex>SmLdIcsyZ&>pb0p{fnx1jD1NM`*82^ zzOvo>dw$qH&H9wN^mMq18MXv#DIxXhO6o~R!mfnSn;pSXlg7O{GQ((l*`|>m?~Dqc z^^{;wG+6b+-so(YWvay|eon6X$j8D!8dX)k}cNcQxHJ_~(A zYNZr?jM_qH-`XrP#!F$tq>8!IZX?Wcp|Fm$hmo^u6e^XYYcZ#pOPapBUve-x>f@U^`6w4w{ZP*4jnb!Y9K*W)~NMMCxDPO+8L97D19K5pOfLfbM9UC{D;M_ArhYFMkH z+4og^jlW31Liu9724Sk^d3jd#8`P2JseZNO$d|{>%hXL}^b|THR#~1s>1JB?^hScD z)3PdqU<Nxs92+12`zZZDpY`*vuUwoA( z7JQq)7Yja@@x_C=8V}}yLGXdY7Z2vt!=C!+B;R0CfZ#{700xO3s7>|@AafomkPkbt z=Y*2ON!afw-o&0mrTLM$hm|`wkb<5c85Cbq2=Bw*A4ij&k3yw{Q%E6J>}O!xNI&!l z>`V)x_(pTr!$0P3heBOWqxh1w|ITj;&N58SJ!)Xpp=2f#329?{{1*@3+)cBq*<`f% z2F@z5TMQQGp+WGo9_PUlkIwV!2U12FyrRWD>`Pwvb!5no1~byIH6wS_Yo@%m%7Ycs$OTy}v@Q z&^eC=9Coc^Ju?n|r-krB0#6oB0Yz>ryRE$D5aO0#oN+M5I5oXFxS=_Fo~lPqaR1$A zOMNYk;Kxl4y%R=OF7W5qNWE-ti#%QWzE%YrT|bdh zv1Y$+YqeQv^BX0iMISCbWgFZu&Nj|pcAw0URP*gU8yw`kU3$K`&iq)Bso!QirWvcS zexz^`yU>6B2ALTV@>f?X_6Dfr@JJdX zn3J7dr3C&mbmyy$^T@WqvtP4IP*l?@?cKaLsN6sjMY z{Zxx>hA$D=zcpA>iRcp>xHp$+Jh8}(c2z?Ax22UCpPds5{FPE;y*(EOj~$iPo*o@A zdxZ*^4yO4QCQ>M@J;CylzlNT*#Q%41KfIL@JEviC}$H?{~` zh)ia1|Ib93J!#;RNoLg(0fl6pmCseSRtFIwqMt`PqA~56&X7IhcrcR`W}-p zjq6D$3fK#ly6WK z1Lb2zlF4*3LpwZ5JCMW-6ns~VB%wPLHGs~b(ft47F$EP35(t0*2!H?xfB*=900@8p z2!H?xfB*=900@8p2!H?xfB*=900@8p2!H?xfB*=900@8p2!H?xfB*=90K3gJ=D__^ zZR~GZvA<#!N5ePtm#1__1eN^P?OJ6~R(8@sOJjeFO60`Wd@HdZU6CwWft02 zwdX8PtvPz3M)c=(+vsu&2V)HyU2a2>tyI88)G7mC)KMX|X_xK0h|-}`=;QNzN&%Tp@k$OXp-8t@$WV@v>)(FH^J~6z;T)i>sz;rtSEib6+WaV&Z#uf34PcB}-7A zOPb?8?jYp;?Am9r!9OaZ*<_XD;k!hIoU!HpIHB?(9_a76@g-ZzH-}gUtUgOqQg>bQ zCT+BLS;>(}(l;)08e>+eO-J>ZrLGn9?FM zYU^%&l&91(@px5P{7<`;+inM4ms#(7D<;`!=9m;KGo9BjtuxIXQJ%8hGu@&REZx6( zW-peS9R4Ow^udnm!nJLQPliYp&PZCPf%23yEEdN+?(MiaSvJG)fYsZvg#ikw<7uxO zQZ40DF6*r&p*&Y+9`Jf$H{E`IW6U|m5ho2KUo5&9<*9t$c6GW>lxIcnv(OaR>wcTl zy>@lgly7VxeJD1Hz9fwu(;F{@B7&mdMYcudt)H~>@|toXV(-in@eqU0v4KSmPQylW zPG=Z_00@8p2!H?xfB*=900@8p2!H?xfB*=900@8p2!H?xfB*=900@8p2!H?xfB*=9 z00@8p2!H?xfB*=900@8p2!H?xfB*=900@8p2!H?xfB*=900{hR1-ShM5;%R0p%(H3 z-Ap|ZL5+h^tco(Jp)ofeDaDRk(EP!5Gxhs25C%G~!r?4X7Ob7|P;7%J%|~M_zQKxc zIAzpy9_v&702*Vd6t&yO=A$m8ajd?seQBYglyFwhZam$(adNYYfi1>OoUO2@i@m!S zhs_|+o8GZAjCW^`b?UGc;0IO6{Jpc{2z`6l3(Dh04m&kdZeHb{+m#;5a^{oFFUH5q z#Jro$-XT>S(M3q*#|#&%GrcwXCS={wEyai8eu-1bT7`X0yu89LhJCsCjIAh9nJs0s^a zXuuesR@3sM%jy>KM#6_;|F$tJmCfYK%DRPu*Ip71 zA`Rpe@>A`a;%spyv~bhurH^+nTC-h8=s->(+!%d@V^#SY4JICM7>TiEBeK>q=AAg| zkDUzEf!R8OkcnBkJS zcWBSb{klip=Icc@TyXq6%C#~RM<6aaqE!t7epYc7j6-(ViNvaSjBhTZ7(dghT? zBd1Po`seQe{{kUI@Vd)TRQQ3v`nYI@5JRn7?U8Q>9v5_f;k;=!vp1AVTXVG#X68%3 zqA#v%wRH`1`{ymxsSJ7l;wUDB4%gqi0}Sd=Vs`-6qtU<};BnQ&{2i5U_v$>GyjoKm zi`v3GLu!;7>fh*!Dz++4cVbl=g6A)Np>olr>{^)8Qs&Xeq4z;PFE0)s@tfj)Ywap*K5% zqb7}eb7Y3m_OeYQJ>D4=KI z0?Juwl zwC}SoRkcq!tnI9|ewK32A2~Cvh;iQoEQ0!C6j$E<^nkgZDJHYyqp6Ym+1h&tG>&qg zdvSOngNN6`wx)|6zq0MeNguVdV~vJNeY)7Pr#rdh>1J)!v-Jb(6bcqkR=izv>*dFS z1l&f=bSLZgfgc}x$h4gOaqno6#XhTnI{<_z?*JAhc!`PQd+yXS_{S7b<6_NKhK-queZvgJ8`Va2_gDPa#=;6QI z0UD?1JeV@!TiDKi?_A2KS$-843vIdAG<5ls63NY29#-I2x6Xl5XXE?g1UbiP)UM>E zZq+Be&*baL*|+8q?R0DiD!BaQUga?RVI71ZEpd&h6VB;QU;oy&<5ttsho!50^uP8I zh-&U^ zc$3s_!d6F#Hz#A-?j()Lmv3LzS@PwnfzNR+7rF)9F23T^yy(m^TUvpz{@yKMP{$Ix1z=Be13v=uA>HYc~Q-Gs}(s?{{@vu?Pt z>HL_|N}RA&&ZEl}FH<|W*|fTKV^UxIu)ss8LR#I~e#oqh4eMh-a8ssfN z$Wg?Toc-MVLwe-5y_qi)M{mnIqDc16&X{=ToSk0696$Ej(E5bpmfWemb#RuuT}NsC zwc`5eUdN2glR~<@!xq$L1_Zm*{gJb%X>!{hlVp>Oe(%xBt=%z8t}WfU;m!jsqe_SJ z@$cV;@Z^-qFu57C>hrUoB@zASp0!W-{GxeF?}Cc=2|LZhV_eSuk@KWz@xm!@X52C^ zTI(#lcK3VHxuVpvdkf4gM0Rzwey39gZUI7n{}w<9lB652|9&LQ-I3NEuTHz`wsF+? zo~)vfX#a~_dp5Q+N_{QWzPs2u=o@F1729@R@Fi+5Xl9v!#^WM5p@ zA-bJDLTW=||0m}%UAtFjVnn9eIW3!Y{Nr@t>Pqe{pq3o@^0;}Kx~YtwLTAJ(%d;ol zOv|3$NRV_|R%H-uLFV29gtlcIx}fFxj8-^YUCE z7=r)^fB*=900@8p2!H?xfB*=900@8p2!H?xfB*=900@8p2!H?xfB*=900@8p2!H?x zfB*=900@8p2!H?xfB*>miv&bD>m}gSS>Ldx1OhX3E zzk>>c?Qbhu8SnPLq8Mv8v>46A<~yUE;oV*ciY3r=tZg1Pe>qbl-28h{ zjJ3BTq5)g}CW^6kOWe2%#aQcKZvH+LWBYRxWybQCpc&YF12kaq8x&*fCvsvLRQS<6 zu^Eba{k&0(hzIgBQH*ViyS)SyW1iR0fUVySVWC~XC2`_AD3(LtiJZ6>#h4Fw`5&Cv zl{0@X+BoKWgACkJj_ML|~PMJqO!YR1H{DSgops1#Njy z?LdFGSv2jR+w!2AAN`lMObVfpnA$!OzQN@1f3N~p^BruS4~a=0WL9?TDnEg_u}a8J zI%q`Xe8)x>Xdt42o1D=XA}0{^N(Mm(N)RWu_f&GoJUW?5p$06#n%r}DgMW-Jbgbk{ zVfaQ+!nH|$eprjA-zNUN{*iv#A+&(s=0y|vv!h1;%y1f&EO`APG<2Yt5uu?Z24#@h z{j^z!Oi)Gm^?0eFz4YMh<{mWWjfbLhiyI3%UYNjO^58hH*${N1#=e@9^|^h0*Y>1` z%Ogh1isn{~HwnM-ssCfA?X1Pj!wTr6FDK~#ofmWvX1~LBlATbCGRw(hpL$Xgz0~8y zB)0YUPYwNQUKh0L<@$2hxhE`Ej^*U@!N&hNFY&+&eDeby$HXNK>PWNiPnaJ;(0eC@ z0;1l?RwuPxUGEU`tS&3ED4#a9_>x&-ZPw7f$9)@peFa?(u&oRx53Bm2X5fOVpKV#q z{cOWVvL&!0!o-|l=Qe!!Pi{r=Y50q|;ABTi1j3khoY&CWQe zhLrz@-XMV9g&?5WIji~&>PYicKS7P&3CwfM!~%&)pEs(X7b2{&_3uvih&6ZLm=?M8 zQ!Qh?N`aue4R*8!lZRFP&{b7X^}BLG)#=oO(f527`i9g>Df$?-h0eaUS!9ft!iGr| zbEnzb0-LEGnst>3w1pv2BUcBwy@lz)zd$OYvhLm_L6GJZaFV{uD;2b|8fs{ufUf z18>5>d82g^(IAWlEMbq3L^N=c7<5;S^Gc75f?3HqsA00z1<}D8A@LP`^*y?NF%8X%t_w z_TTwU!C8hmoNv{kWF`{{q4!LSeqTqqhIV_H zMA*vQ#*7=r6>}E!hjz^pbb01J#QL`oZ{zXUyCSe8^ol*;@K}UBGY)>Ih44ZGPZmxA zMQ$s*t-R(C;+9~XaWKX>HN82wp*ee=sz*+6|J`OweJzdP$4w5s6Gis?cpK+)e5I!M z!ueeVC&z65lCGeLichRVDbhbvb-V#ai z!(P0`p7G~TZdIgGeEvL>J%yWvhJIM`o`UFoi@zjVL^z#>uE7F#^3OlVxKr%V3f0iS zjlFncft3Z@89ZL_#g&y!@O6=u4IyGbjv3k%svr56cV1yz<0ituuU_^UO++)eHpb(+7Nitw$f=O!;ap3RLe*(gj+cRnhXm#`;<1^^zMvR zL+ITZdUwW}nFws;`|sPk^T0I~zJLG-fB*=900@8p2!H?xfB*=900@8p2!H?xfB*=9 z00@8p2!H?xfB*=900@8p2!H?xfB*=900@8p2!H?xfB*=900@8p2!H?xfB*=900@8p z2>cHSh;z0eP^T`!p0|?+RFqW@7WNivHSP zeGQ5w&~)rQ2H57C9gieVT!vyf^qt6w+fa-h5AO0koY<8!-x3`R%=ZQ-7Do8I$44E-Xct-Q z`#ZfsLJ#u@jW3}p%}W0vJtTK{@7owmo_~)nyz$V5cXh{{;PDMB z)+BD0p5;{7Sf`Rtl%H__-?a;G=tEg>kM{q+`cTI1GrVaqeI|3ezvM3WN1w^qU5q#V zzRzUrUd@~SSM{0vM~Bbes0>&i$=p7ZMbO0;i?KeFv2>FYbLZ30m6&(_H1y2DyZi%A zj4eO-?u((%<$rCT%Y?^O6Z3aey4|buZ1QSNZ7gaF^9-p`YN&ssE2`M4INgbRQ^b0| zHZn_^v}OH-S+lCLO#MH?wsedqUYTdUyBx6PS zlHP^6xeKF4>|mNaS`q$uP`zKH8u(YeUyG|RA3pe`PxzHrq|aPAIG+?P28X8ih*bXnE2>$JeR}l)_cj0quRyRU|jk zr@dhv3G6okIuhg0lw_m^EcJ4o-dNQ8ymGMW@0`ec{gj6#G4jtKue9h!#E!8;j4<$dVs9 zqhHc>YFu-uM~zC$<`nCVPfc$7CH$xNLyrAeWG5XoQbB_uC&v0W=Z&xM!~(DB56)ix zEHD$cAynzEq7fdq8hv~|E%0n-tyti7jJ@LPf2_DzVCBVDEHD#VA-2brf2_EIzYBNl z*EMhYx9GxsP(Pc{E9V5BwOGmUSZC?~7rSsrm!n@2bm7i^IU{u8{!6uh00ck)1V8`; zKmY_l00ck)1V8`;KmY_l00ck)1V8`;KmY_l00ck)1V8`;KmY_l00ck)1V8`;KmY_l z00jOw3P^Bv`ronrKN0O4YO{vBhT1nsqA@paMKRV+@DduZ`8!bqK;HK42T_cbi`(A) zBwt*}7gwN|w|#yGih0}r%c8$2=WYLQfMTr8Ab0)dD8}}O+y37jZ5*4=ZU3K)V&3(i zL@{ss|A#2%ZU6rr#aNRLZu@^nR5)x~-0gXwn794^IJ9xz_WuV_%-jASd*1-BPXj0B zw*S{a3uC^E(SUhIqnNk-|1K1xMOgmO{@;ADM9zw7HBGy!sy<3~XY6e&-R2ueqEg8r z^XOzMg&MFRoD@z*T)$2KVeS9f2LJsHu77RuL`0iG=PWB(qfa6lxG`3#xv_No(8acz zcSI@^F3&n{)PJ&hX@Ooq-1|*0ZIUL7OAc#A)xTI^>qC@Ba0?rJv45nWHl5@fObQ?~ zN1=V-O@B0M#!frl#GZq-XTwf7-t+(_W<${(8!tl>Nhqj4)?#Qkuy&4BN!?cG;-!1TQnxH?^LWQ*e^*U8nB1t=Be%1 z_3cya9Arg?j{hM|(VM=NNb(96qM89nOv;!&Ba3%_WoA_6+hH_!6 zAJl=u;meqjWHO!1&<>B%4kR%H$$zmS_Vy!5lyLNGi_W0Y`~_zI?JmPa5C8!X009sH z0T2KI5C8!X009sH0T2KI5C8!X009sH0T2KI5C8!X009sH0T2KI5C8!X009sH0T2KI z5C8!X009vA-y*CX)!nR>zt^6!Kp3ZO-P{ zX)EfV9Fz7qk}*4|047g4MuM-+J!8yPcT}>kMs_-y0`MJ1H0+H(fu05@$?ec7#9p!M}ZR_xxjN5lN`9u3@$<6S1Q zWz${~h_j~+TTNAWFG}tHq}cjmMxnQ{Epsrsc>eV3gORKaN z$;>P`cE#$w_tJugq=OG$D(BC4d_|sX!CfDx7`O5Ge|&-qBpx_523HL3WlHU1XO>5# zM>x^PZC7f$&Y`65*(yP^;$8<#nfBOm%`G1SQP@0mg{o?;jpEdV^6>Z9&diO46g`xLhrOZ3l z0Zx@K9V$lbF?Kzg9J=-=fhab?)K$lS`?8iTk7z?{g>|(YH>*DE|EP2AnXm1-b{atJJ%Mu}Qi!GF8;In7#*NZG{F^45Y`pObP0yn?MxQQTLw? z#$S{|U`_es{2o*(xEEVN-M*uXt$>c-S*7^nV*B58mBQYWqfMduk)zm^_0OLKtYddh z@KYTDxNFMHOyW-|Gw8 zh2MF|kZFJVyT($>o?ic<5^?z! zeceD_bp7_DFiAckWIt`PUjSK~P79&q81 z(RGA3Hb9oV`6r-&H&#T??*sWUPqZF^cT0gcNkYJ@(-~OjpfycW=+x6 z*tY1|YrD9d^2pFtfvn4s+>)z->8JW7nm>?F<+&^`a(W)H+PpAxM7P)1up7ZEUl53f z-^ljQhFW!fIQjXpig0KA@gbyi!=&U=oA;AE4iPuTaGoJQJvujX&k3FG`I)B=DtX*7 zJRU{YTRUUStma*7-CjFH5{OaETct)|#KXhD&s8Vf5C`m54c~ZvPs?(!%Ld85| zFa1g5D$iUn?MX^QZ$iT=0FoE@D@SkzP>{|axi8Po-F5t$+j}RyoVifBclPXiK8JK; z#a~Z767ibwE%$O}v$<7ZyNIIVXpxQLiyrQKcVY590`<{a#a9zQsBuJ~Pww{{pKBC<_Rc)=ArW`eG%As$M zO{M2tQmA{7XLE``lpPXb^sPcMvuKz8{Z@0igslbj_*pYI^vM}$DxP(0i)63o;FG=9 z?sb}bq!d!~ANS1nT{F|Et+mG3HuRXA)Hm()HUe=Z`J-2)3SIHoLaz(`VrvbWo!?<|xZYl7S;M~UL0z7(Dj>YXo_k~mR|U|uPjD3w^lOvwyw}$~$I~Y{W%1-jxva}i z=QMAyiGEXRHG6I{b(sF=rvxF}G^3OW52boEUG}u>sp)6nugT3?Y_s)8s=deZnVWy0 zD!{g?Opvp0FKnu=F|s;TesKArgu6OE+SAK!_V@1WQMjhuO(4p6TieNq*{@6~xtXeU zctzgT>&^3D?dclFP>C)0&@|^Mdp*5P(LaxkuSinxE9|%M5X~!Wj{WGSoAq9%|J?rb zPw*n>v~##|s(d7K!_JV-`AdnL5;KY?q^HL@E3N)~IQNzCM@9B}8toMW+@tSnyX=1$ zOv~G!e@6ChfnmAH+LXAom-6b--_c_~W&PRfvv%h_DUzSLu6V_;qDYV9r?q^~>q{rU zR*N&pV_gNJ$h!K*BZ?12oJy`|h+cNkvRWoP{ITa7$M)yB;?cBD0&(cBn9&5U{wLP% zdQ(Ska}AJ@@742_@*In6zpkZEwzM47-Hi7tfR60ossOr;2(AKxo~49NZAh#8=JHV9 zYcBIx(5MOvW@x||pH|cIqs!}mKDufcB!siniNPx*^_brZ$REx;V%i@UCOY0Oq}{vz z<_5jtu|5M;AiKLY=Go${o8ES6%T< zo6y3(4d`WiTjc4|_q8h6==zD2iZ%OnTdU1Vo8KrAE&6aNx|Y}mH;l85^OxNxGbGh~ zd(Q?3Id7MqZ>}>xR%Gh88INfm)Z-Uc0feO4KbX;R9b5%K&s74Sss;We5ONgpBxgT2 z|BxQ}ZExnw#L?Tbjwq77voj{%IcKMrFvm~m*@8?Q`JjzNo|CLe@`Qq)Pei}RSdBGu zj+ZqXlh*0*iF+Fee>?A$YtD~r=fkhY98w)^;Cu1O1+#tWo7SD~d?tN2w~|0KSJGRg z7cbi-T20MZe>r%+l2&aR!fq9YcJwI%pW_`+BdOF<1Y~Tve{&wi%#ZrA6 zhgjH#Mf|KfuvA&T+S$C^(0!XkV?b(cHi2lfQh!C!L%9|LS@CB3>NN_R=42dv>c3#K zT4iJCkh{v7tV3tDZ^-4?&||wb(hAf}E=nmK-*#=4w$tubgPyRQs(s_oqnFH*c^U6T zm2y54+OrDk4_!4C+1{?F*i@FEdN@AsVBS6Up}SPJv{gDxTXLDUZ~L4OSDM-b-NgA# zGd4Bv5RPsPsSY3zbz5wjbJlEFI>E@fL}%H2seYL!eIDIm3dTA^t)f>!3m00ck)1V8`; zKmY_l00ck)1V8`;KmY_l00ck)1V8`;KmY_l00ck)1V8`;KmY_l00ck)1V8`;KmY_l z00ck)1V8`;KmY_l00ck)1V8`;KmY_l00ck)1Oy65b8-y2{XeHuRPK8N1~$Oy)Kqm~ z0(=1h5C8!X009sH0T2KI5C8!X009sH0T2KI5C8!X009sH0T2KI5C8!X009sH0T2KI z5C8!X009sH0T2KI5C8!X009sH0T2KI5C8!X009sH0T2KI5C8!X009sftiVuC#zD9L zXXnR{{{03B%DIi}0Vl-zhDV<|Xi!1Ja5P}C3t!COi3$4FbNAit%v@M!XruhzI7!+`!T7l8 z`U#ZSQBT)ucIj)4M45RL;c?Z({2i5U_v$>GyjoKmi`v3GLu!;7>fh*!Dz++4cVh3y zABzZub)-FvoL!?(sT?g&I_UUXHGxvNsyd*(@41TPM*6fj#6R-FHxK{;5C8!X009sH z0T2KI5C8!X009sH0T2KI5C8!X009sH0T2KI5C8!X009sH0T2KI5C8!X009sH0T2KI z5C8!X009sH0T2KI5C8!X009sH0T2KI5C8!X_}?jjweiOi0XKyG1$}y>K^P6?SmS?` zDwCi-e<`VKT5o75?S!;L`W#c04{YqTn?+5b{bd;rDL==&Zt{E_g5DED{# zkPxbmFC$z(RF_WmGYE*9PNz=~Wd`~MlY&G1r&D95F{THgh@m&bkQo!L6FGxE!&fhm z6rmq#5Xhi2W(1J+qhrW1VSZs@zF{#j%&0J<7#&7H7!$3AZV>73?-xvtiu5xgEoTH! z4TF6{_31h>%gMo^)4~H{7&8nab#y`kqR0##dRUCFUzkytAw4WCj2RFX6&fC)r(+Zr z5p19r5gMpt80r@g9;Fw)Jdmy*#)vQqpazo-jr>AmA{k+Z`jJedX$-?i=Jb%5m>ChI z=x`EQM~574Fx|*NpFZ7h8a>1yKu0GmBskD_hF%auCs4;Ih7>V*r5lF;0LN;eNvnAr55t2VX>4c}G0W#ff2u)LC2%CqA7b zItz!BAU4ImIC#D)yY^#(b#(|;C2D6D^_a!_lSbWNq|+`D<&`9bEpV1NDLhVkV1q;P zIQD*Vsses0cF6zO`Fg`|H>3V)=L!GB&P$`6=OhWD$c%PXLi@L+l^LI%6AJv5Qe(Y6 z7Y2_VmDZjf9WZ-^3%2uX{@8g;0NeT1_?0+f^VIh1`t~Vy4zeOc$N!L~=uO{aGNw^} zHcn5!wbyEsIv!6Ta5sf#9|-KoG-&IIpTb26>k8rU#q3jpyFXZrYWYCIVsut;6T8sZ zKA?mbw0X`_^>{JXcF=l+xO;9MS$D zapr6L=#u_u+i01rxWrVgbE}tZ@y%Ga>HTcC+lQWyBHD%sP19QTWcjPTug|oeI_#_c zEGlQsr;7=9la_u@+_->KZ*bhgp;5o*ry+G*P~nD(2VRjQ>WZ#r(zeD)4X1MN@&wNJ?)d>E= z@gF%28D*k?!p+ee`qHYU&HT>2lC41zha5*AaIsX%3pmL1uX|A)YxY7=;evxREaiwV z+~sMp#S`nlZgHOR&gItjG2f$~9nAD~OYt{hNaPZBj3O45{8+TwW7hrTxF;9ioH$i8 zvhCpa_D$|4lZu}1u4lYEO`$HQQGChTBtJiG3e}Gs#VIYicl??NJX45h7FG+fiw~B@ zpn*3=A-76kv$0j-xf^1V!^1r#xETGDl<5 zmZaj4>4EP?D8Ch5yzuLd?nVW>1O96A*j6Bc00@8p2!H?xfB*=900@8p2!H?xfB*=9 z00@8p2!H?xfB*=900@8p2!H?xfB*=900@8p2!H?xfB*=900@8p2!H?xfB*=900@8p z2!H?x{09hN?e?)m;55U;8cbsCD8B#%N6k-p_g{@KHsp()_+oFqIQ*9w zf4(KTQ0VFN^!-(viD}LG9Ur_HUnGX$gP+^Ey*@VWsF^Kl3VNaEgo|qS;k#B7GoF*| z9;rTCdDhNsXTjGD+}1lZtw-AyqC6L!@5gCgYwS~4ry)1qXl+G9QYlV;_2x*4oTwwx z!n#SrP@cjv3vH{~bC##p9KBE@`t!PNbh(9tu?CGUx1q>ZD&Qg+ zy}Q3w>${RAD9oV(oZ^a}V%^Z_rWv27`rFEvcBg#{@ zd!}1df~EU6&+Nrglf&P{i9Xm-UAVR_@yQUW!Wl{HG*F&$hQ;ET$GsglC(C9S9{7{-Z$%PpI zOvy9?JMP!F@3z~%U157=3dMR?=jNJRSFKEq+I0;norcl#>x)Iv`#12F&)cp}_lfeX z=zSKN;(Fb0bGp~AuA1_V4WtjnM$wm~(J_St0w4eaAOHd&00JNY0w4eaAOHd&00JNY z0w4eaAOHd&00JNY0w4eaAOHd&00JNY0w4eaAOHd&00JNY0w4eaAOHd&00JNY0w4ea zAOHd&00JNY0wC~jD!^^L|GW18+XmPEpZf{{^x^=%_Vj9eu_0gV#20(>#o>H$5?`Fn z7oXybukyuBeDNE;80&tTxBdS}zIXy(Y|0lq{}Mxce`xRjuWs-EZ+caM-2wp+009sH z0T2KI5C8!X009sH0T2KI5C8!X009sH0T2KI5C8!X009sH0T2KI5C8!X009sH0T2KI z5C8!X009sH0T2KI5C8!X009sH0T2KI5C8!X_^$}aau(um|F4Fo4@ZM^{Lsa=ns-Dh z6E4p>Zq$FWd1--OK-~LHFKv=0i%Sk`#TJ*xJ|KYr2!H?xfB*=900@8p2!H?xfB*=9 z00@8p2!H?xfB*=900@8p2!H?xfB*=900@8p2!H?xfB*=900@8p2!H?xfB*=900@8p z2!H?xfB*=900@8p2!H?xfB*>mmjy<0mgH~$PejwP_WwgvskS4OB$wPgDdTO`=}|hN zVxF;={-kl0XD*obB&7}GzCVDo8W@292!H?xfB*=900@8p2!H?xfB*=900@8p2!H?x zfB*=900@8p2!H?xfB*=900@8p2!H?xfB*=900@8p2!H?xfB*=900@8p2!H?xfB*=9 z00@8p2!H?xfB*>m7X;)u%lxkWKM~Cn)c*e;ZIbYhX25h1009sH0T2KI5C8!X009sH z0T2KI5C8!X009sH0T2KI5C8!X009sH0T2KI5C8!X009sH0T2KI5C8!X009sH0T2KI z5C8!X009sH0T2KI5C8!X009sH0T2LzKMBZlrtr7_7e~{*g>X1JBZ5lS4xt77X~}^F zhyO7PkHZP0h{%~Tk}saX7n}0M&U~>SUmV33Z{myh^2KNQ;v0N%Ghh6UFBV2`G2kta z0$;54ON>9?l3XbC^m+RJs?Ef-=KPKiUW_jiL-4`R?c81;n|9R97Kg)M=sDq{ntk}L z)x?bFB)dnd&sLtbGuv74H3PTx4o&OPwuLCqMd$l*n%5fp6xM0TjW=3b(U4S%lV80# zQX(hnh_tY7(lC^#u*^c+s`i}asWnG0)QJAPZW~>0;b5#mqswh5vXu(BNJe>zIx3_# z?XrCrQ95)AeSDse{Hj2+7Y?z8E2Qsz>6~n%H6P_EUiOUfWs0_g!kw0Jan)4Kv>o4b z?klBFOnmR|uhsglWC_Z1NpsxC9faJUUHc3+_(w%Fo2+s?e3z(@Gq&6xCsaPf!v*Cj z*;2ka#5!R0S(=i%>ykHVqrJ;Yj!criaoOGNQhHgF`dpOf@=>$Wz*n;IH#RhmzYaf&Bef!7MW37ck81(rIv}utIFbk+O6DnJLtO1df!_y$wo8Bq*$5hynbn& zY3_*fl{>?LcvDD=7H*um5c2pOxZA*MIM5=H`(mD;4r<`H2IOcJ0$IZ#I z8HNX}-i|E{P)Hq5d)<&~DVK6tZ!HPsxia&B*9*Jp_VXKK&iU3Ad7GWsCyzIFIb-Vj zF7s-Nj`1>-r^4pY^bXB49**_76-LSH=dRK+ms3_L70$`+Kb2!DT<44OT#fy(VX0-I ze%`*Xr7v&xhZT33)MP%!;ymWT>C+}AF#%0Pp7a(bNQMz4OaH$F)u%q z=1y`U#y?XsjX=j85(t0*2!H?xfB*=900@8p2!H?xfB*=900@8p2!H?xfB*=900@8p z2!H?xfB*=900@8p2!H?xfB*=900@8p2!H?xfB*=900@8p2!H?xfB*=900@AXMYMu1(w+#B+h(oA6?QPZ5u6< z6_=Q*b#C>NExsAcHoc$icKgtCv>jQT(6>2h(!-|*uYA$Zq^}ZNt7)R${Z6s-;@*X; zW{jF!L>IF4U+5LA*j{H8X=;4UZLX@{qnY2=2SwZ6+EMoX4o;|GamjPxO_%H}XR0lW zdz7MMvzTzV`SsGo#N~DyL`ul_?Igt$#a~<8^;bAY33jEsef6FhTQ>P+*W;#-Ie|ZZ zhF2=2kI9ov)(Um75N^MB?bPkUC)r6d+sI4Bww(yte{DlfjlrDwoTK&nB_DULy{H#` zO*)~odYZvOnU%r{Hs2f%o%f5JEU=XEpjtNNFW z^_xCzzH`)b{*Nz8TTg#eQ+Oz2H{m_=e%_B`yYEMB4D>i*`c=N{T3r0?PgSY1i_@rK zBQEYWb}&1GFV>o4mGwI1VVBP%n*&j;5%1?$I{K)2-VnNX@Idvv;C&7^B|J`t-o1Wz zns21XPx+cG&(FFOJ#XHcDCYR^=hhWx-SGW5XSu?B z$v)L55{tAlXTO>lX;m?KrhADkQKV|l{qXkz3)PEP_;e?fEgdCtxcldc@xE`)g#;|` zxG6%(eMKWsKA%7SXufC5^yaHK=XlFMub8nIlEo| zyVRp^P6c*ZYF$+cCaOx!%41{~(dMd*ID6YKnl#Vi$&*kWl}6g(bEKR!$$$}ue$00^ zwVrs`^3*fqpxWDw6STK!*vIHQ6E~YreGv6w(iRt=2U+iP?^Z<_e2;6J)VjI* z_eK6 z-wJB8>o3GGYOfxkIx&d);%jNk_iTJd3Qm80cj}UswZly8 z&9yf@i(4~gfsqtLzLoZ*C-RP@-PjSzpE6EpocZy%&6kmTb?%adS?g06Vl7#3yu*yD z?z)XqwoSLS&Y99U&irLH;ia;^Yt|f}fNCFO->L3jJdH~WRH^!nhfBO>eG($Sw7saS z;I#2(+Xu@R*C*JHbB`LkHRSXC#l8OD&WsZ$)tUBhoLHT>xBi2PydS>xG^Zt{wKb?`dEH-59N?cw|PyNK+D_5Vs+dj7Hu!6?N ziLciAtyDOd@2VO(zj(|hpFE)u;=6|hC+=&9Q1-Q{yKMeqeDzWLx~#=})Fx?e9=iMR z+QKLI7u23SHsZRNkNKLe%{iU3)N@Y;O})Ba)Okx#?yFW&*==6y|6hA|0#?)BH++2W zT?v_z%tJ-TR3y`p$QUve5fV+iY^q)DQZhv(^Oz|zJBBi4CS^D}hLqV!NGOguN97Rj zT6>qf+jH)_-sidA>$&dt`mXC*t@T^$x2DhfuVK6V|7)OshLvw-9qq}>BA<9Ijd>hb zQ9j7hf6s2Ex2fV{?1Kre7dj}l^OZ`GH)F0{8kc2ksj{m6V%Ks*NekbnO#|;w&uX{z z7w@YMX=bPIORHU6UbnwqyT)yA7DT+ea$eHyQG%;!lB%_I&CtrbvZvZj=&&ZnzmZ?4 zR$Z;%TWjsLGhD8HxX*FB1M?f)tx>7ny70;8;`j9)J#BjO-bF?()vA7Ic{HkD;6pDr zg^#&P9u(vwFZdF4LsGoBdR3*KC#7%lj3a;m0tg_000IagfB*srAbSASop-T{kV*Hyrl1OF>74oYJq7*_3nr*0f;lStv2{;5}JTg(}zpFDO@o%Qp_I;HJ%JrrkY*gtEJQ9p~mGGj^H{Jh2i{j0x_O*E_YQ`;6t z+I5`w+V;hjdt)<_X1KTioG&wKHsK3tujG-;HGe`kCP6q6^VqsRiA!^pO;cR@v;ggezyAc z^+sRbU(>bsOXs8(=Z9L2Yu~C#%AsS{@1vGiFW0%03dK&{xP;K`h4iFED%A>ju{OQc!@bzI~dp4j}kBIL{FwKkb9CwJWt-7@t^Kg)>0o|?4@>(dG! zoqFTSj$UK>y-2$K>#5rv$L;N66Xo;9HDLISwce8)Zv7+kP|~N zRxv@xLi_tP52e!I+`~tyQuukB2ZgGOt(Yv8maA}n6FS+onwsm?#KB1N>S1=k@ft_J zOn;iZxqH3c@v5ylCjLQHr`29MI&XB%iX~T(kXp3Ma&cUbv_3ugy~$a( z%dTT}AH7@o_|qVzwxqxMma|K@QqS1u6An(dUcThPto)S0p#=vj?zsFgt?pM}hl34= z$&7T;2gst@1k69;@amJF>!~QG*weFao>;WIg<|b*J!2e=q?#2yy3(+JAK5C4g#no! zi!a-JmDzh#ep4&xM8eLt&nyR=94#}NxuxpqPM4lF%!&C}KlydLm~l>5H`*Lna9~&W z(15M|4^)!&8)jT)2{A77${Ds#6!vjb1(}DqsmufAla-U?(m=VdGDv8Ez)z*{_LF;> z`zSr!d`j9Q&^$-POb@pph0;%$>n*P2?>d?K%FxU)5(eVA?x+bRgNx(hXN!7j2Z`+4 zH>q$n$&BvpGkd+kEXgU?cK>Kq_N2EH=1geeW%Km#!RY3H9UE~aQ%fq**7+)ypyKsO zOD(MbWKKX>|DHd+eth>xqZ^T~>!Md5KVx~^HfMjW@jAov8fzBTu6IphB{3?iwHv8$ zACMWygr;`1q*7s<5^1x-3c@~?B2Wbhjr6Ds>_k0(t9eDc0>%tfzMXBVyymp9&e8WV zCw-=P9bfr=VQyfON6%(OW7gmP96L)gZQJb-_k9m)Omukj-sE)RsEvJGb_?qj3-Y&b z8;;L?p}X2&zsiEhfd!H;kL-f37skOUHXe?NgR_ zwEdbNt*x369OS7C@iQ-ZW=P6$Wk{ux@1;a&zep%6#SIo|N=fmlVSoSv2q1s}0tg_0 z00IagfB*srAb;l@vm5Mxmod0|780rsWX>*85(O}x%8y#Q|Ign9w zU2#vuCUxCfv7#^?0tg_000IagfB*srAbka28Wb-o(|iM69oY+i z#gCSY9~Bqd18@*}J)wUtd?sCtJ5-QJ41}SP(2Jdu5SQeO@hBm#EeyqW0fg`8ntLgH zg5-hfkLOCm-8{d4PFKwRyRM`y0&$+$7J;~)qv}tzi&Ml-Cr%Pxo|XwT*_yAjKOIwB zIqg)-+n<-%cY5b=Wp%@%DcL*LdN?ih>Ka&in{18vJ!FZN+1{*-YI%2i{FQMuZo$Xf z&2>Mv{9-k>?c0x!42(Xsy&2tLrk7!jO-*HKo7P?GXykHy<8`;RvC;2WFG`;tH|fEO zFRIw+7m62NCsp638YK_!{@ea)MJeVLqs~5g{53bi>|ob1b9yIK?X&&X{h{W4u3cC+ zIxQtRa@L+@{nLNd`L=uK7|#_g4{uLC{fnd1Al>=hJzve#H+NWJS+rv8c1e^)_s9CH zlH`l3|FlxB_sfz2IgkBbPJids!aq;Gbdr5!POXkhPW!%U@L*R=h3LdFx6(asU3<7F zPy6CNpEr89vWFZ$d)%<+;K~+`=QH|eFCBKa%jiKfo(<^Wv~9uKnzPscVY~6roete< z{&oGl-S+feC&y#&C<8N&|+5D;IaFh znyo4tW*QrLv3dVSXRM=h?(WNM-+jtrN%quP3vBl{ZfF~M($(=rhpe^BQl@z)rZ~(y zccgNY3n?QF@;v&_%{=3u(Qew*m31qM#Aw?8L#d@%I9VakStJ$hZ+c+LG&Yh&NK7blz@ zHvNm=?lh0}r`AQaw$1Cm#^9dS+t(=<%?)Y|UD?{dO=M-;6%D`C@$axrQfJQ4YQfPh z_L!?W*h*AW*EQ=DnmXKNV!xt(sqS?G#$=fmTsF`BnibJS|M5@bt)Heu*HE60{OwhC z)0QSiJ+>VUb6TqZ%6|Qj(G9o8rG`8|we7W0@Xd^`b|alPz8jSrvM}Q9(uhkj#{;BS zo}H*M#dPCtSxC=n4tF}Vud&JWfowwe#amk zPrmiA&C3-tH+cD+8F4RW@aWGz_lM+cf7&IisQ1@t&#(5*c3Uz>_TgII!GTxzP1g>n zU#QjefQ7?VUz@=JtGk|#OZ41vH)%kWezV?v1B0Ey-mH)W_8suS#wkDMYxEI+JJsbB z&xgrpCYl_#y1lZ8L7#EI&kH|O!)EH4w!ZGKAJ&;YV)FFOVISx3HF{z&X5nj-huVr> z1Dg)@O&Z;M+u1<-rZsb}Y_8rg&MLR?XY1$RzI0WYCJ&f8v-3=49jk3s@^*jv#dC66 zM(0}r6(9C_b3D3oi!Xg_wNeH-J8X^i89dA1&+PKYr^|*E4LG&m)*vp}xNcqPp*ZR2 ze0fBlZL{ru8lF|;6|t@Da(QOz%|!#pC4RcLKi%f%IWaGFs}3z(^Q)E5ikz8A#+FZqTL0mj~$295T92n|&W5WA7Cl z6+ZReKVnPedWm;ChA!wgE_%}f%QTaKeKl9PdKnsoMC@u6U8Bpn`?ku!XPBgU+Rs`kG6KH_$V;x$=S+xb{_YzuKE17mz%=J zTqO?*@{yN(NxkI8{Nu2sDH*R2&wqMsNQxI%ud3AZg;NxY@4U*q(q$Q72l;`f9BW(OYiUxi*ou#wXURBJ*)Josrc@C zcst#hD&@*)ZNe9Qjp*IETVeVsgP~Wt3^^5%HoQ}+B(#sZ{=e*X`)?UBGjiJ5^{VU` zXZ>Zx*sGoACJwLkOF=KUTOXRgx;tb>n*zUMRjTwE^zq?+^Of;uU7Qs|9G>PnoEhEW zjnVr<%hEHAYA4P9sA`UkH=Z^2gQeOShsqVrR_0#+9)qP&kJp0ehZ=z}$+#1lNdVke~p3%BV z2OSH)+_2jdZ)&X*6TLYu-7xoV(uGcI&Te!aFrb3Yf}S;xpAD^adwXo&(l*DEAN2IR zI&_P7!m6NyO+5Vy?StJn-H3^}G4bZ*zt;OI;={)HEibrpXZXW&IU{G^_i1UVfZ<$)BS@#vgf8M*#aPh#%Rwlni=s6!RGCf>v(9%2p zVF8mL?@xX(XLN6aW^#Xk$ zTlr?Q;o1Q^`Uf<0ZBXENY3J58JK}p*RykVVThTsw^(RG9os1T3yZsc|WJ_Uh^H<}2 zT%)h7)LwM?`RlLFTkY%{bkrZ&;l-17ll$);_|?)UHgDynMb5RhdsS(iY2eX6y8ED$ z$s_77*!W9$wYXO&&UD$n&0%nxVYO#lEk2$y=nX+|y+G)Sw&vm!o z2R3&Y?3M6G?cAdKzwPi}Iff! zNy|F-ymw?BUeIw(Zi@Hm`+k2|m`q%&%y?ZGAw6*Yi?hy!?Z!pN2RbdfZETe4lH0G} zuXVIfh9ySoXDH=eR9S(2J%Y2=OP18UbNz(7_PoN`b~YZjEZ?;&N;>fD^@)|E25-)@ ztYV!ftM4>s`t@nesvS;?ZMwPRiSa#UBlM?UNN;b{<%maezc*od1_@U#ov`#-bL02* zji2;v+x8V$O->E!Rp)&0)UcmoUbZr^tM_}h{k)vq zPv_q3Sm?O?2fUpM!jAp9c6n@RvKCLL)G|i zg;hovpSeBfNbV6;=%$QsFBPx$L8SgTc(IGp^O# z-p$9Xrhk>+s%M9`y3xE)FU#HbLGEnp5W5eD=Q>B#GF))?)`brrJ6B9jGplhn<5P|w z_5Vvhix@`$0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0 z009ILKmY**5I_I{1Q0*~0R#|0009L4y+EVl*(LS=r9x~k^p%C)Na)3QqR`hDdLyA1 z<5famdj38k))#ssp%>?;34IlzHxhb}32uJo3O`SIs1W>ZSMtecPld|O-AC?eF8B17 zoBJz$6dqH`x1@T3a&1GM8SL-r79=m77N?uv5e{_W8*5dj1cKmY**5I_I{1Q0*~ z0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY** z5I_I{1Q0*~f$|76E>4n3wAEu`{rw(7Z!Gk-LNCT^eu&@v5cd_5O7dmx`wZD~?d94L zw|?&1_=8nc)j@Tw;;rYkP)w`ycuw;-?ahQ`|F)BH_XLl}q+42X>gc=@>bn|mmO z{etRz*i=+&!K+ciM49ZtrIvAPQk<^ea=AAyYuVkC&jMV0((B*3{i>aAy{vj22MI}k z+m&xK(;Tjgo6**)_R&Fmss9XFa*9P4`yQ!vT`mu9uzX{|?Vsn!^5)64`~2-Tgo)+a z%pY1&*P_H^t$-KHGeQQW*G*|WrMBCSfq$4pC{D~s^UlqGVyw5&zr(Bl(PoM^T-;zJ ztU+^v|JACPjsOA(AbLA?{6XWVtszG z8on4i{16A0ip#9;Zz3d@?#isWui1}(EK+9Wc*B3>GKK&G2q1s}0tg_000IagfB*sr zAb8OWHe_{6if6L%cwUYnRI3EX4JN*huKb^{MX{xv3ULvVdJ9j zZ^f~`aInHAzAtYn#NzQ834H|Z~di9ANAfwZ+@1v>c78bHy?$Y%G^EJW0E}RU#&&B znAM%Tn@V2JRF4U6e&!0#GPA`(sOc<(fl=|eIOrktMnYfG*$X2^#X(s`psW*B7N@xP zesZ6o{&GKspZ9RFaFp!EkHZoh4@ICyup-FZ&C^r$V@^r@W3E?-r@4>PyJYr{!ynTp z$)~7-lz#HE*YBegPLwLx*VipjQO_;uks2bautcs|pI*`hK>#HE~AHyt9r& z_Vt+FNRumd7l(SO-NH4oZn|bDkxHM4hoR}DT9Td?VbZ!$?`^Jl$JJ2nvMvY@&~4h! zC)f4%N6yjjflTKtZ9XVM;u*!6Ou-Ro9Sv*ihWh%kjLe5N%!*47I+Dn>OGKZtxoBZ`Fhsa zCVzE3a^dlt5n~E=)QU2xld#pw_`>QdvAt4-O!X<2%KCIjb-a@ECgzM=uf$%mA%O=r zwr#H++*kQ^OlHSnbsUvat@d#t%6dAzV;3j9K9?7pR%!k(^{czosu%m=#Kj|q}la-Heo0}(fVu=BtTec}gqZ$10y$;!)ppDmr{|1^AUy8im`%Mbkvz6NS}hrTjM zi+Xlpo}Ewjt(cILamu!*PIrsE5mPPyLH+db69@4=;(acfeXuHSOGH!f@0-ILD(Tzt~& z-?{y&oo>CXdL0LeJ0+31gsO+qW`*jZCeUj8DSLChz?dEVCEGrQWO`NF>Cqk(%L-AH>ecv90l zQ)fjkuG(wRiMzLqb$>E<@ntJ*4CSAyJtkMkg1NSA`>dN z`n1sWsFrM&t=HQX&eLqhELE`GvYwXD0}xnm}S4*cHW zy88$ZpBq)`xp#TyJK)M9!F z??zqhXB%{DxcsG^Sn$Q-CuCMy*}&DSzTs8txk~q_NpA6Gstz`K*6(g6?wB9?@zvls zAxkWNLT2Xt+&PORaTm<)b@{-y=Y(vXq2j0eP9QxkA z+SG`&)-Nkg_ly0x;sS>$ma{-v&I*RUQgY;XwYjQ{Uf(IUY(sFGZlQ z`2>Yu(7$=h5N?6RovSc35_P*%kfmes_d@A-!w>O+AL0w8;<6tT(o_h_?1HN(R7+QzVaY9;g0Z6 zoo0@o(os^VwV_f`AKRVqQqT-;;%_o+XO$TV*z>ZwM;-_}CE zKpkm*x{C20p)dPgulU7MDqQCmw=b>L`MEMYBY*zf&$WEM-s$F6eO9eCx0^5hqvy8# z->3Ggb*?scn*OVuk5hzI^o}sAWh*i#-O1cA)WkV< zQo*&%p6$(kn)KkFU0zF_On~SHrzwoz-uR^6#DE)-{R8XMO$2?H33ikPT zcPh;W{NzC)%D_p*>6#){<}cHdMMHyxt96R6+%HJ{JuCbM)6Dpu}-I46It$;u_@7?>?0Dq z>Mxt->M~+dqdLoTS4{C9Fx5dwmEN27bjQ(48*Y4z=#%E-XB@gN#&2(rdHcJ({j2Ko z(?+$$w^d2;sa0wAs!VTj5n&;@Jh0^7Od4w@egCVHtIJ6l2PyrYFW&czgkC(z%(l4~ z+xK7E&(1)nTGOvp6_y=0b!~XJ_5g`x`>ed)i%f+SsY)n1o@G6tOA3LeVOFtNlo<9G z#!JVprQ&iG3*p_dT;)Q@D_6k~@>C&mxxYNnJSfzBf}3hW*-ym~H{l88=j|V;RC@iJ z*A(H<=ny~v0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0 z009If660b7>i_?vzB~2*ssB&?f8oYZQp17z|9|tS_>NQDRsi+?OEuY1uG$CG|EK;x z_5Z2=PyPRY(?C*rDj`$i_@i27}73&iKDm|KI3`H%;pQt3iGh z{!;({d!u;N|EK=HSn*q|8eYDiTGann-$zRQDS1p%|Gyl~U{e3TRLNl!0R#|0009IL zKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009IL_}>&TDc*R=r`p7C zS(SXn%19V5^ZR$j%l|ki`QNoaPA5hH0R#|0009ILKmY**5I_I{1Q0*~0R#|0009IL zKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~ z0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY** z5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0 z009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{ z1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009IL zKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~ z0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY** z5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0 z009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{ z1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009IL zKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~ z0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY** z5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0 z009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{ z1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009IL zKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~ z0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY** z5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0 z009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{ z1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009IL zKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~ z0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY** z5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0 z009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{ z1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009IL zKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~ z0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY** z5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0 z009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{ z1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009IL zKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~ z0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY** z5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0 z009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{ z1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009IL zKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~ z0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY** z5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0 z009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{ z1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009IL zKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~ z0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY** z5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0 z009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{ z1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009IL RKmY**5I_I{1pd Date: Mon, 2 Dec 2024 19:45:08 +0100 Subject: [PATCH 13/21] sqldb+invoices: Optimize invoice fetching when the reference is only a hash The current sqlc GetInvoice query experiences incremental slowdowns during the migration of large invoice databases, primarily due to its complex predicate set. For this specific use case, a streamlined GetInvoiceByHash function provides a more efficient solution, maintaining near-constant lookup times even with extensive table sizes. --- invoices/invoices.go | 5 +++ invoices/sql_store.go | 68 +++++++++++++++++++++++---------- sqldb/sqlc/invoices.sql.go | 32 ++++++++++++++++ sqldb/sqlc/querier.go | 1 + sqldb/sqlc/queries/invoices.sql | 5 +++ 5 files changed, 91 insertions(+), 20 deletions(-) diff --git a/invoices/invoices.go b/invoices/invoices.go index c48629c583a..32164cbe179 100644 --- a/invoices/invoices.go +++ b/invoices/invoices.go @@ -187,6 +187,11 @@ func (r InvoiceRef) Modifier() RefModifier { return r.refModifier } +// IsHashOnly returns true if the invoice ref only contains a payment hash. +func (r InvoiceRef) IsHashOnly() bool { + return r.payHash != nil && r.payAddr == nil && r.setID == nil +} + // String returns a human-readable representation of an InvoiceRef. func (r InvoiceRef) String() string { var ids []string diff --git a/invoices/sql_store.go b/invoices/sql_store.go index 8a819e5ba9d..f7ca0263763 100644 --- a/invoices/sql_store.go +++ b/invoices/sql_store.go @@ -51,6 +51,9 @@ type SQLInvoiceQueries interface { //nolint:interfacebloat GetInvoice(ctx context.Context, arg sqlc.GetInvoiceParams) ([]sqlc.Invoice, error) + GetInvoiceByHash(ctx context.Context, hash []byte) (sqlc.Invoice, + error) + GetInvoiceBySetID(ctx context.Context, setID []byte) ([]sqlc.Invoice, error) @@ -354,22 +357,31 @@ func (i *SQLStore) AddInvoice(ctx context.Context, return newInvoice.AddIndex, nil } -// fetchInvoice fetches the common invoice data and the AMP state for the -// invoice with the given reference. -func fetchInvoice(ctx context.Context, db SQLInvoiceQueries, - ref InvoiceRef) (*Invoice, error) { +// getInvoiceByRef fetches the invoice with the given reference. The reference +// may be a payment hash, a payment address, or a set ID for an AMP sub invoice. +func getInvoiceByRef(ctx context.Context, + db SQLInvoiceQueries, ref InvoiceRef) (sqlc.Invoice, error) { + // If the reference is empty, we can't look up the invoice. if ref.PayHash() == nil && ref.PayAddr() == nil && ref.SetID() == nil { - return nil, ErrInvoiceNotFound + return sqlc.Invoice{}, ErrInvoiceNotFound } - var ( - invoice *Invoice - params sqlc.GetInvoiceParams - ) + // If the reference is a hash only, we can look up the invoice directly + // by the payment hash which is faster. + if ref.IsHashOnly() { + invoice, err := db.GetInvoiceByHash(ctx, ref.PayHash()[:]) + if errors.Is(err, sql.ErrNoRows) { + return sqlc.Invoice{}, ErrInvoiceNotFound + } + + return invoice, err + } + + // Otherwise the reference may include more fields, so we'll need to + // assemble the query parameters based on the fields that are set. + var params sqlc.GetInvoiceParams - // Given all invoices are uniquely identified by their payment hash, - // we can use it to query a specific invoice. if ref.PayHash() != nil { params.Hash = ref.PayHash()[:] } @@ -405,18 +417,34 @@ func fetchInvoice(ctx context.Context, db SQLInvoiceQueries, } else { rows, err = db.GetInvoice(ctx, params) } + switch { case len(rows) == 0: - return nil, ErrInvoiceNotFound + return sqlc.Invoice{}, ErrInvoiceNotFound case len(rows) > 1: // In case the reference is ambiguous, meaning it matches more // than one invoice, we'll return an error. - return nil, fmt.Errorf("ambiguous invoice ref: %s: %s", - ref.String(), spew.Sdump(rows)) + return sqlc.Invoice{}, fmt.Errorf("ambiguous invoice ref: "+ + "%s: %s", ref.String(), spew.Sdump(rows)) case err != nil: - return nil, fmt.Errorf("unable to fetch invoice: %w", err) + return sqlc.Invoice{}, fmt.Errorf("unable to fetch invoice: %w", + err) + } + + return rows[0], nil +} + +// fetchInvoice fetches the common invoice data and the AMP state for the +// invoice with the given reference. +func fetchInvoice(ctx context.Context, db SQLInvoiceQueries, ref InvoiceRef) ( + *Invoice, error) { + + // Fetch the invoice from the database. + sqlInvoice, err := getInvoiceByRef(ctx, db, ref) + if err != nil { + return nil, err } var ( @@ -433,8 +461,8 @@ func fetchInvoice(ctx context.Context, db SQLInvoiceQueries, fetchAmpHtlcs = true case HtlcSetOnlyModifier: - // In this case we'll fetch all AMP HTLCs for the - // specified set id. + // In this case we'll fetch all AMP HTLCs for the specified set + // id. if ref.SetID() == nil { return nil, fmt.Errorf("set ID is required to use " + "the HTLC set only modifier") @@ -454,8 +482,8 @@ func fetchInvoice(ctx context.Context, db SQLInvoiceQueries, } // Fetch the rest of the invoice data and fill the invoice struct. - _, invoice, err = fetchInvoiceData( - ctx, db, rows[0], setID, fetchAmpHtlcs, + _, invoice, err := fetchInvoiceData( + ctx, db, sqlInvoice, setID, fetchAmpHtlcs, ) if err != nil { return nil, err @@ -658,7 +686,7 @@ func fetchAmpState(ctx context.Context, db SQLInvoiceQueries, invoiceID int64, invoiceKeys[key] = struct{}{} - if htlc.State != HtlcStateCanceled { //nolint: ll + if htlc.State != HtlcStateCanceled { amtPaid += htlc.Amt } } diff --git a/sqldb/sqlc/invoices.sql.go b/sqldb/sqlc/invoices.sql.go index b98a78c69ee..13ac8094a12 100644 --- a/sqldb/sqlc/invoices.sql.go +++ b/sqldb/sqlc/invoices.sql.go @@ -255,6 +255,38 @@ func (q *Queries) GetInvoice(ctx context.Context, arg GetInvoiceParams) ([]Invoi return items, nil } +const getInvoiceByHash = `-- name: GetInvoiceByHash :one +SELECT i.id, i.hash, i.preimage, i.settle_index, i.settled_at, i.memo, i.amount_msat, i.cltv_delta, i.expiry, i.payment_addr, i.payment_request, i.payment_request_hash, i.state, i.amount_paid_msat, i.is_amp, i.is_hodl, i.is_keysend, i.created_at +FROM invoices i +WHERE i.hash = $1 +` + +func (q *Queries) GetInvoiceByHash(ctx context.Context, hash []byte) (Invoice, error) { + row := q.db.QueryRowContext(ctx, getInvoiceByHash, hash) + var i Invoice + err := row.Scan( + &i.ID, + &i.Hash, + &i.Preimage, + &i.SettleIndex, + &i.SettledAt, + &i.Memo, + &i.AmountMsat, + &i.CltvDelta, + &i.Expiry, + &i.PaymentAddr, + &i.PaymentRequest, + &i.PaymentRequestHash, + &i.State, + &i.AmountPaidMsat, + &i.IsAmp, + &i.IsHodl, + &i.IsKeysend, + &i.CreatedAt, + ) + return i, err +} + const getInvoiceBySetID = `-- name: GetInvoiceBySetID :many SELECT i.id, i.hash, i.preimage, i.settle_index, i.settled_at, i.memo, i.amount_msat, i.cltv_delta, i.expiry, i.payment_addr, i.payment_request, i.payment_request_hash, i.state, i.amount_paid_msat, i.is_amp, i.is_hodl, i.is_keysend, i.created_at FROM invoices i diff --git a/sqldb/sqlc/querier.go b/sqldb/sqlc/querier.go index 6f05b32c2cd..c63f7fadb85 100644 --- a/sqldb/sqlc/querier.go +++ b/sqldb/sqlc/querier.go @@ -24,6 +24,7 @@ type Querier interface { // from different invoices. It is the caller's responsibility to ensure that // we bubble up an error in those cases. GetInvoice(ctx context.Context, arg GetInvoiceParams) ([]Invoice, error) + GetInvoiceByHash(ctx context.Context, hash []byte) (Invoice, error) GetInvoiceBySetID(ctx context.Context, setID []byte) ([]Invoice, error) GetInvoiceFeatures(ctx context.Context, invoiceID int64) ([]InvoiceFeature, error) GetInvoiceHTLCCustomRecords(ctx context.Context, invoiceID int64) ([]GetInvoiceHTLCCustomRecordsRow, error) diff --git a/sqldb/sqlc/queries/invoices.sql b/sqldb/sqlc/queries/invoices.sql index f57c9ab7659..db1f46e617b 100644 --- a/sqldb/sqlc/queries/invoices.sql +++ b/sqldb/sqlc/queries/invoices.sql @@ -54,6 +54,11 @@ WHERE ( GROUP BY i.id LIMIT 2; +-- name: GetInvoiceByHash :one +SELECT i.* +FROM invoices i +WHERE i.hash = $1; + -- name: GetInvoiceBySetID :many SELECT i.* FROM invoices i From 8d20e2a23be9f950c80ca9e2add927fbcf2641cd Mon Sep 17 00:00:00 2001 From: Andras Banki-Horvath Date: Tue, 17 Sep 2024 15:18:12 +0200 Subject: [PATCH 14/21] lnd: run invoice migration on startup This commit runs the invoice migration if the user has a KV SQL backend configured. --- config_builder.go | 93 ++++++++++++++++++++--------------- invoices/testdata/channel.db | Bin 1048576 -> 1048576 bytes lncfg/db.go | 5 +- sample-lnd.conf | 3 ++ sqldb/migrations.go | 9 ++++ 5 files changed, 70 insertions(+), 40 deletions(-) diff --git a/config_builder.go b/config_builder.go index b5e19e5dc7c..45a923e05a2 100644 --- a/config_builder.go +++ b/config_builder.go @@ -51,6 +51,7 @@ import ( "github.com/lightningnetwork/lnd/rpcperms" "github.com/lightningnetwork/lnd/signal" "github.com/lightningnetwork/lnd/sqldb" + "github.com/lightningnetwork/lnd/sqldb/sqlc" "github.com/lightningnetwork/lnd/sweep" "github.com/lightningnetwork/lnd/walletunlocker" "github.com/lightningnetwork/lnd/watchtower" @@ -60,6 +61,16 @@ import ( "gopkg.in/macaroon-bakery.v2/bakery" ) +const ( + // invoiceMigrationBatchSize is the number of invoices that will be + // migrated in a single batch. + invoiceMigrationBatchSize = 1000 + + // invoiceMigration is the version of the migration that will be used to + // migrate invoices from the kvdb to the sql database. + invoiceMigration = 7 +) + // GrpcRegistrar is an interface that must be satisfied by an external subserver // that wants to be able to register its own gRPC server onto lnd's main // grpc.Server instance. @@ -1038,7 +1049,7 @@ func (d *DefaultDatabaseBuilder) BuildDatabase( if err != nil { cleanUp() - err := fmt.Errorf("unable to open graph DB: %w", err) + err = fmt.Errorf("unable to open graph DB: %w", err) d.logger.Error(err) return nil, nil, err @@ -1072,65 +1083,69 @@ func (d *DefaultDatabaseBuilder) BuildDatabase( case err != nil: cleanUp() - err := fmt.Errorf("unable to open graph DB: %w", err) + err = fmt.Errorf("unable to open graph DB: %w", err) d.logger.Error(err) return nil, nil, err } - // Instantiate a native SQL invoice store if the flag is set. + // Instantiate a native SQL store if the flag is set. if d.cfg.DB.UseNativeSQL { - // We need to apply all migrations to the native SQL store - // before we can use it. - err := dbs.NativeSQLStore.ApplyAllMigrations( - ctx, sqldb.GetMigrations(), - ) - if err != nil { - cleanUp() - err := fmt.Errorf("unable to apply migrations: %w", err) - d.logger.Error(err) - - return nil, nil, err - } + migrations := sqldb.GetMigrations() + + // If the user has not explicitly disabled the SQL invoice + // migration, attach the custom migration function to invoice + // migration (version 7). Even if this custom migration is + // disabled, the regular native SQL store migrations will still + // run. If the database version is already above this custom + // migration's version (7), it will be skipped permanently, + // regardless of the flag. + if !d.cfg.DB.SkipSQLInvoiceMigration { + migrationFn := func(tx *sqlc.Queries) error { + return invoices.MigrateInvoicesToSQL( + ctx, dbs.ChanStateDB.Backend, + dbs.ChanStateDB, tx, + invoiceMigrationBatchSize, + ) + } - // KV invoice db resides in the same database as the channel - // state DB. Let's query the database to see if we have any - // invoices there. If we do, we won't allow the user to start - // lnd with native SQL enabled, as we don't currently migrate - // the invoices to the new database schema. - invoiceSlice, err := dbs.ChanStateDB.QueryInvoices( - ctx, invoices.InvoiceQuery{ - NumMaxInvoices: 1, - }, - ) - if err != nil { - cleanUp() - d.logger.Errorf("Unable to query KV invoice DB: %v", - err) + // Make sure we attach the custom migration function to + // the correct migration version. + for i := 0; i < len(migrations); i++ { + if migrations[i].Version != invoiceMigration { + continue + } - return nil, nil, err + migrations[i].MigrationFn = migrationFn + } } - if len(invoiceSlice.Invoices) > 0 { + // We need to apply all migrations to the native SQL store + // before we can use it. + err = dbs.NativeSQLStore.ApplyAllMigrations(ctx, migrations) + if err != nil { cleanUp() - err := fmt.Errorf("found invoices in the KV invoice " + - "DB, migration to native SQL is not yet " + - "supported") + err = fmt.Errorf("faild to run migrations for the "+ + "native SQL store: %w", err) d.logger.Error(err) return nil, nil, err } + // With the DB ready and migrations applied, we can now create + // the base DB and transaction executor for the native SQL + // invoice store. baseDB := dbs.NativeSQLStore.GetBaseDB() executor := sqldb.NewTransactionExecutor( - baseDB, - func(tx *sql.Tx) invoices.SQLInvoiceQueries { + baseDB, func(tx *sql.Tx) invoices.SQLInvoiceQueries { return baseDB.WithTx(tx) }, ) - dbs.InvoiceDB = invoices.NewSQLStore( + sqlInvoiceDB := invoices.NewSQLStore( executor, clock.NewDefaultClock(), ) + + dbs.InvoiceDB = sqlInvoiceDB } else { dbs.InvoiceDB = dbs.ChanStateDB } @@ -1143,7 +1158,7 @@ func (d *DefaultDatabaseBuilder) BuildDatabase( if err != nil { cleanUp() - err := fmt.Errorf("unable to open %s database: %w", + err = fmt.Errorf("unable to open %s database: %w", lncfg.NSTowerClientDB, err) d.logger.Error(err) return nil, nil, err @@ -1158,7 +1173,7 @@ func (d *DefaultDatabaseBuilder) BuildDatabase( if err != nil { cleanUp() - err := fmt.Errorf("unable to open %s database: %w", + err = fmt.Errorf("unable to open %s database: %w", lncfg.NSTowerServerDB, err) d.logger.Error(err) return nil, nil, err diff --git a/invoices/testdata/channel.db b/invoices/testdata/channel.db index 78741eae9c63a6284b389e9c6b14f71fa4d32aa9..69397f529dd11a2bcab9d1fa8e64e78d18b0d29b 100644 GIT binary patch delta 406 zcmZo@aA;_7n4rL@I8jkvK!cG14C>x)Gc9iXY`8ISz5V3>_KFZ$b*SvcEfaR+D6fbI z$xb&o!^F9nqk*YmGS>kj_<(c=l)o3q7X`8vp!`QbJ`V$fGu(i85CMh|r~*5n?c6~22`HaY zgn%jvt$}Y@1AhYmEhkHc delta 153 zcmZo@aA;_7n4rL@Fi}xnK!uS34EpDMi%e9}jN2Hv-hT3bdqs$>DpdBDgWbW Date: Tue, 17 Sep 2024 15:20:25 +0200 Subject: [PATCH 15/21] itest: add integration test for invoice migration --- itest/list_on_test.go | 4 + itest/lnd_invoice_migration_test.go | 303 ++++++++++++++++++++++++++++ 2 files changed, 307 insertions(+) create mode 100644 itest/lnd_invoice_migration_test.go diff --git a/itest/list_on_test.go b/itest/list_on_test.go index be3244fe5ed..6d4232d723a 100644 --- a/itest/list_on_test.go +++ b/itest/list_on_test.go @@ -682,6 +682,10 @@ var allTestCases = []*lntest.TestCase{ Name: "quiescence", TestFunc: testQuiescence, }, + { + Name: "invoice migration", + TestFunc: testInvoiceMigration, + }, } // appendPrefixed is used to add a prefix to each test name in the subtests diff --git a/itest/lnd_invoice_migration_test.go b/itest/lnd_invoice_migration_test.go new file mode 100644 index 00000000000..fdebfdc862a --- /dev/null +++ b/itest/lnd_invoice_migration_test.go @@ -0,0 +1,303 @@ +package itest + +import ( + "database/sql" + "path" + "time" + + "github.com/lightningnetwork/lnd/channeldb" + "github.com/lightningnetwork/lnd/clock" + "github.com/lightningnetwork/lnd/invoices" + "github.com/lightningnetwork/lnd/kvdb" + "github.com/lightningnetwork/lnd/kvdb/postgres" + "github.com/lightningnetwork/lnd/kvdb/sqlbase" + "github.com/lightningnetwork/lnd/kvdb/sqlite" + "github.com/lightningnetwork/lnd/lncfg" + "github.com/lightningnetwork/lnd/lnrpc" + "github.com/lightningnetwork/lnd/lnrpc/routerrpc" + "github.com/lightningnetwork/lnd/lntest" + "github.com/lightningnetwork/lnd/lntest/node" + "github.com/lightningnetwork/lnd/sqldb" + "github.com/stretchr/testify/require" +) + +func openChannelDB(ht *lntest.HarnessTest, hn *node.HarnessNode) *channeldb.DB { + sqlbase.Init(0) + var ( + backend kvdb.Backend + err error + ) + + switch hn.Cfg.DBBackend { + case node.BackendSqlite: + backend, err = kvdb.Open( + kvdb.SqliteBackendName, + ht.Context(), + &sqlite.Config{ + Timeout: defaultTimeout, + BusyTimeout: defaultTimeout, + }, + hn.Cfg.DBDir(), lncfg.SqliteChannelDBName, + lncfg.NSChannelDB, + ) + require.NoError(ht, err) + + case node.BackendPostgres: + backend, err = kvdb.Open( + kvdb.PostgresBackendName, ht.Context(), + &postgres.Config{ + Dsn: hn.Cfg.PostgresDsn, + Timeout: defaultTimeout, + }, lncfg.NSChannelDB, + ) + require.NoError(ht, err) + } + + db, err := channeldb.CreateWithBackend(backend) + require.NoError(ht, err) + + return db +} + +func openNativeSQLInvoiceDB(ht *lntest.HarnessTest, + hn *node.HarnessNode) invoices.InvoiceDB { + + var db *sqldb.BaseDB + + switch hn.Cfg.DBBackend { + case node.BackendSqlite: + sqliteStore, err := sqldb.NewSqliteStore( + &sqldb.SqliteConfig{ + Timeout: defaultTimeout, + BusyTimeout: defaultTimeout, + }, + path.Join( + hn.Cfg.DBDir(), + lncfg.SqliteNativeDBName, + ), + ) + require.NoError(ht, err) + db = sqliteStore.BaseDB + + case node.BackendPostgres: + postgresStore, err := sqldb.NewPostgresStore( + &sqldb.PostgresConfig{ + Dsn: hn.Cfg.PostgresDsn, + Timeout: defaultTimeout, + }, + ) + require.NoError(ht, err) + db = postgresStore.BaseDB + } + + executor := sqldb.NewTransactionExecutor( + db, func(tx *sql.Tx) invoices.SQLInvoiceQueries { + return db.WithTx(tx) + }, + ) + + return invoices.NewSQLStore( + executor, clock.NewDefaultClock(), + ) +} + +// clampTime truncates the time of the passed invoice to the microsecond level. +func clampTime(invoice *invoices.Invoice) { + trunc := func(t time.Time) time.Time { + return t.Truncate(time.Microsecond) + } + + invoice.CreationDate = trunc(invoice.CreationDate) + + if !invoice.SettleDate.IsZero() { + invoice.SettleDate = trunc(invoice.SettleDate) + } + + if invoice.IsAMP() { + for setID, ampState := range invoice.AMPState { + if ampState.SettleDate.IsZero() { + continue + } + + ampState.SettleDate = trunc(ampState.SettleDate) + invoice.AMPState[setID] = ampState + } + } + + for _, htlc := range invoice.Htlcs { + if !htlc.AcceptTime.IsZero() { + htlc.AcceptTime = trunc(htlc.AcceptTime) + } + + if !htlc.ResolveTime.IsZero() { + htlc.ResolveTime = trunc(htlc.ResolveTime) + } + } +} + +// testInvoiceMigration tests that the invoice migration from the old KV store +// to the new native SQL store works as expected. +func testInvoiceMigration(ht *lntest.HarnessTest) { + alice := ht.NewNodeWithCoins("Alice", nil) + bob := ht.NewNodeWithCoins("Bob", []string{"--accept-amp"}) + + // Make sure we run the test with SQLite or Postgres. + if bob.Cfg.DBBackend != node.BackendSqlite && + bob.Cfg.DBBackend != node.BackendPostgres { + + ht.Skip("node not running with SQLite or Postgres") + } + + // Skip the test if the node is already running with native SQL. + if bob.Cfg.NativeSQL { + ht.Skip("node already running with native SQL") + } + + ht.EnsureConnected(alice, bob) + cp := ht.OpenChannel( + alice, bob, lntest.OpenChannelParams{ + Amt: 1000000, + PushAmt: 500000, + }, + ) + + // Alice and bob should have one channel open with each other now. + ht.AssertNodeNumChannels(alice, 1) + ht.AssertNodeNumChannels(bob, 1) + + // Step 1: Add 10 normal invoices and pay 5 of them. + normalInvoices := make([]*lnrpc.AddInvoiceResponse, 10) + for i := 0; i < 10; i++ { + invoice := &lnrpc.Invoice{ + Value: int64(1000 + i*100), // Varying amounts + IsAmp: false, + } + + resp := bob.RPC.AddInvoice(invoice) + normalInvoices[i] = resp + } + + for _, inv := range normalInvoices { + sendReq := &routerrpc.SendPaymentRequest{ + PaymentRequest: inv.PaymentRequest, + TimeoutSeconds: 60, + FeeLimitMsat: noFeeLimitMsat, + } + + ht.SendPaymentAssertSettled(alice, sendReq) + } + + // Step 2: Add 10 AMP invoices and send multiple payments to 5 of them. + ampInvoices := make([]*lnrpc.AddInvoiceResponse, 10) + for i := 0; i < 10; i++ { + invoice := &lnrpc.Invoice{ + Value: int64(2000 + i*200), // Varying amounts + IsAmp: true, + } + + resp := bob.RPC.AddInvoice(invoice) + ampInvoices[i] = resp + } + + // Select the first 5 invoices to send multiple AMP payments. + for i := 0; i < 5; i++ { + inv := ampInvoices[i] + + // Send 3 payments to each. + for j := 0; j < 3; j++ { + payReq := &routerrpc.SendPaymentRequest{ + PaymentRequest: inv.PaymentRequest, + TimeoutSeconds: 60, + FeeLimitMsat: noFeeLimitMsat, + Amp: true, + } + + // Send a normal AMP payment first, then a spontaneous + // AMP payment. + ht.SendPaymentAssertSettled(alice, payReq) + + // Generate an external payment address when attempting + // to pseudo-reuse an AMP invoice. When using an + // external payment address, we'll also expect an extra + // invoice to appear in the ListInvoices response, since + // a new invoice will be JIT inserted under a different + // payment address than the one in the invoice. + // + // NOTE: This will only work when the peer has + // spontaneous AMP payments enabled otherwise no invoice + // under a different payment_addr will be found. + payReq.PaymentAddr = ht.Random32Bytes() + ht.SendPaymentAssertSettled(alice, payReq) + } + } + + // We can close the channel now. + ht.CloseChannel(alice, cp) + + // Now stop Bob so we can open the DB for examination. + require.NoError(ht, bob.Stop()) + + // Open the KV channel DB. + db := openChannelDB(ht, bob) + + query := invoices.InvoiceQuery{ + IndexOffset: 0, + // As a sanity check, fetch more invoices than we have + // to ensure that we did not add any extra invoices. + NumMaxInvoices: 9999, + } + + // Fetch all invoices and make sure we have 35 in total. + result1, err := db.QueryInvoices(ht.Context(), query) + require.NoError(ht, err) + require.Equal(ht, 35, len(result1.Invoices)) + numInvoices := len(result1.Invoices) + + bob.SetExtraArgs([]string{"--db.use-native-sql"}) + + // Now run the migration flow three times to ensure that each run is + // idempotent. + for i := 0; i < 3; i++ { + // Start bob with the native SQL flag set. This will trigger the + // migration to run. + require.NoError(ht, bob.Start(ht.Context())) + + // At this point the migration should have completed and the + // node should be running with native SQL. Now we'll stop Bob + // again so we can safely examine the database. + require.NoError(ht, bob.Stop()) + + // Now we'll open the database with the native SQL backend and + // fetch the invoices again to ensure that they were migrated + // correctly. + sqlInvoiceDB := openNativeSQLInvoiceDB(ht, bob) + result2, err := sqlInvoiceDB.QueryInvoices(ht.Context(), query) + require.NoError(ht, err) + + require.Equal(ht, numInvoices, len(result2.Invoices)) + + // Simply zero out the add index so we don't fail on that when + // comparing. + for i := 0; i < numInvoices; i++ { + result1.Invoices[i].AddIndex = 0 + result2.Invoices[i].AddIndex = 0 + + // Clamp the precision to microseconds. Note that we + // need to override both invoices as the original + // invoice is coming from KV database, it was stored as + // a binary serialized Go time.Time value which has + // nanosecond precision. The migrated invoice is stored + // in SQL in PostgreSQL has microsecond precision while + // in SQLite it has nanosecond precision if using TEXT + // storage class. + clampTime(&result1.Invoices[i]) + clampTime(&result2.Invoices[i]) + require.Equal( + ht, result1.Invoices[i], result2.Invoices[i], + ) + } + } + + // Start Bob again so the test can complete. + require.NoError(ht, bob.Start(ht.Context())) +} From 0839d4ba7b3576cd2af8c02f708e644ee733fcf0 Mon Sep 17 00:00:00 2001 From: Andras Banki-Horvath Date: Wed, 18 Sep 2024 14:18:09 +0200 Subject: [PATCH 16/21] itest: remove obsolete itest --- itest/list_on_test.go | 4 ---- itest/lnd_misc_test.go | 39 --------------------------------------- 2 files changed, 43 deletions(-) diff --git a/itest/list_on_test.go b/itest/list_on_test.go index 6d4232d723a..38cd56d3503 100644 --- a/itest/list_on_test.go +++ b/itest/list_on_test.go @@ -626,10 +626,6 @@ var allTestCases = []*lntest.TestCase{ Name: "open channel locked balance", TestFunc: testOpenChannelLockedBalance, }, - { - Name: "nativesql no migration", - TestFunc: testNativeSQLNoMigration, - }, { Name: "sweep cpfp anchor outgoing timeout", TestFunc: testSweepCPFPAnchorOutgoingTimeout, diff --git a/itest/lnd_misc_test.go b/itest/lnd_misc_test.go index 30dba0a8782..98b1121c6a6 100644 --- a/itest/lnd_misc_test.go +++ b/itest/lnd_misc_test.go @@ -1,7 +1,6 @@ package itest import ( - "context" "encoding/hex" "fmt" "os" @@ -1245,44 +1244,6 @@ func testSignVerifyMessageWithAddr(ht *lntest.HarnessTest) { require.False(ht, respValid.Valid, "external signature did validate") } -// testNativeSQLNoMigration tests that nodes that have invoices would not start -// up with native SQL enabled, as we don't currently support migration of KV -// invoices to the new SQL schema. -func testNativeSQLNoMigration(ht *lntest.HarnessTest) { - alice := ht.NewNode("Alice", nil) - - // Make sure we run the test with SQLite or Postgres. - if alice.Cfg.DBBackend != node.BackendSqlite && - alice.Cfg.DBBackend != node.BackendPostgres { - - ht.Skip("node not running with SQLite or Postgres") - } - - // Skip the test if the node is already running with native SQL. - if alice.Cfg.NativeSQL { - ht.Skip("node already running with native SQL") - } - - alice.RPC.AddInvoice(&lnrpc.Invoice{ - Value: 10_000, - }) - - alice.SetExtraArgs([]string{"--db.use-native-sql"}) - - // Restart the node manually as we're really only interested in the - // startup error. - require.NoError(ht, alice.Stop()) - require.NoError(ht, alice.StartLndCmd(context.Background())) - - // We expect the node to fail to start up with native SQL enabled, as we - // have an invoice in the KV store. - require.Error(ht, alice.WaitForProcessExit()) - - // Reset the extra args and restart alice. - alice.SetExtraArgs(nil) - require.NoError(ht, alice.Start(ht.Context())) -} - // testSendSelectedCoins tests that we're able to properly send the selected // coins from the wallet to a single target address. func testSendSelectedCoins(ht *lntest.HarnessTest) { From 5e3ef3ec0c62f8954ac6dfbeda32c8afaeb83381 Mon Sep 17 00:00:00 2001 From: Andras Banki-Horvath Date: Fri, 10 Jan 2025 11:18:44 +0100 Subject: [PATCH 17/21] invoices+sql: use the stored AmtPaid value instead of recalculating Previously we'd recalculate the paid amount by summing amounts of settled HTLCs. This approach while correct would stop the SQL migration process as some KV invoices may have incorrectly stored paid amounts. --- invoices/sql_store.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/invoices/sql_store.go b/invoices/sql_store.go index f7ca0263763..55517bfdd48 100644 --- a/invoices/sql_store.go +++ b/invoices/sql_store.go @@ -1576,13 +1576,6 @@ func fetchInvoiceData(ctx context.Context, db SQLInvoiceQueries, if len(htlcs) > 0 { invoice.Htlcs = htlcs - var amountPaid lnwire.MilliSatoshi - for _, htlc := range htlcs { - if htlc.State == HtlcStateSettled { - amountPaid += htlc.Amt - } - } - invoice.AmtPaid = amountPaid } return hash, invoice, nil From ea98933317a0b9b4470aca6494099e6975716d73 Mon Sep 17 00:00:00 2001 From: Andras Banki-Horvath Date: Fri, 10 Jan 2025 11:24:14 +0100 Subject: [PATCH 18/21] invoices: allow migration test to work on kv sqlite channeldb --- invoices/kv_sql_migration_test.go | 63 +++++++++++++++++++++++++++++-- 1 file changed, 60 insertions(+), 3 deletions(-) diff --git a/invoices/kv_sql_migration_test.go b/invoices/kv_sql_migration_test.go index 5d36b5c642b..b3048a17bf4 100644 --- a/invoices/kv_sql_migration_test.go +++ b/invoices/kv_sql_migration_test.go @@ -3,12 +3,18 @@ package invoices_test import ( "context" "database/sql" + "os" + "path" "testing" "time" "github.com/lightningnetwork/lnd/channeldb" "github.com/lightningnetwork/lnd/clock" invpkg "github.com/lightningnetwork/lnd/invoices" + "github.com/lightningnetwork/lnd/kvdb" + "github.com/lightningnetwork/lnd/kvdb/sqlbase" + "github.com/lightningnetwork/lnd/kvdb/sqlite" + "github.com/lightningnetwork/lnd/lncfg" "github.com/lightningnetwork/lnd/sqldb" "github.com/lightningnetwork/lnd/sqldb/sqlc" "github.com/stretchr/testify/require" @@ -132,15 +138,66 @@ func TestMigrationWithChannelDB(t *testing.T) { for _, test := range tests { test := test t.Run(test.name, func(t *testing.T) { - store := channeldb.OpenForTesting(t, test.dbPath) + var kvStore *channeldb.DB + + // First check if we have a channel.sqlite file in the + // testdata directory. If we do, we'll use that as the + // channel db for the migration test. + chanDBPath := path.Join( + test.dbPath, lncfg.SqliteChannelDBName, + ) + + // Just some sane defaults for the sqlite config. + const ( + timeout = 5 * time.Second + maxConns = 50 + ) + + sqliteConfig := &sqlite.Config{ + Timeout: timeout, + BusyTimeout: timeout, + MaxConnections: maxConns, + } + + if fileExists(chanDBPath) { + sqlbase.Init(maxConns) + + sqliteBackend, err := kvdb.Open( + kvdb.SqliteBackendName, + context.Background(), + sqliteConfig, test.dbPath, + lncfg.SqliteChannelDBName, + lncfg.NSChannelDB, + ) + + require.NoError(t, err) + kvStore, err = channeldb.CreateWithBackend( + sqliteBackend, + ) + + require.NoError(t, err) + } else { + kvStore = channeldb.OpenForTesting( + t, test.dbPath, + ) + } t.Run("Postgres", func(t *testing.T) { - migrationTest(t, store, false) + migrationTest(t, kvStore, false) }) t.Run("SQLite", func(t *testing.T) { - migrationTest(t, store, true) + migrationTest(t, kvStore, true) }) }) } } + +func fileExists(filename string) bool { + info, err := os.Stat(filename) + if os.IsNotExist(err) { + return false + } + + return !info.IsDir() +} From 84598b6dc1573f4137c29ee7fba0ee02763bda5b Mon Sep 17 00:00:00 2001 From: Andras Banki-Horvath Date: Tue, 21 Jan 2025 17:00:15 +0100 Subject: [PATCH 19/21] sqldb: ensure schema definitions are fully SQLite compatible Previously, we applied replacements to our schema definitions to make them compatible with both SQLite and Postgres backends, as the files were not fully compatible with either. With this change, the only replacement required for SQLite has been moved to the generator script. This adjustment ensures compatibility by enabling auto-incrementing primary keys that are treated as 64-bit integers by sqlc. --- invoices/sql_migration.go | 4 +-- scripts/gen_sqlc_docker.sh | 28 +++++++++++++++++++ sqldb/postgres.go | 3 +- sqldb/sqlc/invoices.sql.go | 4 +-- sqldb/sqlc/migrations/000001_invoices.up.sql | 8 +++--- .../migrations/000003_invoice_events.up.sql | 2 +- sqldb/sqlc/models.go | 4 +-- sqldb/sqlite.go | 11 +++----- 8 files changed, 44 insertions(+), 20 deletions(-) diff --git a/invoices/sql_migration.go b/invoices/sql_migration.go index af0e4865f47..0d590320dd4 100644 --- a/invoices/sql_migration.go +++ b/invoices/sql_migration.go @@ -100,7 +100,7 @@ func createInvoiceHashIndex(ctx context.Context, db kvdb.Backend, return tx.InsertKVInvoiceKeyAndAddIndex(ctx, sqlc.InsertKVInvoiceKeyAndAddIndexParams{ - ID: int32(invoiceKey), + ID: int64(invoiceKey), AddIndex: int64(addIndexNo), }, ) @@ -132,7 +132,7 @@ func createInvoiceHashIndex(ctx context.Context, db kvdb.Backend, return tx.SetKVInvoicePaymentHash(ctx, sqlc.SetKVInvoicePaymentHashParams{ - ID: int32(invoiceKey), + ID: int64(invoiceKey), Hash: k, }, ) diff --git a/scripts/gen_sqlc_docker.sh b/scripts/gen_sqlc_docker.sh index e011d689e8e..148ca05be21 100755 --- a/scripts/gen_sqlc_docker.sh +++ b/scripts/gen_sqlc_docker.sh @@ -2,12 +2,40 @@ set -e +# restore_files is a function to restore original schema files. +restore_files() { + echo "Restoring SQLite bigint patch..." + for file in sqldb/sqlc/migrations/*.up.sql.bak; do + mv "$file" "${file%.bak}" + done +} + + +# Set trap to call restore_files on script exit. This makes sure the old files +# are always restored. +trap restore_files EXIT + # Directory of the script file, independent of where it's called from. DIR="$(cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)" # Use the user's cache directories GOCACHE=`go env GOCACHE` GOMODCACHE=`go env GOMODCACHE` +# SQLite doesn't support "BIGINT PRIMARY KEY" for auto-incrementing primary +# keys, only "INTEGER PRIMARY KEY". Internally it uses 64-bit integers for +# numbers anyway, independent of the column type. So we can just use +# "INTEGER PRIMARY KEY" and it will work the same under the hood, giving us +# auto incrementing 64-bit integers. +# _BUT_, sqlc will generate Go code with int32 if we use "INTEGER PRIMARY KEY", +# even though we want int64. So before we run sqlc, we need to patch the +# source schema SQL files to use "BIGINT PRIMARY KEY" instead of "INTEGER +# PRIMARY KEY". +echo "Applying SQLite bigint patch..." +for file in sqldb/sqlc/migrations/*.up.sql; do + echo "Patching $file" + sed -i.bak -E 's/INTEGER PRIMARY KEY/BIGINT PRIMARY KEY/g' "$file" +done + echo "Generating sql models and queries in go..." docker run \ diff --git a/sqldb/postgres.go b/sqldb/postgres.go index 455ecb40572..d271d214e4e 100644 --- a/sqldb/postgres.go +++ b/sqldb/postgres.go @@ -29,8 +29,7 @@ var ( // has some differences. postgresSchemaReplacements = map[string]string{ "BLOB": "BYTEA", - "INTEGER PRIMARY KEY": "SERIAL PRIMARY KEY", - "BIGINT PRIMARY KEY": "BIGSERIAL PRIMARY KEY", + "INTEGER PRIMARY KEY": "BIGSERIAL PRIMARY KEY", "TIMESTAMP": "TIMESTAMP WITHOUT TIME ZONE", } diff --git a/sqldb/sqlc/invoices.sql.go b/sqldb/sqlc/invoices.sql.go index 13ac8094a12..1cd7dfff4e2 100644 --- a/sqldb/sqlc/invoices.sql.go +++ b/sqldb/sqlc/invoices.sql.go @@ -591,7 +591,7 @@ INSERT INTO invoice_payment_hashes ( ` type InsertKVInvoiceKeyAndAddIndexParams struct { - ID int32 + ID int64 AddIndex int64 } @@ -675,7 +675,7 @@ WHERE id = $1 ` type SetKVInvoicePaymentHashParams struct { - ID int32 + ID int64 Hash []byte } diff --git a/sqldb/sqlc/migrations/000001_invoices.up.sql b/sqldb/sqlc/migrations/000001_invoices.up.sql index e4c80a26fd5..7f8c653d54c 100644 --- a/sqldb/sqlc/migrations/000001_invoices.up.sql +++ b/sqldb/sqlc/migrations/000001_invoices.up.sql @@ -11,7 +11,7 @@ INSERT INTO invoice_sequences(name, current_value) VALUES ('settle_index', 0); -- invoices table contains all the information shared by all the invoice types. CREATE TABLE IF NOT EXISTS invoices ( -- The id of the invoice. Translates to the AddIndex. - id BIGINT PRIMARY KEY, + id INTEGER PRIMARY KEY, -- The hash for this invoice. The invoice hash will always identify that -- invoice. @@ -102,8 +102,8 @@ CREATE INDEX IF NOT EXISTS invoice_feature_invoice_id_idx ON invoice_features(in CREATE TABLE IF NOT EXISTS invoice_htlcs ( -- The id for this htlc. Used in foreign keys instead of the -- htlc_id/chan_id combination. - id BIGINT PRIMARY KEY, - + id INTEGER PRIMARY KEY, + -- Short chan id indicating the htlc's origin. uint64 stored as text. chan_id TEXT NOT NULL, @@ -111,7 +111,7 @@ CREATE TABLE IF NOT EXISTS invoice_htlcs ( -- int64 in the database. The application layer must check that there is no -- overflow when storing/loading this column. htlc_id BIGINT NOT NULL, - + -- The htlc's amount in millisatoshis. amount_msat BIGINT NOT NULL, diff --git a/sqldb/sqlc/migrations/000003_invoice_events.up.sql b/sqldb/sqlc/migrations/000003_invoice_events.up.sql index a3718c0eb4f..b9280a9d2a7 100644 --- a/sqldb/sqlc/migrations/000003_invoice_events.up.sql +++ b/sqldb/sqlc/migrations/000003_invoice_events.up.sql @@ -29,7 +29,7 @@ VALUES -- AMP sub invoices. This table can be used to create a historical view of what -- happened to the node's invoices. CREATE TABLE IF NOT EXISTS invoice_events ( - id BIGINT PRIMARY KEY, + id INTEGER PRIMARY KEY, -- added_at is the timestamp when this event was added. added_at TIMESTAMP NOT NULL, diff --git a/sqldb/sqlc/models.go b/sqldb/sqlc/models.go index f69d0352bb2..7d322045172 100644 --- a/sqldb/sqlc/models.go +++ b/sqldb/sqlc/models.go @@ -58,7 +58,7 @@ type InvoiceEvent struct { } type InvoiceEventType struct { - ID int32 + ID int64 Description string } @@ -88,7 +88,7 @@ type InvoiceHtlcCustomRecord struct { } type InvoicePaymentHash struct { - ID int32 + ID int64 AddIndex int64 Hash []byte } diff --git a/sqldb/sqlite.go b/sqldb/sqlite.go index 59cb0356929..81e1f26b395 100644 --- a/sqldb/sqlite.go +++ b/sqldb/sqlite.go @@ -28,13 +28,10 @@ const ( ) var ( - // sqliteSchemaReplacements is a map of schema strings that need to be - // replaced for sqlite. This is needed because sqlite doesn't directly - // support the BIGINT type for primary keys, so we need to replace it - // with INTEGER. - sqliteSchemaReplacements = map[string]string{ - "BIGINT PRIMARY KEY": "INTEGER PRIMARY KEY", - } + // sqliteSchemaReplacements maps schema strings to their SQLite + // compatible replacements. Currently, no replacements are needed as our + // SQL schema definition files are designed for SQLite compatibility. + sqliteSchemaReplacements = map[string]string{} // Make sure SqliteStore implements the MigrationExecutor interface. _ MigrationExecutor = (*SqliteStore)(nil) From 97c025f28930fa97b1c15007fa6651ee0bde3620 Mon Sep 17 00:00:00 2001 From: Andras Banki-Horvath Date: Thu, 23 Jan 2025 08:48:32 +0100 Subject: [PATCH 20/21] invoices: raise the number of allowed clients for the Postgres fixture --- sqldb/postgres_fixture.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqldb/postgres_fixture.go b/sqldb/postgres_fixture.go index ce21aab7d43..d7814df7897 100644 --- a/sqldb/postgres_fixture.go +++ b/sqldb/postgres_fixture.go @@ -59,7 +59,7 @@ func NewTestPgFixture(t *testing.T, expiry time.Duration) *TestPgFixture { "postgres", "-c", "log_statement=all", "-c", "log_destination=stderr", - "-c", "max_connections=1000", + "-c", "max_connections=5000", }, }, func(config *docker.HostConfig) { // Set AutoRemove to true so that stopped container goes away From b1a462ddbaa0d12130a2768b6efadaea2d40cae7 Mon Sep 17 00:00:00 2001 From: Andras Banki-Horvath Date: Tue, 17 Sep 2024 16:51:19 +0200 Subject: [PATCH 21/21] docs: update release notes for 0.19.0 --- docs/release-notes/release-notes-0.19.0.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/release-notes/release-notes-0.19.0.md b/docs/release-notes/release-notes-0.19.0.md index 4fc64028d35..21654cee2a0 100644 --- a/docs/release-notes/release-notes-0.19.0.md +++ b/docs/release-notes/release-notes-0.19.0.md @@ -241,6 +241,11 @@ The underlying functionality between those two options remain the same. transactions can run at once, increasing efficiency. Includes several bugfixes to allow this to work properly. +* [Migrate KV invoices to + SQL](https://github.com/lightningnetwork/lnd/pull/8831) as part of a larger + effort to support SQL databases natively in LND. + + ## Code Health * A code refactor that [moves all the graph related DB code out of the @@ -265,6 +270,7 @@ The underlying functionality between those two options remain the same. * Abdullahi Yunus * Alex Akselrod +* Andras Banki-Horvath * Animesh Bilthare * Boris Nagaev * Carla Kirk-Cohen