diff --git a/eclair-core/src/main/scala/fr/acinq/eclair/Eclair.scala b/eclair-core/src/main/scala/fr/acinq/eclair/Eclair.scala index 06aa17d69c..820f06ef9a 100644 --- a/eclair-core/src/main/scala/fr/acinq/eclair/Eclair.scala +++ b/eclair-core/src/main/scala/fr/acinq/eclair/Eclair.scala @@ -135,7 +135,7 @@ trait Eclair { def getInvoice(paymentHash: ByteVector32)(implicit timeout: Timeout): Future[Option[Invoice]] - def pendingInvoices(from: TimestampSecond, to: TimestampSecond)(implicit timeout: Timeout): Future[Seq[Invoice]] + def pendingInvoices(from: TimestampSecond, to: TimestampSecond, paginated_opt: Option[Paginated])(implicit timeout: Timeout): Future[Seq[Invoice]] def allInvoices(from: TimestampSecond, to: TimestampSecond, paginated_opt: Option[Paginated])(implicit timeout: Timeout): Future[Seq[Invoice]] @@ -439,8 +439,8 @@ class EclairImpl(appKit: Kit) extends Eclair with Logging { appKit.nodeParams.db.payments.listIncomingPayments(from.toTimestampMilli, to.toTimestampMilli, paginated_opt).map(_.invoice) } - override def pendingInvoices(from: TimestampSecond, to: TimestampSecond)(implicit timeout: Timeout): Future[Seq[Invoice]] = Future { - appKit.nodeParams.db.payments.listPendingIncomingPayments(from.toTimestampMilli, to.toTimestampMilli).map(_.invoice) + override def pendingInvoices(from: TimestampSecond, to: TimestampSecond, paginated_opt: Option[Paginated])(implicit timeout: Timeout): Future[Seq[Invoice]] = Future { + appKit.nodeParams.db.payments.listPendingIncomingPayments(from.toTimestampMilli, to.toTimestampMilli, paginated_opt).map(_.invoice) } override def getInvoice(paymentHash: ByteVector32)(implicit timeout: Timeout): Future[Option[Invoice]] = Future { diff --git a/eclair-core/src/main/scala/fr/acinq/eclair/Paginated.scala b/eclair-core/src/main/scala/fr/acinq/eclair/Paginated.scala index 166fc96634..a21f9b341c 100644 --- a/eclair-core/src/main/scala/fr/acinq/eclair/Paginated.scala +++ b/eclair-core/src/main/scala/fr/acinq/eclair/Paginated.scala @@ -18,7 +18,9 @@ package fr.acinq.eclair /** * Created by rorp on 03/11/2022. - * + */ + +/** * Simple class for pagination database and API queries. */ case class Paginated(count: Int, skip: Int) { diff --git a/eclair-core/src/main/scala/fr/acinq/eclair/db/DualDatabases.scala b/eclair-core/src/main/scala/fr/acinq/eclair/db/DualDatabases.scala index d5c22e93a7..92bec058cb 100644 --- a/eclair-core/src/main/scala/fr/acinq/eclair/db/DualDatabases.scala +++ b/eclair-core/src/main/scala/fr/acinq/eclair/db/DualDatabases.scala @@ -311,9 +311,9 @@ case class DualPaymentsDb(primary: PaymentsDb, secondary: PaymentsDb) extends Pa primary.listIncomingPayments(from, to, paginated_opt) } - override def listPendingIncomingPayments(from: TimestampMilli, to: TimestampMilli): Seq[IncomingPayment] = { - runAsync(secondary.listPendingIncomingPayments(from, to)) - primary.listPendingIncomingPayments(from, to) + override def listPendingIncomingPayments(from: TimestampMilli, to: TimestampMilli, paginated_opt: Option[Paginated]): Seq[IncomingPayment] = { + runAsync(secondary.listPendingIncomingPayments(from, to, paginated_opt)) + primary.listPendingIncomingPayments(from, to, paginated_opt) } override def listExpiredIncomingPayments(from: TimestampMilli, to: TimestampMilli): Seq[IncomingPayment] = { diff --git a/eclair-core/src/main/scala/fr/acinq/eclair/db/PaymentsDb.scala b/eclair-core/src/main/scala/fr/acinq/eclair/db/PaymentsDb.scala index b8824d1274..de4398403e 100644 --- a/eclair-core/src/main/scala/fr/acinq/eclair/db/PaymentsDb.scala +++ b/eclair-core/src/main/scala/fr/acinq/eclair/db/PaymentsDb.scala @@ -55,7 +55,7 @@ trait IncomingPaymentsDb { def listIncomingPayments(from: TimestampMilli, to: TimestampMilli, paginated_opt: Option[Paginated]): Seq[IncomingPayment] /** List all pending (not paid, not expired) incoming payments in the given time range (milli-seconds). */ - def listPendingIncomingPayments(from: TimestampMilli, to: TimestampMilli): Seq[IncomingPayment] + def listPendingIncomingPayments(from: TimestampMilli, to: TimestampMilli, paginated_opt: Option[Paginated]): Seq[IncomingPayment] /** List all expired (not paid) incoming payments in the given time range (milli-seconds). */ def listExpiredIncomingPayments(from: TimestampMilli, to: TimestampMilli): Seq[IncomingPayment] diff --git a/eclair-core/src/main/scala/fr/acinq/eclair/db/pg/PgPaymentsDb.scala b/eclair-core/src/main/scala/fr/acinq/eclair/db/pg/PgPaymentsDb.scala index d6526f10c3..0b9f1db3e6 100644 --- a/eclair-core/src/main/scala/fr/acinq/eclair/db/pg/PgPaymentsDb.scala +++ b/eclair-core/src/main/scala/fr/acinq/eclair/db/pg/PgPaymentsDb.scala @@ -346,9 +346,9 @@ class PgPaymentsDb(implicit ds: DataSource, lock: PgLock) extends PaymentsDb wit } } - override def listPendingIncomingPayments(from: TimestampMilli, to: TimestampMilli): Seq[IncomingPayment] = withMetrics("payments/list-incoming-pending", DbBackends.Postgres) { + override def listPendingIncomingPayments(from: TimestampMilli, to: TimestampMilli, paginated_opt: Option[Paginated]): Seq[IncomingPayment] = withMetrics("payments/list-incoming-pending", DbBackends.Postgres) { withLock { pg => - using(pg.prepareStatement("SELECT * FROM payments.received WHERE received_msat IS NULL AND created_at > ? AND created_at < ? AND expire_at > ? ORDER BY created_at")) { statement => + using(pg.prepareStatement(limited("SELECT * FROM payments.received WHERE received_msat IS NULL AND created_at > ? AND created_at < ? AND expire_at > ? ORDER BY created_at", paginated_opt))) { statement => statement.setTimestamp(1, from.toSqlTimestamp) statement.setTimestamp(2, to.toSqlTimestamp) statement.setTimestamp(3, Timestamp.from(Instant.now())) diff --git a/eclair-core/src/main/scala/fr/acinq/eclair/db/sqlite/SqlitePaymentsDb.scala b/eclair-core/src/main/scala/fr/acinq/eclair/db/sqlite/SqlitePaymentsDb.scala index ca2297154a..bb7e10edbe 100644 --- a/eclair-core/src/main/scala/fr/acinq/eclair/db/sqlite/SqlitePaymentsDb.scala +++ b/eclair-core/src/main/scala/fr/acinq/eclair/db/sqlite/SqlitePaymentsDb.scala @@ -344,8 +344,8 @@ class SqlitePaymentsDb(val sqlite: Connection) extends PaymentsDb with Logging { } } - override def listPendingIncomingPayments(from: TimestampMilli, to: TimestampMilli): Seq[IncomingPayment] = withMetrics("payments/list-incoming-pending", DbBackends.Sqlite) { - using(sqlite.prepareStatement("SELECT * FROM received_payments WHERE received_msat IS NULL AND created_at > ? AND created_at < ? AND expire_at > ? ORDER BY created_at")) { statement => + override def listPendingIncomingPayments(from: TimestampMilli, to: TimestampMilli, paginated_opt: Option[Paginated]): Seq[IncomingPayment] = withMetrics("payments/list-incoming-pending", DbBackends.Sqlite) { + using(sqlite.prepareStatement(limited("SELECT * FROM received_payments WHERE received_msat IS NULL AND created_at > ? AND created_at < ? AND expire_at > ? ORDER BY created_at", paginated_opt))) { statement => statement.setLong(1, from.toLong) statement.setLong(2, to.toLong) statement.setLong(3, TimestampMilli.now().toLong) diff --git a/eclair-core/src/test/scala/fr/acinq/eclair/db/PaymentsDbSpec.scala b/eclair-core/src/test/scala/fr/acinq/eclair/db/PaymentsDbSpec.scala index 0e8218c7da..57dd15730d 100644 --- a/eclair-core/src/test/scala/fr/acinq/eclair/db/PaymentsDbSpec.scala +++ b/eclair-core/src/test/scala/fr/acinq/eclair/db/PaymentsDbSpec.scala @@ -559,7 +559,7 @@ class PaymentsDbSpec extends AnyFunSuite { assert(db.listIncomingPayments(0 unixms, now, None) == Seq(expiredPayment1, expiredPayment2, expiredPayment3, pendingPayment1, pendingPayment2, pendingPayment3, payment1.copy(status = IncomingPaymentStatus.Pending), payment2.copy(status = IncomingPaymentStatus.Pending), payment3.copy(status = IncomingPaymentStatus.Pending))) assert(db.listExpiredIncomingPayments(0 unixms, now) == Seq(expiredPayment1, expiredPayment2, expiredPayment3)) assert(db.listReceivedIncomingPayments(0 unixms, now) == Nil) - assert(db.listPendingIncomingPayments(0 unixms, now) == Seq(pendingPayment1, pendingPayment2, pendingPayment3, payment1.copy(status = IncomingPaymentStatus.Pending), payment2.copy(status = IncomingPaymentStatus.Pending), payment3.copy(status = IncomingPaymentStatus.Pending))) + assert(db.listPendingIncomingPayments(0 unixms, now, None) == Seq(pendingPayment1, pendingPayment2, pendingPayment3, payment1.copy(status = IncomingPaymentStatus.Pending), payment2.copy(status = IncomingPaymentStatus.Pending), payment3.copy(status = IncomingPaymentStatus.Pending))) db.receiveIncomingPayment(paidInvoice1.paymentHash, 461 msat, receivedAt1) db.receiveIncomingPayment(paidInvoice1.paymentHash, 100 msat, receivedAt2) // adding another payment to this invoice should sum @@ -572,14 +572,14 @@ class PaymentsDbSpec extends AnyFunSuite { assert(db.listIncomingPayments(0 unixms, now, None) == Seq(expiredPayment1, expiredPayment2, expiredPayment3, pendingPayment1, pendingPayment2, pendingPayment3, payment1, payment2, payment3)) assert(db.listIncomingPayments(now - 60.seconds, now, None) == Seq(pendingPayment1, pendingPayment2, pendingPayment3, payment1, payment2, payment3)) - assert(db.listIncomingPayments(0 unixms, now, Some(Paginated(0,0))) == Seq()) - assert(db.listIncomingPayments(0 unixms, now, Some(Paginated(0,3))) == Seq()) - assert(db.listIncomingPayments(0 unixms, now, Some(Paginated(3,0))) == Seq(expiredPayment1, expiredPayment2, expiredPayment3)) - assert(db.listIncomingPayments(0 unixms, now, Some(Paginated(3,3))) == Seq(pendingPayment1, pendingPayment2, pendingPayment3)) - assert(db.listPendingIncomingPayments(0 unixms, now) == Seq(pendingPayment1, pendingPayment2, pendingPayment3)) + assert(db.listIncomingPayments(0 unixms, now, Some(Paginated(0, 0))) == Seq()) + assert(db.listIncomingPayments(0 unixms, now, Some(Paginated(0, 3))) == Seq()) + assert(db.listIncomingPayments(0 unixms, now, Some(Paginated(3, 0))) == Seq(expiredPayment1, expiredPayment2, expiredPayment3)) + assert(db.listIncomingPayments(0 unixms, now, Some(Paginated(3, 3))) == Seq(pendingPayment1, pendingPayment2, pendingPayment3)) + assert(db.listPendingIncomingPayments(0 unixms, now, None) == Seq(pendingPayment1, pendingPayment2, pendingPayment3)) + assert(db.listPendingIncomingPayments(0 unixms, now, Some(Paginated(1, 1))) == Seq(pendingPayment2)) assert(db.listReceivedIncomingPayments(0 unixms, now) == Seq(payment1, payment2, payment3)) - assert(db.removeIncomingPayment(paidInvoice1.paymentHash).isFailure) db.removeIncomingPayment(paidInvoice1.paymentHash).failed.foreach(e => assert(e.getMessage == "Cannot remove a received incoming payment")) assert(db.removeIncomingPayment(paidInvoice3.paymentHash).isFailure) diff --git a/eclair-core/src/test/scala/fr/acinq/eclair/payment/receive/InvoicePurgerSpec.scala b/eclair-core/src/test/scala/fr/acinq/eclair/payment/receive/InvoicePurgerSpec.scala index 77278740b5..0093fb5bde 100644 --- a/eclair-core/src/test/scala/fr/acinq/eclair/payment/receive/InvoicePurgerSpec.scala +++ b/eclair-core/src/test/scala/fr/acinq/eclair/payment/receive/InvoicePurgerSpec.scala @@ -62,7 +62,7 @@ class InvoicePurgerSpec extends ScalaTestWithActorTestKit(ConfigFactory.load("ap val now = TimestampMilli.now() assert(db.listIncomingPayments(0 unixms, now, None) == expiredPayments ++ pendingPayments ++ paidPayments) assert(db.listIncomingPayments(now - 100.days, now, None) == pendingPayments ++ paidPayments) - assert(db.listPendingIncomingPayments(0 unixms, now) == pendingPayments) + assert(db.listPendingIncomingPayments(0 unixms, now, None) == pendingPayments) assert(db.listReceivedIncomingPayments(0 unixms, now) == paidPayments) assert(db.listExpiredIncomingPayments(0 unixms, now) == expiredPayments) diff --git a/eclair-node/src/main/scala/fr/acinq/eclair/api/directives/ExtraDirectives.scala b/eclair-node/src/main/scala/fr/acinq/eclair/api/directives/ExtraDirectives.scala index 8627aabd7f..d0b8e4720e 100644 --- a/eclair-node/src/main/scala/fr/acinq/eclair/api/directives/ExtraDirectives.scala +++ b/eclair-node/src/main/scala/fr/acinq/eclair/api/directives/ExtraDirectives.scala @@ -43,9 +43,6 @@ trait ExtraDirectives extends Directives { val nodeIdFormParam: NameReceptacle[PublicKey] = "nodeId".as[PublicKey] val nodeIdsFormParam: NameUnmarshallerReceptacle[List[PublicKey]] = "nodeIds".as[List[PublicKey]](pubkeyListUnmarshaller) val paymentHashFormParam: NameUnmarshallerReceptacle[ByteVector32] = "paymentHash".as[ByteVector32](sha256HashUnmarshaller) - // we limit default values to avoid accidentally reading too much data from the DB - val fromFormParam: NameDefaultUnmarshallerReceptacle[TimestampSecond] = "from".as[TimestampSecond](timestampSecondUnmarshaller).?(TimestampSecond.now() - 1.day) - val toFormParam: NameDefaultUnmarshallerReceptacle[TimestampSecond] = "to".as[TimestampSecond](timestampSecondUnmarshaller).?(TimestampSecond.now()) val amountMsatFormParam: NameReceptacle[MilliSatoshi] = "amountMsat".as[MilliSatoshi] val invoiceFormParam: NameReceptacle[Bolt11Invoice] = "invoice".as[Bolt11Invoice] val routeFormatFormParam: NameUnmarshallerReceptacle[RouteFormat] = "format".as[RouteFormat](routeFormatUnmarshaller) @@ -55,6 +52,13 @@ trait ExtraDirectives extends Directives { val countFormParam: NameReceptacle[Int] = "count".as[Int] val skipFormParam: NameReceptacle[Int] = "skip".as[Int] + // @formatter:off + // We limit default values to avoid accidentally reading too much data from the DB. + // We cannot use a val, otherwise the default values would not be recomputed for every request. + def fromFormParam(): NameDefaultUnmarshallerReceptacle[TimestampSecond] = "from".as[TimestampSecond](timestampSecondUnmarshaller).?(TimestampSecond.now() - 1.day) + def toFormParam(): NameDefaultUnmarshallerReceptacle[TimestampSecond] = "to".as[TimestampSecond](timestampSecondUnmarshaller).?(TimestampSecond.now()) + // @formatter:on + // custom directive to fail with HTTP 404 (and JSON response) if the element was not found def completeOrNotFound[T](fut: Future[Option[T]])(implicit marshaller: ToResponseMarshaller[T]): Route = onComplete(fut) { case Success(Some(t)) => complete(t) diff --git a/eclair-node/src/main/scala/fr/acinq/eclair/api/handlers/Channel.scala b/eclair-node/src/main/scala/fr/acinq/eclair/api/handlers/Channel.scala index 350c298c9c..0e3341e2b5 100644 --- a/eclair-node/src/main/scala/fr/acinq/eclair/api/handlers/Channel.scala +++ b/eclair-node/src/main/scala/fr/acinq/eclair/api/handlers/Channel.scala @@ -123,7 +123,7 @@ trait Channel { } val channelStats: Route = postRequest("channelstats") { implicit t => - formFields(fromFormParam, toFormParam) { (from, to) => + formFields(fromFormParam(), toFormParam()) { (from, to) => complete(eclairApi.channelStats(from, to)) } } diff --git a/eclair-node/src/main/scala/fr/acinq/eclair/api/handlers/Fees.scala b/eclair-node/src/main/scala/fr/acinq/eclair/api/handlers/Fees.scala index 74f5c96f1f..d83f8a6409 100644 --- a/eclair-node/src/main/scala/fr/acinq/eclair/api/handlers/Fees.scala +++ b/eclair-node/src/main/scala/fr/acinq/eclair/api/handlers/Fees.scala @@ -28,7 +28,7 @@ trait Fees { import fr.acinq.eclair.api.serde.JsonSupport.{formats, marshaller, serialization} val networkFees: Route = postRequest("networkfees") { implicit t => - formFields(fromFormParam, toFormParam) { (from, to) => + formFields(fromFormParam(), toFormParam()) { (from, to) => complete(eclairApi.networkFees(from, to)) } } diff --git a/eclair-node/src/main/scala/fr/acinq/eclair/api/handlers/Invoice.scala b/eclair-node/src/main/scala/fr/acinq/eclair/api/handlers/Invoice.scala index 2d78e261d6..60dd8dd223 100644 --- a/eclair-node/src/main/scala/fr/acinq/eclair/api/handlers/Invoice.scala +++ b/eclair-node/src/main/scala/fr/acinq/eclair/api/handlers/Invoice.scala @@ -43,15 +43,17 @@ trait Invoice { val listInvoices: Route = postRequest("listinvoices") { implicit t => withPaginated { paginated_opt => - formFields(fromFormParam, toFormParam) { (from, to) => + formFields(fromFormParam(), toFormParam()) { (from, to) => complete(eclairApi.allInvoices(from, to, paginated_opt)) } } } val listPendingInvoices: Route = postRequest("listpendinginvoices") { implicit t => - formFields(fromFormParam, toFormParam) { (from, to) => - complete(eclairApi.pendingInvoices(from, to)) + withPaginated { paginated_opt => + formFields(fromFormParam(), toFormParam()) { (from, to) => + complete(eclairApi.pendingInvoices(from, to, paginated_opt)) + } } } diff --git a/eclair-node/src/main/scala/fr/acinq/eclair/api/handlers/Node.scala b/eclair-node/src/main/scala/fr/acinq/eclair/api/handlers/Node.scala index a3a835015b..286c71e903 100644 --- a/eclair-node/src/main/scala/fr/acinq/eclair/api/handlers/Node.scala +++ b/eclair-node/src/main/scala/fr/acinq/eclair/api/handlers/Node.scala @@ -57,7 +57,7 @@ trait Node { } val audit: Route = postRequest("audit") { implicit t => - formFields(fromFormParam, toFormParam) { (from, to) => + formFields(fromFormParam(), toFormParam()) { (from, to) => complete(eclairApi.audit(from, to)) } }