Fix static default from and to pagination

eclair-core/src/main/scala/fr/acinq/eclair/Eclair.scala

@@ -135,7 +135,7 @@ trait Eclair {
 
   def getInvoice(paymentHash: ByteVector32)(implicit timeout: Timeout): Future[Option[Invoice]]
 
-  def pendingInvoices(from: TimestampSecond, to: TimestampSecond)(implicit timeout: Timeout): Future[Seq[Invoice]]
+  def pendingInvoices(from: TimestampSecond, to: TimestampSecond, paginated_opt: Option[Paginated])(implicit timeout: Timeout): Future[Seq[Invoice]]
 
   def allInvoices(from: TimestampSecond, to: TimestampSecond, paginated_opt: Option[Paginated])(implicit timeout: Timeout): Future[Seq[Invoice]]
 
@@ -439,8 +439,8 @@ class EclairImpl(appKit: Kit) extends Eclair with Logging {
     appKit.nodeParams.db.payments.listIncomingPayments(from.toTimestampMilli, to.toTimestampMilli, paginated_opt).map(_.invoice)
   }
 
-  override def pendingInvoices(from: TimestampSecond, to: TimestampSecond)(implicit timeout: Timeout): Future[Seq[Invoice]] = Future {
-    appKit.nodeParams.db.payments.listPendingIncomingPayments(from.toTimestampMilli, to.toTimestampMilli).map(_.invoice)
+  override def pendingInvoices(from: TimestampSecond, to: TimestampSecond, paginated_opt: Option[Paginated])(implicit timeout: Timeout): Future[Seq[Invoice]] = Future {
+    appKit.nodeParams.db.payments.listPendingIncomingPayments(from.toTimestampMilli, to.toTimestampMilli, paginated_opt).map(_.invoice)
   }
 
   override def getInvoice(paymentHash: ByteVector32)(implicit timeout: Timeout): Future[Option[Invoice]] = Future {

eclair-core/src/main/scala/fr/acinq/eclair/Paginated.scala

@@ -18,7 +18,9 @@ package fr.acinq.eclair
 
 /**
  * Created by rorp on 03/11/2022.
- *
+ */
+
+/**
  * Simple class for pagination database and API queries.
  */
 case class Paginated(count: Int, skip: Int) {

eclair-core/src/main/scala/fr/acinq/eclair/db/DualDatabases.scala

@@ -311,9 +311,9 @@ case class DualPaymentsDb(primary: PaymentsDb, secondary: PaymentsDb) extends Pa
     primary.listIncomingPayments(from, to, paginated_opt)
   }
 
-  override def listPendingIncomingPayments(from: TimestampMilli, to: TimestampMilli): Seq[IncomingPayment] = {
-    runAsync(secondary.listPendingIncomingPayments(from, to))
-    primary.listPendingIncomingPayments(from, to)
+  override def listPendingIncomingPayments(from: TimestampMilli, to: TimestampMilli, paginated_opt: Option[Paginated]): Seq[IncomingPayment] = {
+    runAsync(secondary.listPendingIncomingPayments(from, to, paginated_opt))
+    primary.listPendingIncomingPayments(from, to, paginated_opt)
   }
 
   override def listExpiredIncomingPayments(from: TimestampMilli, to: TimestampMilli): Seq[IncomingPayment] = {

eclair-core/src/main/scala/fr/acinq/eclair/db/PaymentsDb.scala

@@ -55,7 +55,7 @@ trait IncomingPaymentsDb {
   def listIncomingPayments(from: TimestampMilli, to: TimestampMilli, paginated_opt: Option[Paginated]): Seq[IncomingPayment]
 
   /** List all pending (not paid, not expired) incoming payments in the given time range (milli-seconds). */
-  def listPendingIncomingPayments(from: TimestampMilli, to: TimestampMilli): Seq[IncomingPayment]
+  def listPendingIncomingPayments(from: TimestampMilli, to: TimestampMilli, paginated_opt: Option[Paginated]): Seq[IncomingPayment]
 
   /** List all expired (not paid) incoming payments in the given time range (milli-seconds). */
   def listExpiredIncomingPayments(from: TimestampMilli, to: TimestampMilli): Seq[IncomingPayment]

eclair-core/src/main/scala/fr/acinq/eclair/db/pg/PgPaymentsDb.scala

@@ -346,9 +346,9 @@ class PgPaymentsDb(implicit ds: DataSource, lock: PgLock) extends PaymentsDb wit
     }
   }
 
-  override def listPendingIncomingPayments(from: TimestampMilli, to: TimestampMilli): Seq[IncomingPayment] = withMetrics("payments/list-incoming-pending", DbBackends.Postgres) {
+  override def listPendingIncomingPayments(from: TimestampMilli, to: TimestampMilli, paginated_opt: Option[Paginated]): Seq[IncomingPayment] = withMetrics("payments/list-incoming-pending", DbBackends.Postgres) {
     withLock { pg =>
-      using(pg.prepareStatement("SELECT * FROM payments.received WHERE received_msat IS NULL AND created_at > ? AND created_at < ? AND expire_at > ? ORDER BY created_at")) { statement =>
+      using(pg.prepareStatement(limited("SELECT * FROM payments.received WHERE received_msat IS NULL AND created_at > ? AND created_at < ? AND expire_at > ? ORDER BY created_at", paginated_opt))) { statement =>
         statement.setTimestamp(1, from.toSqlTimestamp)
         statement.setTimestamp(2, to.toSqlTimestamp)
         statement.setTimestamp(3, Timestamp.from(Instant.now()))

eclair-core/src/main/scala/fr/acinq/eclair/db/sqlite/SqlitePaymentsDb.scala

@@ -344,8 +344,8 @@ class SqlitePaymentsDb(val sqlite: Connection) extends PaymentsDb with Logging {
     }
   }
 
-  override def listPendingIncomingPayments(from: TimestampMilli, to: TimestampMilli): Seq[IncomingPayment] = withMetrics("payments/list-incoming-pending", DbBackends.Sqlite) {
-    using(sqlite.prepareStatement("SELECT * FROM received_payments WHERE received_msat IS NULL AND created_at > ? AND created_at < ? AND expire_at > ? ORDER BY created_at")) { statement =>
+  override def listPendingIncomingPayments(from: TimestampMilli, to: TimestampMilli, paginated_opt: Option[Paginated]): Seq[IncomingPayment] = withMetrics("payments/list-incoming-pending", DbBackends.Sqlite) {
+    using(sqlite.prepareStatement(limited("SELECT * FROM received_payments WHERE received_msat IS NULL AND created_at > ? AND created_at < ? AND expire_at > ? ORDER BY created_at", paginated_opt))) { statement =>
       statement.setLong(1, from.toLong)
       statement.setLong(2, to.toLong)
       statement.setLong(3, TimestampMilli.now().toLong)

eclair-core/src/test/scala/fr/acinq/eclair/db/PaymentsDbSpec.scala

@@ -559,7 +559,7 @@ class PaymentsDbSpec extends AnyFunSuite {
       assert(db.listIncomingPayments(0 unixms, now, None) == Seq(expiredPayment1, expiredPayment2, expiredPayment3, pendingPayment1, pendingPayment2, pendingPayment3, payment1.copy(status = IncomingPaymentStatus.Pending), payment2.copy(status = IncomingPaymentStatus.Pending), payment3.copy(status = IncomingPaymentStatus.Pending)))
       assert(db.listExpiredIncomingPayments(0 unixms, now) == Seq(expiredPayment1, expiredPayment2, expiredPayment3))
       assert(db.listReceivedIncomingPayments(0 unixms, now) == Nil)
-      assert(db.listPendingIncomingPayments(0 unixms, now) == Seq(pendingPayment1, pendingPayment2, pendingPayment3, payment1.copy(status = IncomingPaymentStatus.Pending), payment2.copy(status = IncomingPaymentStatus.Pending), payment3.copy(status = IncomingPaymentStatus.Pending)))
+      assert(db.listPendingIncomingPayments(0 unixms, now, None) == Seq(pendingPayment1, pendingPayment2, pendingPayment3, payment1.copy(status = IncomingPaymentStatus.Pending), payment2.copy(status = IncomingPaymentStatus.Pending), payment3.copy(status = IncomingPaymentStatus.Pending)))
 
       db.receiveIncomingPayment(paidInvoice1.paymentHash, 461 msat, receivedAt1)
       db.receiveIncomingPayment(paidInvoice1.paymentHash, 100 msat, receivedAt2) // adding another payment to this invoice should sum
@@ -572,14 +572,14 @@ class PaymentsDbSpec extends AnyFunSuite {
 
       assert(db.listIncomingPayments(0 unixms, now, None) == Seq(expiredPayment1, expiredPayment2, expiredPayment3, pendingPayment1, pendingPayment2, pendingPayment3, payment1, payment2, payment3))
       assert(db.listIncomingPayments(now - 60.seconds, now, None) == Seq(pendingPayment1, pendingPayment2, pendingPayment3, payment1, payment2, payment3))
-      assert(db.listIncomingPayments(0 unixms, now, Some(Paginated(0,0))) == Seq())
-      assert(db.listIncomingPayments(0 unixms, now, Some(Paginated(0,3))) == Seq())
-      assert(db.listIncomingPayments(0 unixms, now, Some(Paginated(3,0))) == Seq(expiredPayment1, expiredPayment2, expiredPayment3))
-      assert(db.listIncomingPayments(0 unixms, now, Some(Paginated(3,3))) == Seq(pendingPayment1, pendingPayment2, pendingPayment3))
-      assert(db.listPendingIncomingPayments(0 unixms, now) == Seq(pendingPayment1, pendingPayment2, pendingPayment3))
+      assert(db.listIncomingPayments(0 unixms, now, Some(Paginated(0, 0))) == Seq())
+      assert(db.listIncomingPayments(0 unixms, now, Some(Paginated(0, 3))) == Seq())
+      assert(db.listIncomingPayments(0 unixms, now, Some(Paginated(3, 0))) == Seq(expiredPayment1, expiredPayment2, expiredPayment3))
+      assert(db.listIncomingPayments(0 unixms, now, Some(Paginated(3, 3))) == Seq(pendingPayment1, pendingPayment2, pendingPayment3))
+      assert(db.listPendingIncomingPayments(0 unixms, now, None) == Seq(pendingPayment1, pendingPayment2, pendingPayment3))
+      assert(db.listPendingIncomingPayments(0 unixms, now, Some(Paginated(1, 1))) == Seq(pendingPayment2))
       assert(db.listReceivedIncomingPayments(0 unixms, now) == Seq(payment1, payment2, payment3))
 
-
       assert(db.removeIncomingPayment(paidInvoice1.paymentHash).isFailure)
       db.removeIncomingPayment(paidInvoice1.paymentHash).failed.foreach(e => assert(e.getMessage == "Cannot remove a received incoming payment"))
       assert(db.removeIncomingPayment(paidInvoice3.paymentHash).isFailure)

eclair-core/src/test/scala/fr/acinq/eclair/payment/receive/InvoicePurgerSpec.scala

@@ -62,7 +62,7 @@ class InvoicePurgerSpec extends ScalaTestWithActorTestKit(ConfigFactory.load("ap
     val now = TimestampMilli.now()
     assert(db.listIncomingPayments(0 unixms, now, None) == expiredPayments ++ pendingPayments ++ paidPayments)
     assert(db.listIncomingPayments(now - 100.days, now, None) == pendingPayments ++ paidPayments)
-    assert(db.listPendingIncomingPayments(0 unixms, now) == pendingPayments)
+    assert(db.listPendingIncomingPayments(0 unixms, now, None) == pendingPayments)
     assert(db.listReceivedIncomingPayments(0 unixms, now) == paidPayments)
     assert(db.listExpiredIncomingPayments(0 unixms, now) == expiredPayments)
 

eclair-node/src/main/scala/fr/acinq/eclair/api/directives/ExtraDirectives.scala

@@ -43,9 +43,6 @@ trait ExtraDirectives extends Directives {
   val nodeIdFormParam: NameReceptacle[PublicKey] = "nodeId".as[PublicKey]
   val nodeIdsFormParam: NameUnmarshallerReceptacle[List[PublicKey]] = "nodeIds".as[List[PublicKey]](pubkeyListUnmarshaller)
   val paymentHashFormParam: NameUnmarshallerReceptacle[ByteVector32] = "paymentHash".as[ByteVector32](sha256HashUnmarshaller)
-  // we limit default values to avoid accidentally reading too much data from the DB
-  val fromFormParam: NameDefaultUnmarshallerReceptacle[TimestampSecond] = "from".as[TimestampSecond](timestampSecondUnmarshaller).?(TimestampSecond.now() - 1.day)
-  val toFormParam: NameDefaultUnmarshallerReceptacle[TimestampSecond] = "to".as[TimestampSecond](timestampSecondUnmarshaller).?(TimestampSecond.now())
   val amountMsatFormParam: NameReceptacle[MilliSatoshi] = "amountMsat".as[MilliSatoshi]
   val invoiceFormParam: NameReceptacle[Bolt11Invoice] = "invoice".as[Bolt11Invoice]
   val routeFormatFormParam: NameUnmarshallerReceptacle[RouteFormat] = "format".as[RouteFormat](routeFormatUnmarshaller)
@@ -55,6 +52,13 @@ trait ExtraDirectives extends Directives {
   val countFormParam: NameReceptacle[Int] = "count".as[Int]
   val skipFormParam: NameReceptacle[Int] = "skip".as[Int]
 
+  // @formatter:off
+  // We limit default values to avoid accidentally reading too much data from the DB.
+  // We cannot use a val, otherwise the default values would not be recomputed for every request.
+  def fromFormParam(): NameDefaultUnmarshallerReceptacle[TimestampSecond] = "from".as[TimestampSecond](timestampSecondUnmarshaller).?(TimestampSecond.now() - 1.day)
+  def toFormParam(): NameDefaultUnmarshallerReceptacle[TimestampSecond] = "to".as[TimestampSecond](timestampSecondUnmarshaller).?(TimestampSecond.now())
+  // @formatter:on
+
   // custom directive to fail with HTTP 404 (and JSON response) if the element was not found
   def completeOrNotFound[T](fut: Future[Option[T]])(implicit marshaller: ToResponseMarshaller[T]): Route = onComplete(fut) {
     case Success(Some(t)) => complete(t)

eclair-node/src/main/scala/fr/acinq/eclair/api/handlers/Channel.scala

@@ -123,7 +123,7 @@ trait Channel {
   }
 
   val channelStats: Route = postRequest("channelstats") { implicit t =>
-    formFields(fromFormParam, toFormParam) { (from, to) =>
+    formFields(fromFormParam(), toFormParam()) { (from, to) =>
       complete(eclairApi.channelStats(from, to))
     }
   }

eclair-node/src/main/scala/fr/acinq/eclair/api/handlers/Fees.scala

@@ -28,7 +28,7 @@ trait Fees {
   import fr.acinq.eclair.api.serde.JsonSupport.{formats, marshaller, serialization}
 
   val networkFees: Route = postRequest("networkfees") { implicit t =>
-    formFields(fromFormParam, toFormParam) { (from, to) =>
+    formFields(fromFormParam(), toFormParam()) { (from, to) =>
       complete(eclairApi.networkFees(from, to))
     }
   }

eclair-node/src/main/scala/fr/acinq/eclair/api/handlers/Invoice.scala

@@ -43,15 +43,17 @@ trait Invoice {
 
   val listInvoices: Route = postRequest("listinvoices") { implicit t =>
     withPaginated { paginated_opt =>
-      formFields(fromFormParam, toFormParam) { (from, to) =>
+      formFields(fromFormParam(), toFormParam()) { (from, to) =>
         complete(eclairApi.allInvoices(from, to, paginated_opt))
       }
     }
   }
 
   val listPendingInvoices: Route = postRequest("listpendinginvoices") { implicit t =>
-    formFields(fromFormParam, toFormParam) { (from, to) =>
-      complete(eclairApi.pendingInvoices(from, to))
+    withPaginated { paginated_opt =>
+      formFields(fromFormParam(), toFormParam()) { (from, to) =>
+        complete(eclairApi.pendingInvoices(from, to, paginated_opt))
+      }
     }
   }
 

eclair-node/src/main/scala/fr/acinq/eclair/api/handlers/Node.scala

@@ -57,7 +57,7 @@ trait Node {
   }
 
   val audit: Route = postRequest("audit") { implicit t =>
-    formFields(fromFormParam, toFormParam) { (from, to) =>
+    formFields(fromFormParam(), toFormParam()) { (from, to) =>
       complete(eclairApi.audit(from, to))
     }
   }