From 4d5835baf8f50c7c794417fb3e9c80c14cde287e Mon Sep 17 00:00:00 2001 From: Raphael Panic Date: Wed, 16 Apr 2025 13:29:38 +0200 Subject: [PATCH 1/7] Seperated block state tracking and tx execution --- .../sequencing/BlockProductionService.ts | 37 ++---- .../sequencing/TransactionExecutionService.ts | 113 +++++++++++++----- 2 files changed, 93 insertions(+), 57 deletions(-) diff --git a/packages/sequencer/src/protocol/production/sequencing/BlockProductionService.ts b/packages/sequencer/src/protocol/production/sequencing/BlockProductionService.ts index ece360797..385789a25 100644 --- a/packages/sequencer/src/protocol/production/sequencing/BlockProductionService.ts +++ b/packages/sequencer/src/protocol/production/sequencing/BlockProductionService.ts @@ -104,13 +104,12 @@ export class BlockProductionService { const lastResult = lastBlockWithResult.result; const lastBlock = lastBlockWithResult.block; - const executionResults: TransactionExecutionResult[] = []; const incomingMessagesList = new MinaActionsHashList( Field(lastBlock.toMessagesHash) ); - let blockState: BlockTrackers = { + const blockState: BlockTrackers = { blockHashRoot: Field(lastResult.blockHashRoot), eternalTransactionsList: new TransactionHashList( lastBlock.toEternalTransactionsHash @@ -133,28 +132,13 @@ export class BlockProductionService { UntypedStateTransition.fromStateTransition(transition) ); - for (const tx of transactions) { - try { - // Create execution trace - const [newState, executionTrace] = - // eslint-disable-next-line no-await-in-loop - await this.transactionExecutionService.createExecutionTrace( - stateService, - tx, - networkState, - blockState - ); - - blockState = newState; - - // Push result to results and transaction onto bundle-hash - executionResults.push(executionTrace); - } catch (error) { - if (error instanceof Error) { - log.error("Error in inclusion of tx, skipping", error); - } - } - } + const [newBlockState, executionResults] = + await this.transactionExecutionService.createExecutionTraces( + stateService, + transactions, + networkState, + blockState + ); const previousBlockHash = lastResult.blockHash === 0n ? undefined : Field(lastResult.blockHash); @@ -168,9 +152,10 @@ export class BlockProductionService { const block: Omit = { transactions: executionResults, - transactionsHash: blockState.transactionList.commitment, + transactionsHash: newBlockState.transactionList.commitment, fromEternalTransactionsHash: lastBlock.toEternalTransactionsHash, - toEternalTransactionsHash: blockState.eternalTransactionsList.commitment, + toEternalTransactionsHash: + newBlockState.eternalTransactionsList.commitment, height: lastBlock.hash.toBigInt() !== 0n ? lastBlock.height.add(1) : Field(0), fromBlockHashRoot: Field(lastResult.blockHashRoot), diff --git a/packages/sequencer/src/protocol/production/sequencing/TransactionExecutionService.ts b/packages/sequencer/src/protocol/production/sequencing/TransactionExecutionService.ts index 05d25c279..c35c91854 100644 --- a/packages/sequencer/src/protocol/production/sequencing/TransactionExecutionService.ts +++ b/packages/sequencer/src/protocol/production/sequencing/TransactionExecutionService.ts @@ -291,6 +291,57 @@ export class TransactionExecutionService { ); } + public addTransactionToBlockProverState( + state: BlockTrackers, + tx: PendingTransaction + ): BlockTrackers { + const signedTransaction = tx.toProtocolTransaction(); + // Add tx to commitments + return this.blockProver.addTransactionToBundle( + state, + Bool(tx.isMessage), + signedTransaction.transaction + ); + } + + public async createExecutionTraces( + asyncStateService: CachedStateService, + transactions: PendingTransaction[], + networkState: NetworkState, + state: BlockTrackers + ): Promise<[BlockTrackers, TransactionExecutionResult[]]> { + let blockState = state; + const executionResults: TransactionExecutionResult[] = []; + + for (const tx of transactions) { + try { + const newState = this.addTransactionToBlockProverState(state, tx); + + // Create execution trace + const executionTrace = + // eslint-disable-next-line no-await-in-loop + await this.createExecutionTrace( + asyncStateService, + tx, + networkState, + blockState, + newState + ); + + blockState = newState; + + // Push result to results and transaction onto bundle-hash + executionResults.push(executionTrace); + } catch (error) { + if (error instanceof Error) { + log.error("Error in inclusion of tx, skipping", error); + } + } + } + + return [blockState, executionResults]; + } + @trace("block.transaction", ([, tx, networkState]) => ({ height: networkState.block.height.toString(), methodId: tx.methodId.toString(), @@ -300,8 +351,9 @@ export class TransactionExecutionService { asyncStateService: CachedStateService, tx: PendingTransaction, networkState: NetworkState, - state: BlockTrackers - ): Promise<[BlockTrackers, TransactionExecutionResult]> { + state: BlockTrackers, + newState: BlockTrackers + ): Promise { // TODO Use RecordingStateService -> async asProver needed const recordingStateService = new CachedStateService(asyncStateService); @@ -328,10 +380,16 @@ export class TransactionExecutionService { networkState, state ); - const beforeTxHookResult = await this.executeProtocolHooks( - beforeTxArguments, - async (hook, hookArgs) => await hook.beforeTransaction(hookArgs), - "beforeTx" + const beforeTxHookResult = await this.tracer.trace( + "block.transaction.before.execute", + () => + this.executeProtocolHooks( + beforeTxArguments, + async (hook, hookArgs) => { + await hook.beforeTransaction(hookArgs); + }, + "beforeTx" + ) ); const beforeHookEvents = extractEvents(beforeTxHookResult, "beforeTxHook"); @@ -339,10 +397,9 @@ export class TransactionExecutionService { beforeTxHookResult.stateTransitions ); - const runtimeResult = await this.executeRuntimeMethod( - method, - args, - runtimeContextInputs + const runtimeResult = await this.tracer.trace( + "block.transaction.execute", + () => this.executeRuntimeMethod(method, args, runtimeContextInputs) ); traceLogSTs("STs:", runtimeResult.stateTransitions); @@ -354,13 +411,6 @@ export class TransactionExecutionService { ); } - // Add runtime to commitments - const newState = this.blockProver.addTransactionToBundle( - state, - Bool(tx.isMessage), - signedTransaction.transaction - ); - // Execute afterTransaction hook const afterTxArguments = toAfterTransactionHookArgument( signedTransaction, @@ -378,10 +428,14 @@ export class TransactionExecutionService { }) ); - const afterTxHookResult = await this.executeProtocolHooks( - afterTxArguments, - async (hook, hookArgs) => await hook.afterTransaction(hookArgs), - "afterTx" + const afterTxHookResult = await this.tracer.trace( + "block.transaction.after.execute", + () => + this.executeProtocolHooks( + afterTxArguments, + async (hook, hookArgs) => await hook.afterTransaction(hookArgs), + "afterTx" + ) ); const afterHookEvents = extractEvents(afterTxHookResult, "afterTxHook"); await recordingStateService.applyStateTransitions( @@ -407,16 +461,13 @@ export class TransactionExecutionService { runtimeResult.status ); - return [ - state, - { - tx, - status: runtimeResult.status, - statusMessage: runtimeResult.statusMessage, + return { + tx, + status: runtimeResult.status, + statusMessage: runtimeResult.statusMessage, - stateTransitions, - events: beforeHookEvents.concat(runtimeResultEvents, afterHookEvents), - }, - ]; + stateTransitions, + events: beforeHookEvents.concat(runtimeResultEvents, afterHookEvents), + }; } } From b976c584ae51db52d2c3308a2461c6e880037ede Mon Sep 17 00:00:00 2001 From: Raphael Panic Date: Wed, 16 Apr 2025 13:31:59 +0200 Subject: [PATCH 2/7] Automatically caching async retrieved state entries --- packages/sequencer/src/state/state/CachedStateService.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/packages/sequencer/src/state/state/CachedStateService.ts b/packages/sequencer/src/state/state/CachedStateService.ts index 2824ba418..cc1167015 100644 --- a/packages/sequencer/src/state/state/CachedStateService.ts +++ b/packages/sequencer/src/state/state/CachedStateService.ts @@ -88,6 +88,15 @@ export class CachedStateService const remote = await this.parent?.getMany(remoteKeys); + if (remote !== undefined) { + // Update the remotely fetched keys into local cache + await mapSequential(remote, async ({ key, value }) => { + if (this.getNullAware(key) === undefined) { + await this.set(key, value); + } + }); + } + return local.concat(remote ?? []); } From 93ddedb93e998ab2916e00f770c8d3a8b4242138 Mon Sep 17 00:00:00 2001 From: Raphael Panic Date: Thu, 17 Apr 2025 16:17:09 +0200 Subject: [PATCH 3/7] Moved fee hook verifyConfig to startup --- packages/library/src/hooks/TransactionFeeHook.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/library/src/hooks/TransactionFeeHook.ts b/packages/library/src/hooks/TransactionFeeHook.ts index dfa325b8f..20d9b0adc 100644 --- a/packages/library/src/hooks/TransactionFeeHook.ts +++ b/packages/library/src/hooks/TransactionFeeHook.ts @@ -80,12 +80,12 @@ export class TransactionFeeHook extends ProvableTransactionHook Date: Thu, 17 Apr 2025 16:17:54 +0200 Subject: [PATCH 4/7] Optimized db access pattern for inserting blocks --- .../src/services/prisma/PrismaBlockStorage.ts | 21 ++++++++++++------- .../src/services/prisma/PrismaStateService.ts | 1 + 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/packages/persistance/src/services/prisma/PrismaBlockStorage.ts b/packages/persistance/src/services/prisma/PrismaBlockStorage.ts index 51ebaed62..1e8c1302d 100644 --- a/packages/persistance/src/services/prisma/PrismaBlockStorage.ts +++ b/packages/persistance/src/services/prisma/PrismaBlockStorage.ts @@ -7,6 +7,8 @@ import { BlockStorage, BlockWithResult, BlockWithMaybeResult, + Tracer, + trace, } from "@proto-kit/sequencer"; import { log } from "@proto-kit/common"; import { @@ -33,7 +35,8 @@ export class PrismaBlockStorage private readonly transactionResultMapper: TransactionExecutionResultMapper, private readonly transactionMapper: TransactionMapper, private readonly blockResultMapper: BlockResultMapper, - private readonly blockMapper: BlockMapper + private readonly blockMapper: BlockMapper, + @inject("Tracer") public readonly tracer: Tracer ) {} private async getBlockByQuery( @@ -76,6 +79,7 @@ export class PrismaBlockStorage return (await this.getBlockByQuery({ hash }))?.block; } + @trace("db.block.push", ([{ height }]) => ({ height: height.toString() })) public async pushBlock(block: Block): Promise { log.trace( "Pushing block to DB. Txs:", @@ -96,12 +100,15 @@ export class PrismaBlockStorage const { prismaClient } = this.connection; - await prismaClient.transaction.createMany({ - data: block.transactions.map((txr) => - this.transactionMapper.mapOut(txr.tx) - ), - skipDuplicates: true, - }); + // Note: We can assume all transactions are already in the DB here, because the + // mempool shares the same table as this one. But that could change in the future, + // then transaction have to be inserted-if-missing + // await prismaClient.transaction.createMany({ + // data: block.transactions.map((txr) => + // this.transactionMapper.mapOut(txr.tx) + // ), + // skipDuplicates: true, + // }); await prismaClient.block.create({ data: { diff --git a/packages/persistance/src/services/prisma/PrismaStateService.ts b/packages/persistance/src/services/prisma/PrismaStateService.ts index 1aed65a38..3da8c7225 100644 --- a/packages/persistance/src/services/prisma/PrismaStateService.ts +++ b/packages/persistance/src/services/prisma/PrismaStateService.ts @@ -61,6 +61,7 @@ export class PrismaStateService implements AsyncStateService { this.cache = []; } + @trace("db.state.getMany") public async getMany(keys: Field[]): Promise { const records = await this.connection.prismaClient.state.findMany({ where: { From bd06851b480614a6a7c7531e78e4296c56097f83 Mon Sep 17 00:00:00 2001 From: Raphael Panic Date: Thu, 17 Apr 2025 16:18:33 +0200 Subject: [PATCH 5/7] Removed mempool doing a db fetch for logging --- packages/sequencer/src/mempool/private/PrivateMempool.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/sequencer/src/mempool/private/PrivateMempool.ts b/packages/sequencer/src/mempool/private/PrivateMempool.ts index c57553c3d..067205d02 100644 --- a/packages/sequencer/src/mempool/private/PrivateMempool.ts +++ b/packages/sequencer/src/mempool/private/PrivateMempool.ts @@ -78,9 +78,7 @@ export class PrivateMempool const success = await this.transactionStorage.pushUserTransaction(tx); if (success) { this.events.emit("mempool-transaction-added", tx); - log.trace( - `Transaction added to mempool: ${tx.hash().toString()} (${(await this.transactionStorage.getPendingUserTransactions()).length} transactions in mempool)` - ); + log.trace(`Transaction added to mempool: ${tx.hash().toString()}`); } else { log.error( `Transaction ${tx.hash().toString()} rejected: already exists in mempool` From 338fd7d037e48d881c8c226f456ebefefb91e378 Mon Sep 17 00:00:00 2001 From: Raphael Panic Date: Thu, 17 Apr 2025 16:20:10 +0200 Subject: [PATCH 6/7] Added manual traces to console tracer for easier debugging --- .../sequencer/src/logging/ConsoleTracer.ts | 19 +++++++++++++++++++ .../test-integration/benchmarks/tps.test.ts | 7 +++++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/packages/sequencer/src/logging/ConsoleTracer.ts b/packages/sequencer/src/logging/ConsoleTracer.ts index 990177d3c..2b03d79f9 100644 --- a/packages/sequencer/src/logging/ConsoleTracer.ts +++ b/packages/sequencer/src/logging/ConsoleTracer.ts @@ -68,6 +68,25 @@ export class ConsoleTracer implements Tracer { this.clearTraces(); } + activeManualTraceStack: [string, number][] = []; + + public startTrace(name: string) { + const startTime = Date.now(); + + this.activeManualTraceStack.push([name, startTime]); + } + + public endTrace() { + const [name, startTime] = this.activeManualTraceStack.pop()!; + const duration = Date.now() - startTime; + + if (name in this.store) { + this.store[name].push({ duration }); + } else { + this.store[name] = [{ duration }]; + } + } + public async trace( name: string, f: () => Promise, diff --git a/packages/sequencer/test-integration/benchmarks/tps.test.ts b/packages/sequencer/test-integration/benchmarks/tps.test.ts index 64e22ab13..fe4e27436 100644 --- a/packages/sequencer/test-integration/benchmarks/tps.test.ts +++ b/packages/sequencer/test-integration/benchmarks/tps.test.ts @@ -108,7 +108,9 @@ export async function createAppChain() { maximumBlockSize: 100, }, BlockTrigger: {}, - Mempool: {}, + Mempool: { + validationEnabled: false, + }, }, Signer: { signer: PrivateKey.random(), @@ -125,7 +127,7 @@ export async function createAppChain() { const timeout = 600000; -describe("tps", () => { +describe.skip("tps", () => { let appChain: Awaited>; let privateKeys: PrivateKey[] = []; let balances: Balances; @@ -177,6 +179,7 @@ describe("tps", () => { tracer.enableManualOutputs(); await fundKeys(200); + tracer.printSummary(); } catch (e) { console.error(e); throw e; From ce391076e92d6f3be031edba39fdd8649fe39208 Mon Sep 17 00:00:00 2001 From: Raphael Panic Date: Thu, 17 Apr 2025 16:24:21 +0200 Subject: [PATCH 7/7] Bunch of optimizations in the execution service --- .../src/mempool/PendingTransaction.ts | 96 +++++++++++-------- .../sequencing/TransactionExecutionService.ts | 39 ++++++-- 2 files changed, 87 insertions(+), 48 deletions(-) diff --git a/packages/sequencer/src/mempool/PendingTransaction.ts b/packages/sequencer/src/mempool/PendingTransaction.ts index b7b3dcd58..fd1fedfec 100644 --- a/packages/sequencer/src/mempool/PendingTransaction.ts +++ b/packages/sequencer/src/mempool/PendingTransaction.ts @@ -28,45 +28,57 @@ export type UnsignedTransactionBody = { }; export class UnsignedTransaction implements UnsignedTransactionBody { - public methodId: Field; + public readonly methodId: Field; - public nonce: UInt64; + public readonly nonce: UInt64; - public sender: PublicKey; + public readonly sender: PublicKey; - public argsFields: Field[]; + public readonly argsFields: Field[]; - public auxiliaryData: string[]; + public readonly auxiliaryData: string[]; - public isMessage: boolean; + public readonly isMessage: boolean; - public constructor(data: { - methodId: Field; - nonce: UInt64; - sender: PublicKey; - argsFields: Field[]; - auxiliaryData: string[]; - isMessage: boolean; - }) { + public constructor( + data: { + methodId: Field; + nonce: UInt64; + sender: PublicKey; + argsFields: Field[]; + auxiliaryData: string[]; + isMessage: boolean; + }, + memoizedHash?: Field + ) { this.methodId = data.methodId; this.nonce = data.nonce; this.sender = data.sender; this.argsFields = data.argsFields; this.auxiliaryData = data.auxiliaryData; this.isMessage = data.isMessage; + + if (memoizedHash !== undefined) { + this.memoizedHash = memoizedHash; + } } public argsHash(): Field { return Poseidon.hash(this.argsFields); } + private memoizedHash?: Field = undefined; + public hash(): Field { - return Poseidon.hash([ - this.methodId, - ...this.sender.toFields(), - ...this.nonce.toFields(), - this.argsHash(), - ]); + if (this.memoizedHash === undefined) { + this.memoizedHash = Poseidon.hash([ + this.methodId, + ...this.sender.toFields(), + ...this.nonce.toFields(), + this.argsHash(), + ]); + } + return this.memoizedHash; } public getSignatureData(): Field[] { @@ -124,29 +136,35 @@ export class PendingTransaction extends UnsignedTransaction { public static fromJSON( object: PendingTransactionJSONType ): PendingTransaction { - return new PendingTransaction({ - methodId: Field.fromJSON(object.methodId), - nonce: UInt64.from(object.nonce), - sender: PublicKey.fromBase58(object.sender), - argsFields: object.argsFields.map((x) => Field.fromJSON(x)), - signature: Signature.fromJSON(object.signature), - auxiliaryData: object.auxiliaryData.slice(), - isMessage: object.isMessage, - }); + return new PendingTransaction( + { + methodId: Field.fromJSON(object.methodId), + nonce: UInt64.from(object.nonce), + sender: PublicKey.fromBase58(object.sender), + argsFields: object.argsFields.map((x) => Field.fromJSON(x)), + signature: Signature.fromJSON(object.signature), + auxiliaryData: object.auxiliaryData.slice(), + isMessage: object.isMessage, + }, + Field(object.hash) + ); } public signature: Signature; - public constructor(data: { - methodId: Field; - nonce: UInt64; - sender: PublicKey; - signature: Signature; - argsFields: Field[]; - auxiliaryData: string[]; - isMessage: boolean; - }) { - super(data); + public constructor( + data: { + methodId: Field; + nonce: UInt64; + sender: PublicKey; + signature: Signature; + argsFields: Field[]; + auxiliaryData: string[]; + isMessage: boolean; + }, + memoizedHash?: Field + ) { + super(data, memoizedHash); this.signature = data.signature; } diff --git a/packages/sequencer/src/protocol/production/sequencing/TransactionExecutionService.ts b/packages/sequencer/src/protocol/production/sequencing/TransactionExecutionService.ts index c35c91854..0e12cd7a1 100644 --- a/packages/sequencer/src/protocol/production/sequencing/TransactionExecutionService.ts +++ b/packages/sequencer/src/protocol/production/sequencing/TransactionExecutionService.ts @@ -21,6 +21,8 @@ import { MethodPublicOutput, toBeforeTransactionHookArgument, toAfterTransactionHookArgument, + ProvableStateTransition, + DefaultProvableHashList, } from "@proto-kit/protocol"; import { Bool, Field } from "o1js"; import { AreProofsEnabled, log, mapSequential } from "@proto-kit/common"; @@ -30,7 +32,6 @@ import { RuntimeModule, RuntimeModulesRecord, toEventsHash, - toStateTransitionsHash, } from "@proto-kit/module"; // eslint-disable-next-line import/no-extraneous-dependencies import zip from "lodash/zip"; @@ -142,6 +143,18 @@ function extractEvents( ); } +// TODO Also use this in tracing as a replacement of toStateTransitionHash +export function toStateTransitionHashNonProvable( + stateTransitions: StateTransition[] +) { + const reduced = reduceStateTransitions(stateTransitions); + const list = new DefaultProvableHashList(ProvableStateTransition); + + reduced.map((st) => st.toProvable()).forEach((st) => list.push(st)); + + return list.commitment; +} + export async function executeWithExecutionContext( method: () => Promise, contextInputs: RuntimeMethodExecutionData, @@ -313,6 +326,8 @@ export class TransactionExecutionService { let blockState = state; const executionResults: TransactionExecutionResult[] = []; + const networkStateHash = networkState.hash(); + for (const tx of transactions) { try { const newState = this.addTransactionToBlockProverState(state, tx); @@ -323,7 +338,7 @@ export class TransactionExecutionService { await this.createExecutionTrace( asyncStateService, tx, - networkState, + { networkState, hash: networkStateHash }, blockState, newState ); @@ -342,7 +357,7 @@ export class TransactionExecutionService { return [blockState, executionResults]; } - @trace("block.transaction", ([, tx, networkState]) => ({ + @trace("block.transaction", ([, tx, { networkState }]) => ({ height: networkState.block.height.toString(), methodId: tx.methodId.toString(), isMessage: tx.isMessage, @@ -350,7 +365,10 @@ export class TransactionExecutionService { public async createExecutionTrace( asyncStateService: CachedStateService, tx: PendingTransaction, - networkState: NetworkState, + { + networkState, + hash: networkStateHash, + }: { networkState: NetworkState; hash: Field }, state: BlockTrackers, newState: BlockTrackers ): Promise { @@ -411,6 +429,11 @@ export class TransactionExecutionService { ); } + const eventsHash = toEventsHash(runtimeResult.events); + const stateTransitionsHash = toStateTransitionHashNonProvable( + runtimeResult.stateTransitions + ); + // Execute afterTransaction hook const afterTxArguments = toAfterTransactionHookArgument( signedTransaction, @@ -418,13 +441,11 @@ export class TransactionExecutionService { newState, new MethodPublicOutput({ status: runtimeResult.status, - networkStateHash: networkState.hash(), + networkStateHash: networkStateHash, isMessage: Bool(tx.isMessage), transactionHash: tx.hash(), - eventsHash: toEventsHash(runtimeResult.events), - stateTransitionsHash: toStateTransitionsHash( - runtimeResult.stateTransitions - ), + eventsHash, + stateTransitionsHash, }) );