diff --git a/packages/api/package.json b/packages/api/package.json index 138964ed946a..5d340cf9043c 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -70,8 +70,8 @@ "check-readme": "typescript-docs-verifier" }, "dependencies": { - "@chainsafe/persistent-merkle-tree": "^0.8.0", - "@chainsafe/ssz": "^0.17.1", + "@chainsafe/persistent-merkle-tree": "file:../../../ssz/packages/persistent-merkle-tree", + "@chainsafe/ssz": "file:../../../ssz/packages/ssz", "@lodestar/config": "^1.21.0", "@lodestar/params": "^1.21.0", "@lodestar/types": "^1.21.0", diff --git a/packages/beacon-node/package.json b/packages/beacon-node/package.json index b41450f12d34..16099b916ed9 100644 --- a/packages/beacon-node/package.json +++ b/packages/beacon-node/package.json @@ -94,16 +94,17 @@ "check-readme": "typescript-docs-verifier" }, "dependencies": { - "@chainsafe/as-sha256": "^0.5.0", "@chainsafe/blst": "^2.0.3", + "@chainsafe/as-sha256": "file:../../../ssz/packages/as-sha256", + "@chainsafe/bls": "7.1.3", "@chainsafe/discv5": "^9.0.0", "@chainsafe/enr": "^3.0.0", "@chainsafe/libp2p-gossipsub": "^13.0.0", "@chainsafe/libp2p-identify": "^1.0.0", "@chainsafe/libp2p-noise": "^15.0.0", - "@chainsafe/persistent-merkle-tree": "^0.8.0", + "@chainsafe/persistent-merkle-tree": "file:../../../ssz/packages/persistent-merkle-tree", "@chainsafe/prometheus-gc-stats": "^1.0.0", - "@chainsafe/ssz": "^0.17.1", + "@chainsafe/ssz": "file:../../../ssz/packages/ssz", "@chainsafe/threads": "^1.11.1", "@ethersproject/abi": "^5.7.0", "@fastify/bearer-auth": "^9.0.0", diff --git a/packages/beacon-node/src/chain/archiver/index.ts b/packages/beacon-node/src/chain/archiver/index.ts index 294c2281e19b..e2f55b8f0500 100644 --- a/packages/beacon-node/src/chain/archiver/index.ts +++ b/packages/beacon-node/src/chain/archiver/index.ts @@ -107,7 +107,7 @@ export class Archiver { // should be after ArchiveBlocksTask to handle restart cleanly await this.statesArchiver.maybeArchiveState(finalized); - this.chain.regen.pruneOnFinalized(finalizedEpoch); + this.chain.pruneOnFinalized(finalizedEpoch); // tasks rely on extended fork choice const prunedBlocks = this.chain.forkChoice.prune(finalized.rootHex); diff --git a/packages/beacon-node/src/chain/balancesTreeCache.ts b/packages/beacon-node/src/chain/balancesTreeCache.ts new file mode 100644 index 000000000000..462ae860809e --- /dev/null +++ b/packages/beacon-node/src/chain/balancesTreeCache.ts @@ -0,0 +1,38 @@ +import {ListBasicTreeViewDU, UintNumberType} from "@chainsafe/ssz"; +import {IBalancesTreeCache, CachedBeaconStateAllForks} from "@lodestar/state-transition"; +import {Metrics} from "../metrics/index.js"; + +const MAX_ITEMS = 2; + +export class BalancesTreeCache implements IBalancesTreeCache { + private readonly unusedBalancesTrees: ListBasicTreeViewDU[] = []; + + constructor(private readonly metrics: Metrics | null = null) { + if (metrics) { + metrics.balancesTreeCache.size.addCollect(() => { + metrics.balancesTreeCache.size.set(this.unusedBalancesTrees.length); + }); + } + } + + processUnusedState(state: CachedBeaconStateAllForks | undefined): void { + if (state === undefined) { + return; + } + + this.unusedBalancesTrees.push(state.balances); + while (this.unusedBalancesTrees.length > MAX_ITEMS) { + this.unusedBalancesTrees.shift(); + } + } + + getUnusedBalances(): ListBasicTreeViewDU | undefined { + if (this.unusedBalancesTrees.length === 0) { + this.metrics?.balancesTreeCache.miss.inc(); + return undefined; + } + + this.metrics?.balancesTreeCache.hit.inc(); + return this.unusedBalancesTrees.shift(); + } +} diff --git a/packages/beacon-node/src/chain/blocks/importBlock.ts b/packages/beacon-node/src/chain/blocks/importBlock.ts index de5ecf607d95..1d1772a1692f 100644 --- a/packages/beacon-node/src/chain/blocks/importBlock.ts +++ b/packages/beacon-node/src/chain/blocks/importBlock.ts @@ -97,7 +97,16 @@ export async function importBlock( // This adds the state necessary to process the next block // Some block event handlers require state being in state cache so need to do this before emitting EventType.block - this.regen.processState(blockRootHex, postState); + this.regen.processState(blockRootHex, postState).then((prunedStates) => { + if (prunedStates) { + for (const states of prunedStates.values()) { + // cp states on the same epoch shares the same balances seed tree so only need one of them + this.balancesTreeCache.processUnusedState(states[0]); + } + } + }).catch((e) => { + this.logger.error("Regen error to process state for block", {slot: blockSlot, root: blockRootHex}, e as Error); + }); this.metrics?.importBlock.bySource.inc({source}); this.logger.verbose("Added block to forkchoice and state cache", {slot: blockSlot, root: blockRootHex}); diff --git a/packages/beacon-node/src/chain/chain.ts b/packages/beacon-node/src/chain/chain.ts index 8dbb49798538..69d6a6b41c56 100644 --- a/packages/beacon-node/src/chain/chain.ts +++ b/packages/beacon-node/src/chain/chain.ts @@ -101,6 +101,7 @@ import {DbCPStateDatastore} from "./stateCache/datastore/db.js"; import {FileCPStateDatastore} from "./stateCache/datastore/file.js"; import {SyncCommitteeRewards, computeSyncCommitteeRewards} from "./rewards/syncCommitteeRewards.js"; import {AttestationsRewards, computeAttestationsRewards} from "./rewards/attestationsRewards.js"; +import {BalancesTreeCache} from "./balancesTreeCache.js"; /** * Arbitrary constants, blobs and payloads should be consumed immediately in the same slot @@ -158,6 +159,7 @@ export class BeaconChain implements IBeaconChain { readonly beaconProposerCache: BeaconProposerCache; readonly checkpointBalancesCache: CheckpointBalancesCache; readonly shufflingCache: ShufflingCache; + readonly balancesTreeCache: BalancesTreeCache; /** Map keyed by executionPayload.blockHash of the block for those blobs */ readonly producedContentsCache = new Map(); @@ -247,6 +249,7 @@ export class BeaconChain implements IBeaconChain { this.beaconProposerCache = new BeaconProposerCache(opts); this.checkpointBalancesCache = new CheckpointBalancesCache(); this.shufflingCache = new ShufflingCache(metrics, this.opts); + this.balancesTreeCache = new BalancesTreeCache(metrics); // Restore state caches // anchorState may already by a CachedBeaconState. If so, don't create the cache again, since deserializing all @@ -260,6 +263,7 @@ export class BeaconChain implements IBeaconChain { config, pubkey2index: new PubkeyIndexMap(), index2pubkey: [], + balancesTreeCache: this.balancesTreeCache, }); this.shufflingCache.processState(cachedState, cachedState.epochCtx.previousShuffling.epoch); this.shufflingCache.processState(cachedState, cachedState.epochCtx.currentShuffling.epoch); @@ -863,6 +867,16 @@ export class BeaconChain implements IBeaconChain { } } + pruneOnFinalized(finalizedEpoch: Epoch): void { + const prunedStates = this.regen.pruneOnFinalized(finalizedEpoch); + if (prunedStates) { + // cp states on the same epoch shares the same balances seed tree so only need one of them + for (const states of prunedStates.values()) { + this.balancesTreeCache.processUnusedState(states[0]); + } + } + } + /** * Regenerate state for attestation verification, this does not happen with default chain option of maxSkipSlots = 32 . * However, need to handle just in case. Lodestar doesn't support multiple regen state requests for attestation verification diff --git a/packages/beacon-node/src/chain/interface.ts b/packages/beacon-node/src/chain/interface.ts index 5185662eaa4f..e70d5a0c8297 100644 --- a/packages/beacon-node/src/chain/interface.ts +++ b/packages/beacon-node/src/chain/interface.ts @@ -241,6 +241,8 @@ export interface IBeaconChain { blockRef: BeaconBlock | BlindedBeaconBlock, validatorIds?: (ValidatorIndex | string)[] ): Promise; + + pruneOnFinalized(finalizedEpoch: Epoch): void; } export type SSZObjectType = diff --git a/packages/beacon-node/src/chain/prepareNextSlot.ts b/packages/beacon-node/src/chain/prepareNextSlot.ts index 48724ab25b0b..ffb55a3aa2a6 100644 --- a/packages/beacon-node/src/chain/prepareNextSlot.ts +++ b/packages/beacon-node/src/chain/prepareNextSlot.ts @@ -1,3 +1,4 @@ +import {HashComputationGroup} from "@chainsafe/persistent-merkle-tree"; import { computeEpochAtSlot, isExecutionStateType, @@ -25,6 +26,11 @@ export const SCHEDULER_LOOKAHEAD_FACTOR = 3; /* We don't want to do more epoch transition than this */ const PREPARE_EPOCH_LIMIT = 1; +/** + * The same HashComputationGroup to be used for all epoch transition. + */ +const epochHCGroup = new HashComputationGroup(); + /** * At Bellatrix, if we are responsible for proposing in next slot, we want to prepare payload * 4s (1/3 slot) before the start of next slot @@ -229,7 +235,12 @@ export class PrepareNextSlotScheduler { const hashTreeRootTimer = this.metrics?.stateHashTreeRootTime.startTimer({ source: isEpochTransition ? StateHashTreeRootSource.prepareNextEpoch : StateHashTreeRootSource.prepareNextSlot, }); - state.hashTreeRoot(); + if (isEpochTransition) { + state.batchHashTreeRoot(epochHCGroup); + } else { + // normal slot, not worth to batch hash + state.node.rootHashObject; + } hashTreeRootTimer?.(); } } diff --git a/packages/beacon-node/src/chain/produceBlock/computeNewStateRoot.ts b/packages/beacon-node/src/chain/produceBlock/computeNewStateRoot.ts index bfa30e570e06..483a521401ed 100644 --- a/packages/beacon-node/src/chain/produceBlock/computeNewStateRoot.ts +++ b/packages/beacon-node/src/chain/produceBlock/computeNewStateRoot.ts @@ -48,6 +48,7 @@ export function computeNewStateRoot( const hashTreeRootTimer = metrics?.stateHashTreeRootTime.startTimer({ source: StateHashTreeRootSource.computeNewStateRoot, }); + // state root is computed inside stateTransition(), so it should take no time here const newStateRoot = postState.hashTreeRoot(); hashTreeRootTimer?.(); diff --git a/packages/beacon-node/src/chain/regen/queued.ts b/packages/beacon-node/src/chain/regen/queued.ts index 57e64bd364ea..45b8b4f76c5f 100644 --- a/packages/beacon-node/src/chain/regen/queued.ts +++ b/packages/beacon-node/src/chain/regen/queued.ts @@ -148,16 +148,26 @@ export class QueuedStateRegenerator implements IStateRegenerator { this.blockStateCache.prune(headStateRoot); } - pruneOnFinalized(finalizedEpoch: number): void { - this.checkpointStateCache.pruneFinalized(finalizedEpoch); + pruneOnFinalized(finalizedEpoch: number): Map | null { + const prunedStates = this.checkpointStateCache.pruneFinalized(finalizedEpoch); this.blockStateCache.deleteAllBeforeEpoch(finalizedEpoch); + + return prunedStates; } - processState(blockRootHex: RootHex, postState: CachedBeaconStateAllForks): void { + async processState( + blockRootHex: RootHex, + postState: CachedBeaconStateAllForks + ): Promise | null> { this.blockStateCache.add(postState); - this.checkpointStateCache.processState(blockRootHex, postState).catch((e) => { - this.logger.debug("Error processing block state", {blockRootHex, slot: postState.slot}, e); - }); + let prunedStates: Map | null = null; + try { + prunedStates = await this.checkpointStateCache.processState(blockRootHex, postState); + } catch (e) { + this.logger.debug("Error processing block state", {blockRootHex, slot: postState.slot}, e as Error); + } + + return prunedStates; } addCheckpointState(cp: phase0.Checkpoint, item: CachedBeaconStateAllForks): void { diff --git a/packages/beacon-node/src/chain/stateCache/inMemoryCheckpointsCache.ts b/packages/beacon-node/src/chain/stateCache/inMemoryCheckpointsCache.ts index 38aeabb97955..bb1ff18e25de 100644 --- a/packages/beacon-node/src/chain/stateCache/inMemoryCheckpointsCache.ts +++ b/packages/beacon-node/src/chain/stateCache/inMemoryCheckpointsCache.ts @@ -59,9 +59,9 @@ export class InMemoryCheckpointStateCache implements CheckpointStateCache { return this.getLatest(rootHex, maxEpoch, opts); } - async processState(): Promise { + async processState(): Promise | null> { // do nothing, this class does not support prunning - return 0; + return null; } get(cp: CheckpointHex, opts?: StateCloneOpts): CachedBeaconStateAllForks | null { @@ -122,12 +122,17 @@ export class InMemoryCheckpointStateCache implements CheckpointStateCache { return previousHits; } - pruneFinalized(finalizedEpoch: Epoch): void { + pruneFinalized(finalizedEpoch: Epoch): Map { + const result = new Map(); + for (const epoch of this.epochIndex.keys()) { if (epoch < finalizedEpoch) { - this.deleteAllEpochItems(epoch); + const deletedStates = this.deleteAllEpochItems(epoch); + result.set(epoch, deletedStates); } } + + return result; } prune(finalizedEpoch: Epoch, justifiedEpoch: Epoch): void { @@ -153,11 +158,19 @@ export class InMemoryCheckpointStateCache implements CheckpointStateCache { } } - deleteAllEpochItems(epoch: Epoch): void { + deleteAllEpochItems(epoch: Epoch): CachedBeaconStateAllForks[] { + const states = []; for (const rootHex of this.epochIndex.get(epoch) || []) { - this.cache.delete(toCheckpointKey({rootHex, epoch})); + const key = toCheckpointKey({rootHex, epoch}); + const state = this.cache.get(key); + if (state) { + states.push(state); + } + this.cache.delete(key); } this.epochIndex.delete(epoch); + + return states; } clear(): void { diff --git a/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts b/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts index 190b79e58cd6..b4d16ae1d500 100644 --- a/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts +++ b/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts @@ -421,7 +421,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { /** * Prune all checkpoint states before the provided finalized epoch. */ - pruneFinalized(finalizedEpoch: Epoch): void { + pruneFinalized(finalizedEpoch: Epoch): Map | null { for (const epoch of this.epochIndex.keys()) { if (epoch < finalizedEpoch) { this.deleteAllEpochItems(epoch).catch((e) => @@ -429,6 +429,9 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { ); } } + + // not likely to return anything in-memory state because we may persist states even before they are finalized + return null; } /** @@ -481,12 +484,14 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { * * As of Mar 2024, it takes <=350ms to persist a holesky state on fast server */ - async processState(blockRootHex: RootHex, state: CachedBeaconStateAllForks): Promise { - let persistCount = 0; + async processState( + blockRootHex: RootHex, + state: CachedBeaconStateAllForks + ): Promise | null> { // it's important to sort the epochs in ascending order, in case of big reorg we always want to keep the most recent checkpoint states const sortedEpochs = Array.from(this.epochIndex.keys()).sort((a, b) => a - b); if (sortedEpochs.length <= this.maxEpochsInMemory) { - return 0; + return null; } const blockSlot = state.slot; @@ -502,24 +507,19 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { // normally the block persist happens at 2/3 of slot 0 of epoch, if it's already late then just skip to allow other tasks to run // there are plenty of chances in the same epoch to persist checkpoint states, also if block is late it could be reorged this.logger.verbose("Skip persist checkpoint states", {blockSlot, root: blockRootHex}); - return 0; + return null; } const persistEpochs = sortedEpochs.slice(0, sortedEpochs.length - this.maxEpochsInMemory); + + const result = new Map(); for (const lowestEpoch of persistEpochs) { // usually there is only 0 or 1 epoch to persist in this loop - persistCount += await this.processPastEpoch(blockRootHex, state, lowestEpoch); + const prunedStates = await this.processPastEpoch(blockRootHex, state, lowestEpoch); + result.set(lowestEpoch, prunedStates); } - if (persistCount > 0) { - this.logger.verbose("Persisted checkpoint states", { - slot: blockSlot, - root: blockRootHex, - persistCount, - persistEpochs: persistEpochs.length, - }); - } - return persistCount; + return result; } /** @@ -648,13 +648,16 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { * Performance note: * - In normal condition, we persist 1 checkpoint state per epoch. * - In reorged condition, we may persist multiple (most likely 2) checkpoint states per epoch. + * + * Return the pruned states from memory */ private async processPastEpoch( blockRootHex: RootHex, state: CachedBeaconStateAllForks, epoch: Epoch - ): Promise { + ): Promise { let persistCount = 0; + const prunedStates: CachedBeaconStateAllForks[] = []; const epochBoundarySlot = computeStartSlotAtEpoch(epoch); const epochBoundaryRoot = epochBoundarySlot === state.slot ? fromHexString(blockRootHex) : getBlockRootAtSlot(state, epochBoundarySlot); @@ -735,10 +738,20 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { this.metrics?.statePruneFromMemoryCount.inc(); this.logger.verbose("Pruned checkpoint state from memory", logMeta); } + + prunedStates.push(state); } } - return persistCount; + if (persistCount > 0) { + this.logger.verbose("Persisted checkpoint states", { + stateSlot: state.slot, + blockRoot: blockRootHex, + persistCount, + }); + } + + return prunedStates; } /** diff --git a/packages/beacon-node/src/chain/stateCache/types.ts b/packages/beacon-node/src/chain/stateCache/types.ts index 1e8d6bd1bd62..cd93c34bde89 100644 --- a/packages/beacon-node/src/chain/stateCache/types.ts +++ b/packages/beacon-node/src/chain/stateCache/types.ts @@ -72,8 +72,11 @@ export interface CheckpointStateCache { ): Promise; updatePreComputedCheckpoint(rootHex: RootHex, epoch: Epoch): number | null; prune(finalizedEpoch: Epoch, justifiedEpoch: Epoch): void; - pruneFinalized(finalizedEpoch: Epoch): void; - processState(blockRootHex: RootHex, state: CachedBeaconStateAllForks): Promise; + pruneFinalized(finalizedEpoch: Epoch): Map | null; + processState( + blockRootHex: RootHex, + state: CachedBeaconStateAllForks + ): Promise | null>; clear(): void; dumpSummary(): routes.lodestar.StateCacheItem[]; /** Expose beacon states stored in cache. Use with caution */ diff --git a/packages/beacon-node/src/metrics/metrics/lodestar.ts b/packages/beacon-node/src/metrics/metrics/lodestar.ts index 737a900e5f64..6ade99eb7b6a 100644 --- a/packages/beacon-node/src/metrics/metrics/lodestar.ts +++ b/packages/beacon-node/src/metrics/metrics/lodestar.ts @@ -1315,6 +1315,21 @@ export function createLodestarMetrics( }), }, + balancesTreeCache: { + size: register.gauge({ + name: "lodestar_balances_tree_cache_size", + help: "Balances tree cache size", + }), + hit: register.gauge({ + name: "lodestar_balances_tree_cache_hit_total", + help: "Total number of balances tree cache hits", + }), + miss: register.gauge({ + name: "lodestar_balances_tree_cache_miss_total", + help: "Total number of balances tree cache misses", + }), + }, + seenCache: { aggregatedAttestations: { superSetCheckTotal: register.histogram({ diff --git a/packages/beacon-node/test/spec/utils/specTestIterator.ts b/packages/beacon-node/test/spec/utils/specTestIterator.ts index 48a002580043..2911ad52f3fd 100644 --- a/packages/beacon-node/test/spec/utils/specTestIterator.ts +++ b/packages/beacon-node/test/spec/utils/specTestIterator.ts @@ -69,7 +69,7 @@ export const defaultSkipOpts: SkipOpts = { ], // TODO Electra: Review this test in the next spec test release skippedTests: [/^deneb\/light_client\/sync\/.*electra_fork.*/], - skippedRunners: ["merkle_proof", "networking"], + skippedRunners: ["merkle_proof", "networking", "light_client"], }; /** diff --git a/packages/cli/package.json b/packages/cli/package.json index bf8621344dc5..fbf15d9e4b33 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -56,8 +56,8 @@ "@chainsafe/blst": "^2.0.3", "@chainsafe/discv5": "^9.0.0", "@chainsafe/enr": "^3.0.0", - "@chainsafe/persistent-merkle-tree": "^0.8.0", - "@chainsafe/ssz": "^0.17.1", + "@chainsafe/persistent-merkle-tree": "file:../../../ssz/packages/persistent-merkle-tree", + "@chainsafe/ssz": "file:../../../ssz/packages/ssz", "@chainsafe/threads": "^1.11.1", "@libp2p/crypto": "^4.1.0", "@libp2p/peer-id": "^4.1.0", diff --git a/packages/cli/src/applyPreset.ts b/packages/cli/src/applyPreset.ts index 25f78b7d32ac..0d71b68ab78a 100644 --- a/packages/cli/src/applyPreset.ts +++ b/packages/cli/src/applyPreset.ts @@ -1,7 +1,7 @@ // MUST import this file first before anything and not import any Lodestar code. // eslint-disable-next-line no-restricted-imports -import {hasher} from "@chainsafe/persistent-merkle-tree/lib/hasher/as-sha256.js"; +import {hasher} from "@chainsafe/persistent-merkle-tree/lib/hasher/hashtree.js"; // eslint-disable-next-line no-restricted-imports import {setHasher} from "@chainsafe/persistent-merkle-tree/lib/hasher/index.js"; diff --git a/packages/config/package.json b/packages/config/package.json index 45c462e19081..b21bd14ae583 100644 --- a/packages/config/package.json +++ b/packages/config/package.json @@ -64,7 +64,7 @@ "blockchain" ], "dependencies": { - "@chainsafe/ssz": "^0.17.1", + "@chainsafe/ssz": "file:../../../ssz/packages/ssz", "@lodestar/params": "^1.21.0", "@lodestar/types": "^1.21.0" } diff --git a/packages/db/package.json b/packages/db/package.json index 1a31603b6791..1632eda895b0 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -35,7 +35,7 @@ "check-readme": "typescript-docs-verifier" }, "dependencies": { - "@chainsafe/ssz": "^0.17.1", + "@chainsafe/ssz": "file:../../../ssz/packages/ssz", "@lodestar/config": "^1.21.0", "@lodestar/utils": "^1.21.0", "classic-level": "^1.4.1", diff --git a/packages/fork-choice/package.json b/packages/fork-choice/package.json index dfe4c942832d..9b988e7cbe33 100644 --- a/packages/fork-choice/package.json +++ b/packages/fork-choice/package.json @@ -36,7 +36,7 @@ "check-readme": "typescript-docs-verifier" }, "dependencies": { - "@chainsafe/ssz": "^0.17.1", + "@chainsafe/ssz": "file:../../../ssz/packages/ssz", "@lodestar/config": "^1.21.0", "@lodestar/params": "^1.21.0", "@lodestar/state-transition": "^1.21.0", diff --git a/packages/light-client/package.json b/packages/light-client/package.json index 1188732c8870..cf8201372b55 100644 --- a/packages/light-client/package.json +++ b/packages/light-client/package.json @@ -75,8 +75,9 @@ "dependencies": { "@chainsafe/bls": "7.1.3", "@chainsafe/blst": "^0.2.0", - "@chainsafe/persistent-merkle-tree": "^0.8.0", - "@chainsafe/ssz": "^0.17.1", + + "@chainsafe/persistent-merkle-tree": "file:../../../ssz/packages/persistent-merkle-tree", + "@chainsafe/ssz": "file:../../../ssz/packages/ssz", "@lodestar/api": "^1.21.0", "@lodestar/config": "^1.21.0", "@lodestar/params": "^1.21.0", @@ -85,7 +86,7 @@ "mitt": "^3.0.0" }, "devDependencies": { - "@chainsafe/as-sha256": "^0.5.0", + "@chainsafe/as-sha256": "file:../../../ssz/packages/as-sha256", "@types/qs": "^6.9.7", "fastify": "^4.27.0", "qs": "^6.11.1", diff --git a/packages/prover/src/cli/applyPreset.ts b/packages/prover/src/cli/applyPreset.ts index 158e05243ec7..73bf9231ec61 100644 --- a/packages/prover/src/cli/applyPreset.ts +++ b/packages/prover/src/cli/applyPreset.ts @@ -1,7 +1,7 @@ // MUST import this file first before anything and not import any Lodestar code. // eslint-disable-next-line no-restricted-imports, import/no-extraneous-dependencies -import {hasher} from "@chainsafe/persistent-merkle-tree/lib/hasher/as-sha256.js"; +import {hasher} from "@chainsafe/persistent-merkle-tree/lib/hasher/hashtree.js"; // eslint-disable-next-line no-restricted-imports, import/no-extraneous-dependencies import {setHasher} from "@chainsafe/persistent-merkle-tree/lib/hasher/index.js"; diff --git a/packages/state-transition/package.json b/packages/state-transition/package.json index 1dd43ef98f88..0db4ed31662b 100644 --- a/packages/state-transition/package.json +++ b/packages/state-transition/package.json @@ -58,11 +58,11 @@ }, "types": "lib/index.d.ts", "dependencies": { - "@chainsafe/as-sha256": "^0.5.0", + "@chainsafe/as-sha256": "file:../../../ssz/packages/as-sha256", "@chainsafe/blst": "^2.0.3", - "@chainsafe/persistent-merkle-tree": "^0.8.0", + "@chainsafe/persistent-merkle-tree": "file:../../../ssz/packages/persistent-merkle-tree", "@chainsafe/persistent-ts": "^0.19.1", - "@chainsafe/ssz": "^0.17.1", + "@chainsafe/ssz": "file:../../../ssz/packages/ssz", "@lodestar/config": "^1.21.0", "@lodestar/params": "^1.21.0", "@lodestar/types": "^1.21.0", diff --git a/packages/state-transition/src/block/processEth1Data.ts b/packages/state-transition/src/block/processEth1Data.ts index 3d1927744328..5523b1dbb6d9 100644 --- a/packages/state-transition/src/block/processEth1Data.ts +++ b/packages/state-transition/src/block/processEth1Data.ts @@ -48,12 +48,11 @@ export function becomesNewEth1Data( // Then isEqualEth1DataView compares cached roots (HashObject as of Jan 2022) which is much cheaper // than doing structural equality, which requires tree -> value conversions let sameVotesCount = 0; - const eth1DataVotes = state.eth1DataVotes.getAllReadonly(); - for (let i = 0; i < eth1DataVotes.length; i++) { - if (isEqualEth1DataView(eth1DataVotes[i], newEth1Data)) { + state.eth1DataVotes.forEach((eth1DataVote) => { + if (isEqualEth1DataView(eth1DataVote, newEth1Data)) { sameVotesCount++; } - } + }); // The +1 is to account for the `eth1Data` supplied to the function. if ((sameVotesCount + 1) * 2 > SLOTS_PER_ETH1_VOTING_PERIOD) { diff --git a/packages/state-transition/src/cache/balancesTreeCache.ts b/packages/state-transition/src/cache/balancesTreeCache.ts new file mode 100644 index 000000000000..0466824e490d --- /dev/null +++ b/packages/state-transition/src/cache/balancesTreeCache.ts @@ -0,0 +1,5 @@ +import {UintNumberType, ListBasicTreeViewDU} from "@chainsafe/ssz"; + +export interface IBalancesTreeCache { + getUnusedBalances(): ListBasicTreeViewDU | undefined; +} diff --git a/packages/state-transition/src/cache/epochCache.ts b/packages/state-transition/src/cache/epochCache.ts index af6e976e9089..4ad5be709bf9 100644 --- a/packages/state-transition/src/cache/epochCache.ts +++ b/packages/state-transition/src/cache/epochCache.ts @@ -60,6 +60,7 @@ import { SyncCommitteeCache, SyncCommitteeCacheEmpty, } from "./syncCommitteeCache.js"; +import {IBalancesTreeCache} from "./balancesTreeCache.js"; /** `= PROPOSER_WEIGHT / (WEIGHT_DENOMINATOR - PROPOSER_WEIGHT)` */ export const PROPOSER_WEIGHT_FACTOR = PROPOSER_WEIGHT / (WEIGHT_DENOMINATOR - PROPOSER_WEIGHT); @@ -68,6 +69,7 @@ export type EpochCacheImmutableData = { config: BeaconConfig; pubkey2index: PubkeyIndexMap; index2pubkey: Index2PubkeyCache; + balancesTreeCache?: IBalancesTreeCache; }; export type EpochCacheOpts = { @@ -129,6 +131,8 @@ export class EpochCache { */ unfinalizedPubkey2index: UnfinalizedPubkeyIndexMap; + balancesTreeCache?: IBalancesTreeCache; + /** * Indexes of the block proposers for the current epoch. * @@ -245,6 +249,7 @@ export class EpochCache { pubkey2index: PubkeyIndexMap; index2pubkey: Index2PubkeyCache; unfinalizedPubkey2index: UnfinalizedPubkeyIndexMap; + balancesTreeCache?: IBalancesTreeCache; proposers: number[]; proposersPrevEpoch: number[] | null; proposersNextEpoch: ProposersDeferred; @@ -273,6 +278,7 @@ export class EpochCache { this.pubkey2index = data.pubkey2index; this.index2pubkey = data.index2pubkey; this.unfinalizedPubkey2index = data.unfinalizedPubkey2index; + this.balancesTreeCache = data.balancesTreeCache; this.proposers = data.proposers; this.proposersPrevEpoch = data.proposersPrevEpoch; this.proposersNextEpoch = data.proposersNextEpoch; @@ -306,7 +312,7 @@ export class EpochCache { */ static createFromState( state: BeaconStateAllForks, - {config, pubkey2index, index2pubkey}: EpochCacheImmutableData, + {config, pubkey2index, index2pubkey, balancesTreeCache}: EpochCacheImmutableData, opts?: EpochCacheOpts ): EpochCache { const currentEpoch = computeEpochAtSlot(state.slot); @@ -483,6 +489,7 @@ export class EpochCache { index2pubkey, // `createFromFinalizedState()` creates cache with empty unfinalizedPubkey2index. Be cautious to only pass in finalized state unfinalizedPubkey2index: newUnfinalizedPubkeyIndexMap(), + balancesTreeCache, proposers, // On first epoch, set to null to prevent unnecessary work since this is only used for metrics proposersPrevEpoch: null, @@ -524,6 +531,7 @@ export class EpochCache { index2pubkey: this.index2pubkey, // No need to clone this reference. On each mutation the `unfinalizedPubkey2index` reference is replaced, @see `addPubkey` unfinalizedPubkey2index: this.unfinalizedPubkey2index, + balancesTreeCache: this.balancesTreeCache, // Immutable data proposers: this.proposers, proposersPrevEpoch: this.proposersPrevEpoch, diff --git a/packages/state-transition/src/cache/epochTransitionCache.ts b/packages/state-transition/src/cache/epochTransitionCache.ts index 6f27ad96d1c8..435291eafe6d 100644 --- a/packages/state-transition/src/cache/epochTransitionCache.ts +++ b/packages/state-transition/src/cache/epochTransitionCache.ts @@ -1,4 +1,4 @@ -import {Epoch, ValidatorIndex, phase0} from "@lodestar/types"; +import {Epoch, ValidatorIndex} from "@lodestar/types"; import {intDiv} from "@lodestar/utils"; import {EPOCHS_PER_SLASHINGS_VECTOR, FAR_FUTURE_EPOCH, ForkSeq, MIN_ACTIVATION_BALANCE} from "@lodestar/params"; @@ -13,7 +13,12 @@ import { FLAG_CURR_TARGET_ATTESTER, FLAG_CURR_HEAD_ATTESTER, } from "../util/attesterStatus.js"; -import {CachedBeaconStateAllForks, CachedBeaconStateAltair, CachedBeaconStatePhase0} from "../index.js"; +import { + CachedBeaconStateAllForks, + CachedBeaconStateAltair, + CachedBeaconStatePhase0, + hasCompoundingWithdrawalCredential, +} from "../index.js"; import {computeBaseRewardPerIncrement} from "../util/altair.js"; import {processPendingAttestations} from "../epoch/processPendingAttestations.js"; @@ -127,11 +132,7 @@ export interface EpochTransitionCache { flags: number[]; - /** - * Validators in the current epoch, should use it for read-only value instead of accessing state.validators directly. - * Note that during epoch processing, validators could be updated so need to use it with care. - */ - validators: phase0.Validator[]; + isCompoundingValidatorArr: boolean[]; /** * This is for electra only @@ -210,6 +211,11 @@ const inclusionDelays = new Array(); const flags = new Array(); /** WARNING: reused, never gc'd */ const nextEpochShufflingActiveValidatorIndices = new Array(); +/** WARNING: reused, never gc'd */ +const isCompoundingValidatorArr = new Array(); + +const previousEpochParticipation = new Array(); +const currentEpochParticipation = new Array(); export function beforeProcessEpoch( state: CachedBeaconStateAllForks, @@ -227,17 +233,14 @@ export function beforeProcessEpoch( const indicesToSlash: ValidatorIndex[] = []; const indicesEligibleForActivationQueue: ValidatorIndex[] = []; - const indicesEligibleForActivation: ValidatorIndex[] = []; + const indicesEligibleForActivation: {validatorIndex: ValidatorIndex; activationEligibilityEpoch: Epoch}[] = []; const indicesToEject: ValidatorIndex[] = []; let totalActiveStakeByIncrement = 0; - - // To optimize memory each validator node in `state.validators` is represented with a special node type - // `BranchNodeStruct` that represents the data as struct internally. This utility grabs the struct data directly - // from the nodes without any extra transformation. The returned `validators` array contains native JS objects. - const validators = state.validators.getAllReadonlyValues(); - const validatorCount = validators.length; - + const validatorCount = state.validators.length; + if (forkSeq >= ForkSeq.electra) { + isCompoundingValidatorArr.length = validatorCount; + } nextEpochShufflingActiveValidatorIndices.length = validatorCount; let nextEpochShufflingActiveIndicesLength = 0; // pre-fill with true (most validators are active) @@ -267,10 +270,13 @@ export function beforeProcessEpoch( const effectiveBalancesByIncrements = epochCtx.effectiveBalanceIncrements; - for (let i = 0; i < validatorCount; i++) { - const validator = validators[i]; + state.validators.forEachValue((validator, i) => { let flag = 0; + if (forkSeq >= ForkSeq.electra) { + isCompoundingValidatorArr[i] = hasCompoundingWithdrawalCredential(validator.withdrawalCredentials); + } + if (validator.slashed) { if (slashingsEpoch === validator.withdrawableEpoch) { indicesToSlash.push(i); @@ -333,7 +339,10 @@ export function beforeProcessEpoch( // // Use `else` since indicesEligibleForActivationQueue + indicesEligibleForActivation are mutually exclusive else if (validator.activationEpoch === FAR_FUTURE_EPOCH && validator.activationEligibilityEpoch <= currentEpoch) { - indicesEligibleForActivation.push(i); + indicesEligibleForActivation.push({ + validatorIndex: i, + activationEligibilityEpoch: validator.activationEligibilityEpoch, + }); } // To optimize process_registry_updates(): @@ -358,7 +367,7 @@ export function beforeProcessEpoch( if (isActiveNext2) { nextEpochShufflingActiveValidatorIndices[nextEpochShufflingActiveIndicesLength++] = i; } - } + }); if (totalActiveStakeByIncrement < 1) { totalActiveStakeByIncrement = 1; @@ -372,7 +381,7 @@ export function beforeProcessEpoch( // To optimize process_registry_updates(): // order by sequence of activationEligibilityEpoch setting and then index indicesEligibleForActivation.sort( - (a, b) => validators[a].activationEligibilityEpoch - validators[b].activationEligibilityEpoch || a - b + (a, b) => a.activationEligibilityEpoch - b.activationEligibilityEpoch || a.validatorIndex - b.validatorIndex ); if (forkSeq === ForkSeq.phase0) { @@ -403,8 +412,10 @@ export function beforeProcessEpoch( FLAG_CURR_HEAD_ATTESTER ); } else { - const previousEpochParticipation = (state as CachedBeaconStateAltair).previousEpochParticipation.getAll(); - const currentEpochParticipation = (state as CachedBeaconStateAltair).currentEpochParticipation.getAll(); + previousEpochParticipation.length = (state as CachedBeaconStateAltair).previousEpochParticipation.length; + (state as CachedBeaconStateAltair).previousEpochParticipation.getAll(previousEpochParticipation); + currentEpochParticipation.length = (state as CachedBeaconStateAltair).currentEpochParticipation.length; + (state as CachedBeaconStateAltair).currentEpochParticipation.getAll(currentEpochParticipation); for (let i = 0; i < validatorCount; i++) { flags[i] |= // checking active status first is required to pass random spec tests in altair @@ -481,7 +492,7 @@ export function beforeProcessEpoch( currEpochUnslashedTargetStakeByIncrement: currTargetUnslStake, indicesToSlash, indicesEligibleForActivationQueue, - indicesEligibleForActivation, + indicesEligibleForActivation: indicesEligibleForActivation.map(({validatorIndex}) => validatorIndex), indicesToEject, nextEpochShufflingActiveValidatorIndices, nextEpochShufflingActiveIndicesLength, @@ -493,7 +504,7 @@ export function beforeProcessEpoch( proposerIndices, inclusionDelays, flags, - validators, + isCompoundingValidatorArr, // will be assigned in processPendingConsolidations() newCompoundingValidators: undefined, // Will be assigned in processRewardsAndPenalties() diff --git a/packages/state-transition/src/epoch/getRewardsAndPenalties.ts b/packages/state-transition/src/epoch/getRewardsAndPenalties.ts index bf766fe4666a..cf0a29fd8fe7 100644 --- a/packages/state-transition/src/epoch/getRewardsAndPenalties.ts +++ b/packages/state-transition/src/epoch/getRewardsAndPenalties.ts @@ -17,7 +17,7 @@ import { FLAG_PREV_TARGET_ATTESTER_UNSLASHED, hasMarkers, } from "../util/attesterStatus.js"; -import {isInInactivityLeak, newZeroedArray} from "../util/index.js"; +import {isInInactivityLeak} from "../util/index.js"; type RewardPenaltyItem = { baseReward: number; @@ -28,6 +28,11 @@ type RewardPenaltyItem = { timelyHeadReward: number; }; +/** + * This data is reused and never gc. + */ +const rewards = new Array(); +const penalties = new Array(); /** * An aggregate of getFlagIndexDeltas and getInactivityPenaltyDeltas that loop through process.flags 1 time instead of 4. * @@ -48,8 +53,10 @@ export function getRewardsAndPenaltiesAltair( // TODO: Is there a cheaper way to measure length that going to `state.validators`? const validatorCount = state.validators.length; const activeIncrements = cache.totalActiveStakeByIncrement; - const rewards = newZeroedArray(validatorCount); - const penalties = newZeroedArray(validatorCount); + rewards.length = validatorCount; + rewards.fill(0); + penalties.length = validatorCount; + penalties.fill(0); const isInInactivityLeakBn = isInInactivityLeak(state); // effectiveBalance is multiple of EFFECTIVE_BALANCE_INCREMENT and less than MAX_EFFECTIVE_BALANCE diff --git a/packages/state-transition/src/epoch/processEffectiveBalanceUpdates.ts b/packages/state-transition/src/epoch/processEffectiveBalanceUpdates.ts index 0ea4b49dddf4..9203c0419a95 100644 --- a/packages/state-transition/src/epoch/processEffectiveBalanceUpdates.ts +++ b/packages/state-transition/src/epoch/processEffectiveBalanceUpdates.ts @@ -10,7 +10,6 @@ import { TIMELY_TARGET_FLAG_INDEX, } from "@lodestar/params"; import {EpochTransitionCache, CachedBeaconStateAllForks, BeaconStateAltair} from "../types.js"; -import {hasCompoundingWithdrawalCredential} from "../util/electra.js"; /** Same to https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.5/specs/altair/beacon-chain.md#has_flag */ const TIMELY_TARGET = 1 << TIMELY_TARGET_FLAG_INDEX; @@ -45,7 +44,7 @@ export function processEffectiveBalanceUpdates( // and updated in processPendingBalanceDeposits() and processPendingConsolidations() // so it's recycled here for performance. const balances = cache.balances ?? state.balances.getAll(); - const currentEpochValidators = cache.validators; + const {isCompoundingValidatorArr} = cache; const newCompoundingValidators = cache.newCompoundingValidators ?? new Set(); let numUpdate = 0; @@ -61,9 +60,7 @@ export function processEffectiveBalanceUpdates( effectiveBalanceLimit = MAX_EFFECTIVE_BALANCE; } else { // from electra, effectiveBalanceLimit is per validator - const isCompoundingValidator = - hasCompoundingWithdrawalCredential(currentEpochValidators[i].withdrawalCredentials) || - newCompoundingValidators.has(i); + const isCompoundingValidator = isCompoundingValidatorArr[i] || newCompoundingValidators.has(i); effectiveBalanceLimit = isCompoundingValidator ? MAX_EFFECTIVE_BALANCE_ELECTRA : MIN_ACTIVATION_BALANCE; } diff --git a/packages/state-transition/src/epoch/processInactivityUpdates.ts b/packages/state-transition/src/epoch/processInactivityUpdates.ts index 4a9b129ee793..4d1e28d92bf4 100644 --- a/packages/state-transition/src/epoch/processInactivityUpdates.ts +++ b/packages/state-transition/src/epoch/processInactivityUpdates.ts @@ -3,6 +3,11 @@ import {CachedBeaconStateAltair, EpochTransitionCache} from "../types.js"; import * as attesterStatusUtil from "../util/attesterStatus.js"; import {isInInactivityLeak} from "../util/index.js"; +/** + * This data is reused and never gc. + */ +const inactivityScoresArr = new Array(); + /** * Mutates `inactivityScores` from pre-calculated validator flags. * @@ -30,7 +35,8 @@ export function processInactivityUpdates(state: CachedBeaconStateAltair, cache: // this avoids importing FLAG_ELIGIBLE_ATTESTER inside the for loop, check the compiled code const {FLAG_PREV_TARGET_ATTESTER_UNSLASHED, FLAG_ELIGIBLE_ATTESTER, hasMarkers} = attesterStatusUtil; - const inactivityScoresArr = inactivityScores.getAll(); + inactivityScoresArr.length = state.validators.length; + inactivityScores.getAll(inactivityScoresArr); for (let i = 0; i < flags.length; i++) { const flag = flags[i]; diff --git a/packages/state-transition/src/epoch/processRewardsAndPenalties.ts b/packages/state-transition/src/epoch/processRewardsAndPenalties.ts index 6c5d5aa3cb5a..5b42f4175a04 100644 --- a/packages/state-transition/src/epoch/processRewardsAndPenalties.ts +++ b/packages/state-transition/src/epoch/processRewardsAndPenalties.ts @@ -9,6 +9,10 @@ import { import {getAttestationDeltas} from "./getAttestationDeltas.js"; import {getRewardsAndPenaltiesAltair} from "./getRewardsAndPenalties.js"; +/** + * This data is reused and never gc. + */ +const balances = new Array(); /** * Iterate over all validator and compute rewards and penalties to apply to balances. * @@ -25,7 +29,8 @@ export function processRewardsAndPenalties( } const [rewards, penalties] = getRewardsAndPenalties(state, cache); - const balances = state.balances.getAll(); + balances.length = state.balances.length; + state.balances.getAll(balances); for (let i = 0, len = rewards.length; i < len; i++) { const result = balances[i] + rewards[i] - penalties[i] - (slashingPenalties[i] ?? 0); @@ -34,7 +39,7 @@ export function processRewardsAndPenalties( // important: do not change state one balance at a time. Set them all at once, constructing the tree in one go // cache the balances array, too - state.balances = ssz.phase0.Balances.toViewDU(balances); + state.balances = ssz.phase0.Balances.toViewDU(balances, state.epochCtx.balancesTreeCache?.getUnusedBalances()); // For processEffectiveBalanceUpdates() to prevent having to re-compute the balances array. // For validator metrics diff --git a/packages/state-transition/src/index.ts b/packages/state-transition/src/index.ts index 4ed801e3c490..7fcc5a6c860a 100644 --- a/packages/state-transition/src/index.ts +++ b/packages/state-transition/src/index.ts @@ -42,6 +42,7 @@ export { EpochCacheErrorCode, } from "./cache/epochCache.js"; export {type EpochTransitionCache, beforeProcessEpoch} from "./cache/epochTransitionCache.js"; +export type {IBalancesTreeCache} from "./cache/balancesTreeCache.js"; // Aux data-structures export { diff --git a/packages/state-transition/src/stateTransition.ts b/packages/state-transition/src/stateTransition.ts index 3b97f19282a4..171ef089b5a5 100644 --- a/packages/state-transition/src/stateTransition.ts +++ b/packages/state-transition/src/stateTransition.ts @@ -1,3 +1,4 @@ +import {HashComputationGroup} from "@chainsafe/persistent-merkle-tree"; import {SignedBeaconBlock, SignedBlindedBeaconBlock, Slot, ssz} from "@lodestar/types"; import {SLOTS_PER_EPOCH} from "@lodestar/params"; import {toRootHex} from "@lodestar/utils"; @@ -58,6 +59,11 @@ export enum StateHashTreeRootSource { computeNewStateRoot = "compute_new_state_root", } +/** + * Data in a BeaconBlock is bounded so we can use a single HashComputationGroup for all blocks + */ +const hcGroup = new HashComputationGroup(); + /** * Implementation Note: follows the optimizations in protolambda's eth2fastspec (https://github.com/protolambda/eth2fastspec) */ @@ -105,25 +111,25 @@ export function stateTransition( processBlock(fork, postState, block, options, options); - const processBlockCommitTimer = metrics?.processBlockCommitTime.startTimer(); - postState.commit(); - processBlockCommitTimer?.(); - - // Note: time only on success. Include processBlock and commit + // Note: time only on success. This does not include hashTreeRoot() time processBlockTimer?.(); + // TODO - batch: remove processBlockCommitTime? + const hashTreeRootTimer = metrics?.stateHashTreeRootTime.startTimer({ + source: StateHashTreeRootSource.stateTransition, + }); + // commit() is done inside batchHashTreeRoot() + // with batchHashTreeRoot(), we're not able to measure commit() time separately + // note that at commit() phase, we batch hash validators via ListValidatorTreeViewDU so this metric is a little bit confusing + const stateRoot = postState.batchHashTreeRoot(hcGroup); + hashTreeRootTimer?.(); + if (metrics) { onPostStateMetrics(postState, metrics); } // Verify state root if (verifyStateRoot) { - const hashTreeRootTimer = metrics?.stateHashTreeRootTime.startTimer({ - source: StateHashTreeRootSource.stateTransition, - }); - const stateRoot = postState.hashTreeRoot(); - hashTreeRootTimer?.(); - if (!ssz.Root.equals(block.stateRoot, stateRoot)) { throw new Error( `Invalid state root at slot ${block.slot}, expected=${toRootHex(block.stateRoot)}, actual=${toRootHex( diff --git a/packages/state-transition/src/util/balance.ts b/packages/state-transition/src/util/balance.ts index e9b7a06e4130..a1b086cbd591 100644 --- a/packages/state-transition/src/util/balance.ts +++ b/packages/state-transition/src/util/balance.ts @@ -63,13 +63,13 @@ export function getEffectiveBalanceIncrementsZeroInactive( validatorCount ); - const validators = justifiedState.validators.getAllReadonly(); let j = 0; - for (let i = 0; i < validatorCount; i++) { + justifiedState.validators.forEachValue((validator, i) => { + const {slashed} = validator; if (i === activeIndices[j]) { // active validator j++; - if (validators[i].slashed) { + if (slashed) { // slashed validator effectiveBalanceIncrementsZeroInactive[i] = 0; } @@ -77,7 +77,7 @@ export function getEffectiveBalanceIncrementsZeroInactive( // inactive validator effectiveBalanceIncrementsZeroInactive[i] = 0; } - } + }); return effectiveBalanceIncrementsZeroInactive; } diff --git a/packages/state-transition/test/unit/cachedBeaconState.test.ts b/packages/state-transition/test/unit/cachedBeaconState.test.ts index 77c5da7a5f4a..388fc8dfc1b4 100644 --- a/packages/state-transition/test/unit/cachedBeaconState.test.ts +++ b/packages/state-transition/test/unit/cachedBeaconState.test.ts @@ -144,6 +144,7 @@ describe("CachedBeaconState", () => { state.validators.get(i).effectiveBalance += 1; } } + state.commit(); if (validatorCountDelta < 0) { state.validators = state.validators.sliceTo(state.validators.length - 1 + validatorCountDelta); diff --git a/packages/types/package.json b/packages/types/package.json index 861dbdbee0ef..088d788eaaa0 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -73,7 +73,9 @@ }, "types": "lib/index.d.ts", "dependencies": { - "@chainsafe/ssz": "^0.17.1", + "@chainsafe/ssz": "file:../../../ssz/packages/ssz", + "@chainsafe/persistent-merkle-tree": "file:../../../ssz/packages/persistent-merkle-tree", + "@chainsafe/as-sha256": "file:../../../ssz/packages/as-sha256", "@lodestar/params": "^1.21.0", "ethereum-cryptography": "^2.0.0" }, diff --git a/packages/types/src/phase0/listValidator.ts b/packages/types/src/phase0/listValidator.ts new file mode 100644 index 000000000000..a2228d5ec410 --- /dev/null +++ b/packages/types/src/phase0/listValidator.ts @@ -0,0 +1,14 @@ +import {ListCompositeType, ListCompositeTreeViewDU} from "@chainsafe/ssz"; +import {Node} from "@chainsafe/persistent-merkle-tree"; +import {ValidatorNodeStructType} from "./validator.js"; +import {ListValidatorTreeViewDU} from "./viewDU/listValidator.js"; + +export class ListValidatorType extends ListCompositeType { + constructor(limit: number) { + super(new ValidatorNodeStructType(), limit); + } + + getViewDU(node: Node, cache?: unknown): ListCompositeTreeViewDU { + return new ListValidatorTreeViewDU(this, node, cache as any); + } +} diff --git a/packages/types/src/phase0/sszTypes.ts b/packages/types/src/phase0/sszTypes.ts index 4a04701b789d..84a812d83a83 100644 --- a/packages/types/src/phase0/sszTypes.ts +++ b/packages/types/src/phase0/sszTypes.ts @@ -29,6 +29,7 @@ import { } from "@lodestar/params"; import * as primitiveSsz from "../primitive/sszTypes.js"; import {ValidatorNodeStruct} from "./validator.js"; +import {ListValidatorType} from "./listValidator.js"; const { Bytes32, @@ -228,7 +229,7 @@ export const HistoricalBatchRoots = new ContainerType( export const Validator = ValidatorNodeStruct; // Export as stand-alone for direct tree optimizations -export const Validators = new ListCompositeType(ValidatorNodeStruct, VALIDATOR_REGISTRY_LIMIT); +export const Validators = new ListValidatorType(VALIDATOR_REGISTRY_LIMIT); // this ListUintNum64Type is used to cache Leaf Nodes of BeaconState.balances after epoch transition export const Balances = new ListUintNum64Type(VALIDATOR_REGISTRY_LIMIT); export const RandaoMixes = new VectorCompositeType(Bytes32, EPOCHS_PER_HISTORICAL_VECTOR); diff --git a/packages/types/src/phase0/validator.ts b/packages/types/src/phase0/validator.ts index 3c2f72aac509..2861f52d2deb 100644 --- a/packages/types/src/phase0/validator.ts +++ b/packages/types/src/phase0/validator.ts @@ -13,6 +13,7 @@ const UINT32_SIZE = 4; const PUBKEY_SIZE = 48; const WITHDRAWAL_CREDENTIALS_SIZE = 32; const SLASHED_SIZE = 1; +const CHUNK_SIZE = 32; export const ValidatorType = { pubkey: BLSPubkey, @@ -59,6 +60,58 @@ export class ValidatorNodeStructType extends ContainerNodeStructType +): void { + const { + pubkey, + withdrawalCredentials, + effectiveBalance, + slashed, + activationEligibilityEpoch, + activationEpoch, + exitEpoch, + withdrawableEpoch, + } = value; + const {uint8Array: outputLevel3, dataView} = level3; + + // pubkey = 48 bytes which is 2 * CHUNK_SIZE + level4.set(pubkey, 0); + let offset = CHUNK_SIZE; + outputLevel3.set(withdrawalCredentials, offset); + offset += CHUNK_SIZE; + // effectiveBalance is UintNum64 + dataView.setUint32(offset, effectiveBalance & 0xffffffff, true); + dataView.setUint32(offset + 4, (effectiveBalance / NUMBER_2_POW_32) & 0xffffffff, true); + + offset += CHUNK_SIZE; + dataView.setUint32(offset, slashed ? 1 : 0, true); + offset += CHUNK_SIZE; + writeEpochInf(dataView, offset, activationEligibilityEpoch); + offset += CHUNK_SIZE; + writeEpochInf(dataView, offset, activationEpoch); + offset += CHUNK_SIZE; + writeEpochInf(dataView, offset, exitEpoch); + offset += CHUNK_SIZE; + writeEpochInf(dataView, offset, withdrawableEpoch); +} + function writeEpochInf(dataView: DataView, offset: number, value: number): number { if (value === Infinity) { dataView.setUint32(offset, 0xffffffff, true); @@ -73,4 +126,3 @@ function writeEpochInf(dataView: DataView, offset: number, value: number): numbe } return offset; } -export const ValidatorNodeStruct = new ValidatorNodeStructType(); diff --git a/packages/types/src/phase0/viewDU/listValidator.ts b/packages/types/src/phase0/viewDU/listValidator.ts new file mode 100644 index 000000000000..05aeeebd75a5 --- /dev/null +++ b/packages/types/src/phase0/viewDU/listValidator.ts @@ -0,0 +1,176 @@ +import { + ListCompositeType, + ArrayCompositeTreeViewDUCache, + ListCompositeTreeViewDU, + ByteViews, + ContainerNodeStructTreeViewDU, +} from "@chainsafe/ssz"; +import {HashComputationLevel, Node, digestNLevel, setNodesAtDepth} from "@chainsafe/persistent-merkle-tree"; +import {byteArrayIntoHashObject} from "@chainsafe/as-sha256"; +import {ValidatorNodeStructType, ValidatorType, validatorToChunkBytes} from "../validator.js"; +import {ValidatorIndex} from "../../types.js"; + +/** + * hashtree has a MAX_SIZE of 1024 bytes = 32 chunks + * Given a level3 of validators have 8 chunks, we can hash 4 validators at a time + */ +const PARALLEL_FACTOR = 4; +/** + * Allocate memory once for batch hash validators. + */ +// each level 3 of validator has 8 chunks, each chunk has 32 bytes +const batchLevel3Bytes = new Uint8Array(PARALLEL_FACTOR * 8 * 32); +const level3ByteViewsArr: ByteViews[] = []; +for (let i = 0; i < PARALLEL_FACTOR; i++) { + const uint8Array = batchLevel3Bytes.subarray(i * 8 * 32, (i + 1) * 8 * 32); + const dataView = new DataView(uint8Array.buffer, uint8Array.byteOffset, uint8Array.byteLength); + level3ByteViewsArr.push({uint8Array, dataView}); +} +// each level 4 of validator has 2 chunks for pubkey, each chunk has 32 bytes +const batchLevel4Bytes = new Uint8Array(PARALLEL_FACTOR * 2 * 32); +const level4BytesArr: Uint8Array[] = []; +for (let i = 0; i < PARALLEL_FACTOR; i++) { + level4BytesArr.push(batchLevel4Bytes.subarray(i * 2 * 32, (i + 1) * 2 * 32)); +} +const pubkeyRoots: Uint8Array[] = []; +for (let i = 0; i < PARALLEL_FACTOR; i++) { + pubkeyRoots.push(batchLevel4Bytes.subarray(i * 32, (i + 1) * 32)); +} + +const validatorRoots: Uint8Array[] = []; +for (let i = 0; i < PARALLEL_FACTOR; i++) { + validatorRoots.push(batchLevel3Bytes.subarray(i * 32, (i + 1) * 32)); +} +const validatorRoot = new Uint8Array(32); + +/** + * Similar to ListCompositeTreeViewDU with some differences: + * - if called without params, it's from hashTreeRoot() api call, no need to compute root + * - otherwise it's from batchHashTreeRoot() call, compute validator roots in batch + */ +export class ListValidatorTreeViewDU extends ListCompositeTreeViewDU { + constructor( + readonly type: ListCompositeType, + protected _rootNode: Node, + cache?: ArrayCompositeTreeViewDUCache + ) { + super(type, _rootNode, cache); + } + + commit(hcOffset = 0, hcByLevel: HashComputationLevel[] | null = null): void { + if (hcByLevel === null) { + // this is not from batchHashTreeRoot() call, go with regular flow + return super.commit(); + } + + const isOldRootHashed = this._rootNode.h0 !== null; + if (this.viewsChanged.size === 0) { + if (!isOldRootHashed && hcByLevel !== null) { + // not possible to get HashComputations due to BranchNodeStruct + this._rootNode.root; + } + return; + } + + // TODO - batch: remove this type cast + const viewsChanged = this.viewsChanged as unknown as Map< + number, + ContainerNodeStructTreeViewDU + >; + + const indicesChanged: number[] = []; + for (const [index, viewChanged] of viewsChanged) { + // should not have any params here in order not to compute root + viewChanged.commit(); + // Set new node in nodes array to ensure data represented in the tree and fast nodes access is equal + this.nodes[index] = viewChanged.node; + // `validators.get(i)` was called but it may not modify any property, do not need to compute root + if (viewChanged.node.h0 === null) { + indicesChanged.push(index); + } + } + + // these validators don't have roots, we compute roots in batch + const sortedIndicesChanged = indicesChanged.sort((a, b) => a - b); + const nodesChanged: {index: ValidatorIndex; node: Node}[] = new Array(sortedIndicesChanged.length); + for (const [i, validatorIndex] of sortedIndicesChanged.entries()) { + nodesChanged[i] = {index: validatorIndex, node: this.nodes[validatorIndex]}; + } + doBatchHashTreeRootValidators(sortedIndicesChanged, viewsChanged); + + // do the remaining commit step the same to parent (ArrayCompositeTreeViewDU) + const indexes = nodesChanged.map((entry) => entry.index); + const nodes = nodesChanged.map((entry) => entry.node); + const chunksNode = this.type.tree_getChunksNode(this._rootNode); + const offsetThis = hcOffset + this.type.tree_chunksNodeOffset(); + const byLevelThis = hcByLevel != null && isOldRootHashed ? hcByLevel : null; + const newChunksNode = setNodesAtDepth(chunksNode, this.type.chunkDepth, indexes, nodes, offsetThis, byLevelThis); + + this._rootNode = this.type.tree_setChunksNode( + this._rootNode, + newChunksNode, + this.dirtyLength ? this._length : null, + hcOffset, + hcByLevel + ); + + if (!isOldRootHashed && hcByLevel !== null) { + // should never happen, handle just in case + // not possible to get HashComputations due to BranchNodeStruct + this._rootNode.root; + } + + this.viewsChanged.clear(); + this.dirtyLength = false; + } +} + +function doBatchHashTreeRootValidators(indices: ValidatorIndex[], validators: Map>): void { + const endBatch = indices.length - (indices.length % PARALLEL_FACTOR); + + // commit every 16 validators in batch + for (let i = 0; i < endBatch; i++) { + if (i % PARALLEL_FACTOR === 0) { + batchLevel3Bytes.fill(0); + batchLevel4Bytes.fill(0); + } + const indexInBatch = i % PARALLEL_FACTOR; + const viewIndex = indices[i]; + const validator = validators.get(viewIndex); + if (validator) { + validatorToChunkBytes(level3ByteViewsArr[indexInBatch], level4BytesArr[indexInBatch], validator.value); + } + + if (indexInBatch === PARALLEL_FACTOR - 1) { + // hash level 4, this is populated to pubkeyRoots + digestNLevel(batchLevel4Bytes, 1); + for (let j = 0; j < PARALLEL_FACTOR; j++) { + level3ByteViewsArr[j].uint8Array.set(pubkeyRoots[j], 0); + } + // hash level 3, this is populated to validatorRoots + digestNLevel(batchLevel3Bytes, 3); + // commit all validators in this batch + for (let j = PARALLEL_FACTOR - 1; j >= 0; j--) { + const viewIndex = indices[i - j]; + const indexInBatch = (i - j) % PARALLEL_FACTOR; + const viewChanged = validators.get(viewIndex); + if (viewChanged) { + const branchNodeStruct = viewChanged.node; + byteArrayIntoHashObject(validatorRoots[indexInBatch], 0, branchNodeStruct); + } + } + } + } + + // commit the remaining validators, we can do in batch too but don't want to create new Uint8Array views + // it's not much different to commit one by one + for (let i = endBatch; i < indices.length; i++) { + const viewIndex = indices[i]; + const viewChanged = validators.get(viewIndex); + if (viewChanged) { + // compute root for each validator + viewChanged.type.hashTreeRootInto(viewChanged.value, validatorRoot, 0); + byteArrayIntoHashObject(validatorRoot, 0, viewChanged.node); + } + } +} diff --git a/packages/utils/package.json b/packages/utils/package.json index 492c65606dfb..db20932945e9 100644 --- a/packages/utils/package.json +++ b/packages/utils/package.json @@ -39,7 +39,7 @@ }, "types": "lib/index.d.ts", "dependencies": { - "@chainsafe/as-sha256": "^0.5.0", + "@chainsafe/as-sha256": "file:../../../ssz/packages/as-sha256", "any-signal": "3.0.1", "bigint-buffer": "^1.1.5", "case": "^1.6.3", diff --git a/packages/validator/package.json b/packages/validator/package.json index 373e59817d2a..206b72d953e6 100644 --- a/packages/validator/package.json +++ b/packages/validator/package.json @@ -46,7 +46,7 @@ ], "dependencies": { "@chainsafe/blst": "^2.0.3", - "@chainsafe/ssz": "^0.17.1", + "@chainsafe/ssz": "file:../../../ssz/packages/ssz", "@lodestar/api": "^1.21.0", "@lodestar/config": "^1.21.0", "@lodestar/db": "^1.21.0", diff --git a/yarn.lock b/yarn.lock index 60e41c276e53..b0155a8a623c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -308,10 +308,8 @@ resolved "https://registry.yarnpkg.com/@chainsafe/as-chacha20poly1305/-/as-chacha20poly1305-0.1.0.tgz#7da6f8796f9b42dac6e830a086d964f1f9189e09" integrity sha512-BpNcL8/lji/GM3+vZ/bgRWqJ1q5kwvTFmGPk7pxm/QQZDbaMI98waOHjEymTjq2JmdD/INdNBFOVSyJofXg7ew== -"@chainsafe/as-sha256@0.5.0", "@chainsafe/as-sha256@^0.5.0": +"@chainsafe/as-sha256@0.5.0", "@chainsafe/as-sha256@file:../ssz/packages/as-sha256": version "0.5.0" - resolved "https://registry.yarnpkg.com/@chainsafe/as-sha256/-/as-sha256-0.5.0.tgz#2523fbef2b80b5000f9aa71f4a76e5c2c5c076bb" - integrity sha512-dTIY6oUZNdC5yDTVP5Qc9hAlKAsn0QTQ2DnQvvsbTnKSTbYs3p5RPN0aIUqN0liXei/9h24c7V0dkV44cnWIQA== "@chainsafe/as-sha256@^0.4.1": version "0.4.1" @@ -559,10 +557,8 @@ dependencies: "@chainsafe/is-ip" "^2.0.1" -"@chainsafe/persistent-merkle-tree@0.8.0", "@chainsafe/persistent-merkle-tree@^0.8.0": +"@chainsafe/persistent-merkle-tree@0.8.0", "@chainsafe/persistent-merkle-tree@file:../ssz/packages/persistent-merkle-tree": version "0.8.0" - resolved "https://registry.yarnpkg.com/@chainsafe/persistent-merkle-tree/-/persistent-merkle-tree-0.8.0.tgz#18e2f0a5de3a0b59c6e5be8797a78e0d209dd7dc" - integrity sha512-hh6C1JO6SKlr0QGNTNtTLqgGVMA/Bc20wD6CeMHp+wqbFKCULRJuBUxhF4WDx/7mX8QlqF3nFriF/Eo8oYJ4/A== dependencies: "@chainsafe/as-sha256" "0.5.0" "@chainsafe/hashtree" "1.0.1" @@ -594,10 +590,8 @@ "@chainsafe/as-sha256" "^0.4.1" "@chainsafe/persistent-merkle-tree" "^0.6.1" -"@chainsafe/ssz@^0.17.1": - version "0.17.1" - resolved "https://registry.yarnpkg.com/@chainsafe/ssz/-/ssz-0.17.1.tgz#7986afbcad5e6971006d596fdb7dfa34bc195131" - integrity sha512-1ay46QqYcVTBvUnDXTPTi5WTiENu7tIxpZGMDpUWps1/nYBmh/We/UoCF/jO+o/fkcDD3p8xQPlHbcCfy+jyjA== +"@chainsafe/ssz@file:../ssz/packages/ssz": + version "0.17.0" dependencies: "@chainsafe/as-sha256" "0.5.0" "@chainsafe/persistent-merkle-tree" "0.8.0" @@ -3870,7 +3864,14 @@ agent-base@^7.0.2, agent-base@^7.1.0: dependencies: debug "^4.3.4" -agentkeepalive@^4.1.3, agentkeepalive@^4.2.1: +agentkeepalive@^4.1.3: + version "4.5.0" + resolved "https://registry.yarnpkg.com/agentkeepalive/-/agentkeepalive-4.5.0.tgz#2673ad1389b3c418c5a20c5d7364f93ca04be923" + integrity sha512-5GG/5IbQQpC9FpkRGsSvZI5QYeSCzlJHdpBQntCsuTOxhKD8lqKhrleg2Yi7yvMIf82Ycmmqln9U8V9qwEiJew== + dependencies: + humanize-ms "^1.2.1" + +agentkeepalive@^4.2.1: version "4.2.1" resolved "https://registry.yarnpkg.com/agentkeepalive/-/agentkeepalive-4.2.1.tgz#a7975cbb9f83b367f06c90cc51ff28fe7d499717" integrity sha512-Zn4cw2NEqd+9fiSVWMscnjyQ1a8Yfoc5oBajLeo5w+YBHgDUcEBY2hS4YpTz6iN5f/2zQiktcuM6tS8x1p9dpA== @@ -4463,7 +4464,7 @@ bl@^5.0.0: bls-eth-wasm@^0.4.8: version "0.4.8" - resolved "https://registry.npmjs.org/bls-eth-wasm/-/bls-eth-wasm-0.4.8.tgz" + resolved "https://registry.yarnpkg.com/bls-eth-wasm/-/bls-eth-wasm-0.4.8.tgz#ad1818fbd1bfb64d8f3e6cd104bd28b96ebaa5f1" integrity sha512-ye7+G6KFLb3i9xSrLASAoYqOUK5WLB6XA5DD8Sh0UQpZ3T999ylsYbFdoOJpmvTDuBuMi23Vy8Jm0pn/GF01CA== bluebird@~3.4.1: