diff --git a/packages/ssz/src/index.ts b/packages/ssz/src/index.ts index 360b8ce9..c5a70480 100644 --- a/packages/ssz/src/index.ts +++ b/packages/ssz/src/index.ts @@ -31,12 +31,20 @@ export {CompositeType, CompositeTypeAny, CompositeView, CompositeViewDU, isCompo export {TreeView} from "./view/abstract.js"; export {ValueOfFields} from "./view/container.js"; export {TreeViewDU} from "./viewDU/abstract.js"; +export {ListCompositeTreeViewDU} from "./viewDU/listComposite.js"; +export {ListBasicTreeViewDU} from "./viewDU/listBasic.js"; +export {ArrayCompositeTreeViewDUCache} from "./viewDU/arrayComposite.js"; +export {ContainerNodeStructTreeViewDU} from "./viewDU/containerNodeStruct.js"; // Values export {BitArray, getUint8ByteToBitBooleanArray} from "./value/bitArray.js"; // Utils export {fromHexString, toHexString, byteArrayEquals} from "./util/byteArray.js"; + export {Snapshot} from "./util/types.js"; export {hash64, symbolCachedPermanentRoot} from "./util/merkleize.js"; export {upgradeToNewType} from "./util/upgrade.js"; + +// others +export {BranchNodeStruct} from "./branchNodeStruct.js"; diff --git a/packages/ssz/test/lodestarTypes/phase0/listValidator.ts b/packages/ssz/test/lodestarTypes/phase0/listValidator.ts new file mode 100644 index 00000000..a06abf98 --- /dev/null +++ b/packages/ssz/test/lodestarTypes/phase0/listValidator.ts @@ -0,0 +1,19 @@ +import {Node} from "@chainsafe/persistent-merkle-tree"; +import {ListCompositeType} from "../../../src/type/listComposite.js"; +import {ListCompositeTreeViewDU} from "../../../src/viewDU/listComposite.js"; +import {ValidatorNodeStructType} from "./validator.js"; +import {ListValidatorTreeViewDU} from "./viewDU/listValidator.js"; + +/** + * Model ssz type for a list of validators in ethereum consensus layer. + * This defines ListValidatorTreeViewDU to work with validators in batch. + */ +export class ListValidatorType extends ListCompositeType { + constructor(limit: number) { + super(new ValidatorNodeStructType(), limit); + } + + getViewDU(node: Node, cache?: unknown): ListCompositeTreeViewDU { + return new ListValidatorTreeViewDU(this, node, cache as any); + } +} diff --git a/packages/ssz/test/lodestarTypes/phase0/sszTypes.ts b/packages/ssz/test/lodestarTypes/phase0/sszTypes.ts index 077359d2..f1afe4d2 100644 --- a/packages/ssz/test/lodestarTypes/phase0/sszTypes.ts +++ b/packages/ssz/test/lodestarTypes/phase0/sszTypes.ts @@ -2,7 +2,6 @@ import { BitListType, BitVectorType, ContainerType, - ContainerNodeStructType, ListBasicType, ListCompositeType, VectorBasicType, @@ -18,6 +17,10 @@ import { ATTESTATION_SUBNET_COUNT, } from "../params.js"; import * as primitiveSsz from "../primitive/sszTypes.js"; +import {ListValidatorType} from "./listValidator.js"; +import {ValidatorNodeStruct} from "./validator.js"; + +export {ValidatorNodeStruct}; const { EPOCHS_PER_ETH1_VOTING_PERIOD, @@ -261,12 +264,14 @@ export const ValidatorContainer = new ContainerType( {typeName: "Validator", jsonCase: "eth2"} ); -export const ValidatorNodeStruct = new ContainerNodeStructType(ValidatorContainer.fields, ValidatorContainer.opts); // The main Validator type is the 'ContainerNodeStructType' version export const Validator = ValidatorNodeStruct; // Export as stand-alone for direct tree optimizations -export const Validators = new ListCompositeType(ValidatorNodeStruct, VALIDATOR_REGISTRY_LIMIT); +// since Mar 2025 instead of using ListCompositeType +// export const Validators = new ListCompositeType(ValidatorNodeStruct, VALIDATOR_REGISTRY_LIMIT); +// we switch to ListValidatorType which support batch hash +export const Validators = new ListValidatorType(VALIDATOR_REGISTRY_LIMIT); export const Balances = new ListUintNum64Type(VALIDATOR_REGISTRY_LIMIT); export const RandaoMixes = new VectorCompositeType(Bytes32, EPOCHS_PER_HISTORICAL_VECTOR); export const Slashings = new VectorBasicType(Gwei, EPOCHS_PER_SLASHINGS_VECTOR); diff --git a/packages/ssz/test/lodestarTypes/phase0/validator.ts b/packages/ssz/test/lodestarTypes/phase0/validator.ts new file mode 100644 index 00000000..81b2b5a4 --- /dev/null +++ b/packages/ssz/test/lodestarTypes/phase0/validator.ts @@ -0,0 +1,130 @@ +import {ByteViews} from "../../../src/type/abstract.js"; +import {ContainerNodeStructType} from "../../../src/type/containerNodeStruct.js"; +import {ValueOfFields} from "../../../src/view/container.js"; +import * as primitiveSsz from "../primitive/sszTypes.js"; + +const {Boolean, Bytes32, UintNum64, BLSPubkey, EpochInf} = primitiveSsz; + +// this is to work with uint32, see https://github.com/ChainSafe/ssz/blob/ssz-v0.15.1/packages/ssz/src/type/uint.ts +const NUMBER_2_POW_32 = 2 ** 32; + +/* + * Below constants are respective to their ssz type in `ValidatorType`. + */ +const UINT32_SIZE = 4; +const PUBKEY_SIZE = 48; +const WITHDRAWAL_CREDENTIALS_SIZE = 32; +const SLASHED_SIZE = 1; +const CHUNK_SIZE = 32; + +export const ValidatorType = { + pubkey: BLSPubkey, + withdrawalCredentials: Bytes32, + effectiveBalance: UintNum64, + slashed: Boolean, + activationEligibilityEpoch: EpochInf, + activationEpoch: EpochInf, + exitEpoch: EpochInf, + withdrawableEpoch: EpochInf, +}; + +/** + * Improve serialization performance for state.validators.serialize(); + */ +export class ValidatorNodeStructType extends ContainerNodeStructType { + constructor() { + super(ValidatorType, {typeName: "Validator", jsonCase: "eth2"}); + } + + value_serializeToBytes( + {uint8Array: output, dataView}: ByteViews, + offset: number, + validator: ValueOfFields + ): number { + output.set(validator.pubkey, offset); + offset += PUBKEY_SIZE; + output.set(validator.withdrawalCredentials, offset); + offset += WITHDRAWAL_CREDENTIALS_SIZE; + const {effectiveBalance, activationEligibilityEpoch, activationEpoch, exitEpoch, withdrawableEpoch} = validator; + // effectiveBalance is UintNum64 + dataView.setUint32(offset, effectiveBalance & 0xffffffff, true); + offset += UINT32_SIZE; + dataView.setUint32(offset, (effectiveBalance / NUMBER_2_POW_32) & 0xffffffff, true); + offset += UINT32_SIZE; + output[offset] = validator.slashed ? 1 : 0; + offset += SLASHED_SIZE; + offset = writeEpochInf(dataView, offset, activationEligibilityEpoch); + offset = writeEpochInf(dataView, offset, activationEpoch); + offset = writeEpochInf(dataView, offset, exitEpoch); + offset = writeEpochInf(dataView, offset, withdrawableEpoch); + + return offset; + } +} + +export const ValidatorNodeStruct = new ValidatorNodeStructType(); + +/** + * Write to level3 and level4 bytes to compute merkle root. Note that this is to compute + * merkle root and it's different from serialization (which is more compressed). + * pub0 + pub1 are at level4, they will be hashed to 1st chunked of level 3 + * then use 8 chunks of level 3 to compute the root hash. + * reserved withdr eff sla actElig act exit with + * level 3 |----------|----------|----------|----------|----------|----------|----------|----------| + * + * pub0 pub1 + * level4 |----------|----------| + * + */ +export function validatorToChunkBytes( + level3: ByteViews, + level4: Uint8Array, + value: ValueOfFields +): void { + const { + pubkey, + withdrawalCredentials, + effectiveBalance, + slashed, + activationEligibilityEpoch, + activationEpoch, + exitEpoch, + withdrawableEpoch, + } = value; + const {uint8Array: outputLevel3, dataView} = level3; + + // pubkey = 48 bytes which is 2 * CHUNK_SIZE + level4.set(pubkey, 0); + let offset = CHUNK_SIZE; + outputLevel3.set(withdrawalCredentials, offset); + offset += CHUNK_SIZE; + // effectiveBalance is UintNum64 + dataView.setUint32(offset, effectiveBalance & 0xffffffff, true); + dataView.setUint32(offset + 4, (effectiveBalance / NUMBER_2_POW_32) & 0xffffffff, true); + + offset += CHUNK_SIZE; + dataView.setUint32(offset, slashed ? 1 : 0, true); + offset += CHUNK_SIZE; + writeEpochInf(dataView, offset, activationEligibilityEpoch); + offset += CHUNK_SIZE; + writeEpochInf(dataView, offset, activationEpoch); + offset += CHUNK_SIZE; + writeEpochInf(dataView, offset, exitEpoch); + offset += CHUNK_SIZE; + writeEpochInf(dataView, offset, withdrawableEpoch); +} + +function writeEpochInf(dataView: DataView, offset: number, value: number): number { + if (value === Infinity) { + dataView.setUint32(offset, 0xffffffff, true); + offset += UINT32_SIZE; + dataView.setUint32(offset, 0xffffffff, true); + offset += UINT32_SIZE; + } else { + dataView.setUint32(offset, value & 0xffffffff, true); + offset += UINT32_SIZE; + dataView.setUint32(offset, (value / NUMBER_2_POW_32) & 0xffffffff, true); + offset += UINT32_SIZE; + } + return offset; +} diff --git a/packages/ssz/test/lodestarTypes/phase0/viewDU/listValidator.ts b/packages/ssz/test/lodestarTypes/phase0/viewDU/listValidator.ts new file mode 100644 index 00000000..16fbeb67 --- /dev/null +++ b/packages/ssz/test/lodestarTypes/phase0/viewDU/listValidator.ts @@ -0,0 +1,179 @@ +import {byteArrayIntoHashObject} from "@chainsafe/as-sha256"; +import {HashComputationLevel, Node, digestNLevel, setNodesAtDepth} from "@chainsafe/persistent-merkle-tree"; +import {ListCompositeType} from "../../../../src/type/listComposite.js"; +import {ArrayCompositeTreeViewDUCache} from "../../../../src/viewDU/arrayComposite.js"; +import {ListCompositeTreeViewDU} from "../../../../src/viewDU/listComposite.js"; +import {ValidatorNodeStructType, ValidatorType, validatorToChunkBytes} from "../validator.js"; +import {ByteViews} from "../../../../src/type/abstract.js"; +import {ContainerNodeStructTreeViewDU} from "../../../../src/viewDU/containerNodeStruct.js"; +import {ValidatorIndex} from "../../primitive/types.js"; + +/** + * hashtree has a MAX_SIZE of 1024 bytes = 32 chunks + * Given a level3 of validators have 8 chunks, we can hash 4 validators at a time + */ +const PARALLEL_FACTOR = 4; +/** + * Allocate memory once for batch hash validators. + */ +// each level 3 of validator has 8 chunks, each chunk has 32 bytes +const batchLevel3Bytes = new Uint8Array(PARALLEL_FACTOR * 8 * 32); +const level3ByteViewsArr: ByteViews[] = []; +for (let i = 0; i < PARALLEL_FACTOR; i++) { + const uint8Array = batchLevel3Bytes.subarray(i * 8 * 32, (i + 1) * 8 * 32); + const dataView = new DataView(uint8Array.buffer, uint8Array.byteOffset, uint8Array.byteLength); + level3ByteViewsArr.push({uint8Array, dataView}); +} +// each level 4 of validator has 2 chunks for pubkey, each chunk has 32 bytes +const batchLevel4Bytes = new Uint8Array(PARALLEL_FACTOR * 2 * 32); +const level4BytesArr: Uint8Array[] = []; +for (let i = 0; i < PARALLEL_FACTOR; i++) { + level4BytesArr.push(batchLevel4Bytes.subarray(i * 2 * 32, (i + 1) * 2 * 32)); +} +const pubkeyRoots: Uint8Array[] = []; +for (let i = 0; i < PARALLEL_FACTOR; i++) { + pubkeyRoots.push(batchLevel4Bytes.subarray(i * 32, (i + 1) * 32)); +} + +const validatorRoots: Uint8Array[] = []; +for (let i = 0; i < PARALLEL_FACTOR; i++) { + validatorRoots.push(batchLevel3Bytes.subarray(i * 32, (i + 1) * 32)); +} +const validatorRoot = new Uint8Array(32); + +/** + * Similar to ListCompositeTreeViewDU with some differences: + * - if called without params, it's from hashTreeRoot() api call, no need to compute root + * - otherwise it's from batchHashTreeRoot() call, compute validator roots in batch + */ +export class ListValidatorTreeViewDU extends ListCompositeTreeViewDU { + constructor( + readonly type: ListCompositeType, + protected _rootNode: Node, + cache?: ArrayCompositeTreeViewDUCache + ) { + super(type, _rootNode, cache); + } + + commit(hcOffset = 0, hcByLevel: HashComputationLevel[] | null = null): void { + if (hcByLevel === null) { + // this is not from batchHashTreeRoot() call, go with regular flow + return super.commit(); + } + + const isOldRootHashed = this._rootNode.h0 !== null; + if (this.viewsChanged.size === 0) { + if (!isOldRootHashed && hcByLevel !== null) { + // not possible to get HashComputations due to BranchNodeStruct + this._rootNode.root; + } + return; + } + + // TODO - batch: remove this type cast + const viewsChanged = this.viewsChanged as unknown as Map< + number, + ContainerNodeStructTreeViewDU + >; + + const indicesChanged: number[] = []; + for (const [index, viewChanged] of viewsChanged) { + // should not have any params here in order not to compute root + viewChanged.commit(); + // Set new node in nodes array to ensure data represented in the tree and fast nodes access is equal + this.nodes[index] = viewChanged.node; + // `validators.get(i)` was called but it may not modify any property, do not need to compute root + if (viewChanged.node.h0 === null) { + indicesChanged.push(index); + } + } + + // these validators don't have roots, we compute roots in batch + const sortedIndicesChanged = indicesChanged.sort((a, b) => a - b); + const nodesChanged: {index: ValidatorIndex; node: Node}[] = new Array<{index: ValidatorIndex; node: Node}>( + sortedIndicesChanged.length + ); + for (const [i, validatorIndex] of sortedIndicesChanged.entries()) { + nodesChanged[i] = {index: validatorIndex, node: this.nodes[validatorIndex]}; + } + doBatchHashTreeRootValidators(sortedIndicesChanged, viewsChanged); + + // do the remaining commit step the same to parent (ArrayCompositeTreeViewDU) + const indexes = nodesChanged.map((entry) => entry.index); + const nodes = nodesChanged.map((entry) => entry.node); + const chunksNode = this.type.tree_getChunksNode(this._rootNode); + const offsetThis = hcOffset + this.type.tree_chunksNodeOffset(); + const byLevelThis = hcByLevel != null && isOldRootHashed ? hcByLevel : null; + const newChunksNode = setNodesAtDepth(chunksNode, this.type.chunkDepth, indexes, nodes, offsetThis, byLevelThis); + + this._rootNode = this.type.tree_setChunksNode( + this._rootNode, + newChunksNode, + this.dirtyLength ? this._length : null, + hcOffset, + hcByLevel + ); + + if (!isOldRootHashed && hcByLevel !== null) { + // should never happen, handle just in case + // not possible to get HashComputations due to BranchNodeStruct + this._rootNode.root; + } + + this.viewsChanged.clear(); + this.dirtyLength = false; + } +} + +export function doBatchHashTreeRootValidators( + indices: ValidatorIndex[], + validators: Map> +): void { + const endBatch = indices.length - (indices.length % PARALLEL_FACTOR); + + // commit every 16 validators in batch + for (let i = 0; i < endBatch; i++) { + if (i % PARALLEL_FACTOR === 0) { + batchLevel3Bytes.fill(0); + batchLevel4Bytes.fill(0); + } + const indexInBatch = i % PARALLEL_FACTOR; + const viewIndex = indices[i]; + const validator = validators.get(viewIndex); + if (validator) { + validatorToChunkBytes(level3ByteViewsArr[indexInBatch], level4BytesArr[indexInBatch], validator.value); + } + + if (indexInBatch === PARALLEL_FACTOR - 1) { + // hash level 4, this is populated to pubkeyRoots + digestNLevel(batchLevel4Bytes, 1); + for (let j = 0; j < PARALLEL_FACTOR; j++) { + level3ByteViewsArr[j].uint8Array.set(pubkeyRoots[j], 0); + } + // hash level 3, this is populated to validatorRoots + digestNLevel(batchLevel3Bytes, 3); + // commit all validators in this batch + for (let j = PARALLEL_FACTOR - 1; j >= 0; j--) { + const viewIndex = indices[i - j]; + const indexInBatch = (i - j) % PARALLEL_FACTOR; + const viewChanged = validators.get(viewIndex); + if (viewChanged) { + const branchNodeStruct = viewChanged.node; + byteArrayIntoHashObject(validatorRoots[indexInBatch], 0, branchNodeStruct); + } + } + } + } + + // commit the remaining validators, we can do in batch too but don't want to create new Uint8Array views + // it's not much different to commit one by one + for (let i = endBatch; i < indices.length; i++) { + const viewIndex = indices[i]; + const viewChanged = validators.get(viewIndex); + if (viewChanged) { + // compute root for each validator + viewChanged.type.hashTreeRootInto(viewChanged.value, validatorRoot, 0); + byteArrayIntoHashObject(validatorRoot, 0, viewChanged.node); + } + } +} diff --git a/packages/ssz/test/perf/eth2/beaconState.test.ts b/packages/ssz/test/perf/eth2/beaconState.test.ts index aba8ff48..17953f79 100644 --- a/packages/ssz/test/perf/eth2/beaconState.test.ts +++ b/packages/ssz/test/perf/eth2/beaconState.test.ts @@ -6,9 +6,9 @@ import {preset} from "../../lodestarTypes/params.js"; const {SLOTS_PER_HISTORICAL_ROOT, EPOCHS_PER_ETH1_VOTING_PERIOD, SLOTS_PER_EPOCH} = preset; const vc = 200_000; -const numModified = vc / 20; +const numModified = vc / 2; // every we increase vc, need to change this value from "recursive hash" test -const expectedRoot = "0x759d635af161ac1e4f4af11aa7721fd4996253af50f8a81e5003bbb4cbcaae42"; +const expectedRoot = "0xb0780ec0d44bff1ae8a351e98e37a9d8c3e28edb38c9d5a6312656e0cba915d9"; /** * This simulates a BeaconState being modified after an epoch transition in lodestar @@ -22,71 +22,73 @@ describe(`BeaconState ViewDU partially modified tree vc=${vc} numModified=${numM minMs: 20_000, }); + const hc = new HashComputationGroup(); bench({ - id: `BeaconState ViewDU hashTreeRoot() vc=${vc}`, + id: `BeaconState ViewDU batchHashTreeRoot vc=${vc} mod=${numModified}`, beforeEach: () => createPartiallyModifiedDenebState(), fn: (state: CompositeViewDU) => { - state.hashTreeRoot(); - if (toHexString(state.node.root) !== expectedRoot) { - throw new Error("hashTreeRoot does not match expectedRoot"); + // commit() step is inside hashTreeRoot(), reuse HashComputationGroup + if (toHexString(state.batchHashTreeRoot(hc)) !== expectedRoot) { + throw new Error( + `batchHashTreeRoot ${toHexString(state.batchHashTreeRoot(hc))} does not match expectedRoot ${expectedRoot}` + ); } + state.batchHashTreeRoot(hc); }, }); bench({ - id: `BeaconState ViewDU recursive hash - commit step vc=${vc}`, + id: `BeaconState ViewDU batchHashTreeRoot - commit step vc=${vc} mod=${numModified}`, beforeEach: () => createPartiallyModifiedDenebState(), fn: (state: CompositeViewDU) => { - state.commit(); + state.commit(0, []); }, }); bench({ - id: `BeaconState ViewDU validator tree creation vc=${numModified}`, + id: `BeaconState ViewDU batchHashTreeRoot - hash step vc=${vc} mod=${numModified}`, beforeEach: () => { const state = createPartiallyModifiedDenebState(); - state.commit(); - return state; + const hcByLevel: HashComputationLevel[] = []; + state.commit(0, hcByLevel); + return hcByLevel; }, - fn: (state: CompositeViewDU) => { - const validators = state.validators; - for (let i = 0; i < numModified; i++) { - validators.getReadonly(i).node.left; - } + fn: (hcByLevel) => { + executeHashComputations(hcByLevel); }, }); - const hc = new HashComputationGroup(); bench({ - id: `BeaconState ViewDU batchHashTreeRoot vc=${vc}`, + id: `BeaconState ViewDU hashTreeRoot() vc=${vc} mod=${numModified}`, beforeEach: () => createPartiallyModifiedDenebState(), fn: (state: CompositeViewDU) => { - // commit() step is inside hashTreeRoot(), reuse HashComputationGroup - if (toHexString(state.batchHashTreeRoot(hc)) !== expectedRoot) { - throw new Error("batchHashTreeRoot does not match expectedRoot"); + state.hashTreeRoot(); + if (toHexString(state.node.root) !== expectedRoot) { + throw new Error(`hashTreeRoot ${toHexString(state.node.root)} does not match expectedRoot ${expectedRoot}`); } - state.batchHashTreeRoot(hc); }, }); bench({ - id: `BeaconState ViewDU hashTreeRoot - commit step vc=${vc}`, + id: `BeaconState ViewDU hashTreeRoot - commit step vc=${vc} mod=${numModified}`, beforeEach: () => createPartiallyModifiedDenebState(), fn: (state: CompositeViewDU) => { - state.commit(0, []); + state.commit(); }, }); bench({ - id: `BeaconState ViewDU hashTreeRoot - hash step vc=${vc}`, + id: `BeaconState ViewDU hashTreeRoot - validator tree creation vc=${numModified} mod=${numModified}`, beforeEach: () => { const state = createPartiallyModifiedDenebState(); - const hcByLevel: HashComputationLevel[] = []; - state.commit(0, hcByLevel); - return hcByLevel; + state.commit(); + return state; }, - fn: (hcByLevel) => { - executeHashComputations(hcByLevel); + fn: (state: CompositeViewDU) => { + const validators = state.validators; + for (let i = 0; i < numModified; i++) { + validators.getReadonly(i).node.left; + } }, }); }); @@ -116,7 +118,12 @@ function createPartiallyModifiedDenebState(): CompositeViewDU 32e9)); + // remaining validators are accessed with no modification + for (let i = numModified; i < vc; i++) { + state.validators.get(i); + } state.eth1Data = BeaconState.fields.eth1Data.toViewDU({ depositRoot: Buffer.alloc(32, 0x02), diff --git a/packages/ssz/test/perf/eth2/hashTreeRoot.test.ts b/packages/ssz/test/perf/eth2/hashTreeRoot.test.ts index 689890a4..05c3b2fb 100644 --- a/packages/ssz/test/perf/eth2/hashTreeRoot.test.ts +++ b/packages/ssz/test/perf/eth2/hashTreeRoot.test.ts @@ -74,6 +74,8 @@ describe("HashTreeRoot frequent eth2 objects", () => { before: () => getStateViewDU().serialize(), beforeEach: (bytes) => sszAltair.BeaconState.deserializeToViewDU(bytes), fn: (state) => { + // performance is not great due to memory allocation of BranchNodeStruct left, right access + // it's good to have an idea on batchHashTreeRoot() at the 1st time state.batchHashTreeRoot(hc); }, }); diff --git a/packages/ssz/test/perf/eth2/validators.test.ts b/packages/ssz/test/perf/eth2/validators.test.ts index 07ca01fb..571fa23c 100644 --- a/packages/ssz/test/perf/eth2/validators.test.ts +++ b/packages/ssz/test/perf/eth2/validators.test.ts @@ -1,7 +1,10 @@ import {describe, bench} from "@chainsafe/benchmark"; import {Validator} from "../../lodestarTypes/phase0/types.js"; -import {ValidatorContainer, ValidatorNodeStruct} from "../../lodestarTypes/phase0/sszTypes.js"; -import {CompositeViewDU} from "../../../src/index.js"; +import {ValidatorContainer, ValidatorNodeStruct, Validators} from "../../lodestarTypes/phase0/sszTypes.js"; +import {BranchNodeStruct, CompositeViewDU, ContainerNodeStructTreeViewDU} from "../../../src/index.js"; +import {ValidatorIndex} from "../../lodestarTypes/types.js"; +import {ValidatorType} from "../../lodestarTypes/phase0/validator.js"; +import {doBatchHashTreeRootValidators} from "../../lodestarTypes/phase0/viewDU/listValidator.js"; const validatorStruct: Validator = { pubkey: Buffer.alloc(48, 0xdd), @@ -49,3 +52,58 @@ describe("Validator vs ValidatorLeafNodeStruct", () => { } } }); + +/** + * This is almost 6x faster on a Mac machine + * batch hash validators + * ✔ doBatchHashTreeRootValidators 478927.2 ops/s 2.088000 us/op - 226628 runs 0.505 s + * ✔ ContainerNodeStructViewDU hashTreeRoot 82182.77 ops/s 12.16800 us/op - 39987 runs 0.505 s + */ +describe("batch hash validators", () => { + // ListValidatorTreeViewDU commits every 4 validators in batch + const listValidator = Validators.toViewDU(Array.from({length: 4}, () => validatorStruct)); + const nodes: BranchNodeStruct[] = []; + const validatorsMap: Map> = new Map(); + for (let i = 0; i < listValidator.length; i++) { + nodes.push(listValidator.get(i).node as BranchNodeStruct); + validatorsMap.set(i, listValidator.get(i) as unknown as ContainerNodeStructTreeViewDU); + } + + // this does not create validator tree every time, and it compute roots in batch + bench({ + id: "doBatchHashTreeRootValidators", + beforeEach: () => { + for (let i = 0; i < listValidator.length; i++) { + const validator = listValidator.get(i); + validator.exitEpoch = 20242024; + validator.node.h0 = null as unknown as number; + } + }, + fn: () => { + doBatchHashTreeRootValidators([0, 1, 2, 3], validatorsMap); + + // make sure all validators' root is computed + for (let i = 0; i < listValidator.length; i++) { + if (listValidator.get(i).node.h0 == null) { + throw Error("root not computed"); + } + } + }, + }); + + // this needs to create validator tree every time + bench({ + id: "ContainerNodeStructViewDU hashTreeRoot", + beforeEach: () => { + for (const node of nodes) { + node.value.exitEpoch = 20242024; + node.h0 = null as unknown as number; + } + }, + fn: () => { + for (const node of nodes) { + node.root; + } + }, + }); +}); diff --git a/packages/ssz/test/unit/lodestarTypes/phase0/listValidator.test.ts b/packages/ssz/test/unit/lodestarTypes/phase0/listValidator.test.ts new file mode 100644 index 00000000..c26c779a --- /dev/null +++ b/packages/ssz/test/unit/lodestarTypes/phase0/listValidator.test.ts @@ -0,0 +1,89 @@ +import {ListCompositeType} from "../../../../src/type/listComposite.js"; +import {ValidatorType} from "../../../lodestarTypes/phase0/validator.js"; +import {preset} from "../../../lodestarTypes/params.js"; +import {ssz} from "../../../lodestarTypes/index.js"; +import {describe, it, expect} from "vitest"; +import {ContainerType} from "../../../../src/type/container.js"; +import {Validator} from "../../../lodestarTypes/phase0/index.js"; +const {VALIDATOR_REGISTRY_LIMIT} = preset; + +describe("ListValidator ssz type", function () { + const seedValidator = { + activationEligibilityEpoch: 10, + activationEpoch: 11, + exitEpoch: Infinity, + slashed: false, + withdrawableEpoch: 13, + pubkey: Buffer.alloc(48, 100), + withdrawalCredentials: Buffer.alloc(32, 100), + effectiveBalance: 32000000000, + }; + + const testCases = [32, 33, 34, 35]; + const ValidatorContainer = new ContainerType(ValidatorType, {typeName: "Validator", jsonCase: "eth2"}); + const oldValidatorsType = new ListCompositeType(ValidatorContainer, VALIDATOR_REGISTRY_LIMIT); + for (const numValidators of testCases) { + it(`should commit ${numValidators} validators`, () => { + const validators = Array.from({length: numValidators}, (_, i) => ({ + ...seedValidator, + withdrawableEpoch: seedValidator.withdrawableEpoch + i, + })); + const oldViewDU = oldValidatorsType.toViewDU(validators); + const newViewDU = ssz.phase0.Validators.toViewDU(validators); + // modify all validators + for (let i = 0; i < numValidators; i++) { + oldViewDU.get(i).activationEpoch = 2024; + newViewDU.get(i).activationEpoch = 2024; + } + expect(newViewDU.batchHashTreeRoot()).to.be.deep.equal(oldViewDU.batchHashTreeRoot()); + expect(newViewDU.serialize()).to.be.deep.equal(oldViewDU.serialize()); + }); + } + + const testCases2 = [[1], [3, 5], [1, 9, 7]]; + const numValidator = 33; + for (const modifiedIndices of testCases2) { + it(`should modify ${modifiedIndices.length} validators`, () => { + const validators = Array.from({length: numValidator}, (_, i) => ({ + ...seedValidator, + withdrawableEpoch: seedValidator.withdrawableEpoch + i, + })); + const oldViewDU = oldValidatorsType.toViewDU(validators); + const newViewDU = ssz.phase0.Validators.toViewDU(validators); + for (const index of modifiedIndices) { + oldViewDU.get(index).activationEpoch = 2024; + newViewDU.get(index).activationEpoch = 2024; + } + expect(newViewDU.batchHashTreeRoot()).to.be.deep.equal(oldViewDU.batchHashTreeRoot()); + expect(newViewDU.serialize()).to.be.deep.equal(oldViewDU.serialize()); + }); + } + + const testCases3 = [1, 3, 5, 7]; + for (const numPush of testCases3) { + it(`should push ${numPush} validators`, () => { + const validators = Array.from({length: numValidator}, (_, i) => ({ + ...seedValidator, + withdrawableEpoch: seedValidator.withdrawableEpoch + i, + })); + const oldViewDU = oldValidatorsType.toViewDU(validators); + const newViewDU = ssz.phase0.Validators.toViewDU(validators); + const newValidators: Validator[] = []; + // this ensure the commit() should update nodes array + newViewDU.getAllReadonlyValues(); + for (let i = 0; i < numPush; i++) { + const validator = {...seedValidator, withdrawableEpoch: seedValidator.withdrawableEpoch + numValidator + i}; + newValidators.push(validator); + oldViewDU.push(ValidatorContainer.toViewDU(validator)); + newViewDU.push(ssz.phase0.Validator.toViewDU(validator)); + } + oldViewDU.commit(); + expect(newViewDU.batchHashTreeRoot()).to.be.deep.equal(oldViewDU.node.root); + expect(newViewDU.serialize()).to.be.deep.equal(oldViewDU.serialize()); + const allValidators = newViewDU.getAllReadonlyValues(); + for (let i = 0; i < numPush; i++) { + expect(allValidators[numValidator + i]).to.be.deep.equal(newValidators[i]); + } + }); + } +}); diff --git a/packages/ssz/test/unit/lodestarTypes/phase0/validator.test.ts b/packages/ssz/test/unit/lodestarTypes/phase0/validator.test.ts new file mode 100644 index 00000000..f66b6536 --- /dev/null +++ b/packages/ssz/test/unit/lodestarTypes/phase0/validator.test.ts @@ -0,0 +1,80 @@ +import {digestNLevel} from "@chainsafe/persistent-merkle-tree"; +import {ContainerType} from "../../../../../ssz/src/type/container.js"; +import {ssz} from "../../../lodestarTypes/index.js"; +import {ValidatorNodeStruct, ValidatorType, validatorToChunkBytes} from "../../../lodestarTypes/phase0/validator.js"; +import {describe, it, expect} from "vitest"; +import {Validator} from "../../../lodestarTypes/phase0/sszTypes.js"; + +const ValidatorContainer = new ContainerType(ValidatorType, {typeName: "Validator", jsonCase: "eth2"}); + +describe("Validator ssz types", function () { + const seedValidator = { + activationEligibilityEpoch: 10, + activationEpoch: 11, + exitEpoch: Infinity, + slashed: false, + withdrawableEpoch: 13, + pubkey: Buffer.alloc(48, 100), + withdrawalCredentials: Buffer.alloc(32, 100), + effectiveBalance: 32000000000, + }; + + const validators = [ + {...seedValidator, effectiveBalance: 31000000000, slashed: false}, + {...seedValidator, effectiveBalance: 32000000000, slashed: true}, + ]; + + it("should serialize and hash to the same value", () => { + for (const validator of validators) { + const serialized = ValidatorContainer.serialize(validator); + const serialized2 = ssz.phase0.Validator.serialize(validator); + const serialized3 = ssz.phase0.Validator.toViewDU(validator).serialize(); + expect(serialized2).to.be.deep.equal(serialized); + expect(serialized3).to.be.deep.equal(serialized); + + const root = ValidatorContainer.hashTreeRoot(validator); + const root2 = ssz.phase0.Validator.hashTreeRoot(validator); + const root3 = ssz.phase0.Validator.toViewDU(validator).hashTreeRoot(); + const root4 = ssz.phase0.Validator.toViewDU(validator).batchHashTreeRoot(); + expect(root2).to.be.deep.equal(root); + expect(root3).to.be.deep.equal(root); + expect(root4).to.be.deep.equal(root); + } + }); +}); + +describe("validatorToChunkBytes", function () { + const seedValidator = { + activationEligibilityEpoch: 10, + activationEpoch: 11, + exitEpoch: Infinity, + slashed: false, + withdrawableEpoch: 13, + pubkey: Buffer.alloc(48, 100), + withdrawalCredentials: Buffer.alloc(32, 100), + }; + + const validators = [ + {...seedValidator, effectiveBalance: 31000000000, slashed: false}, + {...seedValidator, effectiveBalance: 32000000000, slashed: true}, + ]; + + it("should populate validator value to merkle bytes", () => { + for (const validator of validators) { + const expectedRoot0 = ValidatorNodeStruct.hashTreeRoot(validator); + // validator has 8 fields + const level3 = new Uint8Array(32 * 8); + const dataView = new DataView(level3.buffer, level3.byteOffset, level3.byteLength); + // pubkey takes 2 chunks, has to go to another level + const level4 = new Uint8Array(32 * 2); + validatorToChunkBytes({uint8Array: level3, dataView}, level4, validator); + // additional slice() call make it easier to debug + const pubkeyRoot = digestNLevel(level4, 1).slice(); + level3.set(pubkeyRoot, 0); + const root = digestNLevel(level3, 3).slice(); + const expectedRootNode2 = Validator.value_toTree(validator); + expect(root).to.be.deep.equals(expectedRoot0); + expect(root).to.be.deep.equals(expectedRootNode2.root); + } + }); +}); diff --git a/packages/ssz/test/unit/merkleize.test.ts b/packages/ssz/test/unit/merkleize.test.ts index 6fea23c4..e3c054d9 100644 --- a/packages/ssz/test/unit/merkleize.test.ts +++ b/packages/ssz/test/unit/merkleize.test.ts @@ -1,6 +1,6 @@ import {describe, it, expect} from "vitest"; import {bitLength, maxChunksToDepth, merkleize, mixInLength, nextPowerOf2} from "../../src/util/merkleize.js"; -import {merkleizeBlocksBytes, LeafNode, zeroHash} from "@chainsafe/persistent-merkle-tree"; +import {merkleizeBlocksBytes, LeafNode, zeroHash, merkleizeBlockArray} from "@chainsafe/persistent-merkle-tree"; describe("util / merkleize / bitLength", () => { const bitLengthByIndex = [0, 1, 2, 2, 3, 3, 3, 3, 4, 4]; @@ -48,7 +48,7 @@ describe("util / merkleize / mixInLength", () => { } }); -describe("merkleize should be equal to merkleizeInto of hasher", () => { +describe("merkleize should be equal to merkleizeBlocksBytes of hasher", () => { const numNodes = [0, 1, 2, 3, 4, 5, 6, 7, 8]; for (const numNode of numNodes) { it(`merkleize for ${numNode} nodes`, () => { @@ -63,3 +63,31 @@ describe("merkleize should be equal to merkleizeInto of hasher", () => { }); } }); + +// same to the above but with merkleizeBlockArray() method +describe("merkleize should be equal to merkleizeBlockArray of hasher", () => { + // hashtree has a buffer of 16 * 64 bytes = 32 nodes + const numNodes = [64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79]; + for (const numNode of numNodes) { + it(`merkleize for ${numNode} nodes`, () => { + const nodes = Array.from({length: numNode}, (_, i) => LeafNode.fromRoot(Buffer.alloc(32, i))); + const data = Buffer.concat(nodes.map((node) => node.root)); + const padData = numNode % 2 === 1 ? Buffer.concat([data, zeroHash(0)]) : data; + expect(padData.length % 64).to.equal(0); + const blocks: Uint8Array[] = []; + for (let i = 0; i < padData.length; i += 64) { + blocks.push(padData.slice(i, i + 64)); + } + const expectedRoot = Buffer.alloc(32); + // depth of 79 nodes are 7, make it 10 to test the padding + const chunkCount = Math.max(numNode, 10); + // add redundant blocks, should not affect the result + const blockLimit = blocks.length; + blocks.push(Buffer.alloc(64, 1)); + blocks.push(Buffer.alloc(64, 2)); + merkleizeBlockArray(blocks, blockLimit, chunkCount, expectedRoot, 0); + const roots = nodes.map((node) => node.root); + expect(merkleize(roots, chunkCount)).to.be.deep.equal(expectedRoot); + }); + } +});