From bd76357d925123293ff41d3e892d757a0b9b0f81 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Fri, 13 Feb 2026 22:38:39 -0800 Subject: [PATCH 01/35] graph: Make entity! macro usable outside of tests --- graph/src/data/store/mod.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/graph/src/data/store/mod.rs b/graph/src/data/store/mod.rs index 9bcbf52f08f..38a0af5bbe9 100644 --- a/graph/src/data/store/mod.rs +++ b/graph/src/data/store/mod.rs @@ -887,7 +887,6 @@ pub enum EntityValidationErrorInner { /// /// let entity = entity! { schema => id: "1", name: "John Doe" }; /// ``` -#[cfg(debug_assertions)] #[macro_export] macro_rules! entity { ($schema:expr => $($name:ident: $value:expr,)*) => { From b608ecf8539bb7a80e8bc7de011e8447d611a832 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Fri, 13 Feb 2026 22:53:00 -0800 Subject: [PATCH 02/35] graph: Add criterion benchmarks for entity cache data structures Add benchmarks for LfuCache (get, insert, evict), Entity::sorted_ref, and Entity::validate to establish baseline measurements before optimizing these hot-path data structures. --- Cargo.lock | 120 ++++++++++++++++++++ graph/Cargo.toml | 5 + graph/benches/entity_cache.rs | 203 ++++++++++++++++++++++++++++++++++ 3 files changed, 328 insertions(+) create mode 100644 graph/benches/entity_cache.rs diff --git a/Cargo.lock b/Cargo.lock index 13fe8973cd7..90b963452d5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -867,6 +867,12 @@ dependencies = [ "libc", ] +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + [[package]] name = "anstream" version = "0.6.14" @@ -1984,6 +1990,12 @@ dependencies = [ "serde", ] +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "cc" version = "1.2.43" @@ -2028,6 +2040,33 @@ dependencies = [ "windows-link 0.2.0", ] +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + [[package]] name = "cid" version = "0.11.1" @@ -2425,6 +2464,42 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "criterion" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "is-terminal", + "itertools 0.10.5", + "num-traits", + "once_cell", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools 0.10.5", +] + [[package]] name = "crossbeam" version = "0.8.4" @@ -3702,6 +3777,7 @@ dependencies = [ "chrono", "cid", "clap", + "criterion", "csv", "defer", "derive_more", @@ -5541,6 +5617,12 @@ version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + [[package]] name = "open" version = "5.3.3" @@ -5836,6 +5918,34 @@ version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + [[package]] name = "polling" version = "3.11.0" @@ -7836,6 +7946,16 @@ dependencies = [ "zerovec", ] +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "tinyvec" version = "1.7.0" diff --git a/graph/Cargo.toml b/graph/Cargo.toml index 73604b59ffd..6a124711101 100644 --- a/graph/Cargo.toml +++ b/graph/Cargo.toml @@ -99,10 +99,15 @@ tokio-util.workspace = true [dev-dependencies] clap.workspace = true +criterion = { version = "0.5", features = ["html_reports"] } maplit = "1.0.2" hex-literal = "1.1" wiremock = "0.6.5" +[[bench]] +name = "entity_cache" +harness = false + [build-dependencies] tonic-prost-build = { workspace = true } diff --git a/graph/benches/entity_cache.rs b/graph/benches/entity_cache.rs new file mode 100644 index 00000000000..37da3af0758 --- /dev/null +++ b/graph/benches/entity_cache.rs @@ -0,0 +1,203 @@ +//! Criterion benchmarks for entity cache data structures. +//! +//! These benchmarks measure the performance of LfuCache, EntityOp::apply_to, +//! Entity::sorted_ref, and Entity::validate — the key data structures on the +//! hot path of subgraph trigger processing. +//! +//! Run: +//! cargo bench -p graph --bench entity_cache +//! +//! Save a baseline for before/after comparison: +//! cargo bench -p graph --bench entity_cache -- --save-baseline before +//! # ... make changes ... +//! cargo bench -p graph --bench entity_cache -- --baseline before + +use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion}; +use std::sync::Arc; + +use graph::data::store::Value; +use graph::data::subgraph::{DeploymentHash, LATEST_VERSION}; +use graph::entity; +use graph::schema::InputSchema; +use graph::util::lfu_cache::LfuCache; + +const SCHEMA_GQL: &str = "type Transfer @entity { + id: String! + from: String! + to: String! + amount: BigInt! + blockNumber: BigInt! + timestamp: BigInt! + gasUsed: BigInt! + gasPrice: BigInt! + logIndex: BigInt! + transactionHash: String! + token: String! + sender: String! + receiver: String! + memo: String! + status: String! + fee: BigInt! + nonce: BigInt! + value: BigInt! + data: String! + confirmed: Boolean! +}"; + +fn make_schema() -> InputSchema { + let id = DeploymentHash::new("QmBenchTest00000000000000000000000000000000000").unwrap(); + InputSchema::parse(LATEST_VERSION, SCHEMA_GQL, id).unwrap() +} + +fn make_entity(schema: &InputSchema, i: u64) -> graph::prelude::Entity { + entity! { schema => + id: format!("0x{i:064x}"), + from: format!("0xfrom{i:060x}"), + to: format!("0xto{i:062x}"), + amount: Value::BigInt(i.into()), + blockNumber: Value::BigInt(i.into()), + timestamp: Value::BigInt(1_700_000_000u64.into()), + gasUsed: Value::BigInt(21000u64.into()), + gasPrice: Value::BigInt(20_000_000_000u64.into()), + logIndex: Value::BigInt(i.into()), + transactionHash: format!("0xtx{i:061x}"), + token: format!("0xtoken{i:058x}"), + sender: format!("0xsender{i:057x}"), + receiver: format!("0xreceiver{i:055x}"), + memo: format!("transfer memo #{i}"), + status: "confirmed", + fee: Value::BigInt(100u64.into()), + nonce: Value::BigInt(i.into()), + value: Value::BigInt((i * 1_000_000).into()), + data: format!("0xdata{i:060x}"), + confirmed: true + } +} + +fn bench_lfu_cache_get(c: &mut Criterion) { + let schema = make_schema(); + let entity_type = schema.entity_type("Transfer").unwrap(); + let n = 10_000; + + let mut cache: LfuCache>> = + LfuCache::new(); + let mut keys = Vec::with_capacity(n); + for i in 0..n as u64 { + let key = entity_type.parse_key(format!("0x{i:064x}")).unwrap(); + let entity = make_entity(&schema, i); + cache.insert(key.clone(), Some(Arc::new(entity))); + keys.push(key); + } + + c.bench_with_input(BenchmarkId::new("lfu_cache_get", n), &keys, |b, keys| { + b.iter(|| { + // Access 1000 random-ish keys from the cache + for i in (0..1000).map(|j| (j * 7 + 13) % keys.len()) { + black_box(cache.get(&keys[i])); + } + }); + }); +} + +fn bench_lfu_cache_insert(c: &mut Criterion) { + let schema = make_schema(); + let entity_type = schema.entity_type("Transfer").unwrap(); + let n = 1_000; + + let entries: Vec<_> = (0..n as u64) + .map(|i| { + let key = entity_type.parse_key(format!("0x{i:064x}")).unwrap(); + let entity = make_entity(&schema, i); + (key, Some(Arc::new(entity))) + }) + .collect(); + + c.bench_with_input( + BenchmarkId::new("lfu_cache_insert", n), + &entries, + |b, entries| { + b.iter(|| { + let mut cache: LfuCache< + graph::schema::EntityKey, + Option>, + > = LfuCache::new(); + for (key, entity) in entries { + cache.insert(key.clone(), entity.clone()); + } + black_box(&cache); + }); + }, + ); +} + +fn bench_lfu_cache_evict(c: &mut Criterion) { + let schema = make_schema(); + let entity_type = schema.entity_type("Transfer").unwrap(); + let n = 10_000; + + let entries: Vec<_> = (0..n as u64) + .map(|i| { + let key = entity_type.parse_key(format!("0x{i:064x}")).unwrap(); + let entity = make_entity(&schema, i); + (key, Some(Arc::new(entity))) + }) + .collect(); + + c.bench_with_input( + BenchmarkId::new("lfu_cache_evict", n), + &entries, + |b, entries| { + b.iter_batched( + || { + let mut cache: LfuCache< + graph::schema::EntityKey, + Option>, + > = LfuCache::new(); + for (key, entity) in entries { + cache.insert(key.clone(), entity.clone()); + } + cache + }, + |mut cache| { + // Evict with a small target weight to force significant eviction + black_box(cache.evict(1)); + }, + criterion::BatchSize::SmallInput, + ); + }, + ); +} + +fn bench_entity_sorted_ref(c: &mut Criterion) { + let schema = make_schema(); + let entity = make_entity(&schema, 42); + + c.bench_function("entity_sorted_ref_20_fields", |b| { + b.iter(|| { + black_box(entity.sorted_ref()); + }); + }); +} + +fn bench_entity_validate(c: &mut Criterion) { + let schema = make_schema(); + let entity_type = schema.entity_type("Transfer").unwrap(); + let entity = make_entity(&schema, 42); + let key = entity_type.parse_key("0x42").unwrap(); + + c.bench_function("entity_validate_20_fields", |b| { + b.iter(|| { + black_box(entity.validate(black_box(&key))).unwrap(); + }); + }); +} + +criterion_group!( + benches, + bench_lfu_cache_get, + bench_lfu_cache_insert, + bench_lfu_cache_evict, + bench_entity_sorted_ref, + bench_entity_validate, +); +criterion_main!(benches); From 98db9e8f299034c873cddc314ec81a0004e1a52f Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Fri, 13 Feb 2026 22:59:45 -0800 Subject: [PATCH 03/35] runtime, core: Add stopwatch sub-sections to hot path Add stopwatch instrumentation for per-trigger WASM instantiation and start function costs, and DDS loop iteration tracking: - "instantiate_async" section around wasmtime instance creation - "wasm_start" section around _start() and custom start functions - "refetch_block" section around block refetching in DDS loop - DDS loop iteration counter with info log on completion --- core/src/subgraph/runner/mod.rs | 15 ++++++++++- runtime/wasm/src/module/instance.rs | 42 ++++++++++++++++------------- 2 files changed, 38 insertions(+), 19 deletions(-) diff --git a/core/src/subgraph/runner/mod.rs b/core/src/subgraph/runner/mod.rs index aea1ef6aaf1..ec47ce48240 100644 --- a/core/src/subgraph/runner/mod.rs +++ b/core/src/subgraph/runner/mod.rs @@ -923,7 +923,9 @@ where .stopwatch .start_section(HANDLE_CREATED_DS_SECTION_NAME); + let mut dds_iterations: u32 = 0; while block_state.has_created_data_sources() { + dds_iterations += 1; // Instantiate dynamic data sources, removing them from the block state. let (data_sources, runtime_hosts) = self.create_dynamic_data_sources(block_state.drain_created_data_sources())?; @@ -944,7 +946,10 @@ where // It's also not clear why refetching needs to happen inside // the loop; will firehose really return something diffrent // each time even though the cursor doesn't change? - let block = self.refetch_block(logger, block, firehose_cursor).await?; + let block = { + let _section = self.metrics.stream.stopwatch.start_section("refetch_block"); + self.refetch_block(logger, block, firehose_cursor).await? + }; // Reprocess the triggers from this block that match the new data sources let block_with_triggers = self @@ -1005,6 +1010,14 @@ where })?; } + if dds_iterations > 0 { + info!( + logger, + "Dynamic data source processing complete"; + "iterations" => dds_iterations, + ); + } + Ok(block_state) } diff --git a/runtime/wasm/src/module/instance.rs b/runtime/wasm/src/module/instance.rs index a4bf1f34d81..1e81b455f73 100644 --- a/runtime/wasm/src/module/instance.rs +++ b/runtime/wasm/src/module/instance.rs @@ -630,10 +630,13 @@ impl WasmInstance { // See also: runtime-timeouts store.set_epoch_deadline(2); - let instance = valid_module - .instance_pre - .instantiate_async(store.as_context_mut()) - .await?; + let instance = { + let _section = host_metrics.stopwatch.start_section("instantiate_async"); + valid_module + .instance_pre + .instantiate_async(store.as_context_mut()) + .await? + }; let asc_heap = AscHeapCtx::new( &instance, @@ -645,25 +648,28 @@ impl WasmInstance { // See start_function comment for more information // TL;DR; we need the wasmtime::Instance to create the heap, therefore // we cannot execute anything that requires access to the heap before it's created. - if let Some(start_func) = valid_module.start_function.as_ref() { - instance - .get_func(store.as_context_mut(), start_func) - .context(format!("`{start_func}` function not found"))? - .typed::<(), ()>(store.as_context_mut())? - .call_async(store.as_context_mut(), ()) - .await?; - } - - match api_version { - version if version <= Version::new(0, 0, 4) => {} - _ => { + { + let _section = host_metrics.stopwatch.start_section("wasm_start"); + if let Some(start_func) = valid_module.start_function.as_ref() { instance - .get_func(store.as_context_mut(), "_start") - .context("`_start` function not found")? + .get_func(store.as_context_mut(), start_func) + .context(format!("`{start_func}` function not found"))? .typed::<(), ()>(store.as_context_mut())? .call_async(store.as_context_mut(), ()) .await?; } + + match api_version { + version if version <= Version::new(0, 0, 4) => {} + _ => { + instance + .get_func(store.as_context_mut(), "_start") + .context("`_start` function not found")? + .typed::<(), ()>(store.as_context_mut())? + .call_async(store.as_context_mut(), ()) + .await?; + } + } } Ok(WasmInstance { From cb1885574f21adc72a39ebbae43fe6f82ffa601c Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Fri, 13 Feb 2026 23:19:05 -0800 Subject: [PATCH 04/35] tests: Add synthetic stress test with 200 triggers per block Add a runner test that exercises the hot path at scale: a single block with 200 trigger events where each handler does a store.get + store.set on a shared counter entity plus creates a per-event entity. This serves as the canonical benchmark for measuring per-trigger performance improvements in subsequent optimization phases. --- pnpm-lock.yaml | 9 +++ .../stress-test/abis/Contract.abi | 15 ++++ tests/runner-tests/stress-test/package.json | 13 +++ tests/runner-tests/stress-test/schema.graphql | 6 ++ tests/runner-tests/stress-test/src/mapping.ts | 26 ++++++ tests/runner-tests/stress-test/subgraph.yaml | 23 ++++++ tests/tests/runner_tests.rs | 81 +++++++++++++++---- 7 files changed, 157 insertions(+), 16 deletions(-) create mode 100644 tests/runner-tests/stress-test/abis/Contract.abi create mode 100644 tests/runner-tests/stress-test/package.json create mode 100644 tests/runner-tests/stress-test/schema.graphql create mode 100644 tests/runner-tests/stress-test/src/mapping.ts create mode 100644 tests/runner-tests/stress-test/subgraph.yaml diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d361bbe9c56..35506e45667 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -258,6 +258,15 @@ importers: specifier: 0.31.0 version: 0.31.0 + tests/runner-tests/stress-test: + devDependencies: + '@graphprotocol/graph-cli': + specifier: 0.60.0 + version: 0.60.0(@types/node@24.3.0)(bufferutil@4.0.9)(encoding@0.1.13)(node-fetch@2.7.0(encoding@0.1.13))(typescript@5.9.2)(utf-8-validate@5.0.10) + '@graphprotocol/graph-ts': + specifier: 0.31.0 + version: 0.31.0 + tests/runner-tests/typename: devDependencies: '@graphprotocol/graph-cli': diff --git a/tests/runner-tests/stress-test/abis/Contract.abi b/tests/runner-tests/stress-test/abis/Contract.abi new file mode 100644 index 00000000000..9d9f56b9263 --- /dev/null +++ b/tests/runner-tests/stress-test/abis/Contract.abi @@ -0,0 +1,15 @@ +[ + { + "anonymous": false, + "inputs": [ + { + "indexed": false, + "internalType": "string", + "name": "testCommand", + "type": "string" + } + ], + "name": "TestEvent", + "type": "event" + } +] diff --git a/tests/runner-tests/stress-test/package.json b/tests/runner-tests/stress-test/package.json new file mode 100644 index 00000000000..05680673223 --- /dev/null +++ b/tests/runner-tests/stress-test/package.json @@ -0,0 +1,13 @@ +{ + "name": "stress-test", + "version": "0.0.0", + "private": true, + "scripts": { + "codegen": "graph codegen --skip-migrations", + "deploy:test": "graph deploy test/stress-test --version-label v0.0.1 --ipfs $IPFS_URI --node $GRAPH_NODE_ADMIN_URI" + }, + "devDependencies": { + "@graphprotocol/graph-cli": "0.60.0", + "@graphprotocol/graph-ts": "0.31.0" + } +} diff --git a/tests/runner-tests/stress-test/schema.graphql b/tests/runner-tests/stress-test/schema.graphql new file mode 100644 index 00000000000..8feb3ffbe8f --- /dev/null +++ b/tests/runner-tests/stress-test/schema.graphql @@ -0,0 +1,6 @@ +type Counter @entity { + id: ID! + count: Int8! + value: BigInt! + label: String! +} diff --git a/tests/runner-tests/stress-test/src/mapping.ts b/tests/runner-tests/stress-test/src/mapping.ts new file mode 100644 index 00000000000..ce0b4fd34ce --- /dev/null +++ b/tests/runner-tests/stress-test/src/mapping.ts @@ -0,0 +1,26 @@ +import { BigInt } from "@graphprotocol/graph-ts"; +import { TestEvent } from "../generated/Contract/Contract"; +import { Counter } from "../generated/schema"; + +export function handleTestEvent(event: TestEvent): void { + let id = event.params.testCommand; + + // Create a per-event entity + let entity = new Counter(id); + entity.count = 1; + entity.value = event.block.number; + entity.label = "event_" + id; + entity.save(); + + // Read-modify-write a shared counter to exercise store.get + store.set + let global = Counter.load("global"); + if (global == null) { + global = new Counter("global"); + global.count = 0; + global.value = BigInt.fromI32(0); + global.label = "global"; + } + global.count = global.count + 1; + global.value = global.value.plus(BigInt.fromI32(1)); + global.save(); +} diff --git a/tests/runner-tests/stress-test/subgraph.yaml b/tests/runner-tests/stress-test/subgraph.yaml new file mode 100644 index 00000000000..8f95fe2abf9 --- /dev/null +++ b/tests/runner-tests/stress-test/subgraph.yaml @@ -0,0 +1,23 @@ +specVersion: 0.0.4 +schema: + file: ./schema.graphql +dataSources: + - kind: ethereum/contract + name: Contract + network: test + source: + address: "0x0000000000000000000000000000000000000000" + abi: Contract + mapping: + kind: ethereum/events + apiVersion: 0.0.6 + language: wasm/assemblyscript + abis: + - name: Contract + file: ./abis/Contract.abi + entities: + - Counter + eventHandlers: + - event: TestEvent(string) + handler: handleTestEvent + file: ./src/mapping.ts diff --git a/tests/tests/runner_tests.rs b/tests/tests/runner_tests.rs index f1dbda62c41..727b219abb1 100644 --- a/tests/tests/runner_tests.rs +++ b/tests/tests/runner_tests.rs @@ -1337,22 +1337,71 @@ async fn aggregation_current_bucket() { assert_eq!( query_res, Some(object! { - tokens: vec![ - object! { - id: "0xaa", - stats: vec![ - object! { sum: "40" }, - object! { sum: "12" }, - ] - }, - object! { - id: "0xbb", - stats: vec![ - object! { sum: "60" }, - object! { sum: "40" }, - ] - }, - ] + tokens: vec![ + object! { + id: "0xaa", + stats: vec![ + object! { sum: "40" }, + object! { sum: "12" }, + ] + }, + object! { + id: "0xbb", + stats: vec![ + object! { sum: "60" }, + object! { sum: "40" }, + ] + }, + ]}) + ); +} + +/// Synthetic stress test: processes a block with 200 triggers where each handler +/// does a store.get + store.set on a shared entity, plus creates a per-event entity. +/// This exercises the hot path at scale for meaningful per-trigger timings. +#[graph::test] +async fn stress_test_many_triggers() { + let RunnerTestRecipe { stores, test_info } = + RunnerTestRecipe::new("stress_test_many_triggers", "stress-test").await; + + const NUM_TRIGGERS: u32 = 200; + + let blocks = { + let block_0 = genesis(); + let mut block_1 = empty_block(block_0.ptr(), test_ptr(1)); + for i in 0..NUM_TRIGGERS { + push_test_log(&mut block_1, format!("{i}")); + } + vec![block_0, block_1] + }; + + let stop_block = blocks.last().unwrap().block.ptr(); + let chain = chain(&test_info.test_name, blocks, &stores, None).await; + let ctx = fixture::setup(&test_info, &stores, &chain, None, None).await; + + let start = std::time::Instant::now(); + ctx.start_and_sync_to(stop_block).await; + let elapsed = start.elapsed(); + + let per_trigger_us = elapsed.as_micros() / NUM_TRIGGERS as u128; + eprintln!( + "stress_test_many_triggers: {NUM_TRIGGERS} triggers in {:.2}s ({per_trigger_us} us/trigger)", + elapsed.as_secs_f64() + ); + + // Verify the global counter was incremented by every trigger + let query_res = ctx + .query(r#"{ counter(id: "global") { id count } }"#) + .await + .unwrap(); + + assert_json_eq!( + query_res, + Some(object! { + counter: object! { + id: "global", + count: NUM_TRIGGERS.to_string(), + } }) ); } From 4226880aaa229ef43ca2da6e07f797817fac6a00 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Fri, 13 Feb 2026 23:24:56 -0800 Subject: [PATCH 05/35] core: Hoist refetch_block() out of DDS processing loop The firehose cursor doesn't change between DDS loop iterations, so refetching the block on every iteration returns the same result. Move the refetch_block() call before the loop to eliminate N-1 redundant network round-trips for blocks that spawn N batches of dynamic data sources. --- core/src/subgraph/runner/mod.rs | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/core/src/subgraph/runner/mod.rs b/core/src/subgraph/runner/mod.rs index ec47ce48240..e581da47806 100644 --- a/core/src/subgraph/runner/mod.rs +++ b/core/src/subgraph/runner/mod.rs @@ -924,6 +924,17 @@ where .start_section(HANDLE_CREATED_DS_SECTION_NAME); let mut dds_iterations: u32 = 0; + + // Refetch the block once before entering the DDS loop. The + // firehose cursor doesn't change between iterations, so there + // is no point in refetching on every iteration. + let block = if block_state.has_created_data_sources() { + let _section = self.metrics.stream.stopwatch.start_section("refetch_block"); + self.refetch_block(logger, block, firehose_cursor).await? + } else { + block.cheap_clone() + }; + while block_state.has_created_data_sources() { dds_iterations += 1; // Instantiate dynamic data sources, removing them from the block state. @@ -937,20 +948,6 @@ where vec![], )); - // TODO: We have to pass a reference to `block` to - // `refetch_block`, otherwise the call to - // handle_offchain_triggers below gets an error that `block` - // has moved. That is extremely fishy since it means that - // `handle_offchain_triggers` uses the non-refetched block - // - // It's also not clear why refetching needs to happen inside - // the loop; will firehose really return something diffrent - // each time even though the cursor doesn't change? - let block = { - let _section = self.metrics.stream.stopwatch.start_section("refetch_block"); - self.refetch_block(logger, block, firehose_cursor).await? - }; - // Reprocess the triggers from this block that match the new data sources let block_with_triggers = self .inputs From 14d4b4c5a0877aa56e4eda7c3b1ad129999ea9de Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Fri, 13 Feb 2026 23:33:25 -0800 Subject: [PATCH 06/35] runtime: Enable wasmtime pooling allocator Configure wasmtime's PoolingAllocationConfig to pre-allocate a pool of instance slots. This replaces per-trigger mmap/munmap with copy-on-write page reuse (madvise), significantly reducing instantiation overhead for high-trigger blocks. Pool size is configurable via GRAPH_WASM_INSTANCE_POOL_SIZE (default: 1000). --- Cargo.toml | 2 +- graph/src/env/mappings.rs | 10 ++++++++++ runtime/wasm/src/mapping.rs | 11 +++++++++++ 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 65aad95bc71..8f0befad21b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -107,7 +107,7 @@ tonic-prost-build = "0.14" tower-http = { version = "0.6.8", features = ["cors"] } tower = { version = "0.5.1", features = ["full"] } wasmparser = "0.118.1" -wasmtime = { version = "38.0.4", features = ["async"] } +wasmtime = { version = "38.0.4", features = ["async", "pooling-allocator"] } rand = { version = "0.9.2", features = ["os_rng"] } prometheus = "0.14.0" url = "2.5.8" diff --git a/graph/src/env/mappings.rs b/graph/src/env/mappings.rs index c56bea2bac4..152a78e8189 100644 --- a/graph/src/env/mappings.rs +++ b/graph/src/env/mappings.rs @@ -89,6 +89,13 @@ pub struct EnvVarsMapping { /// are `none`, `speed`, and `speed_and_size`. The default value is /// `speed`. pub wasm_opt_level: WasmOptLevel, + + /// Size of the wasmtime instance pool. Controls the maximum number of + /// concurrent WASM instances across all subgraphs. Each slot + /// pre-reserves virtual address space for one linear memory. + /// + /// Set by `GRAPH_WASM_INSTANCE_POOL_SIZE`. Defaults to 1000. + pub wasm_instance_pool_size: u32, } /// Cranelift optimization level for WASM compilation. Maps to @@ -164,6 +171,7 @@ impl TryFrom for EnvVarsMapping { store_errors_are_nondeterministic: x.store_errors_are_nondeterministic.0, fds_max_backoff: Duration::from_secs(x.fds_max_backoff), wasm_opt_level: x.wasm_opt_level, + wasm_instance_pool_size: x.wasm_instance_pool_size, }; Ok(vars) } @@ -209,6 +217,8 @@ pub struct InnerMappingHandlers { fds_max_backoff: u64, #[envconfig(from = "GRAPH_WASM_OPT_LEVEL", default = "speed")] wasm_opt_level: WasmOptLevel, + #[envconfig(from = "GRAPH_WASM_INSTANCE_POOL_SIZE", default = "1000")] + wasm_instance_pool_size: u32, } fn validate_ipfs_cache_location(path: PathBuf) -> Result { diff --git a/runtime/wasm/src/mapping.rs b/runtime/wasm/src/mapping.rs index e2613d5a9fb..bb28ab887bf 100644 --- a/runtime/wasm/src/mapping.rs +++ b/runtime/wasm/src/mapping.rs @@ -318,6 +318,17 @@ impl ValidModule { config.max_wasm_stack(ENV_VARS.mappings.max_stack_size); config.async_support(true); + // Use the pooling allocator to reuse pre-allocated instance slots + // instead of mmap/munmap per trigger. This significantly reduces + // per-trigger instantiation cost. + let pool_size = ENV_VARS.mappings.wasm_instance_pool_size; + let mut pool = wasmtime::PoolingAllocationConfig::new(); + pool.total_core_instances(pool_size); + pool.total_memories(pool_size); + pool.total_tables(pool_size); + pool.total_stacks(pool_size); + config.allocation_strategy(wasmtime::InstanceAllocationStrategy::Pooling(pool)); + let engine = &wasmtime::Engine::new(&config)?; let module = wasmtime::Module::from_binary(engine, &raw_module)?; From 98637191b357e07a9c04e7544dada41e1847d1fb Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Fri, 13 Feb 2026 23:38:36 -0800 Subject: [PATCH 07/35] runtime: Share wasmtime Engine across all ValidModules Replace per-module Engine creation with a single shared Engine initialized via OnceLock on first use. This means all WASM modules share: - One pooling allocator pool (single virtual address reservation) - One epoch counter task (instead of one per module) - One set of compiled Cranelift settings The shared engine is created with identical config to what each module previously used. The epoch counter task now lives for the process lifetime (trivial cost) instead of being tied to module Drop. --- runtime/wasm/src/mapping.rs | 126 +++++++++++++++------------- runtime/wasm/src/module/context.rs | 9 +- runtime/wasm/src/module/instance.rs | 18 ++-- 3 files changed, 85 insertions(+), 68 deletions(-) diff --git a/runtime/wasm/src/mapping.rs b/runtime/wasm/src/mapping.rs index bb28ab887bf..b99cc157e0b 100644 --- a/runtime/wasm/src/mapping.rs +++ b/runtime/wasm/src/mapping.rs @@ -15,7 +15,7 @@ use parity_wasm::elements::ExportEntry; use std::collections::{BTreeMap, HashMap}; use std::panic::AssertUnwindSafe; use std::sync::atomic::{AtomicUsize, Ordering}; -use std::sync::Arc; +use std::sync::{Arc, OnceLock}; use std::{panic, thread}; /// Spawn a wasm module in its own thread. @@ -220,6 +220,69 @@ impl MappingContext { // See the start_index comment below for more information. const GN_START_FUNCTION_NAME: &str = "gn::start"; +/// Ensure the epoch counter task is running. The first call with a timeout +/// starts the task; subsequent calls are no-ops (the interval is locked in +/// by the first caller). +fn ensure_epoch_counter(engine: &wasmtime::Engine, timeout: Duration) { + static STARTED: OnceLock<()> = OnceLock::new(); + STARTED.get_or_init(|| { + let engine = engine.clone(); + graph::spawn(async move { + loop { + tokio::time::sleep(timeout).await; + engine.increment_epoch(); + } + }); + }); +} + +/// Returns a shared wasmtime Engine used by all ValidModules. The engine is +/// created once on first access and reused for the lifetime of the process. +/// Sharing the engine means all modules share a single pooling allocator +/// pool and a single epoch counter task. +fn shared_engine() -> &'static wasmtime::Engine { + static ENGINE: OnceLock = OnceLock::new(); + + ENGINE.get_or_init(|| { + let opt_level = match ENV_VARS.mappings.wasm_opt_level { + graph::env::WasmOptLevel::None => wasmtime::OptLevel::None, + graph::env::WasmOptLevel::Speed => wasmtime::OptLevel::Speed, + graph::env::WasmOptLevel::SpeedAndSize => wasmtime::OptLevel::SpeedAndSize, + }; + + let mut config = wasmtime::Config::new(); + config.strategy(wasmtime::Strategy::Cranelift); + config.epoch_interruption(true); + config.cranelift_nan_canonicalization(true); // For NaN determinism. + config.cranelift_opt_level(opt_level); + config.max_wasm_stack(ENV_VARS.mappings.max_stack_size); + config.async_support(true); + + // Use the pooling allocator to reuse pre-allocated instance slots + // instead of mmap/munmap per trigger. This significantly reduces + // per-trigger instantiation cost. + let pool_size = ENV_VARS.mappings.wasm_instance_pool_size; + let mut pool = wasmtime::PoolingAllocationConfig::new(); + pool.total_core_instances(pool_size); + pool.total_memories(pool_size); + pool.total_tables(pool_size); + pool.total_stacks(pool_size); + config.allocation_strategy(wasmtime::InstanceAllocationStrategy::Pooling(pool)); + + let engine = wasmtime::Engine::new(&config).expect("failed to create wasmtime engine"); + + // Start the global epoch counter task if a timeout is configured. + // The epoch is incremented at the configured timeout interval; + // Stores set their deadline to 2 epochs so effective timeout is + // between 1x and 2x the interval. See also: runtime-timeouts + if let Some(timeout) = ENV_VARS.mappings.timeout { + ensure_epoch_counter(&engine, timeout); + } + + engine + }) +} + /// A pre-processed and valid WASM module, ready to be started as a WasmModule. pub struct ValidModule { pub module: wasmtime::Module, @@ -248,9 +311,6 @@ pub struct ValidModule { // The timeout for the module. pub timeout: Option, - // Used as a guard to terminate this task dependency. - epoch_counter_abort_handle: Option, - /// Cache for asc_type_id results. Maps IndexForAscTypeId to their WASM runtime /// type IDs. Populated lazily on first use; deterministic per compiled module. asc_type_id_cache: RwLock>, @@ -302,34 +362,10 @@ impl ValidModule { .map_err(|_| anyhow!("Failed to inject gas counter"))?; let raw_module = parity_module.into_bytes()?; - // We use Cranelift as a compilation engine. Cranelift is an optimizing compiler, but that - // should not cause determinism issues since it adheres to the Wasm spec and NaN - // canonicalization is enabled below. The optimization level is configurable via - // GRAPH_WASM_OPT_LEVEL (default: speed). - let mut config = wasmtime::Config::new(); - config.strategy(wasmtime::Strategy::Cranelift); - config.epoch_interruption(true); - config.cranelift_nan_canonicalization(true); // For NaN determinism. - config.cranelift_opt_level(match ENV_VARS.mappings.wasm_opt_level { - graph::env::WasmOptLevel::None => wasmtime::OptLevel::None, - graph::env::WasmOptLevel::Speed => wasmtime::OptLevel::Speed, - graph::env::WasmOptLevel::SpeedAndSize => wasmtime::OptLevel::SpeedAndSize, - }); - config.max_wasm_stack(ENV_VARS.mappings.max_stack_size); - config.async_support(true); - - // Use the pooling allocator to reuse pre-allocated instance slots - // instead of mmap/munmap per trigger. This significantly reduces - // per-trigger instantiation cost. - let pool_size = ENV_VARS.mappings.wasm_instance_pool_size; - let mut pool = wasmtime::PoolingAllocationConfig::new(); - pool.total_core_instances(pool_size); - pool.total_memories(pool_size); - pool.total_tables(pool_size); - pool.total_stacks(pool_size); - config.allocation_strategy(wasmtime::InstanceAllocationStrategy::Pooling(pool)); - - let engine = &wasmtime::Engine::new(&config)?; + let engine = shared_engine(); + if let Some(timeout) = timeout { + ensure_epoch_counter(engine, timeout); + } let module = wasmtime::Module::from_binary(engine, &raw_module)?; let mut import_name_to_modules: BTreeMap> = BTreeMap::new(); @@ -345,23 +381,6 @@ impl ValidModule { .push(module.to_string()); } - let mut epoch_counter_abort_handle = None; - if let Some(timeout) = timeout { - let engine = engine.clone(); - - // The epoch counter task will perpetually increment the epoch every `timeout` seconds. - // Timeouts on instantiated modules will trigger on epoch deltas. - // Note: The epoch is an u64 so it will never overflow. - // See also: runtime-timeouts - let epoch_counter = async move { - loop { - tokio::time::sleep(timeout).await; - engine.increment_epoch(); - } - }; - epoch_counter_abort_handle = Some(graph::spawn(epoch_counter).abort_handle()); - } - let linker = crate::module::build_linker(engine, &import_name_to_modules)?; let instance_pre = linker.instantiate_pre(&module)?; @@ -371,7 +390,6 @@ impl ValidModule { import_name_to_modules, start_function, timeout, - epoch_counter_abort_handle, asc_type_id_cache: RwLock::new(HashMap::new()), }) } @@ -384,11 +402,3 @@ impl ValidModule { self.asc_type_id_cache.write().insert(idx, type_id); } } - -impl Drop for ValidModule { - fn drop(&mut self) { - if let Some(handle) = self.epoch_counter_abort_handle.take() { - handle.abort(); - } - } -} diff --git a/runtime/wasm/src/module/context.rs b/runtime/wasm/src/module/context.rs index 3e4f26cfd58..7a503ad9e05 100644 --- a/runtime/wasm/src/module/context.rs +++ b/runtime/wasm/src/module/context.rs @@ -54,12 +54,17 @@ impl WasmInstanceContext<'_> { pub fn suspend_timeout(&mut self) { // See also: runtime-timeouts - self.inner.set_epoch_deadline(u64::MAX); + self.inner.set_epoch_deadline(u64::MAX / 2); } pub fn start_timeout(&mut self) { + // Only re-arm the epoch deadline when this module actually has a + // timeout configured; otherwise leave it at the suspended value so + // the shared epoch counter does not interrupt no-timeout modules. // See also: runtime-timeouts - self.inner.set_epoch_deadline(2); + if self.as_ref().valid_module.timeout.is_some() { + self.inner.set_epoch_deadline(2); + } } } diff --git a/runtime/wasm/src/module/instance.rs b/runtime/wasm/src/module/instance.rs index 1e81b455f73..ac4a11638b0 100644 --- a/runtime/wasm/src/module/instance.rs +++ b/runtime/wasm/src/module/instance.rs @@ -620,15 +620,17 @@ impl WasmInstance { ); let mut store = Store::new(engine, wasm_ctx); - // The epoch on the engine will only ever be incremeted if increment_epoch() is explicitly - // called, we only do so if a timeout has been set, it will run forever. When a timeout is - // set, the timeout duration is used as the duration of one epoch. - // - // Therefore, the setting of 2 here means that if a `timeout` is provided, then this - // interrupt will be triggered between a duration of `timeout` and `timeout * 2`. - // + // When a timeout is configured the epoch counter increments every `timeout` and + // a deadline of 2 means the interrupt fires between 1x and 2x the interval. + // When no timeout is set, use a very large deadline so the shared epoch + // counter does not interrupt this module. We cannot use u64::MAX because + // wasmtime adds `current_epoch + deadline` without overflow protection. // See also: runtime-timeouts - store.set_epoch_deadline(2); + if valid_module.timeout.is_some() { + store.set_epoch_deadline(2); + } else { + store.set_epoch_deadline(u64::MAX / 2); + } let instance = { let _section = host_metrics.stopwatch.start_section("instantiate_async"); From 3cd655da315c5dd09231a405fc4750b84550bf70 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Fri, 13 Feb 2026 23:46:07 -0800 Subject: [PATCH 08/35] graph: Replace LfuCache PriorityQueue with HashMap Replace the PriorityQueue-based LfuCache with a plain HashMap. This eliminates the O(log n) heap sift and key clone on every cache access (get/insert/contains_key), replacing them with O(1) HashMap lookups that borrow the key directly. The eviction sort is moved from per-access (via heap maintenance) to per-eviction (a single O(n log n) sort when the cache exceeds its weight limit). Since eviction happens once per block while accesses happen thousands of times per block, this is a net win. Also removes the priority-queue dependency. --- Cargo.lock | 12 -- graph/Cargo.toml | 1 - graph/src/util/lfu_cache.rs | 268 ++++++++++++++++++------------------ 3 files changed, 132 insertions(+), 149 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 90b963452d5..eae924d5b8e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3809,7 +3809,6 @@ dependencies = [ "parking_lot", "petgraph", "portable-atomic", - "priority-queue", "prometheus", "prost", "prost-types", @@ -6107,17 +6106,6 @@ dependencies = [ "uint 0.9.5", ] -[[package]] -name = "priority-queue" -version = "2.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93980406f12d9f8140ed5abe7155acb10bb1e69ea55c88960b9c2f117445ef96" -dependencies = [ - "equivalent", - "indexmap 2.11.4", - "serde", -] - [[package]] name = "proc-macro-crate" version = "3.1.0" diff --git a/graph/Cargo.toml b/graph/Cargo.toml index 6a124711101..e8b72eb91ee 100644 --- a/graph/Cargo.toml +++ b/graph/Cargo.toml @@ -69,7 +69,6 @@ tokio-retry = { workspace = true } toml = "1.0.3" url = { workspace = true } prometheus = "0.14.0" -priority-queue = "2.7.0" tonic = { workspace = true } tonic-prost = { workspace = true } prost = { workspace = true } diff --git a/graph/src/util/lfu_cache.rs b/graph/src/util/lfu_cache.rs index 12712350a01..ef01df66580 100644 --- a/graph/src/util/lfu_cache.rs +++ b/graph/src/util/lfu_cache.rs @@ -1,63 +1,22 @@ use crate::env::ENV_VARS; use crate::prelude::CacheWeight; -use priority_queue::PriorityQueue; -use std::cmp::Reverse; +use std::collections::HashMap; use std::fmt::Debug; -use std::hash::{Hash, Hasher}; +use std::hash::Hash; use std::time::{Duration, Instant}; // The number of `evict` calls without access after which an entry is considered stale. const STALE_PERIOD: u64 = 100; -/// `PartialEq` and `Hash` are delegated to the `key`. #[derive(Clone, Debug)] -pub struct CacheEntry { +pub struct CacheEntry { + pub value: V, weight: usize, - key: K, - value: V, + freq: u64, + stale: bool, will_stale: bool, } -impl PartialEq for CacheEntry { - fn eq(&self, other: &Self) -> bool { - self.key.eq(&other.key) - } -} - -impl Eq for CacheEntry {} - -impl Hash for CacheEntry { - fn hash(&self, state: &mut H) { - self.key.hash(state) - } -} - -impl CacheEntry { - fn cache_key(key: K) -> Self { - // Only the key matters for finding an entry in the cache. - CacheEntry { - key, - value: V::default(), - weight: 0, - will_stale: false, - } - } -} - -impl CacheEntry { - /// Estimate the size of a `CacheEntry` with the given key and value. Do - /// not count the size of `Self` since that is memory that is not freed - /// when the cache entry is dropped as its storage is embedded in the - /// `PriorityQueue` - fn weight(key: &K, value: &V) -> usize { - value.indirect_weight() + key.indirect_weight() - } -} - -// The priorities are `(stale, frequency)` tuples, first all stale entries will be popped and -// then non-stale entries by least frequency. -type Priority = (bool, Reverse); - /// Statistics about what happened during cache eviction pub struct EvictStats { /// The weight of the cache after eviction @@ -96,7 +55,7 @@ impl EvictStats { /// evictions entities are checked for staleness. #[derive(Debug)] pub struct LfuCache { - queue: PriorityQueue, Priority>, + entries: HashMap>, total_weight: usize, stale_counter: u64, dead_weight: bool, @@ -104,10 +63,10 @@ pub struct LfuCache { hits: usize, } -impl Default for LfuCache { +impl Default for LfuCache { fn default() -> Self { LfuCache { - queue: PriorityQueue::new(), + entries: HashMap::new(), total_weight: 0, stale_counter: 0, dead_weight: false, @@ -117,10 +76,10 @@ impl Default for LfuCache { } } -impl LfuCache { +impl LfuCache { pub fn new() -> Self { LfuCache { - queue: PriorityQueue::new(), + entries: HashMap::new(), total_weight: 0, stale_counter: 0, dead_weight: ENV_VARS.mappings.entity_cache_dead_weight, @@ -129,93 +88,84 @@ impl } } - /// Updates and bumps freceny if already present. + fn entry_weight(key: &K, value: &V) -> usize { + value.indirect_weight() + key.indirect_weight() + } + + /// Updates and bumps frequency if already present. pub fn insert(&mut self, key: K, value: V) { - let weight = CacheEntry::weight(&key, &value); - match self.get_mut(key.clone()) { + let weight = Self::entry_weight(&key, &value); + match self.entries.get_mut(&key) { + Some(entry) => { + self.total_weight -= entry.weight; + self.total_weight += weight; + entry.weight = weight; + entry.value = value; + entry.freq += 1; + entry.will_stale = false; + } None => { self.total_weight += weight; - self.queue.push( + self.entries.insert( + key, CacheEntry { - weight, - key, value, + weight, + freq: 1, + stale: false, will_stale: false, }, - (false, Reverse(1)), ); } - Some(entry) => { - let old_weight = entry.weight; - entry.weight = weight; - entry.value = value; - self.total_weight -= old_weight; - self.total_weight += weight; - } } } #[cfg(test)] - fn weight(&self, key: K) -> usize { - let key_entry = CacheEntry::cache_key(key); - self.queue - .get(&key_entry) - .map(|(entry, _)| entry.weight) - .unwrap_or(0) + fn weight(&self, key: &K) -> usize { + self.entries.get(key).map(|e| e.weight).unwrap_or(0) } - fn get_mut(&mut self, key: K) -> Option<&mut CacheEntry> { - // Increment the frequency by 1 - let key_entry = CacheEntry::cache_key(key); - self.queue - .change_priority_by(&key_entry, |(_, Reverse(f))| { - *f += 1; - }); + pub fn iter(&self) -> impl Iterator { + self.entries.iter().map(|(k, e)| (k, &e.value)) + } + + pub fn get(&mut self, key: &K) -> Option<&V> { self.accesses += 1; - self.queue.get_mut(&key_entry).map(|x| { + self.entries.get_mut(key).map(|entry| { self.hits += 1; - x.0.will_stale = false; - x.0 + entry.freq += 1; + entry.will_stale = false; + &entry.value }) } - pub fn iter(&self) -> impl Iterator { - self.queue - .iter() - .map(|entry| (&entry.0.key, &entry.0.value)) - } - - pub fn get(&mut self, key: &K) -> Option<&V> { - self.get_mut(key.clone()).map(|x| &x.value) + pub fn get_mut(&mut self, key: &K) -> Option<&mut V> { + self.accesses += 1; + self.entries.get_mut(key).map(|entry| { + self.hits += 1; + entry.freq += 1; + entry.will_stale = false; + &mut entry.value + }) } pub fn remove(&mut self, key: &K) -> Option { - // `PriorityQueue` doesn't have a remove method, so emulate that by setting the priority to - // the absolute minimum and popping. - let key_entry = CacheEntry::cache_key(key.clone()); - self.queue - .change_priority(&key_entry, (true, Reverse(u64::MIN))) - .and_then(|_| { - self.queue.pop().map(|(e, _)| { - assert_eq!(e.key, key_entry.key); - self.total_weight -= e.weight; - e.value - }) - }) + self.entries.remove(key).map(|entry| { + self.total_weight -= entry.weight; + entry.value + }) } pub fn contains_key(&self, key: &K) -> bool { - self.queue - .get(&CacheEntry::cache_key(key.clone())) - .is_some() + self.entries.contains_key(key) } pub fn is_empty(&self) -> bool { - self.queue.is_empty() + self.entries.is_empty() } pub fn len(&self) -> usize { - self.queue.len() + self.entries.len() } pub fn evict_and_stats(&mut self, max_weight: usize) -> EvictStats { @@ -267,34 +217,75 @@ impl self.hits = 0; // Entries marked `will_stale` were not accessed in this period. Properly mark them as - // stale in their priorities. Also mark all entities as `will_stale` for the _next_ - // period so that they will be marked stale next time unless they are updated or looked - // up between now and then. - for (e, p) in self.queue.iter_mut() { - p.0 = e.will_stale; - e.will_stale = true; + // stale. Also mark all entities as `will_stale` for the _next_ period so that they + // will be marked stale next time unless they are updated or looked up between now and + // then. + for entry in self.entries.values_mut() { + entry.stale = entry.will_stale; + entry.will_stale = true; } } - let mut evicted = 0; let old_len = self.len(); let dead_weight = if self.dead_weight { - self.len() * (std::mem::size_of::>() + 40) + old_len * (std::mem::size_of::>() + 40) } else { 0 }; - while self.total_weight + dead_weight > max_weight { - let entry = self - .queue - .pop() - .expect("empty cache but total_weight > max_weight") - .0; - evicted += entry.weight; - self.total_weight -= entry.weight; + + // Determine a frequency threshold below which entries should be + // evicted. We sort (stale, freq, weight) tuples to find the cutoff, + // then use `retain` to remove entries that fall at or below it. + let mut evict_order: Vec<(bool, u64, usize)> = self + .entries + .values() + .map(|e| (e.stale, e.freq, e.weight)) + .collect(); + // Evict priority: stale entries first, then lowest frequency first. + evict_order.sort_unstable_by(|a, b| { + b.0.cmp(&a.0) // stale=true before stale=false + .then(a.1.cmp(&b.1)) // lowest freq first + }); + + let target_evict = self.total_weight + dead_weight - max_weight; + let mut accumulated = 0; + let mut evict_count = 0; + for &(_, _, w) in &evict_order { + if accumulated >= target_evict { + break; + } + accumulated += w; + evict_count += 1; + } + + let mut evicted_weight = 0; + if evict_count > 0 { + let threshold_stale = evict_order[evict_count - 1].0; + let threshold_freq = evict_order[evict_count - 1].1; + let mut remaining = evict_count; + + self.entries.retain(|_, entry| { + if remaining == 0 { + return true; + } + // An entry should be evicted if it sorts at or before the + // threshold in eviction order (stale first, low freq first). + let dominated = (entry.stale, threshold_stale) == (true, false) + || (entry.stale == threshold_stale && entry.freq <= threshold_freq); + if dominated { + remaining -= 1; + evicted_weight += entry.weight; + false + } else { + true + } + }); } + self.total_weight -= evicted_weight; + Some(EvictStats { new_weight: self.total_weight, - evicted_weight: evicted, + evicted_weight, new_count: self.len(), evicted_count: old_len - self.len(), stale_update: self.stale_counter == 0, @@ -305,18 +296,23 @@ impl } } -impl IntoIterator for LfuCache { - type Item = (CacheEntry, Priority); - type IntoIter = Box>; +impl IntoIterator for LfuCache { + type Item = (K, CacheEntry); + type IntoIter = std::collections::hash_map::IntoIter>; fn into_iter(self) -> Self::IntoIter { - Box::new(self.queue.into_iter()) + self.entries.into_iter() } } -impl Extend<(CacheEntry, Priority)> for LfuCache { - fn extend, Priority)>>(&mut self, iter: T) { - self.queue.extend(iter); +impl Extend<(K, CacheEntry)> + for LfuCache +{ + fn extend)>>(&mut self, iter: T) { + for (key, entry) in iter { + self.total_weight += entry.weight; + self.entries.insert(key, entry); + } } } @@ -338,8 +334,8 @@ fn entity_lru_cache() { let mut cache: LfuCache<&'static str, Weight> = LfuCache::new(); cache.insert("panda", Weight(2)); cache.insert("cow", Weight(1)); - let panda_weight = cache.weight("panda"); - let cow_weight = cache.weight("cow"); + let panda_weight = cache.weight(&"panda"); + let cow_weight = cache.weight(&"cow"); assert_eq!(cache.get(&"cow"), Some(&Weight(1))); assert_eq!(cache.get(&"panda"), Some(&Weight(2))); @@ -354,7 +350,7 @@ fn entity_lru_cache() { assert!(cache.get(&"panda").is_none()); cache.insert("alligator", Weight(2)); - let alligator_weight = cache.weight("alligator"); + let alligator_weight = cache.weight(&"alligator"); // Give "cow" and "alligator" a high frequency. for _ in 0..1000 { @@ -365,10 +361,10 @@ fn entity_lru_cache() { // Insert a lion and make it weigh the same as the cow and the alligator // together. cache.insert("lion", Weight(0)); - let lion_weight = cache.weight("lion"); + let lion_weight = cache.weight(&"lion"); let lion_inner_weight = cow_weight + alligator_weight - lion_weight; cache.insert("lion", Weight(lion_inner_weight)); - let lion_weight = cache.weight("lion"); + let lion_weight = cache.weight(&"lion"); // Make "cow" and "alligator" stale and remove them. for _ in 0..(2 * STALE_PERIOD) { From 9b1e0137e9aab383e64ce71bb0d62a48dc1cae8c Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Fri, 13 Feb 2026 23:49:50 -0800 Subject: [PATCH 09/35] graph: Avoid EntityOp clone in entity cache get() and load_related() Add apply_to_ref() that borrows the EntityOp instead of consuming it, eliminating the .cloned() call that deep-clones the entire Entity (with all its fields) on every cache lookup that has pending updates. For the common Update case, this reduces from 2 full entity clones (one for the EntityOp, one for the merge) to 1 (just the merge). For Remove, zero clones instead of one. --- graph/src/components/store/entity_cache.rs | 68 +++++++++++++++------- 1 file changed, 46 insertions(+), 22 deletions(-) diff --git a/graph/src/components/store/entity_cache.rs b/graph/src/components/store/entity_cache.rs index 7353ad17709..ab027c1563b 100644 --- a/graph/src/components/store/entity_cache.rs +++ b/graph/src/components/store/entity_cache.rs @@ -38,6 +38,24 @@ enum EntityOp { } impl EntityOp { + /// Apply this operation by reference, avoiding a clone of the `EntityOp` + /// itself. The base entity is cloned only when merging an `Update`. + fn apply_to_ref>( + &self, + entity: &Option, + ) -> Result, InternError> { + use EntityOp::*; + match (self, entity) { + (Remove, _) => Ok(None), + (Overwrite(new), _) | (Update(new), None) => Ok(Some(new.clone())), + (Update(updates), Some(entity)) => { + let mut e = entity.borrow().clone(); + e.merge_remove_null_fields(updates.clone())?; + Ok(Some(e)) + } + } + } + fn apply_to>( self, entity: &Option, @@ -218,15 +236,15 @@ impl EntityCache { GetScope::InBlock => true, }); - if let Some(op) = self.updates.get(key).cloned() { + if let Some(op) = self.updates.get(key) { entity = op - .apply_to(&entity) + .apply_to_ref(&entity) .map_err(|e| key.unknown_attribute(e))? .map(Arc::new); } - if let Some(op) = self.handler_updates.get(key).cloned() { + if let Some(op) = self.handler_updates.get(key) { entity = op - .apply_to(&entity) + .apply_to_ref(&entity) .map_err(|e| key.unknown_attribute(e))? .map(Arc::new); } @@ -260,21 +278,28 @@ impl EntityCache { // Apply updates from `updates` and `handler_updates` directly to entities in `entity_map` that match the query for (key, entity) in entity_map.iter_mut() { - let op = match ( - self.updates.get(key).cloned(), - self.handler_updates.get(key).cloned(), - ) { - (Some(op), None) | (None, Some(op)) => op, - (Some(mut op), Some(op2)) => { - op.accumulate(op2); - op - } - (None, None) => continue, - }; + let (has_update, has_handler) = ( + self.updates.contains_key(key), + self.handler_updates.contains_key(key), + ); + if !has_update && !has_handler { + continue; + } + + // Apply the main update first, then the handler update on top. + let mut updated: Option = Some(entity.clone()); + if let Some(op) = self.updates.get(key) { + updated = op + .apply_to_ref(&updated) + .map_err(|e| key.unknown_attribute(e))?; + } + if let Some(op) = self.handler_updates.get(key) { + updated = op + .apply_to_ref(&updated) + .map_err(|e| key.unknown_attribute(e))?; + } - let updated_entity = op - .apply_to(&Some(&*entity)) - .map_err(|e| key.unknown_attribute(e))?; + let updated_entity = updated; if let Some(updated_entity) = updated_entity { *entity = updated_entity; @@ -311,12 +336,11 @@ impl EntityCache { for (key, op) in self.updates.iter() { if !entity_map.contains_key(key) { if let Some(entity) = matches_query(op, &query, key)? { - if let Some(handler_op) = self.handler_updates.get(key).cloned() { + if let Some(handler_op) = self.handler_updates.get(key) { // If there's a corresponding update in handler_updates, apply it to the entity // and insert the updated entity into entity_map - let mut entity = Some(entity); - entity = handler_op - .apply_to(&entity) + let entity = handler_op + .apply_to_ref(&Some(entity)) .map_err(|e| key.unknown_attribute(e))?; if let Some(updated_entity) = entity { From 055c8752b6b9ffc739e33ad4cdff88aa1d22b3fc Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Sat, 14 Feb 2026 02:07:49 -0800 Subject: [PATCH 10/35] graph: Pre-sort entity fields on cache insertion Sort entity fields by key name when they enter the entity cache. This makes subsequent sorted_ref() and sorted() calls effectively O(n) since Rust's timsort is optimized for already-sorted data, eliminating the O(n log n) sort per store.get return. Add Object::sort_by_key() to sort entries by their interned string keys (with tombstones sorted to the end) and Entity::sort_fields() as a convenience wrapper. All entity cache insertion points now call sort_fields(): - get() when loading from store - load_related() when caching derived entities - as_modifications() when merging updates and fetching missing entities --- graph/src/components/store/entity_cache.rs | 36 +++++++++------------- graph/src/data/store/mod.rs | 7 +++++ graph/src/util/intern.rs | 26 ++++++++++++++++ 3 files changed, 47 insertions(+), 22 deletions(-) diff --git a/graph/src/components/store/entity_cache.rs b/graph/src/components/store/entity_cache.rs index ab027c1563b..24dd17867cd 100644 --- a/graph/src/components/store/entity_cache.rs +++ b/graph/src/components/store/entity_cache.rs @@ -56,22 +56,6 @@ impl EntityOp { } } - fn apply_to>( - self, - entity: &Option, - ) -> Result, InternError> { - use EntityOp::*; - match (self, entity) { - (Remove, _) => Ok(None), - (Overwrite(new), _) | (Update(new), None) => Ok(Some(new)), - (Update(updates), Some(entity)) => { - let mut e = entity.borrow().clone(); - e.merge_remove_null_fields(updates)?; - Ok(Some(e)) - } - } - } - fn accumulate(&mut self, next: EntityOp) { use EntityOp::*; let update = match next { @@ -215,8 +199,11 @@ impl EntityCache { let mut entity: Option> = match scope { GetScope::Store => { if !self.current.contains_key(key) { - let entity = self.store.get(key).await?; - self.current.insert(key.clone(), entity.map(Arc::new)); + let entity = self.store.get(key).await?.map(|mut e| { + e.sort_fields(); + Arc::new(e) + }); + self.current.insert(key.clone(), entity); } // Unwrap: we just inserted the entity self.current.get(key).unwrap().cheap_clone() @@ -269,8 +256,9 @@ impl EntityCache { for (key, entity) in entity_map.iter() { // Only insert to the cache if it's not already there if !self.current.contains_key(key) { - self.current - .insert(key.clone(), Some(Arc::new(entity.clone()))); + let mut sorted = entity.clone(); + sorted.sort_fields(); + self.current.insert(key.clone(), Some(Arc::new(sorted))); } } @@ -517,7 +505,8 @@ impl EntityCache { // violation in the database, ensuring correctness let missing = missing.filter(|key| !key.entity_type.is_immutable()); - for (entity_key, entity) in self.store.get_many(missing.cloned().collect()).await? { + for (entity_key, mut entity) in self.store.get_many(missing.cloned().collect()).await? { + entity.sort_fields(); self.current.insert(entity_key, Some(Arc::new(entity))); } @@ -531,6 +520,7 @@ impl EntityCache { (None, EntityOp::Update(mut updates)) | (None, EntityOp::Overwrite(mut updates)) => { updates.remove_null_fields(); + updates.sort_fields(); let data = Arc::new(updates); self.current.insert(key.clone(), Some(data.cheap_clone())); Some(Insert { @@ -547,6 +537,7 @@ impl EntityCache { let changed = data .merge_remove_null_fields(updates) .map_err(|e| key.unknown_attribute(e))?; + data.sort_fields(); let data = Arc::new(data); self.current.insert(key.clone(), Some(data.cheap_clone())); if changed { @@ -561,7 +552,8 @@ impl EntityCache { } } // Entity was removed and then updated, so it will be overwritten - (Some(current), EntityOp::Overwrite(data)) => { + (Some(current), EntityOp::Overwrite(mut data)) => { + data.sort_fields(); let data = Arc::new(data); self.current.insert(key.clone(), Some(data.cheap_clone())); if current != data { diff --git a/graph/src/data/store/mod.rs b/graph/src/data/store/mod.rs index 38a0af5bbe9..afdab8512fc 100644 --- a/graph/src/data/store/mod.rs +++ b/graph/src/data/store/mod.rs @@ -1039,6 +1039,13 @@ impl Entity { self.0.retain(|_, value| !value.is_null()) } + /// Sort the entity's fields by key name. This makes subsequent calls to + /// `sorted_ref()` and `sorted()` effectively O(n) since Rust's sort is + /// optimized for pre-sorted data. + pub fn sort_fields(&mut self) { + self.0.sort_by_key(); + } + /// Add the key/value pairs from `iter` to this entity. This is the same /// as an implementation of `std::iter::Extend` would be, except that /// this operation is fallible because one of the keys from the iterator diff --git a/graph/src/util/intern.rs b/graph/src/util/intern.rs index 884e4cb7e3d..ba47e146f2f 100644 --- a/graph/src/util/intern.rs +++ b/graph/src/util/intern.rs @@ -310,6 +310,32 @@ impl Object { pub fn atoms(&self) -> AtomIter<'_, V> { AtomIter::new(self) } + + /// Sort entries by their string key. Tombstone entries are moved to the + /// end. This makes subsequent iteration in key order O(n) instead of + /// requiring an O(n log n) sort. + pub fn sort_by_key(&mut self) { + let pool = &self.pool; + self.entries.sort_by(|a, b| { + let a_key = if a.key == TOMBSTONE_KEY { + None + } else { + pool.get(a.key) + }; + let b_key = if b.key == TOMBSTONE_KEY { + None + } else { + pool.get(b.key) + }; + // None (tombstones) sort to the end + match (a_key, b_key) { + (Some(a), Some(b)) => a.cmp(b), + (Some(_), None) => std::cmp::Ordering::Less, + (None, Some(_)) => std::cmp::Ordering::Greater, + (None, None) => std::cmp::Ordering::Equal, + } + }); + } } impl Object { From 9f3b17bb713c1f2756524e21092c79561e0a3d16 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Sat, 14 Feb 2026 04:24:02 -0800 Subject: [PATCH 11/35] graph: Defer entity validation to as_modifications() Move Entity::validate() from EntityCache::set() to as_modifications(), where entities are already fully merged. This eliminates a wasted validation + potential DB fetch on every store.set() call for partial updates. The change also makes set() synchronous (no longer async) since the DB fetch for re-validation is removed, simplifying all callers. --- .../amp_subgraph/runner/data_processing.rs | 1 - core/src/subgraph/runner/mod.rs | 7 ++- graph/src/components/store/entity_cache.rs | 35 +++++-------- runtime/test/src/test.rs | 52 +++++-------------- runtime/wasm/src/host_exports.rs | 45 +++++++--------- runtime/wasm/src/module/context.rs | 26 +++++----- store/test-store/tests/graph/entity_cache.rs | 26 +++------- 7 files changed, 70 insertions(+), 122 deletions(-) diff --git a/core/src/amp_subgraph/runner/data_processing.rs b/core/src/amp_subgraph/runner/data_processing.rs index 8c403de2b7f..e68217fab87 100644 --- a/core/src/amp_subgraph/runner/data_processing.rs +++ b/core/src/amp_subgraph/runner/data_processing.rs @@ -231,7 +231,6 @@ async fn process_record_batch( entity_cache .set(key, entity, block_number.compat(), None) - .await .map_err(|e| { Error::Deterministic(e.context(format!( "failed to store a new entity of type '{entity_name}' with id '{entity_id}'" diff --git a/core/src/subgraph/runner/mod.rs b/core/src/subgraph/runner/mod.rs index e581da47806..8d0295c3754 100644 --- a/core/src/subgraph/runner/mod.rs +++ b/core/src/subgraph/runner/mod.rs @@ -1700,7 +1700,7 @@ async fn update_proof_of_indexing( entity_cache: &mut EntityCache, ) -> Result<(), Error> { // Helper to store the digest as a PoI entity in the cache - async fn store_poi_entity( + fn store_poi_entity( entity_cache: &mut EntityCache, key: EntityKey, digest: Bytes, @@ -1720,7 +1720,7 @@ async fn update_proof_of_indexing( data.push((entity_cache.schema.poi_block_time(), block_time)); } let poi = entity_cache.make_entity(data)?; - entity_cache.set(key, poi, block, None).await + entity_cache.set(key, poi, block, None) } let _section_guard = stopwatch.start_section("update_proof_of_indexing"); @@ -1763,8 +1763,7 @@ async fn update_proof_of_indexing( updated_proof_of_indexing, block_time, block_number, - ) - .await?; + )?; } Ok(()) diff --git a/graph/src/components/store/entity_cache.rs b/graph/src/components/store/entity_cache.rs index 24dd17867cd..22dd6bbbbe2 100644 --- a/graph/src/components/store/entity_cache.rs +++ b/graph/src/components/store/entity_cache.rs @@ -374,18 +374,14 @@ impl EntityCache { /// Store the `entity` under the given `key`. The `entity` may be only a /// partial entity; the cache will ensure partial updates get merged /// with existing data. The entity will be validated against the - /// subgraph schema, and any errors will result in an `Err` being - /// returned. - pub async fn set( + /// subgraph schema when `as_modifications()` is called. + pub fn set( &mut self, key: EntityKey, entity: Entity, block: BlockNumber, write_capacity_remaining: Option<&mut usize>, ) -> Result<(), anyhow::Error> { - // check the validate for derived fields - let is_valid = entity.validate(&key).is_ok(); - if let Some(write_capacity_remaining) = write_capacity_remaining { let weight = entity.weight(); if !self.current.contains_key(&key) && weight > *write_capacity_remaining { @@ -413,21 +409,7 @@ impl EntityCache { ); } - self.entity_op(key.clone(), EntityOp::Update(entity)); - - // The updates we were given are not valid by themselves; force a - // lookup in the database and check again with an entity that merges - // the existing entity with the changes - if !is_valid { - let entity = self.get(&key, GetScope::Store).await?.ok_or_else(|| { - anyhow!( - "Failed to read entity {}[{}] back from cache", - key.entity_type, - key.entity_id - ) - })?; - entity.validate(&key)?; - } + self.entity_op(key, EntityOp::Update(entity)); Ok(()) } @@ -576,6 +558,17 @@ impl EntityCache { (None, EntityOp::Remove) => None, }; if let Some(modification) = modification { + // Validate the final merged entity before committing. + // This catches type mismatches and missing non-null fields + // on the fully assembled entity rather than on partial + // updates during handler execution. + match &modification { + Insert { key, data, .. } | Overwrite { key, data, .. } => { + data.validate(key) + .map_err(|e| StoreError::Unknown(e.into()))?; + } + Remove { .. } => {} + } mods.push(modification) } } diff --git a/runtime/test/src/test.rs b/runtime/test/src/test.rs index c656a2703f1..e2426ff948c 100644 --- a/runtime/test/src/test.rs +++ b/runtime/test/src/test.rs @@ -1337,17 +1337,17 @@ impl Host { } } - async fn store_set( + fn store_set( &mut self, entity_type: &str, id: &str, data: Vec<(&str, &str)>, ) -> Result<(), HostExportError> { let data: Vec<_> = data.into_iter().map(|(k, v)| (k, Value::from(v))).collect(); - self.store_setv(entity_type, id, data).await + self.store_setv(entity_type, id, data) } - async fn store_setv( + fn store_setv( &mut self, entity_type: &str, id: &str, @@ -1355,19 +1355,17 @@ impl Host { ) -> Result<(), HostExportError> { let id = String::from(id); let data = HashMap::from_iter(data.into_iter().map(|(k, v)| (Word::from(k), v))); - self.host_exports - .store_set( - &self.ctx.logger, - 12, // Arbitrary block number - &mut self.ctx.state, - &self.ctx.proof_of_indexing, - entity_type.to_string(), - id, - data, - &self.stopwatch, - &self.gas, - ) - .await + self.host_exports.store_set( + &self.ctx.logger, + 12, // Arbitrary block number + &mut self.ctx.state, + &self.ctx.proof_of_indexing, + entity_type.to_string(), + id, + data, + &self.stopwatch, + &self.gas, + ) } async fn store_get( @@ -1415,17 +1413,14 @@ async fn test_store_set_id() { let mut host = Host::new(schema, "hostStoreSetId", "boolean.wasm", None).await; host.store_set(USER, UID, vec![("id", "u1"), ("name", "user1")]) - .await .expect("setting with same id works"); let err = host .store_set(USER, UID, vec![("id", "ux"), ("name", "user1")]) - .await .expect_err("setting with different id fails"); err_says(err, "conflicts with ID passed"); host.store_set(USER, UID, vec![("name", "user2")]) - .await .expect("setting with no id works"); let entity = host.store_get(USER, UID).await.unwrap().unwrap(); @@ -1438,7 +1433,6 @@ async fn test_store_set_id() { let beef = Value::Bytes("0xbeef".parse().unwrap()); let err = host .store_setv(USER, "0xbeef", vec![("id", beef)]) - .await .expect_err("setting with Bytes id fails"); err_says( err, @@ -1446,7 +1440,6 @@ async fn test_store_set_id() { ); host.store_setv(USER, UID, vec![("id", Value::Int(32))]) - .await .expect_err("id must be a string"); // @@ -1456,7 +1449,6 @@ async fn test_store_set_id() { let err = host .store_set(BINARY, BID, vec![("id", BID), ("name", "user1")]) - .await .expect_err("setting with string id in values fails"); err_says( err, @@ -1468,18 +1460,15 @@ async fn test_store_set_id() { BID, vec![("id", bid_bytes), ("name", Value::from("user1"))], ) - .await .expect("setting with bytes id in values works"); let beef = Value::Bytes("0xbeef".parse().unwrap()); let err = host .store_setv(BINARY, BID, vec![("id", beef)]) - .await .expect_err("setting with different id fails"); err_says(err, "conflicts with ID passed"); host.store_set(BINARY, BID, vec![("name", "user2")]) - .await .expect("setting with no id works"); let entity = host.store_get(BINARY, BID).await.unwrap().unwrap(); @@ -1491,7 +1480,6 @@ async fn test_store_set_id() { let err = host .store_setv(BINARY, BID, vec![("id", Value::Int(32))]) - .await .expect_err("id must be Bytes"); err_says( err, @@ -1526,7 +1514,6 @@ async fn test_store_set_invalid_fields() { .await; host.store_set(USER, UID, vec![("id", "u1"), ("name", "user1")]) - .await .unwrap(); let err = host @@ -1540,7 +1527,6 @@ async fn test_store_set_invalid_fields() { ("test2", "invalid_field"), ], ) - .await .err() .unwrap(); @@ -1555,7 +1541,6 @@ async fn test_store_set_invalid_fields() { UID, vec![("id", "u1"), ("name", "user1"), ("test3", "invalid_field")], ) - .await .err() .unwrap(); @@ -1581,7 +1566,6 @@ async fn test_store_set_invalid_fields() { ("test2", "invalid_field"), ], ) - .await .err() .is_none(); @@ -1612,16 +1596,12 @@ async fn generate_id() { // new id. Note that the types of the ids have an incorrect type, but // that doesn't matter since they get overwritten. host.store_set(INT8, AUTO, vec![("id", "u1"), ("name", "int1")]) - .await .expect("setting auto works"); host.store_set(INT8, AUTO, vec![("id", "u1"), ("name", "int2")]) - .await .expect("setting auto works"); host.store_set(BINARY, AUTO, vec![("id", "u1"), ("name", "bin1")]) - .await .expect("setting auto works"); host.store_set(BINARY, AUTO, vec![("id", "u1"), ("name", "bin2")]) - .await .expect("setting auto works"); let entity_cache = host.ctx.state.entity_cache; @@ -1678,11 +1658,9 @@ async fn test_store_intf() { let mut host = Host::new(schema, "hostStoreSetIntf", "boolean.wasm", None).await; host.store_set(PERSON, UID, vec![("id", "u1"), ("name", "user1")]) - .await .expect_err("can not use store_set with an interface"); host.store_set(USER, UID, vec![("id", "u1"), ("name", "user1")]) - .await .expect("storing user works"); host.store_get(PERSON, UID) @@ -1727,7 +1705,6 @@ async fn test_store_ts() { ("amount", b20.clone()), ], ) - .await .expect("Setting 'Data' is allowed"); // This is very backhanded: we generate an id the same way that @@ -1744,7 +1721,6 @@ async fn test_store_ts() { let err = host .store_setv(STATS, SID, vec![("amount", b20)]) - .await .expect_err("store_set must fail for aggregations"); err_says( err, diff --git a/runtime/wasm/src/host_exports.rs b/runtime/wasm/src/host_exports.rs index 0d9c0d2b3bc..722795bf5f7 100644 --- a/runtime/wasm/src/host_exports.rs +++ b/runtime/wasm/src/host_exports.rs @@ -221,7 +221,7 @@ impl HostExports { ))) } - pub(crate) async fn store_set( + pub(crate) fn store_set( &self, logger: &Logger, block: BlockNumber, @@ -336,15 +336,12 @@ impl HostExports { state.metrics.track_entity_write(&entity_type, &entity); - state - .entity_cache - .set( - key, - entity, - block, - Some(&mut state.write_capacity_remaining), - ) - .await?; + state.entity_cache.set( + key, + entity, + block, + Some(&mut state.write_capacity_remaining), + )?; Ok(()) } @@ -1327,7 +1324,7 @@ pub mod test_support { } } - pub async fn store_set( + pub fn store_set( &self, logger: &Logger, block: BlockNumber, @@ -1339,20 +1336,18 @@ pub mod test_support { stopwatch: &StopwatchMetrics, gas: &GasCounter, ) -> Result<(), HostExportError> { - self.host_exports - .store_set( - logger, - block, - state, - proof_of_indexing, - self.block_time, - entity_type, - entity_id, - data, - stopwatch, - gas, - ) - .await + self.host_exports.store_set( + logger, + block, + state, + proof_of_indexing, + self.block_time, + entity_type, + entity_id, + data, + stopwatch, + gas, + ) } pub async fn store_get( diff --git a/runtime/wasm/src/module/context.rs b/runtime/wasm/src/module/context.rs index 7a503ad9e05..e5494c95803 100644 --- a/runtime/wasm/src/module/context.rs +++ b/runtime/wasm/src/module/context.rs @@ -277,20 +277,18 @@ impl WasmInstanceContext<'_> { let host_exports = self.as_ref().ctx.host_exports.cheap_clone(); let ctx = &mut self.as_mut().ctx; - host_exports - .store_set( - &logger, - block_number, - &mut ctx.state, - &ctx.proof_of_indexing, - ctx.timestamp, - entity, - id, - data, - &stopwatch, - gas, - ) - .await?; + host_exports.store_set( + &logger, + block_number, + &mut ctx.state, + &ctx.proof_of_indexing, + ctx.timestamp, + entity, + id, + data, + &stopwatch, + gas, + )?; Ok(()) } diff --git a/store/test-store/tests/graph/entity_cache.rs b/store/test-store/tests/graph/entity_cache.rs index 7e10827548b..1b45efa488c 100644 --- a/store/test-store/tests/graph/entity_cache.rs +++ b/store/test-store/tests/graph/entity_cache.rs @@ -212,14 +212,12 @@ async fn insert_modifications() { let mogwai_key = make_band_key("mogwai"); cache .set(mogwai_key.clone(), mogwai_data.clone(), 0, None) - .await .unwrap(); let mut sigurros_data = entity! { SCHEMA => id: "sigurros", name: "Sigur Ros" }; let sigurros_key = make_band_key("sigurros"); cache .set(sigurros_key.clone(), sigurros_data.clone(), 0, None) - .await .unwrap(); mogwai_data.set_vid(100).unwrap(); @@ -263,14 +261,12 @@ async fn overwrite_modifications() { let mogwai_key = make_band_key("mogwai"); cache .set(mogwai_key.clone(), mogwai_data.clone(), 0, None) - .await .unwrap(); let mut sigurros_data = entity! { SCHEMA => id: "sigurros", name: "Sigur Ros", founded: 1994}; let sigurros_key = make_band_key("sigurros"); cache .set(sigurros_key.clone(), sigurros_data.clone(), 0, None) - .await .unwrap(); mogwai_data.set_vid(100).unwrap(); @@ -304,15 +300,12 @@ async fn consecutive_modifications() { let update_data = entity! { SCHEMA => id: "mogwai", founded: 1995, label: "Rock Action Records" }; let update_key = make_band_key("mogwai"); - cache.set(update_key, update_data, 0, None).await.unwrap(); + cache.set(update_key, update_data, 0, None).unwrap(); // Then, just reset the "label". let update_data = entity! { SCHEMA => id: "mogwai", label: Value::Null }; let update_key = make_band_key("mogwai"); - cache - .set(update_key.clone(), update_data, 0, None) - .await - .unwrap(); + cache.set(update_key.clone(), update_data, 0, None).unwrap(); // We expect a single overwrite modification for the above that leaves "id" // and "name" untouched, sets "founded" and removes the "label" field. @@ -340,7 +333,6 @@ async fn check_vid_sequence() { let mogwai_data = entity! { SCHEMA => id: id, name: name }; cache .set(mogwai_key.clone(), mogwai_data.clone(), 0, None) - .await .unwrap(); } @@ -888,10 +880,7 @@ fn scoped_get() { let account5 = ACCOUNT_TYPE.parse_id("5").unwrap(); let mut wallet5 = create_wallet_entity_no_vid("5", &account5, 100); let key5 = WALLET_TYPE.parse_key("5").unwrap(); - cache - .set(key5.clone(), wallet5.clone(), 0, None) - .await - .unwrap(); + cache.set(key5.clone(), wallet5.clone(), 0, None).unwrap(); wallet5.set_vid(100).unwrap(); // For the new entity, we can retrieve it with either scope @@ -916,10 +905,7 @@ fn scoped_get() { // But if it gets updated, it becomes visible with either scope let mut wallet1 = wallet1; wallet1.set("balance", 70).unwrap(); - cache - .set(key1.clone(), wallet1.clone(), 0, None) - .await - .unwrap(); + cache.set(key1.clone(), wallet1.clone(), 0, None).unwrap(); wallet1a = wallet1; wallet1a.set_vid(101).unwrap(); let act1 = cache.get(&key1, GetScope::InBlock).await.unwrap(); @@ -968,6 +954,8 @@ fn no_interface_mods() { let entity = entity! { LOAD_RELATED_SUBGRAPH => id: "1", balance: 100 }; - cache.set(key, entity, 0, None).await.unwrap_err(); + // set() no longer validates; the error surfaces in as_modifications() + cache.set(key, entity, 0, None).unwrap(); + assert!(cache.as_modifications(0).await.is_err()); }) } From f955ffcc7963a3295a3948a2d4ce501c789ec568 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Sat, 14 Feb 2026 04:41:11 -0800 Subject: [PATCH 12/35] graph: Scope deferred validation to entities from set() Only validate entities in as_modifications() that were added through set() (the WASM handler path). Entities from append() (used by test helpers, POI, and internal code paths) skip validation since they may store values with different type conventions. Tracks which entity keys need validation via a HashSet populated in set() and propagated through extend(). --- graph/src/components/store/entity_cache.rs | 25 ++++++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/graph/src/components/store/entity_cache.rs b/graph/src/components/store/entity_cache.rs index 22dd6bbbbe2..1bea3289c47 100644 --- a/graph/src/components/store/entity_cache.rs +++ b/graph/src/components/store/entity_cache.rs @@ -1,6 +1,6 @@ use anyhow::{anyhow, bail}; use std::borrow::Borrow; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::fmt::{self, Debug}; use std::sync::Arc; @@ -105,6 +105,12 @@ pub struct EntityCache { pub schema: InputSchema, + /// Entity keys that were added through `set()` and need validation + /// in `as_modifications()`. Entities added through `append()` are not + /// validated since they come from internal paths (test helpers, POI) + /// that may not conform to the subgraph schema's type requirements. + needs_validation: HashSet, + /// A sequence number for generating entity IDs. We use one number for /// all id's as the id's are scoped by block and a u32 has plenty of /// room for all changes in one block. To ensure reproducability of @@ -139,6 +145,7 @@ impl EntityCache { updates: HashMap::new(), handler_updates: HashMap::new(), in_handler: false, + needs_validation: HashSet::new(), schema: store.input_schema(), store, seq: 0, @@ -160,6 +167,7 @@ impl EntityCache { updates: HashMap::new(), handler_updates: HashMap::new(), in_handler: false, + needs_validation: HashSet::new(), schema: store.input_schema(), store, seq: 0, @@ -409,6 +417,7 @@ impl EntityCache { ); } + self.needs_validation.insert(key.clone()); self.entity_op(key, EntityOp::Update(entity)); Ok(()) @@ -448,6 +457,7 @@ impl EntityCache { assert!(!other.in_handler); self.current.extend(other.current); + self.needs_validation.extend(other.needs_validation); for (key, op) in other.updates { self.entity_op(key, op); } @@ -558,16 +568,17 @@ impl EntityCache { (None, EntityOp::Remove) => None, }; if let Some(modification) = modification { - // Validate the final merged entity before committing. - // This catches type mismatches and missing non-null fields - // on the fully assembled entity rather than on partial - // updates during handler execution. + // Validate entities that came through set() (handler path). + // Entities from append() skip validation since they come + // from internal paths that may store values differently. match &modification { - Insert { key, data, .. } | Overwrite { key, data, .. } => { + Insert { key, data, .. } | Overwrite { key, data, .. } + if self.needs_validation.contains(key) => + { data.validate(key) .map_err(|e| StoreError::Unknown(e.into()))?; } - Remove { .. } => {} + _ => {} } mods.push(modification) } From 967a8f9cf2d71e484c3da90e9fe0a9ad99a06972 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Sat, 14 Feb 2026 04:41:17 -0800 Subject: [PATCH 13/35] runtime: Merge store_set field iterations into single pass Replace separate check_invalid_fields() call (which iterated fields up to 3 times) and the subsequent filter with a single-pass iteration that both collects invalid field names and filters valid fields simultaneously. --- runtime/wasm/src/host_exports.rs | 94 +++++++++++--------------------- 1 file changed, 32 insertions(+), 62 deletions(-) diff --git a/runtime/wasm/src/host_exports.rs b/runtime/wasm/src/host_exports.rs index 722795bf5f7..b5c7e37bf95 100644 --- a/runtime/wasm/src/host_exports.rs +++ b/runtime/wasm/src/host_exports.rs @@ -162,54 +162,6 @@ impl HostExports { ))) } - fn check_invalid_fields( - &self, - api_version: Version, - data: &HashMap, - state: &BlockState, - entity_type: &EntityType, - ) -> Result<(), HostExportError> { - if api_version >= API_VERSION_0_0_8 { - let has_invalid_fields = data.iter().any(|(field_name, _)| { - !state - .entity_cache - .schema - .has_field_with_name(entity_type, field_name) - }); - - if has_invalid_fields { - let mut invalid_fields: Vec = data - .iter() - .filter_map(|(field_name, _)| { - if !state - .entity_cache - .schema - .has_field_with_name(entity_type, field_name) - { - Some(field_name.clone()) - } else { - None - } - }) - .collect(); - - invalid_fields.sort(); - - return Err(HostExportError::Deterministic(anyhow!( - "Attempted to set undefined fields [{}] for the entity type `{}`. Make sure those fields are defined in the schema.", - invalid_fields - .iter() - .map(|f| f.as_str()) - .collect::>() - .join(", "), - entity_type - ))); - } - } - - Ok(()) - } - /// Ensure that `entity_type` is of the right kind fn expect_object_type(entity_type: &EntityType, op: &str) -> Result<(), HostExportError> { if entity_type.is_object_type() { @@ -302,24 +254,42 @@ impl HostExports { } } - self.check_invalid_fields( - self.data_source.api_version.clone(), - &data, - state, - &key.entity_type, - )?; + // Filter out fields not in the schema, and reject if API >= 0.0.8 + // has any invalid fields. Single pass replaces separate + // check_invalid_fields() + filter iterations. + let mut invalid_fields: Vec = Vec::new(); + let filtered_data: Vec<(Word, Value)> = data + .into_iter() + .filter(|(field_name, _)| { + if state + .entity_cache + .schema + .has_field_with_name(&key.entity_type, field_name) + { + true + } else { + invalid_fields.push(field_name.clone()); + false + } + }) + .collect(); - // Filter out fields that are not in the schema - let filtered_entity_data = data.into_iter().filter(|(field_name, _)| { - state - .entity_cache - .schema - .has_field_with_name(&key.entity_type, field_name) - }); + if !invalid_fields.is_empty() && self.data_source.api_version >= API_VERSION_0_0_8 { + invalid_fields.sort(); + return Err(HostExportError::Deterministic(anyhow!( + "Attempted to set undefined fields [{}] for the entity type `{}`. Make sure those fields are defined in the schema.", + invalid_fields + .iter() + .map(|f| f.as_str()) + .collect::>() + .join(", "), + entity_type + ))); + } let entity = state .entity_cache - .make_entity(filtered_entity_data) + .make_entity(filtered_data.into_iter()) .map_err(|e| HostExportError::Deterministic(anyhow!(e)))?; let poi_section = stopwatch.start_section("host_export_store_set__proof_of_indexing"); From ac891f6079db8e7e30437cbea6990d29c8cc6d1b Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Sat, 14 Feb 2026 12:18:19 -0800 Subject: [PATCH 14/35] runtime: Reduce unbounded_loop test timeout from 3s to 100ms The test only verifies that epoch-based interruption fires; it doesn't care about the exact duration. 100ms is plenty to exercise the mechanism while cutting test wall time from ~7s to ~1s. --- runtime/test/src/test/abi.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runtime/test/src/test/abi.rs b/runtime/test/src/test/abi.rs index 8b81b014027..93e52b0ac72 100644 --- a/runtime/test/src/test/abi.rs +++ b/runtime/test/src/test/abi.rs @@ -6,7 +6,7 @@ use graph_runtime_wasm::asc_abi::class::{ use super::*; async fn test_unbounded_loop(api_version: Version) { - // Set handler timeout to 3 seconds. + // Set handler timeout to 100ms. let mut instance = test_valid_module_and_store_with_timeout( "unboundedLoop", mock_data_source( @@ -14,7 +14,7 @@ async fn test_unbounded_loop(api_version: Version) { api_version.clone(), ), api_version, - Some(Duration::from_secs(3)), + Some(Duration::from_millis(100)), ) .await .0; From 101608c899ae96206a36a94b7cce6a85ac8d2126 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Sat, 14 Feb 2026 13:06:39 -0800 Subject: [PATCH 15/35] tests: Extract shared integration test code into library crate Move shared types (TestCase, TestContext, TestResult, etc.) and shared test functions (test_int8, test_block_handlers, etc.) from integration_tests.rs into tests/src/integration.rs and tests/src/integration_cases.rs. Replace `mod integration_tests;` in gnd_cli_tests.rs and gnd_tests.rs with imports from the library crate. This fixes a race condition where both gnd_cli_tests and gnd_tests compiled integration_tests.rs as a submodule, pulling in its #[graph::test] entry point and causing concurrent database resets. --- tests/src/integration.rs | 381 ++++++++++++++++++ tests/src/integration_cases.rs | 288 +++++++++++++ tests/src/lib.rs | 2 + tests/tests/gnd_cli_tests.rs | 8 +- tests/tests/gnd_tests.rs | 11 +- tests/tests/integration_tests.rs | 668 +------------------------------ 6 files changed, 685 insertions(+), 673 deletions(-) create mode 100644 tests/src/integration.rs create mode 100644 tests/src/integration_cases.rs diff --git a/tests/src/integration.rs b/tests/src/integration.rs new file mode 100644 index 00000000000..294e75a88db --- /dev/null +++ b/tests/src/integration.rs @@ -0,0 +1,381 @@ +//! Shared integration test infrastructure. +//! +//! This module contains the test harness types and helper functions used by +//! all integration test binaries (`integration_tests`, `gnd_cli_tests`, +//! `gnd_tests`). + +use std::future::Future; +use std::pin::Pin; + +use anyhow::{anyhow, bail, Context, Result}; +use graph::prelude::serde_json::Value; +use tokio::process::Child; +use tokio::task::JoinError; + +use crate::contract::Contract; +use crate::subgraph::Subgraph; +use crate::{error, status}; + +pub type TestFn = Box< + dyn FnOnce(TestContext) -> Pin> + Send>> + + Sync + + Send, +>; + +pub struct TestContext { + pub subgraph: Subgraph, +} + +pub enum TestStatus { + Ok, + Err(anyhow::Error), + Panic(JoinError), +} + +pub struct TestResult { + pub name: String, + pub subgraph: Option, + pub status: TestStatus, +} + +impl TestResult { + pub fn success(&self) -> bool { + matches!(self.status, TestStatus::Ok) + } + + fn print_subgraph(&self) { + if let Some(subgraph) = &self.subgraph { + println!(" Subgraph: {}", subgraph.deployment); + } + } + + pub fn print(&self) { + // ANSI escape sequences; see the comment in macros.rs about better colorization + const GREEN: &str = "\x1b[1;32m"; + const RED: &str = "\x1b[1;31m"; + const NC: &str = "\x1b[0m"; + + match &self.status { + TestStatus::Ok => { + println!("* {GREEN}Test {} succeeded{NC}", self.name); + self.print_subgraph(); + } + TestStatus::Err(e) => { + println!("* {RED}Test {} failed{NC}", self.name); + self.print_subgraph(); + println!(" {:?}", e); + } + TestStatus::Panic(e) => { + if e.is_cancelled() { + println!("* {RED}Test {} was cancelled{NC}", self.name) + } else if e.is_panic() { + println!("* {RED}Test {} failed{NC}", self.name); + } else { + println!("* {RED}Test {} exploded mysteriously{NC}", self.name) + } + self.print_subgraph(); + } + } + } +} + +#[derive(Debug, Clone)] +pub enum SourceSubgraph { + Subgraph(String), + WithAlias((String, String)), // (alias, test_name) +} + +impl SourceSubgraph { + fn new(s: &str) -> Self { + if let Some((alias, subgraph)) = s.split_once(':') { + Self::WithAlias((alias.to_string(), subgraph.to_string())) + } else { + Self::Subgraph(s.to_string()) + } + } + + pub fn test_name(&self) -> &str { + match self { + Self::Subgraph(name) => name, + Self::WithAlias((_, name)) => name, + } + } + + pub fn alias(&self) -> Option<&str> { + match self { + Self::Subgraph(_) => None, + Self::WithAlias((alias, _)) => Some(alias), + } + } +} + +pub struct TestCase { + pub name: String, + pub test: TestFn, + pub source_subgraph: Option>, +} + +impl TestCase { + pub fn new(name: &str, test: fn(TestContext) -> T) -> Self + where + T: Future> + Send + 'static, + { + Self { + name: name.to_string(), + test: Box::new(move |ctx| Box::pin(test(ctx))), + source_subgraph: None, + } + } + + pub fn new_with_grafting(name: &str, test: fn(TestContext) -> T, base_subgraph: &str) -> Self + where + T: Future> + Send + 'static, + { + let mut test_case = Self::new(name, test); + test_case.source_subgraph = Some(vec![SourceSubgraph::new(base_subgraph)]); + test_case + } + + pub fn new_with_source_subgraphs( + name: &str, + test: fn(TestContext) -> T, + source_subgraphs: Vec<&str>, + ) -> Self + where + T: Future> + Send + 'static, + { + let mut test_case = Self::new(name, test); + test_case.source_subgraph = Some( + source_subgraphs + .into_iter() + .map(SourceSubgraph::new) + .collect(), + ); + test_case + } + + async fn deploy_and_wait( + &self, + subgraph_name: &str, + contracts: &[Contract], + ) -> Result { + status!(&self.name, "Deploying subgraph"); + let subgraph_name = match Subgraph::deploy(subgraph_name, contracts, None).await { + Ok(name) => name, + Err(e) => { + error!(&self.name, "Deploy failed"); + return Err(anyhow!(e.context("Deploy failed"))); + } + }; + + status!(&self.name, "Waiting for subgraph to become ready"); + let subgraph = match Subgraph::wait_ready(&subgraph_name).await { + Ok(subgraph) => subgraph, + Err(e) => { + error!(&self.name, "Subgraph never synced or failed"); + return Err(anyhow!(e.context("Subgraph never synced or failed"))); + } + }; + + if subgraph.healthy { + status!(&self.name, "Subgraph ({}) is synced", subgraph.deployment); + } else { + status!(&self.name, "Subgraph ({}) has failed", subgraph.deployment); + } + + Ok(subgraph) + } + + pub async fn prepare(&self, contracts: &[Contract]) -> anyhow::Result { + // If a subgraph has subgraph datasources, prepare them first and collect their deployment hashes + let source_mappings = if let Some(_subgraphs) = &self.source_subgraph { + match self.prepare_multiple_sources(contracts).await { + Ok(mappings) => Some(mappings), + Err(e) => { + error!(&self.name, "source subgraph deployment failed: {:?}", e); + return Err(e); + } + } + } else { + None + }; + + status!(&self.name, "Preparing subgraph"); + let (_, subgraph_name, _) = + match Subgraph::prepare(&self.name, contracts, source_mappings.as_deref()).await { + Ok(name) => name, + Err(e) => { + error!(&self.name, "Prepare failed: {:?}", e); + return Err(e); + } + }; + + Ok(subgraph_name) + } + + pub async fn check_health_and_test(self, subgraph_name: String) -> TestResult { + status!( + &self.name, + "Waiting for subgraph ({}) to become ready", + subgraph_name + ); + let subgraph = match Subgraph::wait_ready(&subgraph_name).await { + Ok(subgraph) => subgraph, + Err(e) => { + error!(&self.name, "Subgraph never synced or failed"); + return TestResult { + name: self.name.clone(), + subgraph: None, + status: TestStatus::Err(e.context("Subgraph never synced or failed")), + }; + } + }; + + if subgraph.healthy { + status!(&self.name, "Subgraph ({}) is synced", subgraph.deployment); + } else { + status!(&self.name, "Subgraph ({}) has failed", subgraph.deployment); + } + + let ctx = TestContext { + subgraph: subgraph.clone(), + }; + + status!(&self.name, "Starting test"); + let subgraph2 = subgraph.clone(); + let res = tokio::spawn(async move { (self.test)(ctx).await }).await; + let status = match res { + Ok(Ok(())) => { + status!(&self.name, "Test succeeded"); + TestStatus::Ok + } + Ok(Err(e)) => { + error!(&self.name, "Test failed"); + TestStatus::Err(e) + } + Err(e) => { + error!(&self.name, "Test panicked"); + TestStatus::Panic(e) + } + }; + TestResult { + name: self.name.clone(), + subgraph: Some(subgraph2), + status, + } + } + + pub async fn run(self, contracts: &[Contract]) -> TestResult { + // If a subgraph has subgraph datasources, deploy them first and collect their deployment hashes + let source_mappings = if let Some(_subgraphs) = &self.source_subgraph { + match self.deploy_multiple_sources(contracts).await { + Ok(mappings) => Some(mappings), + Err(e) => { + error!(&self.name, "source subgraph deployment failed"); + return TestResult { + name: self.name.clone(), + subgraph: None, + status: TestStatus::Err(e), + }; + } + } + } else { + None + }; + + status!(&self.name, "Deploying subgraph"); + let subgraph_name = + match Subgraph::deploy(&self.name, contracts, source_mappings.as_deref()).await { + Ok(name) => name, + Err(e) => { + error!(&self.name, "Deploy failed"); + return TestResult { + name: self.name.clone(), + subgraph: None, + status: TestStatus::Err(e.context("Deploy failed")), + }; + } + }; + + self.check_health_and_test(subgraph_name).await + } + + async fn prepare_multiple_sources( + &self, + contracts: &[Contract], + ) -> Result> { + let mut mappings = Vec::new(); + if let Some(sources) = &self.source_subgraph { + for source in sources { + // Source subgraphs don't have their own sources, so pass None + let _ = Subgraph::prepare(source.test_name(), contracts, None).await?; + // If the source has an alias (pre-known IPFS hash), use it for the mapping + if let Some(alias) = source.alias() { + mappings.push((source.test_name().to_string(), alias.to_string())); + } + } + } + Ok(mappings) + } + + async fn deploy_multiple_sources( + &self, + contracts: &[Contract], + ) -> Result> { + let mut mappings = Vec::new(); + if let Some(sources) = &self.source_subgraph { + for source in sources { + let subgraph = self.deploy_and_wait(source.test_name(), contracts).await?; + status!( + source.test_name(), + "Source subgraph deployed with hash {}", + subgraph.deployment + ); + // Use the test_name as the placeholder key + mappings.push((source.test_name().to_string(), subgraph.deployment.clone())); + } + } + Ok(mappings) + } +} + +/// Run the given `query` against the `subgraph` and check that the result +/// has no errors and that the `data` portion of the response matches the +/// `exp` value. +pub async fn query_succeeds( + title: &str, + subgraph: &Subgraph, + query: &str, + exp: Value, +) -> anyhow::Result<()> { + let resp = subgraph.query(query).await?; + match resp.get("errors") { + None => { /* nothing to do */ } + Some(errors) => { + bail!( + "query for `{}` returned GraphQL errors: {:?}", + title, + errors + ); + } + } + match resp.get("data") { + None => { + bail!("query for `{}` returned no data", title); + } + Some(data) => { + if &exp != data { + bail!( + "query for `{title}` returned unexpected data: \nexpected: {exp:?}\n returned: {data:?}", + ); + } + } + } + Ok(()) +} + +pub async fn stop_graph_node(child: &mut Child) -> anyhow::Result<()> { + child.kill().await.context("Failed to kill graph-node")?; + + Ok(()) +} diff --git a/tests/src/integration_cases.rs b/tests/src/integration_cases.rs new file mode 100644 index 00000000000..3c896f6fc00 --- /dev/null +++ b/tests/src/integration_cases.rs @@ -0,0 +1,288 @@ +//! Shared integration test case functions. +//! +//! These test functions are used by multiple integration test binaries +//! (`integration_tests`, `gnd_cli_tests`, `gnd_tests`). + +use graph::prelude::serde_json::{json, Value}; + +use crate::integration::{query_succeeds, TestContext}; +use crate::subgraph::Subgraph; + +pub async fn test_int8(ctx: TestContext) -> anyhow::Result<()> { + let subgraph = ctx.subgraph; + assert!(subgraph.healthy); + + let resp = subgraph + .query( + "{ + foos_0: foos(orderBy: id, block: { number: 0 }) { id } + foos(orderBy: id) { id value } + }", + ) + .await?; + + let exp = json!({ + "foos_0": [], + "foos": [ + { + "id": "0", + "value": "9223372036854775807", + }, + ], + }); + assert_eq!(None, resp.get("errors")); + assert_eq!(exp, resp["data"]); + + Ok(()) +} + +pub async fn test_timestamp(ctx: TestContext) -> anyhow::Result<()> { + let subgraph = ctx.subgraph; + assert!(subgraph.healthy); + + let resp = subgraph + .query( + "{ + foos_0: foos(orderBy: id, block: { number: 0 }) { id } + foos(orderBy: id) { id value } + }", + ) + .await?; + + let exp = json!({ + "foos_0": [], + "foos": [ + { + "id": "0", + "value": "1710837304040956", + }, + ], + }); + assert_eq!(None, resp.get("errors")); + assert_eq!(exp, resp["data"]); + + Ok(()) +} + +pub async fn test_block_handlers(ctx: TestContext) -> anyhow::Result<()> { + let subgraph = ctx.subgraph; + assert!(subgraph.healthy); + + // test non-filtered blockHandler + let exp = json!({ + "blocks": [ + { "id": "1", "number": "1" }, + { "id": "2", "number": "2" }, + { "id": "3", "number": "3" }, + { "id": "4", "number": "4" }, + { "id": "5", "number": "5" }, + { "id": "6", "number": "6" }, + { "id": "7", "number": "7" }, + { "id": "8", "number": "8" }, + { "id": "9", "number": "9" }, + { "id": "10", "number": "10" }, + ] + }); + query_succeeds( + "test non-filtered blockHandler", + &subgraph, + "{ blocks(orderBy: number, first: 10) { id number } }", + exp, + ) + .await?; + + // test query + let mut values = Vec::new(); + for i in 0..=10 { + values.push(json!({ "id": i.to_string(), "value": i.to_string() })); + } + let exp = json!({ "foos": Value::Array(values) }); + query_succeeds( + "test query", + &subgraph, + "{ foos(orderBy: value, skip: 1) { id value } }", + exp, + ) + .await?; + + // should call intialization handler first + let exp = json!({ + "foo": { "id": "initialize", "value": "-1" }, + }); + query_succeeds( + "should call intialization handler first", + &subgraph, + "{ foo( id: \"initialize\" ) { id value } }", + exp, + ) + .await?; + + // test blockHandler with polling filter + let exp = json!({ + "blockFromPollingHandlers": [ + { "id": "1", "number": "1" }, + { "id": "4", "number": "4" }, + { "id": "7", "number": "7" }, + ] + }); + query_succeeds( + "test blockHandler with polling filter", + &subgraph, + "{ blockFromPollingHandlers(orderBy: number, first: 3) { id number } }", + exp, + ) + .await?; + + // test other blockHandler with polling filter + let exp = json!({ + "blockFromOtherPollingHandlers": [ + { "id": "2", "number": "2" }, + { "id": "4", "number": "4" }, + { "id": "6", "number": "6" }, + ] + }); + query_succeeds( + "test other blockHandler with polling filter", + &subgraph, + "{ blockFromOtherPollingHandlers(orderBy: number, first: 3) { id number } }", + exp, + ) + .await?; + + // test initialization handler + let exp = json!({ + "initializes": [ + { "id": "1", "block": "1" }, + ] + }); + query_succeeds( + "test initialization handler", + &subgraph, + "{ initializes(orderBy: block, first: 10) { id block } }", + exp, + ) + .await?; + + // test subgraphFeatures endpoint returns handlers correctly + let subgraph_features = Subgraph::query_with_vars( + "query GetSubgraphFeatures($deployment: String!) { + subgraphFeatures(subgraphId: $deployment) { + specVersion + apiVersion + features + dataSources + network + handlers + } + }", + json!({ "deployment": subgraph.deployment }), + ) + .await?; + let handlers = &subgraph_features["data"]["subgraphFeatures"]["handlers"]; + assert!( + handlers.is_array(), + "subgraphFeatures.handlers must be an array" + ); + let handlers = handlers.as_array().unwrap(); + for handler in [ + "block_filter_polling", + "block_filter_once", + "block", + "event", + ] { + assert!( + handlers.contains(&Value::String(handler.to_string())), + "handlers {:?} must contain {}", + handlers, + handler + ); + } + + Ok(()) +} + +pub async fn subgraph_data_sources(ctx: TestContext) -> anyhow::Result<()> { + let subgraph = ctx.subgraph; + assert!(subgraph.healthy); + let expected_response = json!({ + "mirrorBlocks": [ + { "id": "1-v1", "number": "1", "testMessage": null }, + { "id": "1-v2", "number": "1", "testMessage": null }, + { "id": "1-v3", "number": "1", "testMessage": "1-message" }, + { "id": "2-v1", "number": "2", "testMessage": null }, + { "id": "2-v2", "number": "2", "testMessage": null }, + { "id": "2-v3", "number": "2", "testMessage": "2-message" }, + { "id": "3-v1", "number": "3", "testMessage": null }, + { "id": "3-v2", "number": "3", "testMessage": null }, + { "id": "3-v3", "number": "3", "testMessage": "3-message" }, + ] + }); + + query_succeeds( + "Query all blocks with testMessage", + &subgraph, + "{ mirrorBlocks(where: {number_lte: 3}, orderBy: number) { id, number, testMessage } }", + expected_response, + ) + .await?; + + let expected_response = json!({ + "mirrorBlock": { "id": "1-v3", "number": "1", "testMessage": "1-message" }, + }); + + query_succeeds( + "Query specific block with testMessage", + &subgraph, + "{ mirrorBlock(id: \"1-v3\") { id, number, testMessage } }", + expected_response, + ) + .await?; + + Ok(()) +} + +pub async fn test_value_roundtrip(ctx: TestContext) -> anyhow::Result<()> { + let subgraph = ctx.subgraph; + assert!(subgraph.healthy); + + let exp = json!({ + "foos": [{ "id": "0", "value": "bla" }], + "foos_0": [] + }); + + let query = "{ + foos_0: foos(orderBy: id, block: { number: 0 }) { id } + foos(orderBy: id) { id value } + }"; + + query_succeeds("test query", &subgraph, query, exp).await?; + + Ok(()) +} + +pub async fn test_multiple_subgraph_datasources(ctx: TestContext) -> anyhow::Result<()> { + let subgraph = ctx.subgraph; + assert!(subgraph.healthy); + + // Test querying data aggregated from multiple sources + let exp = json!({ + "aggregatedDatas": [ + { + "id": "0", + "sourceA": "from source A", + "sourceB": "from source B", + "first": "sourceA" + }, + ] + }); + + query_succeeds( + "should aggregate data from multiple sources", + &subgraph, + "{ aggregatedDatas(first: 1) { id sourceA sourceB first } }", + exp, + ) + .await?; + + Ok(()) +} diff --git a/tests/src/lib.rs b/tests/src/lib.rs index 33c6246c8b4..1c61329318f 100644 --- a/tests/src/lib.rs +++ b/tests/src/lib.rs @@ -2,6 +2,8 @@ pub mod config; pub mod contract; pub mod fixture; pub mod helpers; +pub mod integration; +pub mod integration_cases; #[macro_use] pub mod macros; pub mod output; diff --git a/tests/tests/gnd_cli_tests.rs b/tests/tests/gnd_cli_tests.rs index 43676cbed11..c4b3137af20 100644 --- a/tests/tests/gnd_cli_tests.rs +++ b/tests/tests/gnd_cli_tests.rs @@ -29,14 +29,10 @@ use std::path::PathBuf; use anyhow::anyhow; use graph::futures03::StreamExt; use graph_tests::contract::Contract; +use graph_tests::integration::{stop_graph_node, TestCase, TestResult}; +use graph_tests::integration_cases::{test_block_handlers, test_int8, test_value_roundtrip}; use graph_tests::{error, status, CONFIG}; -mod integration_tests; - -use integration_tests::{ - stop_graph_node, test_block_handlers, test_int8, test_value_roundtrip, TestCase, TestResult, -}; - /// Get the path to the gnd binary fn gnd_binary_path() -> PathBuf { // The gnd binary should be at target/debug/gnd relative to the workspace root diff --git a/tests/tests/gnd_tests.rs b/tests/tests/gnd_tests.rs index 0ed1ac5ec8a..47d13143267 100644 --- a/tests/tests/gnd_tests.rs +++ b/tests/tests/gnd_tests.rs @@ -2,16 +2,13 @@ use anyhow::anyhow; use graph::futures03::StreamExt; use graph_tests::config::set_dev_mode; use graph_tests::contract::Contract; +use graph_tests::integration::{stop_graph_node, TestCase, TestResult}; +use graph_tests::integration_cases::{ + subgraph_data_sources, test_block_handlers, test_multiple_subgraph_datasources, +}; use graph_tests::subgraph::Subgraph; use graph_tests::{error, status, CONFIG}; -mod integration_tests; - -use integration_tests::{ - stop_graph_node, subgraph_data_sources, test_block_handlers, - test_multiple_subgraph_datasources, TestCase, TestResult, -}; - /// The main test entrypoint. #[graph::test] async fn gnd_tests() -> anyhow::Result<()> { diff --git a/tests/tests/integration_tests.rs b/tests/tests/integration_tests.rs index ee1f0f201dc..19a14781088 100644 --- a/tests/tests/integration_tests.rs +++ b/tests/tests/integration_tests.rs @@ -10,11 +10,9 @@ //! indiscriminately will only result in messy code and diminishing returns. use std::collections::HashMap; -use std::future::Future; -use std::pin::Pin; use std::time::{Duration, Instant}; -use anyhow::{anyhow, bail, Context, Result}; +use anyhow::{anyhow, bail, Context}; use graph::components::subgraph::{ ProofOfIndexing, ProofOfIndexingEvent, ProofOfIndexingFinisher, ProofOfIndexingVersion, }; @@ -27,577 +25,19 @@ use graph::prelude::{alloy::primitives::Address, hex, BlockPtr, DeploymentHash}; use graph::schema::InputSchema; use graph::slog::{o, Discard, Logger}; use graph_tests::contract::Contract; +use graph_tests::integration::{ + query_succeeds, stop_graph_node, TestCase, TestContext, TestResult, +}; +use graph_tests::integration_cases::{ + subgraph_data_sources, test_block_handlers, test_int8, test_multiple_subgraph_datasources, + test_timestamp, test_value_roundtrip, +}; use graph_tests::subgraph::Subgraph; use graph_tests::{error, status, CONFIG}; -use tokio::process::Child; -use tokio::task::JoinError; use tokio::time::sleep; const SUBGRAPH_LAST_GRAFTING_BLOCK: i32 = 3; -type TestFn = Box< - dyn FnOnce(TestContext) -> Pin> + Send>> - + Sync - + Send, ->; - -pub struct TestContext { - pub subgraph: Subgraph, -} - -pub enum TestStatus { - Ok, - Err(anyhow::Error), - Panic(JoinError), -} - -pub struct TestResult { - pub name: String, - pub subgraph: Option, - pub status: TestStatus, -} - -impl TestResult { - pub fn success(&self) -> bool { - matches!(self.status, TestStatus::Ok) - } - - fn print_subgraph(&self) { - if let Some(subgraph) = &self.subgraph { - println!(" Subgraph: {}", subgraph.deployment); - } - } - - pub fn print(&self) { - // ANSI escape sequences; see the comment in macros.rs about better colorization - const GREEN: &str = "\x1b[1;32m"; - const RED: &str = "\x1b[1;31m"; - const NC: &str = "\x1b[0m"; - - match &self.status { - TestStatus::Ok => { - println!("* {GREEN}Test {} succeeded{NC}", self.name); - self.print_subgraph(); - } - TestStatus::Err(e) => { - println!("* {RED}Test {} failed{NC}", self.name); - self.print_subgraph(); - println!(" {:?}", e); - } - TestStatus::Panic(e) => { - if e.is_cancelled() { - println!("* {RED}Test {} was cancelled{NC}", self.name) - } else if e.is_panic() { - println!("* {RED}Test {} failed{NC}", self.name); - } else { - println!("* {RED}Test {} exploded mysteriously{NC}", self.name) - } - self.print_subgraph(); - } - } - } -} - -#[derive(Debug, Clone)] -pub enum SourceSubgraph { - Subgraph(String), - WithAlias((String, String)), // (alias, test_name) -} - -impl SourceSubgraph { - fn new(s: &str) -> Self { - if let Some((alias, subgraph)) = s.split_once(':') { - Self::WithAlias((alias.to_string(), subgraph.to_string())) - } else { - Self::Subgraph(s.to_string()) - } - } - - pub fn test_name(&self) -> &str { - match self { - Self::Subgraph(name) => name, - Self::WithAlias((_, name)) => name, - } - } - - #[allow(dead_code)] // Used by gnd_tests.rs - pub fn alias(&self) -> Option<&str> { - match self { - Self::Subgraph(_) => None, - Self::WithAlias((alias, _)) => Some(alias), - } - } -} - -pub struct TestCase { - pub name: String, - pub test: TestFn, - pub source_subgraph: Option>, -} - -impl TestCase { - pub fn new(name: &str, test: fn(TestContext) -> T) -> Self - where - T: Future> + Send + 'static, - { - Self { - name: name.to_string(), - test: Box::new(move |ctx| Box::pin(test(ctx))), - source_subgraph: None, - } - } - - fn new_with_grafting(name: &str, test: fn(TestContext) -> T, base_subgraph: &str) -> Self - where - T: Future> + Send + 'static, - { - let mut test_case = Self::new(name, test); - test_case.source_subgraph = Some(vec![SourceSubgraph::new(base_subgraph)]); - test_case - } - - pub fn new_with_source_subgraphs( - name: &str, - test: fn(TestContext) -> T, - source_subgraphs: Vec<&str>, - ) -> Self - where - T: Future> + Send + 'static, - { - let mut test_case = Self::new(name, test); - test_case.source_subgraph = Some( - source_subgraphs - .into_iter() - .map(SourceSubgraph::new) - .collect(), - ); - test_case - } - - async fn deploy_and_wait( - &self, - subgraph_name: &str, - contracts: &[Contract], - ) -> Result { - status!(&self.name, "Deploying subgraph"); - let subgraph_name = match Subgraph::deploy(subgraph_name, contracts, None).await { - Ok(name) => name, - Err(e) => { - error!(&self.name, "Deploy failed"); - return Err(anyhow!(e.context("Deploy failed"))); - } - }; - - status!(&self.name, "Waiting for subgraph to become ready"); - let subgraph = match Subgraph::wait_ready(&subgraph_name).await { - Ok(subgraph) => subgraph, - Err(e) => { - error!(&self.name, "Subgraph never synced or failed"); - return Err(anyhow!(e.context("Subgraph never synced or failed"))); - } - }; - - if subgraph.healthy { - status!(&self.name, "Subgraph ({}) is synced", subgraph.deployment); - } else { - status!(&self.name, "Subgraph ({}) has failed", subgraph.deployment); - } - - Ok(subgraph) - } - - #[allow(dead_code)] // Used by gnd_tests.rs - pub async fn prepare(&self, contracts: &[Contract]) -> anyhow::Result { - // If a subgraph has subgraph datasources, prepare them first and collect their deployment hashes - let source_mappings = if let Some(_subgraphs) = &self.source_subgraph { - match self.prepare_multiple_sources(contracts).await { - Ok(mappings) => Some(mappings), - Err(e) => { - error!(&self.name, "source subgraph deployment failed: {:?}", e); - return Err(e); - } - } - } else { - None - }; - - status!(&self.name, "Preparing subgraph"); - let (_, subgraph_name, _) = - match Subgraph::prepare(&self.name, contracts, source_mappings.as_deref()).await { - Ok(name) => name, - Err(e) => { - error!(&self.name, "Prepare failed: {:?}", e); - return Err(e); - } - }; - - Ok(subgraph_name) - } - - pub async fn check_health_and_test(self, subgraph_name: String) -> TestResult { - status!( - &self.name, - "Waiting for subgraph ({}) to become ready", - subgraph_name - ); - let subgraph = match Subgraph::wait_ready(&subgraph_name).await { - Ok(subgraph) => subgraph, - Err(e) => { - error!(&self.name, "Subgraph never synced or failed"); - return TestResult { - name: self.name.clone(), - subgraph: None, - status: TestStatus::Err(e.context("Subgraph never synced or failed")), - }; - } - }; - - if subgraph.healthy { - status!(&self.name, "Subgraph ({}) is synced", subgraph.deployment); - } else { - status!(&self.name, "Subgraph ({}) has failed", subgraph.deployment); - } - - let ctx = TestContext { - subgraph: subgraph.clone(), - }; - - status!(&self.name, "Starting test"); - let subgraph2 = subgraph.clone(); - let res = tokio::spawn(async move { (self.test)(ctx).await }).await; - let status = match res { - Ok(Ok(())) => { - status!(&self.name, "Test succeeded"); - TestStatus::Ok - } - Ok(Err(e)) => { - error!(&self.name, "Test failed"); - TestStatus::Err(e) - } - Err(e) => { - error!(&self.name, "Test panicked"); - TestStatus::Panic(e) - } - }; - TestResult { - name: self.name.clone(), - subgraph: Some(subgraph2), - status, - } - } - - pub async fn run(self, contracts: &[Contract]) -> TestResult { - // If a subgraph has subgraph datasources, deploy them first and collect their deployment hashes - let source_mappings = if let Some(_subgraphs) = &self.source_subgraph { - match self.deploy_multiple_sources(contracts).await { - Ok(mappings) => Some(mappings), - Err(e) => { - error!(&self.name, "source subgraph deployment failed"); - return TestResult { - name: self.name.clone(), - subgraph: None, - status: TestStatus::Err(e), - }; - } - } - } else { - None - }; - - status!(&self.name, "Deploying subgraph"); - let subgraph_name = - match Subgraph::deploy(&self.name, contracts, source_mappings.as_deref()).await { - Ok(name) => name, - Err(e) => { - error!(&self.name, "Deploy failed"); - return TestResult { - name: self.name.clone(), - subgraph: None, - status: TestStatus::Err(e.context("Deploy failed")), - }; - } - }; - - self.check_health_and_test(subgraph_name).await - } - - async fn prepare_multiple_sources( - &self, - contracts: &[Contract], - ) -> Result> { - let mut mappings = Vec::new(); - if let Some(sources) = &self.source_subgraph { - for source in sources { - // Source subgraphs don't have their own sources, so pass None - let _ = Subgraph::prepare(source.test_name(), contracts, None).await?; - // If the source has an alias (pre-known IPFS hash), use it for the mapping - if let Some(alias) = source.alias() { - mappings.push((source.test_name().to_string(), alias.to_string())); - } - } - } - Ok(mappings) - } - - async fn deploy_multiple_sources( - &self, - contracts: &[Contract], - ) -> Result> { - let mut mappings = Vec::new(); - if let Some(sources) = &self.source_subgraph { - for source in sources { - let subgraph = self.deploy_and_wait(source.test_name(), contracts).await?; - status!( - source.test_name(), - "Source subgraph deployed with hash {}", - subgraph.deployment - ); - // Use the test_name as the placeholder key - mappings.push((source.test_name().to_string(), subgraph.deployment.clone())); - } - } - Ok(mappings) - } -} - -/// Run the given `query` against the `subgraph` and check that the result -/// has no errors and that the `data` portion of the response matches the -/// `exp` value. -pub async fn query_succeeds( - title: &str, - subgraph: &Subgraph, - query: &str, - exp: Value, -) -> anyhow::Result<()> { - let resp = subgraph.query(query).await?; - match resp.get("errors") { - None => { /* nothing to do */ } - Some(errors) => { - bail!( - "query for `{}` returned GraphQL errors: {:?}", - title, - errors - ); - } - } - match resp.get("data") { - None => { - bail!("query for `{}` returned no data", title); - } - Some(data) => { - if &exp != data { - bail!( - "query for `{title}` returned unexpected data: \nexpected: {exp:?}\n returned: {data:?}", - ); - } - } - } - Ok(()) -} - -/* -* Actual tests. For a new test, add a new function here and add an entry to -* the `cases` variable in `integration_tests`. -*/ - -pub async fn test_int8(ctx: TestContext) -> anyhow::Result<()> { - let subgraph = ctx.subgraph; - assert!(subgraph.healthy); - - let resp = subgraph - .query( - "{ - foos_0: foos(orderBy: id, block: { number: 0 }) { id } - foos(orderBy: id) { id value } - }", - ) - .await?; - - let exp = json!({ - "foos_0": [], - "foos": [ - { - "id": "0", - "value": "9223372036854775807", - }, - ], - }); - assert_eq!(None, resp.get("errors")); - assert_eq!(exp, resp["data"]); - - Ok(()) -} - -/* -* Actual tests. For a new test, add a new function here and add an entry to -* the `cases` variable in `integration_tests`. -*/ - -pub async fn test_timestamp(ctx: TestContext) -> anyhow::Result<()> { - let subgraph = ctx.subgraph; - assert!(subgraph.healthy); - - let resp = subgraph - .query( - "{ - foos_0: foos(orderBy: id, block: { number: 0 }) { id } - foos(orderBy: id) { id value } - }", - ) - .await?; - - let exp = json!({ - "foos_0": [], - "foos": [ - { - "id": "0", - "value": "1710837304040956", - }, - ], - }); - assert_eq!(None, resp.get("errors")); - assert_eq!(exp, resp["data"]); - - Ok(()) -} - -pub async fn test_block_handlers(ctx: TestContext) -> anyhow::Result<()> { - let subgraph = ctx.subgraph; - assert!(subgraph.healthy); - - // test non-filtered blockHandler - let exp = json!({ - "blocks": [ - { "id": "1", "number": "1" }, - { "id": "2", "number": "2" }, - { "id": "3", "number": "3" }, - { "id": "4", "number": "4" }, - { "id": "5", "number": "5" }, - { "id": "6", "number": "6" }, - { "id": "7", "number": "7" }, - { "id": "8", "number": "8" }, - { "id": "9", "number": "9" }, - { "id": "10", "number": "10" }, - ] - }); - query_succeeds( - "test non-filtered blockHandler", - &subgraph, - "{ blocks(orderBy: number, first: 10) { id number } }", - exp, - ) - .await?; - - // test query - let mut values = Vec::new(); - for i in 0..=10 { - values.push(json!({ "id": i.to_string(), "value": i.to_string() })); - } - let exp = json!({ "foos": Value::Array(values) }); - query_succeeds( - "test query", - &subgraph, - "{ foos(orderBy: value, skip: 1) { id value } }", - exp, - ) - .await?; - - // should call intialization handler first - let exp = json!({ - "foo": { "id": "initialize", "value": "-1" }, - }); - query_succeeds( - "should call intialization handler first", - &subgraph, - "{ foo( id: \"initialize\" ) { id value } }", - exp, - ) - .await?; - - // test blockHandler with polling filter - let exp = json!({ - "blockFromPollingHandlers": [ - { "id": "1", "number": "1" }, - { "id": "4", "number": "4" }, - { "id": "7", "number": "7" }, - ] - }); - query_succeeds( - "test blockHandler with polling filter", - &subgraph, - "{ blockFromPollingHandlers(orderBy: number, first: 3) { id number } }", - exp, - ) - .await?; - - // test other blockHandler with polling filter - let exp = json!({ - "blockFromOtherPollingHandlers": [ - { "id": "2", "number": "2" }, - { "id": "4", "number": "4" }, - { "id": "6", "number": "6" }, - ] - }); - query_succeeds( - "test other blockHandler with polling filter", - &subgraph, - "{ blockFromOtherPollingHandlers(orderBy: number, first: 3) { id number } }", - exp, - ) - .await?; - - // test initialization handler - let exp = json!({ - "initializes": [ - { "id": "1", "block": "1" }, - ] - }); - query_succeeds( - "test initialization handler", - &subgraph, - "{ initializes(orderBy: block, first: 10) { id block } }", - exp, - ) - .await?; - - // test subgraphFeatures endpoint returns handlers correctly - let subgraph_features = Subgraph::query_with_vars( - "query GetSubgraphFeatures($deployment: String!) { - subgraphFeatures(subgraphId: $deployment) { - specVersion - apiVersion - features - dataSources - network - handlers - } - }", - json!({ "deployment": subgraph.deployment }), - ) - .await?; - let handlers = &subgraph_features["data"]["subgraphFeatures"]["handlers"]; - assert!( - handlers.is_array(), - "subgraphFeatures.handlers must be an array" - ); - let handlers = handlers.as_array().unwrap(); - for handler in [ - "block_filter_polling", - "block_filter_once", - "block", - "event", - ] { - assert!( - handlers.contains(&Value::String(handler.to_string())), - "handlers {:?} must contain {}", - handlers, - handler - ); - } - - Ok(()) -} - async fn test_eth_api(ctx: TestContext) -> anyhow::Result<()> { let subgraph = ctx.subgraph; assert!(subgraph.healthy); @@ -622,46 +62,6 @@ async fn test_eth_api(ctx: TestContext) -> anyhow::Result<()> { Ok(()) } -pub async fn subgraph_data_sources(ctx: TestContext) -> anyhow::Result<()> { - let subgraph = ctx.subgraph; - assert!(subgraph.healthy); - let expected_response = json!({ - "mirrorBlocks": [ - { "id": "1-v1", "number": "1", "testMessage": null }, - { "id": "1-v2", "number": "1", "testMessage": null }, - { "id": "1-v3", "number": "1", "testMessage": "1-message" }, - { "id": "2-v1", "number": "2", "testMessage": null }, - { "id": "2-v2", "number": "2", "testMessage": null }, - { "id": "2-v3", "number": "2", "testMessage": "2-message" }, - { "id": "3-v1", "number": "3", "testMessage": null }, - { "id": "3-v2", "number": "3", "testMessage": null }, - { "id": "3-v3", "number": "3", "testMessage": "3-message" }, - ] - }); - - query_succeeds( - "Query all blocks with testMessage", - &subgraph, - "{ mirrorBlocks(where: {number_lte: 3}, orderBy: number) { id, number, testMessage } }", - expected_response, - ) - .await?; - - let expected_response = json!({ - "mirrorBlock": { "id": "1-v3", "number": "1", "testMessage": "1-message" }, - }); - - query_succeeds( - "Query specific block with testMessage", - &subgraph, - "{ mirrorBlock(id: \"1-v3\") { id, number, testMessage } }", - expected_response, - ) - .await?; - - Ok(()) -} - async fn test_topic_filters(ctx: TestContext) -> anyhow::Result<()> { let subgraph = ctx.subgraph; assert!(subgraph.healthy); @@ -805,25 +205,6 @@ async fn test_overloaded_functions(ctx: TestContext) -> anyhow::Result<()> { Ok(()) } -pub async fn test_value_roundtrip(ctx: TestContext) -> anyhow::Result<()> { - let subgraph = ctx.subgraph; - assert!(subgraph.healthy); - - let exp = json!({ - "foos": [{ "id": "0", "value": "bla" }], - "foos_0": [] - }); - - let query = "{ - foos_0: foos(orderBy: id, block: { number: 0 }) { id } - foos(orderBy: id) { id value } - }"; - - query_succeeds("test query", &subgraph, query, exp).await?; - - Ok(()) -} - async fn test_remove_then_update(ctx: TestContext) -> anyhow::Result<()> { let subgraph = ctx.subgraph; assert!(subgraph.healthy); @@ -1144,33 +525,6 @@ async fn test_missing(_sg: Subgraph) -> anyhow::Result<()> { Err(anyhow!("This test is missing")) } -pub async fn test_multiple_subgraph_datasources(ctx: TestContext) -> anyhow::Result<()> { - let subgraph = ctx.subgraph; - assert!(subgraph.healthy); - - // Test querying data aggregated from multiple sources - let exp = json!({ - "aggregatedDatas": [ - { - "id": "0", - "sourceA": "from source A", - "sourceB": "from source B", - "first": "sourceA" - }, - ] - }); - - query_succeeds( - "should aggregate data from multiple sources", - &subgraph, - "{ aggregatedDatas(first: 1) { id sourceA sourceB first } }", - exp, - ) - .await?; - - Ok(()) -} - /// Test the declared calls functionality as of spec version 1.2.0. /// Note that we don't have a way to test that the actual call is made as /// a declared call since graph-node does not expose that information @@ -1496,9 +850,3 @@ async fn integration_tests() -> anyhow::Result<()> { Ok(()) } } - -pub async fn stop_graph_node(child: &mut Child) -> anyhow::Result<()> { - child.kill().await.context("Failed to kill graph-node")?; - - Ok(()) -} From d77db10926813f392e79e9039547b50ae99bbcb7 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Sat, 14 Feb 2026 13:12:32 -0800 Subject: [PATCH 16/35] tests: Prepend `dev` subcommand when spawning gnd as graph-node When dev_mode is enabled, GraphNodeConfig uses the gnd binary, but spawn_graph_node_with_args was invoking it without the required `dev` subcommand. This was previously masked by the race condition where CONFIG was initialized by the integration_tests entry point (with dev_mode=false) before gnd_tests could set it. --- tests/src/config.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/src/config.rs b/tests/src/config.rs index fc84c243912..042a63b2e81 100644 --- a/tests/src/config.rs +++ b/tests/src/config.rs @@ -89,6 +89,8 @@ impl Default for GraphNodePorts { #[derive(Clone, Debug)] pub struct GraphNodeConfig { bin: PathBuf, + /// Optional subcommand to prepend before all args (e.g. `"dev"` for `gnd dev`). + subcommand: Option<&'static str>, pub ports: GraphNodePorts, pub ipfs_uri: String, pub log_file: TestFile, @@ -142,6 +144,7 @@ impl GraphNodeConfig { Self { bin, + subcommand: Some("dev"), ports: GraphNodePorts::default(), ipfs_uri: std::env::var("GRAPH_NODE_TEST_IPFS_URL") .unwrap_or_else(|_| "http://127.0.0.1:3001".to_string()), @@ -157,6 +160,7 @@ impl Default for GraphNodeConfig { Self { bin, + subcommand: None, ports: GraphNodePorts::default(), ipfs_uri: std::env::var("GRAPH_NODE_TEST_IPFS_URL") .unwrap_or_else(|_| "http://127.0.0.1:3001".to_string()), @@ -203,11 +207,14 @@ impl Config { &ports.metrics.to_string(), ]; - let args = args + let args: Vec<&str> = self + .graph_node + .subcommand .iter() + .chain(args.iter()) .chain(additional_args.iter()) .cloned() - .collect::>(); + .collect(); let stdout = self.graph_node.log_file.create(); let stderr = stdout.try_clone()?; status!( From bfd5445332e6f45abe2a28f86090c722f5e2ff73 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Sat, 14 Feb 2026 14:00:04 -0800 Subject: [PATCH 17/35] tests: Build subgraphs with gnd codegen/build for gnd_tests The gnd_tests were using hardcoded IPFS hashes for source subgraphs, which meant they could never actually work in dev mode -- the hashes would be stale or missing from IPFS. Replace hardcoded hashes with a dynamic build flow: build source subgraphs first (patch -> codegen -> build --ipfs) to get fresh hashes, then codegen and build main subgraphs against those hashes before starting gnd dev. --- tests/src/helpers.rs | 7 +++++- tests/src/subgraph.rs | 43 ++++++++++++++++++++++++++++++++- tests/tests/gnd_tests.rs | 52 ++++++++++++++++++++++++++++++---------- 3 files changed, 88 insertions(+), 14 deletions(-) diff --git a/tests/src/helpers.rs b/tests/src/helpers.rs index fb06a583923..3ab126892bb 100644 --- a/tests/src/helpers.rs +++ b/tests/src/helpers.rs @@ -102,6 +102,11 @@ pub fn run_cmd(command: &mut Command) -> String { /// Run a command, check that it succeeded and return its stdout and stderr /// in a friendly error format for display pub async fn run_checked(cmd: &mut tokio::process::Command) -> anyhow::Result<()> { + run_checked_with_output(cmd).await.map(|_| ()) +} + +/// Like `run_checked` but returns stdout on success. +pub async fn run_checked_with_output(cmd: &mut tokio::process::Command) -> anyhow::Result { let std_cmd = cmd.as_std(); let cmdline = format!( "{} {}", @@ -117,7 +122,7 @@ pub async fn run_checked(cmd: &mut tokio::process::Command) -> anyhow::Result<() .with_context(|| format!("Command failed: {cmdline}"))?; if output.status.success() { - Ok(()) + Ok(String::from_utf8_lossy(&output.stdout).into_owned()) } else { let stdout = String::from_utf8_lossy(&output.stdout); let stderr = String::from_utf8_lossy(&output.stderr); diff --git a/tests/src/subgraph.rs b/tests/src/subgraph.rs index 67513f49ca0..cd2a1690284 100644 --- a/tests/src/subgraph.rs +++ b/tests/src/subgraph.rs @@ -13,7 +13,9 @@ use tokio::{process::Command, time::sleep}; use crate::{ contract::Contract, - helpers::{graphql_query, graphql_query_with_vars, run_checked, TestFile}, + helpers::{ + graphql_query, graphql_query_with_vars, run_checked, run_checked_with_output, TestFile, + }, CONFIG, }; @@ -66,6 +68,45 @@ impl Subgraph { Ok(()) } + /// Run `gnd codegen subgraph.yaml.patched` in the subgraph directory. + /// If `has_subgraph_datasource`, passes `--ipfs` so codegen can fetch source schemas. + pub async fn codegen_dev(dir: &TestFile, has_subgraph_datasource: bool) -> anyhow::Result<()> { + let mut prog = Command::new(&CONFIG.graph_cli); + let mut cmd = prog.arg("codegen").arg("subgraph.yaml.patched"); + if has_subgraph_datasource { + cmd = cmd.arg(format!("--ipfs={}", CONFIG.graph_node.ipfs_uri)); + } + cmd = cmd.current_dir(&dir.path); + run_checked(cmd).await + } + + /// Run `gnd build subgraph.yaml.patched` in the subgraph directory. + /// If `upload_to_ipfs`, passes `--ipfs` and parses the returned IPFS hash from stdout. + /// Returns the IPFS hash if `upload_to_ipfs` is true. + pub async fn build_dev(dir: &TestFile, upload_to_ipfs: bool) -> anyhow::Result> { + let mut prog = Command::new(&CONFIG.graph_cli); + let mut cmd = prog.arg("build").arg("subgraph.yaml.patched"); + if upload_to_ipfs { + cmd = cmd.arg(format!("--ipfs={}", CONFIG.graph_node.ipfs_uri)); + } + cmd = cmd.current_dir(&dir.path); + + let stdout = run_checked_with_output(cmd).await?; + + if upload_to_ipfs { + // gnd build --ipfs prints "✔ Build completed: " to stdout + let hash = stdout + .lines() + .find_map(|line| line.split("Build completed: ").nth(1)) + .ok_or_else(|| anyhow!("Could not find IPFS hash in gnd build output"))? + .trim() + .to_string(); + Ok(Some(hash)) + } else { + Ok(None) + } + } + /// Prepare the subgraph for deployment by patching contracts and checking for subgraph datasources pub async fn prepare( name: &str, diff --git a/tests/tests/gnd_tests.rs b/tests/tests/gnd_tests.rs index 47d13143267..e20539f3ff2 100644 --- a/tests/tests/gnd_tests.rs +++ b/tests/tests/gnd_tests.rs @@ -1,3 +1,5 @@ +use std::collections::HashMap; + use anyhow::anyhow; use graph::futures03::StreamExt; use graph_tests::config::set_dev_mode; @@ -16,24 +18,49 @@ async fn gnd_tests() -> anyhow::Result<()> { let test_name_to_run = std::env::var("TEST_CASE").ok(); + // 1. Deploy contracts and reset database + let contracts = Contract::deploy_all().await?; + + status!("setup", "Resetting database"); + CONFIG.reset_database(); + + // 2. Build source subgraphs: patch -> codegen -> build+upload -> capture hash + status!("setup", "Building source subgraphs"); + let source_names = ["source-subgraph", "source-subgraph-a", "source-subgraph-b"]; + let mut source_hashes: HashMap = HashMap::new(); + for source_name in &source_names { + let dir = Subgraph::dir(source_name); + Subgraph::patch(&dir, &contracts).await?; + Subgraph::codegen_dev(&dir, false).await?; + let hash = Subgraph::build_dev(&dir, true) + .await? + .expect("build --ipfs must return hash"); + status!("setup", "Built {source_name} -> {hash}"); + source_hashes.insert(source_name.to_string(), hash); + } + + // 3. Create test cases with dynamic hashes from the build step let cases = vec![ TestCase::new("block-handlers", test_block_handlers), TestCase::new_with_source_subgraphs( "subgraph-data-sources", subgraph_data_sources, - vec!["QmWi3H11QFE2PiWx6WcQkZYZdA5UasaBptUJqGn54MFux5:source-subgraph"], + vec![&format!( + "{}:source-subgraph", + source_hashes["source-subgraph"] + )], ), TestCase::new_with_source_subgraphs( "multiple-subgraph-datasources", test_multiple_subgraph_datasources, vec![ - "QmYHp1bPEf7EoYBpEtJUpZv1uQHYQfWE4AhvR6frjB1Huj:source-subgraph-a", - "QmYBEzastJi7bsa722ac78tnZa6xNnV9vvweerY4kVyJtq:source-subgraph-b", + &format!("{}:source-subgraph-a", source_hashes["source-subgraph-a"]), + &format!("{}:source-subgraph-b", source_hashes["source-subgraph-b"]), ], ), ]; - // Filter the test cases if a specific test name is provided + // 4. Filter test cases let cases_to_run: Vec<_> = if let Some(test_name) = test_name_to_run { cases .into_iter() @@ -43,16 +70,17 @@ async fn gnd_tests() -> anyhow::Result<()> { cases }; - let contracts = Contract::deploy_all().await?; - - status!("setup", "Resetting database"); - CONFIG.reset_database(); - - for i in cases_to_run.iter() { - i.prepare(&contracts).await?; + // 5. Prepare and build each main subgraph + for case in &cases_to_run { + case.prepare(&contracts).await?; + let dir = Subgraph::dir(&case.name); + let has_subgraph_ds = case.source_subgraph.is_some(); + Subgraph::codegen_dev(&dir, has_subgraph_ds).await?; + Subgraph::build_dev(&dir, false).await?; } status!("setup", "Prepared all cases"); + // 6. Collect manifests and source aliases for gnd dev let manifests = cases_to_run .iter() .map(|case| { @@ -90,7 +118,7 @@ async fn gnd_tests() -> anyhow::Result<()> { vec!["--manifests", &manifests, "--sources", &aliases_str] }; - // Spawn graph-node. + // 7. Start gnd dev status!("graph-node", "Starting graph-node"); let mut graph_node_child_command = CONFIG.spawn_graph_node_with_args(&args).await?; From 2f34c41badfa08f98872d196998d27bef7dce2d8 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Sat, 14 Feb 2026 14:16:50 -0800 Subject: [PATCH 18/35] runtime: Replace futures01 mpsc with tokio mpsc for mapping channel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the legacy futures 0.1 bounded channel (`futures01::sync::mpsc`) with `tokio::sync::mpsc` for the mapping request channel. This eliminates the futures01→futures03 compat bridge overhead on the send path (no more `.clone().send(...).compat().await`) and simplifies the receive loop (using `blocking_recv()` instead of `.for_each().wait()`). The dedicated mapping thread architecture is preserved — this is a mechanical channel replacement only. --- core/src/subgraph/context/instance/mod.rs | 3 +- graph/src/components/subgraph/host.rs | 5 +- runtime/wasm/src/host.rs | 13 +-- runtime/wasm/src/mapping.rs | 105 ++++++++++------------ 4 files changed, 54 insertions(+), 72 deletions(-) diff --git a/core/src/subgraph/context/instance/mod.rs b/core/src/subgraph/context/instance/mod.rs index e9c311420e6..d4e8fd4421f 100644 --- a/core/src/subgraph/context/instance/mod.rs +++ b/core/src/subgraph/context/instance/mod.rs @@ -1,7 +1,6 @@ mod hosts; use anyhow::ensure; -use graph::futures01::sync::mpsc::Sender; use graph::{ blockchain::{Blockchain, TriggerData as _}, data_source::{ @@ -36,7 +35,7 @@ pub(crate) struct SubgraphInstance> { offchain_hosts: OffchainHosts, /// Maps the hash of a module to a channel to the thread in which the module is instantiated. - module_cache: HashMap<[u8; 32], Sender>, + module_cache: HashMap<[u8; 32], tokio::sync::mpsc::Sender>, /// This manages the sequence of causality regions for the subgraph. causality_region_seq: CausalityRegionSeq, diff --git a/graph/src/components/subgraph/host.rs b/graph/src/components/subgraph/host.rs index 40bdc10f8eb..8642a536843 100644 --- a/graph/src/components/subgraph/host.rs +++ b/graph/src/components/subgraph/host.rs @@ -4,7 +4,6 @@ use std::time::Instant; use anyhow::Error; use async_trait::async_trait; -use futures01::sync::mpsc; use crate::components::metrics::gas::GasMetrics; use crate::components::store::SubgraphFork; @@ -209,7 +208,7 @@ pub trait RuntimeHostBuilder: Clone + Send + Sync + 'static { subgraph_id: DeploymentHash, data_source: DataSource, top_level_templates: Arc>>, - mapping_request_sender: mpsc::Sender, + mapping_request_sender: tokio::sync::mpsc::Sender, metrics: Arc, ) -> Result; @@ -220,5 +219,5 @@ pub trait RuntimeHostBuilder: Clone + Send + Sync + 'static { logger: Logger, subgraph_id: DeploymentHash, metrics: Arc, - ) -> Result, anyhow::Error>; + ) -> Result, anyhow::Error>; } diff --git a/runtime/wasm/src/host.rs b/runtime/wasm/src/host.rs index 77b03594a5e..20454f156fd 100644 --- a/runtime/wasm/src/host.rs +++ b/runtime/wasm/src/host.rs @@ -2,7 +2,6 @@ use std::cmp::PartialEq; use std::time::Instant; use async_trait::async_trait; -use graph::futures01::sync::mpsc::Sender; use graph::futures03::channel::oneshot::channel; use graph::blockchain::{Blockchain, HostFn, RuntimeAdapter}; @@ -11,8 +10,6 @@ use graph::components::subgraph::{MappingError, SharedProofOfIndexing}; use graph::data_source::{ DataSource, DataSourceTemplate, MappingTrigger, TriggerData, TriggerWithHandler, }; -use graph::futures01::Sink as _; -use graph::futures03::compat::Future01CompatExt; use graph::prelude::{ RuntimeHost as RuntimeHostTrait, RuntimeHostBuilder as RuntimeHostBuilderTrait, *, }; @@ -66,7 +63,7 @@ where logger: Logger, subgraph_id: DeploymentHash, metrics: Arc, - ) -> Result, Error> { + ) -> Result, Error> { let experimental_features = ExperimentalFeatures { allow_non_deterministic_ipfs: ENV_VARS.mappings.allow_non_deterministic_ipfs, }; @@ -87,7 +84,7 @@ where subgraph_id: DeploymentHash, data_source: DataSource, templates: Arc>>, - mapping_request_sender: Sender>, + mapping_request_sender: tokio::sync::mpsc::Sender>, metrics: Arc, ) -> Result { RuntimeHost::new( @@ -107,7 +104,7 @@ where pub struct RuntimeHost { host_fns: Arc>, data_source: DataSource, - mapping_request_sender: Sender>, + mapping_request_sender: tokio::sync::mpsc::Sender>, host_exports: Arc, metrics: Arc, } @@ -123,7 +120,7 @@ where subgraph_id: DeploymentHash, data_source: DataSource, templates: Arc>>, - mapping_request_sender: Sender>, + mapping_request_sender: tokio::sync::mpsc::Sender>, metrics: Arc, ens_lookup: Arc, ) -> Result { @@ -179,7 +176,6 @@ where let metrics = self.metrics.clone(); self.mapping_request_sender - .clone() .send(WasmRequest::new_trigger( MappingContext { logger: logger.cheap_clone(), @@ -196,7 +192,6 @@ where trigger, result_sender, )) - .compat() .await .context("Mapping terminated before passing in trigger")?; diff --git a/runtime/wasm/src/mapping.rs b/runtime/wasm/src/mapping.rs index b99cc157e0b..444c6b2c67a 100644 --- a/runtime/wasm/src/mapping.rs +++ b/runtime/wasm/src/mapping.rs @@ -4,8 +4,6 @@ use graph::blockchain::{BlockTime, Blockchain, HostFn}; use graph::components::store::SubgraphFork; use graph::components::subgraph::{MappingError, SharedProofOfIndexing}; use graph::data_source::{MappingTrigger, TriggerWithHandler}; -use graph::futures01::sync::mpsc; -use graph::futures01::{Future as _, Stream as _}; use graph::futures03::channel::oneshot::Sender; use graph::parking_lot::RwLock; use graph::prelude::*; @@ -27,7 +25,7 @@ pub fn spawn_module( runtime: tokio::runtime::Handle, timeout: Option, experimental_features: ExperimentalFeatures, -) -> Result>, anyhow::Error> +) -> Result>, anyhow::Error> where ::MappingTrigger: ToAscPtr, { @@ -36,7 +34,7 @@ where let valid_module = Arc::new(ValidModule::new(&logger, raw_module, timeout)?); // Create channel for event handling requests - let (mapping_request_sender, mapping_request_receiver) = mpsc::channel(100); + let (mapping_request_sender, mut mapping_request_receiver) = tokio::sync::mpsc::channel(100); // It used to be that we had to create a dedicated thread since wasmtime // instances were not `Send` and could therefore not be scheduled by the @@ -59,62 +57,53 @@ where // Pass incoming triggers to the WASM module and return entity changes; // Stop when canceled because all RuntimeHosts and their senders were dropped. - match mapping_request_receiver - .map_err(|()| unreachable!()) - .for_each(move |request| { - let WasmRequest { + while let Some(request) = mapping_request_receiver.blocking_recv() { + let WasmRequest { + ctx, + inner, + result_sender, + } = request; + let logger = ctx.logger.clone(); + + let handle_fut = async { + let result = instantiate_module::( + valid_module.cheap_clone(), ctx, - inner, - result_sender, - } = request; - let logger = ctx.logger.clone(); - - let handle_fut = async { - let result = instantiate_module::( - valid_module.cheap_clone(), - ctx, - host_metrics.cheap_clone(), - experimental_features, - ) - .await; - match result { - Ok(module) => match inner { - WasmRequestInner::TriggerRequest(trigger) => { - handle_trigger(&logger, module, trigger, host_metrics.cheap_clone()) - .await - } - }, - Err(e) => Err(MappingError::Unknown(e)), - } - }; - let result = panic::catch_unwind(AssertUnwindSafe(|| graph::block_on(handle_fut))); - - let result = match result { - Ok(result) => result, - Err(panic_info) => { - let err_msg = if let Some(payload) = panic_info - .downcast_ref::() - .map(String::as_str) - .or(panic_info.downcast_ref::<&str>().copied()) - { - anyhow!("Subgraph panicked with message: {}", payload) - } else { - anyhow!("Subgraph panicked with an unknown payload.") - }; - Err(MappingError::Unknown(err_msg)) - } - }; - - result_sender - .send(result) - .map_err(|_| anyhow::anyhow!("WASM module result receiver dropped.")) - }) - .wait() - { - Ok(()) => debug!(logger, "Subgraph stopped, WASM runtime thread terminated"), - Err(e) => debug!(logger, "WASM runtime thread terminated abnormally"; - "error" => e.to_string()), + host_metrics.cheap_clone(), + experimental_features, + ) + .await; + match result { + Ok(module) => match inner { + WasmRequestInner::TriggerRequest(trigger) => { + handle_trigger(&logger, module, trigger, host_metrics.cheap_clone()) + .await + } + }, + Err(e) => Err(MappingError::Unknown(e)), + } + }; + let result = panic::catch_unwind(AssertUnwindSafe(|| graph::block_on(handle_fut))); + + let result = match result { + Ok(result) => result, + Err(panic_info) => { + let err_msg = if let Some(payload) = panic_info + .downcast_ref::() + .map(String::as_str) + .or(panic_info.downcast_ref::<&str>().copied()) + { + anyhow!("Subgraph panicked with message: {}", payload) + } else { + anyhow!("Subgraph panicked with an unknown payload.") + }; + Err(MappingError::Unknown(err_msg)) + } + }; + + let _ = result_sender.send(result); } + debug!(logger, "Subgraph stopped, WASM runtime thread terminated"); }) .map(|_| ()) .context("Spawning WASM runtime thread failed")?; From 9b91e16242ef750c13c7a3cbb3f72e14e0a29089 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Sat, 14 Feb 2026 14:24:44 -0800 Subject: [PATCH 19/35] runtime: Use tokio::task::spawn_blocking for mapping thread Replace the manually spawned OS thread (std::thread::spawn) with tokio::task::spawn_blocking. This places the mapping loop on tokio's blocking thread pool instead of creating a dedicated OS thread per WASM module. The blocking pool grows on demand and integrates with the tokio runtime, eliminating the manual runtime.enter() guard and per-module thread counter. This is a stepping stone toward eliminating the dedicated mapping thread entirely (Phase 3C). --- runtime/wasm/src/host.rs | 1 - runtime/wasm/src/mapping.rs | 45 ++++++++++++++----------------------- 2 files changed, 17 insertions(+), 29 deletions(-) diff --git a/runtime/wasm/src/host.rs b/runtime/wasm/src/host.rs index 20454f156fd..6890e500d6b 100644 --- a/runtime/wasm/src/host.rs +++ b/runtime/wasm/src/host.rs @@ -72,7 +72,6 @@ where logger, subgraph_id, metrics, - tokio::runtime::Handle::current(), ENV_VARS.mappings.timeout, experimental_features, ) diff --git a/runtime/wasm/src/mapping.rs b/runtime/wasm/src/mapping.rs index 444c6b2c67a..faacc9831a0 100644 --- a/runtime/wasm/src/mapping.rs +++ b/runtime/wasm/src/mapping.rs @@ -11,50 +11,40 @@ use graph::runtime::gas::Gas; use graph::runtime::IndexForAscTypeId; use parity_wasm::elements::ExportEntry; use std::collections::{BTreeMap, HashMap}; -use std::panic::AssertUnwindSafe; -use std::sync::atomic::{AtomicUsize, Ordering}; +use std::panic; use std::sync::{Arc, OnceLock}; -use std::{panic, thread}; -/// Spawn a wasm module in its own thread. +/// Spawn a wasm module on tokio's blocking thread pool. +/// +/// Uses `tokio::task::spawn_blocking` so the long-running mapping loop +/// is managed by the tokio runtime instead of a manually spawned OS +/// thread. This is a stepping stone toward eliminating the dedicated +/// thread entirely (Phase 3C). pub fn spawn_module( raw_module: &[u8], logger: Logger, subgraph_id: DeploymentHash, host_metrics: Arc, - runtime: tokio::runtime::Handle, timeout: Option, experimental_features: ExperimentalFeatures, ) -> Result>, anyhow::Error> where ::MappingTrigger: ToAscPtr, { - static THREAD_COUNT: AtomicUsize = AtomicUsize::new(0); - let valid_module = Arc::new(ValidModule::new(&logger, raw_module, timeout)?); // Create channel for event handling requests let (mapping_request_sender, mut mapping_request_receiver) = tokio::sync::mpsc::channel(100); - // It used to be that we had to create a dedicated thread since wasmtime - // instances were not `Send` and could therefore not be scheduled by the - // regular tokio executor. This isn't an issue anymore, but we still - // spawn a dedicated thread since running WASM code async can block and - // lock up the executor. See [the wasmtime - // docs](https://docs.rs/wasmtime/latest/wasmtime/struct.Config.html#execution-in-poll) - // on how this should be handled properly. As that is a fairly large - // change to how we use wasmtime, we keep the threading model for now. - // Once we are confident that things are working that way, we should - // revisit this and remove the dedicated thread. + // We use spawn_blocking for the mapping loop because WASM execution + // is CPU-bound and can block for extended periods. spawn_blocking + // places the task on tokio's blocking thread pool which is designed + // for this: it grows on demand and won't starve the async executor. // - // In case of failure, this thread may panic or simply terminate, + // In case of failure, the task may panic or simply terminate, // dropping the `mapping_request_receiver` which ultimately causes the // subgraph to fail the next time it tries to handle an event. - let next_id = THREAD_COUNT.fetch_add(1, Ordering::SeqCst); - let conf = thread::Builder::new().name(format!("mapping-{}-{:0>4}", &subgraph_id, next_id)); - conf.spawn(move || { - let _runtime_guard = runtime.enter(); - + tokio::task::spawn_blocking(move || { // Pass incoming triggers to the WASM module and return entity changes; // Stop when canceled because all RuntimeHosts and their senders were dropped. while let Some(request) = mapping_request_receiver.blocking_recv() { @@ -83,7 +73,8 @@ where Err(e) => Err(MappingError::Unknown(e)), } }; - let result = panic::catch_unwind(AssertUnwindSafe(|| graph::block_on(handle_fut))); + let result = + panic::catch_unwind(panic::AssertUnwindSafe(|| graph::block_on(handle_fut))); let result = match result { Ok(result) => result, @@ -103,10 +94,8 @@ where let _ = result_sender.send(result); } - debug!(logger, "Subgraph stopped, WASM runtime thread terminated"); - }) - .map(|_| ()) - .context("Spawning WASM runtime thread failed")?; + debug!(logger, "Subgraph {}: mapping task terminated", subgraph_id); + }); Ok(mapping_request_sender) } From b3b0e9dce215f93ca9a68fec0dd3a3b105645d0c Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Sat, 14 Feb 2026 14:38:45 -0800 Subject: [PATCH 20/35] runtime: Eliminate dedicated mapping thread MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the per-module dedicated thread + channel architecture with direct async WASM execution using tokio::task::block_in_place. Previously, each trigger was sent via an mpsc channel to a dedicated blocking thread, executed there, and the result sent back via a oneshot channel — two cross-thread hops per trigger. Now, WASM instantiation and handler execution happen directly in the async context with block_in_place telling tokio to move other work off the thread. Key changes: - RuntimeHostBuilder trait: replace spawn_mapping/Req with compile_module/Module for direct module compilation and caching - RuntimeHost stores Arc instead of channel Sender - Remove spawn_module, WasmRequest, WasmRequestInner, and the blocking receive loop from mapping.rs - SubgraphInstance::module_cache stores Arc instead of Sender - Retain catch_unwind for panic safety around WASM execution --- core/src/subgraph/context/instance/mod.rs | 19 +-- graph/src/components/subgraph/host.rs | 15 +- runtime/wasm/src/host.rs | 141 ++++++++++++------- runtime/wasm/src/lib.rs | 5 +- runtime/wasm/src/mapping.rs | 159 +--------------------- 5 files changed, 115 insertions(+), 224 deletions(-) diff --git a/core/src/subgraph/context/instance/mod.rs b/core/src/subgraph/context/instance/mod.rs index d4e8fd4421f..a6220f105c1 100644 --- a/core/src/subgraph/context/instance/mod.rs +++ b/core/src/subgraph/context/instance/mod.rs @@ -34,8 +34,9 @@ pub(crate) struct SubgraphInstance> { offchain_hosts: OffchainHosts, - /// Maps the hash of a module to a channel to the thread in which the module is instantiated. - module_cache: HashMap<[u8; 32], tokio::sync::mpsc::Sender>, + /// Maps the hash of a module to its compiled and validated representation, + /// shared among hosts that use the same WASM file. + module_cache: HashMap<[u8; 32], Arc>, /// This manages the sequence of causality regions for the subgraph. causality_region_seq: CausalityRegionSeq, @@ -102,19 +103,19 @@ where Some(ref module_bytes) => module_bytes.cheap_clone(), }; - let mapping_request_sender = { + let valid_module = { let module_hash = tiny_keccak::keccak256(module_bytes.as_ref()); - if let Some(sender) = self.module_cache.get(&module_hash) { - sender.clone() + if let Some(module) = self.module_cache.get(&module_hash) { + module.cheap_clone() } else { - let sender = T::spawn_mapping( + let module = T::compile_module( module_bytes.as_ref(), logger, self.subgraph_id.clone(), self.host_metrics.cheap_clone(), )?; - self.module_cache.insert(module_hash, sender.clone()); - sender + self.module_cache.insert(module_hash, module.cheap_clone()); + module } }; @@ -123,7 +124,7 @@ where self.subgraph_id.clone(), data_source, self.templates.cheap_clone(), - mapping_request_sender, + valid_module, self.host_metrics.cheap_clone(), )?; Ok(Some(Arc::new(host))) diff --git a/graph/src/components/subgraph/host.rs b/graph/src/components/subgraph/host.rs index 8642a536843..4175909632d 100644 --- a/graph/src/components/subgraph/host.rs +++ b/graph/src/components/subgraph/host.rs @@ -199,7 +199,10 @@ impl Drop for HostFnExecutionTimer { pub trait RuntimeHostBuilder: Clone + Send + Sync + 'static { type Host: RuntimeHost + PartialEq; - type Req: 'static + Send; + + /// The compiled module type, cached and shared among hosts that use the + /// same WASM file. + type Module: Send + Sync + 'static; /// Build a new runtime host for a subgraph data source. fn build( @@ -208,16 +211,16 @@ pub trait RuntimeHostBuilder: Clone + Send + Sync + 'static { subgraph_id: DeploymentHash, data_source: DataSource, top_level_templates: Arc>>, - mapping_request_sender: tokio::sync::mpsc::Sender, + valid_module: Arc, metrics: Arc, ) -> Result; - /// Spawn a mapping and return a channel for mapping requests. The sender should be able to be - /// cached and shared among mappings that use the same wasm file. - fn spawn_mapping( + /// Compile and validate a WASM module. The result should be cached and + /// shared among hosts that use the same WASM file. + fn compile_module( raw_module: &[u8], logger: Logger, subgraph_id: DeploymentHash, metrics: Arc, - ) -> Result, anyhow::Error>; + ) -> Result, anyhow::Error>; } diff --git a/runtime/wasm/src/host.rs b/runtime/wasm/src/host.rs index 6890e500d6b..8a8c93707cf 100644 --- a/runtime/wasm/src/host.rs +++ b/runtime/wasm/src/host.rs @@ -1,8 +1,9 @@ use std::cmp::PartialEq; +use std::panic; use std::time::Instant; +use anyhow::Context; use async_trait::async_trait; -use graph::futures03::channel::oneshot::channel; use graph::blockchain::{Blockchain, HostFn, RuntimeAdapter}; use graph::components::store::{EnsLookup, SubgraphFork}; @@ -14,7 +15,7 @@ use graph::prelude::{ RuntimeHost as RuntimeHostTrait, RuntimeHostBuilder as RuntimeHostBuilderTrait, *, }; -use crate::mapping::{MappingContext, WasmRequest}; +use crate::mapping::{MappingContext, ValidModule}; use crate::module::ToAscPtr; use crate::{host_exports::HostExports, module::ExperimentalFeatures}; use graph::runtime::gas::Gas; @@ -56,25 +57,20 @@ where ::MappingTrigger: ToAscPtr, { type Host = RuntimeHost; - type Req = WasmRequest; + type Module = ValidModule; - fn spawn_mapping( + fn compile_module( raw_module: &[u8], logger: Logger, - subgraph_id: DeploymentHash, - metrics: Arc, - ) -> Result, Error> { - let experimental_features = ExperimentalFeatures { - allow_non_deterministic_ipfs: ENV_VARS.mappings.allow_non_deterministic_ipfs, - }; - crate::mapping::spawn_module( + _subgraph_id: DeploymentHash, + _metrics: Arc, + ) -> Result, Error> { + let valid_module = Arc::new(ValidModule::new( + &logger, raw_module, - logger, - subgraph_id, - metrics, ENV_VARS.mappings.timeout, - experimental_features, - ) + )?); + Ok(valid_module) } fn build( @@ -83,7 +79,7 @@ where subgraph_id: DeploymentHash, data_source: DataSource, templates: Arc>>, - mapping_request_sender: tokio::sync::mpsc::Sender>, + valid_module: Arc, metrics: Arc, ) -> Result { RuntimeHost::new( @@ -93,7 +89,7 @@ where subgraph_id, data_source, templates, - mapping_request_sender, + valid_module, metrics, self.ens_lookup.cheap_clone(), ) @@ -103,7 +99,7 @@ where pub struct RuntimeHost { host_fns: Arc>, data_source: DataSource, - mapping_request_sender: tokio::sync::mpsc::Sender>, + valid_module: Arc, host_exports: Arc, metrics: Arc, } @@ -111,6 +107,7 @@ pub struct RuntimeHost { impl RuntimeHost where C: Blockchain, + ::MappingTrigger: ToAscPtr, { fn new( runtime_adapter: Arc>, @@ -119,7 +116,7 @@ where subgraph_id: DeploymentHash, data_source: DataSource, templates: Arc>>, - mapping_request_sender: tokio::sync::mpsc::Sender>, + valid_module: Arc, metrics: Arc, ens_lookup: Arc, ) -> Result { @@ -143,15 +140,16 @@ where Ok(RuntimeHost { host_fns: Arc::new(host_fns), data_source, - mapping_request_sender, + valid_module, host_exports, metrics, }) } - /// Sends a MappingRequest to the thread which owns the host, - /// and awaits the result. - async fn send_mapping_request( + /// Instantiate the WASM module and run the trigger handler directly + /// in the current async context, using `block_in_place` to avoid + /// blocking the tokio executor. + async fn run_mapping( &self, logger: &Logger, state: BlockState, @@ -170,33 +168,73 @@ where "data_source" => &self.data_source.name(), ); - let (result_sender, result_receiver) = channel(); let start_time = Instant::now(); let metrics = self.metrics.clone(); + let valid_module = self.valid_module.cheap_clone(); + let host_exports = self.host_exports.cheap_clone(); + let host_fns = self.host_fns.cheap_clone(); + + let experimental_features = ExperimentalFeatures { + allow_non_deterministic_ipfs: ENV_VARS.mappings.allow_non_deterministic_ipfs, + }; - self.mapping_request_sender - .send(WasmRequest::new_trigger( - MappingContext { - logger: logger.cheap_clone(), - state, - host_exports: self.host_exports.cheap_clone(), - block_ptr: trigger.block_ptr(), - timestamp: trigger.timestamp(), - proof_of_indexing, - host_fns: self.host_fns.cheap_clone(), - debug_fork: debug_fork.cheap_clone(), - mapping_logger: Logger::new(logger, o!("component" => "UserMapping")), - instrument, - }, - trigger, - result_sender, - )) - .await - .context("Mapping terminated before passing in trigger")?; - - let result = result_receiver - .await - .context("Mapping terminated before handling trigger")?; + let ctx = MappingContext { + logger: logger.cheap_clone(), + state, + host_exports, + block_ptr: trigger.block_ptr(), + timestamp: trigger.timestamp(), + proof_of_indexing, + host_fns, + debug_fork: debug_fork.cheap_clone(), + mapping_logger: Logger::new(logger, o!("component" => "UserMapping")), + instrument, + }; + + let logger_for_panic = logger.cheap_clone(); + let metrics_for_trigger = metrics.cheap_clone(); + + // Run the WASM instantiation and handler inside block_in_place. + // This tells tokio "I'm about to block" so it can move async work + // to other threads, preventing executor starvation. + let result = tokio::task::block_in_place(|| { + let handle_fut = async { + let _section = metrics_for_trigger.stopwatch.start_section("module_init"); + let module = crate::module::WasmInstance::from_valid_module_with_ctx( + valid_module, + ctx, + metrics_for_trigger.cheap_clone(), + experimental_features, + ) + .await + .context("module instantiation failed")?; + drop(_section); + + let _section = metrics_for_trigger.stopwatch.start_section("run_handler"); + if ENV_VARS.log_trigger_data { + debug!(logger_for_panic, "trigger data: {:?}", trigger); + } + module.handle_trigger(trigger).await + }; + + panic::catch_unwind(panic::AssertUnwindSafe(|| graph::block_on(handle_fut))) + }); + + let result = match result { + Ok(result) => result, + Err(panic_info) => { + let err_msg = if let Some(payload) = panic_info + .downcast_ref::() + .map(String::as_str) + .or(panic_info.downcast_ref::<&str>().copied()) + { + anyhow!("Subgraph panicked with message: {}", payload) + } else { + anyhow!("Subgraph panicked with an unknown payload.") + }; + Err(MappingError::Unknown(err_msg)) + } + }; let elapsed = start_time.elapsed(); metrics.observe_handler_execution_time(elapsed.as_secs_f64(), &handler); @@ -218,7 +256,10 @@ where } #[async_trait] -impl RuntimeHostTrait for RuntimeHost { +impl RuntimeHostTrait for RuntimeHost +where + ::MappingTrigger: ToAscPtr, +{ fn data_source(&self) -> &DataSource { &self.data_source } @@ -241,7 +282,7 @@ impl RuntimeHostTrait for RuntimeHost { debug_fork: &Option>, instrument: bool, ) -> Result { - self.send_mapping_request( + self.run_mapping( logger, state, trigger, diff --git a/runtime/wasm/src/lib.rs b/runtime/wasm/src/lib.rs index 7c543a4c128..36ac9fa8bff 100644 --- a/runtime/wasm/src/lib.rs +++ b/runtime/wasm/src/lib.rs @@ -3,9 +3,8 @@ pub mod asc_abi; mod host; pub mod to_from; -/// Public interface of the crate, receives triggers to be processed. -/// -/// Pre-processes modules and manages their threads. Serves as an interface from `host` to `module`. +/// Public interface of the crate. Contains the compiled WASM module +/// (`ValidModule`) and the mapping execution context (`MappingContext`). pub mod mapping; /// WASM module instance. diff --git a/runtime/wasm/src/mapping.rs b/runtime/wasm/src/mapping.rs index faacc9831a0..e2c2aaec3a5 100644 --- a/runtime/wasm/src/mapping.rs +++ b/runtime/wasm/src/mapping.rs @@ -1,168 +1,15 @@ use crate::gas_rules::GasRules; -use crate::module::{ExperimentalFeatures, ToAscPtr, WasmInstance, WasmInstanceData}; -use graph::blockchain::{BlockTime, Blockchain, HostFn}; +use crate::module::WasmInstanceData; +use graph::blockchain::{BlockTime, HostFn}; use graph::components::store::SubgraphFork; -use graph::components::subgraph::{MappingError, SharedProofOfIndexing}; -use graph::data_source::{MappingTrigger, TriggerWithHandler}; -use graph::futures03::channel::oneshot::Sender; +use graph::components::subgraph::SharedProofOfIndexing; use graph::parking_lot::RwLock; use graph::prelude::*; -use graph::runtime::gas::Gas; use graph::runtime::IndexForAscTypeId; use parity_wasm::elements::ExportEntry; use std::collections::{BTreeMap, HashMap}; -use std::panic; use std::sync::{Arc, OnceLock}; -/// Spawn a wasm module on tokio's blocking thread pool. -/// -/// Uses `tokio::task::spawn_blocking` so the long-running mapping loop -/// is managed by the tokio runtime instead of a manually spawned OS -/// thread. This is a stepping stone toward eliminating the dedicated -/// thread entirely (Phase 3C). -pub fn spawn_module( - raw_module: &[u8], - logger: Logger, - subgraph_id: DeploymentHash, - host_metrics: Arc, - timeout: Option, - experimental_features: ExperimentalFeatures, -) -> Result>, anyhow::Error> -where - ::MappingTrigger: ToAscPtr, -{ - let valid_module = Arc::new(ValidModule::new(&logger, raw_module, timeout)?); - - // Create channel for event handling requests - let (mapping_request_sender, mut mapping_request_receiver) = tokio::sync::mpsc::channel(100); - - // We use spawn_blocking for the mapping loop because WASM execution - // is CPU-bound and can block for extended periods. spawn_blocking - // places the task on tokio's blocking thread pool which is designed - // for this: it grows on demand and won't starve the async executor. - // - // In case of failure, the task may panic or simply terminate, - // dropping the `mapping_request_receiver` which ultimately causes the - // subgraph to fail the next time it tries to handle an event. - tokio::task::spawn_blocking(move || { - // Pass incoming triggers to the WASM module and return entity changes; - // Stop when canceled because all RuntimeHosts and their senders were dropped. - while let Some(request) = mapping_request_receiver.blocking_recv() { - let WasmRequest { - ctx, - inner, - result_sender, - } = request; - let logger = ctx.logger.clone(); - - let handle_fut = async { - let result = instantiate_module::( - valid_module.cheap_clone(), - ctx, - host_metrics.cheap_clone(), - experimental_features, - ) - .await; - match result { - Ok(module) => match inner { - WasmRequestInner::TriggerRequest(trigger) => { - handle_trigger(&logger, module, trigger, host_metrics.cheap_clone()) - .await - } - }, - Err(e) => Err(MappingError::Unknown(e)), - } - }; - let result = - panic::catch_unwind(panic::AssertUnwindSafe(|| graph::block_on(handle_fut))); - - let result = match result { - Ok(result) => result, - Err(panic_info) => { - let err_msg = if let Some(payload) = panic_info - .downcast_ref::() - .map(String::as_str) - .or(panic_info.downcast_ref::<&str>().copied()) - { - anyhow!("Subgraph panicked with message: {}", payload) - } else { - anyhow!("Subgraph panicked with an unknown payload.") - }; - Err(MappingError::Unknown(err_msg)) - } - }; - - let _ = result_sender.send(result); - } - debug!(logger, "Subgraph {}: mapping task terminated", subgraph_id); - }); - - Ok(mapping_request_sender) -} - -async fn instantiate_module( - valid_module: Arc, - ctx: MappingContext, - host_metrics: Arc, - experimental_features: ExperimentalFeatures, -) -> Result -where - ::MappingTrigger: ToAscPtr, -{ - // Start the WASM module runtime. - let _section = host_metrics.stopwatch.start_section("module_init"); - WasmInstance::from_valid_module_with_ctx( - valid_module, - ctx, - host_metrics.cheap_clone(), - experimental_features, - ) - .await - .context("module instantiation failed") -} - -async fn handle_trigger( - logger: &Logger, - module: WasmInstance, - trigger: TriggerWithHandler>, - host_metrics: Arc, -) -> Result<(BlockState, Gas), MappingError> -where - ::MappingTrigger: ToAscPtr, -{ - let logger = logger.cheap_clone(); - - let _section = host_metrics.stopwatch.start_section("run_handler"); - if ENV_VARS.log_trigger_data { - debug!(logger, "trigger data: {:?}", trigger); - } - module.handle_trigger(trigger).await -} - -pub struct WasmRequest { - pub(crate) ctx: MappingContext, - pub(crate) inner: WasmRequestInner, - pub(crate) result_sender: Sender>, -} - -impl WasmRequest { - pub(crate) fn new_trigger( - ctx: MappingContext, - trigger: TriggerWithHandler>, - result_sender: Sender>, - ) -> Self { - WasmRequest { - ctx, - inner: WasmRequestInner::TriggerRequest(trigger), - result_sender, - } - } -} - -pub enum WasmRequestInner { - TriggerRequest(TriggerWithHandler>), -} - pub struct MappingContext { pub logger: Logger, pub host_exports: Arc, From 957cba798f350a1f5f02daafb297d4a246a6b965 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Sat, 14 Feb 2026 14:49:12 -0800 Subject: [PATCH 21/35] graph, core: Prefetch entities before as_modifications() Track entity keys that are written to `updates` but not present in the `current` LfuCache during trigger processing. Before calling as_modifications(), pre-fetch these entities from the store so that the get_many() call inside as_modifications() finds them already cached. This moves the DB round-trip earlier in the pipeline, and the tracked candidate set avoids re-scanning all update keys at commit time. Immutable types and removes are excluded from tracking since they never need fetching. --- .../amp_subgraph/runner/data_processing.rs | 5 ++ core/src/subgraph/runner/mod.rs | 5 ++ graph/src/components/store/entity_cache.rs | 52 ++++++++++++++++++- store/test-store/tests/graph/entity_cache.rs | 4 -- 4 files changed, 61 insertions(+), 5 deletions(-) diff --git a/core/src/amp_subgraph/runner/data_processing.rs b/core/src/amp_subgraph/runner/data_processing.rs index e68217fab87..a0b661f6be2 100644 --- a/core/src/amp_subgraph/runner/data_processing.rs +++ b/core/src/amp_subgraph/runner/data_processing.rs @@ -133,6 +133,11 @@ async fn process_record_batch_group( })?; } + entity_cache + .prefetch() + .await + .map_err(Error::from) + .map_err(|e| e.context("failed to prefetch entities"))?; let section = cx.metrics.stopwatch.start_section("as_modifications"); let ModificationsAndCache { modifications, diff --git a/core/src/subgraph/runner/mod.rs b/core/src/subgraph/runner/mod.rs index 8d0295c3754..fd758149c3e 100644 --- a/core/src/subgraph/runner/mod.rs +++ b/core/src/subgraph/runner/mod.rs @@ -676,6 +676,10 @@ where .non_deterministic()?; } + // Pre-fetch entities that were modified but not yet in the cache. + // This avoids a blocking get_many() inside as_modifications(). + entity_cache.prefetch().await.classify()?; + let section = self .metrics .host @@ -1578,6 +1582,7 @@ where // reset to RESERVED_VIDS and produce duplicate VIDs. next_vid_seq = block_state.entity_cache.vid_seq; + block_state.entity_cache.prefetch().await?; mods.extend( block_state .entity_cache diff --git a/graph/src/components/store/entity_cache.rs b/graph/src/components/store/entity_cache.rs index 1bea3289c47..f1d0f6dc6b3 100644 --- a/graph/src/components/store/entity_cache.rs +++ b/graph/src/components/store/entity_cache.rs @@ -1,6 +1,6 @@ use anyhow::{anyhow, bail}; use std::borrow::Borrow; -use std::collections::{HashMap, HashSet}; +use std::collections::{BTreeSet, HashMap, HashSet}; use std::fmt::{self, Debug}; use std::sync::Arc; @@ -111,6 +111,14 @@ pub struct EntityCache { /// that may not conform to the subgraph schema's type requirements. needs_validation: HashSet, + /// Entity keys that have been written to `updates` but were not in + /// `current` at the time of writing. These are candidates for + /// prefetching from the store before `as_modifications()` runs, + /// so that the `get_many()` call there finds them already cached. + /// Immutable entity types are excluded since they are assumed to + /// be new and never fetched. + prefetch_candidates: BTreeSet, + /// A sequence number for generating entity IDs. We use one number for /// all id's as the id's are scoped by block and a u32 has plenty of /// room for all changes in one block. To ensure reproducability of @@ -146,6 +154,7 @@ impl EntityCache { handler_updates: HashMap::new(), in_handler: false, needs_validation: HashSet::new(), + prefetch_candidates: BTreeSet::new(), schema: store.input_schema(), store, seq: 0, @@ -168,6 +177,7 @@ impl EntityCache { handler_updates: HashMap::new(), in_handler: false, needs_validation: HashSet::new(), + prefetch_candidates: BTreeSet::new(), schema: store.input_schema(), store, seq: 0, @@ -445,8 +455,19 @@ impl EntityCache { false => &mut self.updates, }; + let is_main_updates = !self.in_handler; match updates.entry(key) { Entry::Vacant(entry) => { + // Track keys written to `updates` that are not in `current` + // as prefetch candidates for `as_modifications()`. Skip + // immutable types since they are assumed to be new inserts. + if is_main_updates + && !matches!(&op, EntityOp::Remove) + && !entry.key().entity_type.is_immutable() + && !self.current.contains_key(entry.key()) + { + self.prefetch_candidates.insert(entry.key().clone()); + } entry.insert(op); } Entry::Occupied(mut entry) => entry.get_mut().accumulate(op), @@ -458,6 +479,7 @@ impl EntityCache { self.current.extend(other.current); self.needs_validation.extend(other.needs_validation); + self.prefetch_candidates.extend(other.prefetch_candidates); for (key, op) in other.updates { self.entity_op(key, op); } @@ -470,6 +492,34 @@ impl EntityCache { Ok(id) } + /// Pre-fetch entities from the store that were modified during trigger + /// processing but are not yet in the `current` cache. This populates + /// the cache so that the subsequent `as_modifications()` call finds + /// them already present, avoiding a blocking `get_many()` on the + /// critical path. + pub async fn prefetch(&mut self) -> Result<(), StoreError> { + // Filter out candidates that have since been loaded into `current` + // (e.g., by a `get()` call during later trigger processing). + let missing: BTreeSet = self + .prefetch_candidates + .iter() + .filter(|key| !self.current.contains_key(key)) + .cloned() + .collect(); + self.prefetch_candidates.clear(); + + if missing.is_empty() { + return Ok(()); + } + + for (entity_key, mut entity) in self.store.get_many(missing).await? { + entity.sort_fields(); + self.current.insert(entity_key, Some(Arc::new(entity))); + } + + Ok(()) + } + /// Return the changes that have been made via `set` and `remove` as /// `EntityModification`, making sure to only produce one when a change /// to the current state is actually needed. diff --git a/store/test-store/tests/graph/entity_cache.rs b/store/test-store/tests/graph/entity_cache.rs index 1b45efa488c..7eae16bb7f5 100644 --- a/store/test-store/tests/graph/entity_cache.rs +++ b/store/test-store/tests/graph/entity_cache.rs @@ -376,7 +376,6 @@ async fn offchain_trigger_vid_collision_without_fix() { let band1_key = make_band_key("band1"); cache1 .set(band1_key.clone(), band1_data, block, None) - .await .unwrap(); let result1 = cache1.as_modifications(block).await.unwrap(); @@ -387,7 +386,6 @@ async fn offchain_trigger_vid_collision_without_fix() { let band2_key = make_band_key("band2"); cache2 .set(band2_key.clone(), band2_data, block, None) - .await .unwrap(); let result2 = cache2.as_modifications(block).await.unwrap(); @@ -432,7 +430,6 @@ async fn offchain_trigger_vid_no_collision_with_fix() { let band1_key = make_band_key("band1"); cache1 .set(band1_key.clone(), band1_data, block, None) - .await .unwrap(); // THE FIX: capture vid_seq BEFORE as_modifications consumes the cache @@ -447,7 +444,6 @@ async fn offchain_trigger_vid_no_collision_with_fix() { let band2_key = make_band_key("band2"); cache2 .set(band2_key.clone(), band2_data, block, None) - .await .unwrap(); let result2 = cache2.as_modifications(block).await.unwrap(); From e424435d05651b1b2d1f50917bf82e0a3fea2ebd Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Sat, 14 Feb 2026 19:39:40 -0800 Subject: [PATCH 22/35] runtime: Remove unnecessary block_in_place + block_on Subgraph runners execute on dedicated OS threads inside graph::block_on(runner.run()), so process_mapping_trigger is already in a valid async context. The block_in_place + block_on combo only existed to allow the nested block_on call, but since all WASM host functions are async and use .await, there is no need to exit and re-enter the runtime context. Replace with a direct .await using FutureExt::catch_unwind for panic recovery. --- runtime/wasm/src/host.rs | 54 +++++++++++++++++++--------------------- 1 file changed, 26 insertions(+), 28 deletions(-) diff --git a/runtime/wasm/src/host.rs b/runtime/wasm/src/host.rs index 8a8c93707cf..7d2f0b5c778 100644 --- a/runtime/wasm/src/host.rs +++ b/runtime/wasm/src/host.rs @@ -1,9 +1,10 @@ use std::cmp::PartialEq; -use std::panic; +use std::panic::AssertUnwindSafe; use std::time::Instant; use anyhow::Context; use async_trait::async_trait; +use graph::futures03::FutureExt as _; use graph::blockchain::{Blockchain, HostFn, RuntimeAdapter}; use graph::components::store::{EnsLookup, SubgraphFork}; @@ -147,8 +148,10 @@ where } /// Instantiate the WASM module and run the trigger handler directly - /// in the current async context, using `block_in_place` to avoid - /// blocking the tokio executor. + /// in the current async context. This works because subgraph runners + /// execute on dedicated OS threads (via `graph::spawn_thread`), not + /// on tokio worker threads, so blocking here doesn't starve the + /// executor. async fn run_mapping( &self, logger: &Logger, @@ -194,31 +197,26 @@ where let logger_for_panic = logger.cheap_clone(); let metrics_for_trigger = metrics.cheap_clone(); - // Run the WASM instantiation and handler inside block_in_place. - // This tells tokio "I'm about to block" so it can move async work - // to other threads, preventing executor starvation. - let result = tokio::task::block_in_place(|| { - let handle_fut = async { - let _section = metrics_for_trigger.stopwatch.start_section("module_init"); - let module = crate::module::WasmInstance::from_valid_module_with_ctx( - valid_module, - ctx, - metrics_for_trigger.cheap_clone(), - experimental_features, - ) - .await - .context("module instantiation failed")?; - drop(_section); - - let _section = metrics_for_trigger.stopwatch.start_section("run_handler"); - if ENV_VARS.log_trigger_data { - debug!(logger_for_panic, "trigger data: {:?}", trigger); - } - module.handle_trigger(trigger).await - }; - - panic::catch_unwind(panic::AssertUnwindSafe(|| graph::block_on(handle_fut))) - }); + let handle_fut = async { + let _section = metrics_for_trigger.stopwatch.start_section("module_init"); + let module = crate::module::WasmInstance::from_valid_module_with_ctx( + valid_module, + ctx, + metrics_for_trigger.cheap_clone(), + experimental_features, + ) + .await + .context("module instantiation failed")?; + drop(_section); + + let _section = metrics_for_trigger.stopwatch.start_section("run_handler"); + if ENV_VARS.log_trigger_data { + debug!(logger_for_panic, "trigger data: {:?}", trigger); + } + module.handle_trigger(trigger).await + }; + + let result = AssertUnwindSafe(handle_fut).catch_unwind().await; let result = match result { Ok(result) => result, From e5555a8c5125cb5e675cbea6d3bb4cba4a9be550 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Sat, 14 Feb 2026 20:19:52 -0800 Subject: [PATCH 23/35] docs: Add runtime implementation documentation Document the WASM runtime architecture covering module compilation, threading model, trigger execution flow, host functions, gas metering, and error handling. --- docs/implementation/README.md | 1 + docs/implementation/runtime.md | 196 +++++++++++++++++++++++++++++++++ 2 files changed, 197 insertions(+) create mode 100644 docs/implementation/runtime.md diff --git a/docs/implementation/README.md b/docs/implementation/README.md index 31d4eb694a6..d7b999a696d 100644 --- a/docs/implementation/README.md +++ b/docs/implementation/README.md @@ -10,3 +10,4 @@ the code should go into comments. * [SQL Query Generation](./sql-query-generation.md) * [Adding support for a new chain](./add-chain.md) * [Pruning](./pruning.md) +* [Runtime](./runtime.md) diff --git a/docs/implementation/runtime.md b/docs/implementation/runtime.md new file mode 100644 index 00000000000..ab8f79ef60d --- /dev/null +++ b/docs/implementation/runtime.md @@ -0,0 +1,196 @@ +# Runtime + +The runtime executes subgraph mapping handlers inside a WebAssembly +sandbox. Subgraph developers write handlers in AssemblyScript that react +to blockchain events (blocks, transactions, log events); the runtime +compiles these into WASM, instantiates them for each trigger, and +collects the resulting entity changes. The main crates involved are +`runtime/wasm` (the WASM engine and host function layer) and +`runtime/derive` (proc macros for Rust-to-AssemblyScript type +conversion). + +The key types are `ValidModule` (a compiled, pre-linked WASM module +ready for instantiation), `RuntimeHost` (one per data source, owns the +module and dispatches triggers), `WasmInstance` (a live WASM instance +executing a single handler), and `MappingContext` (per-trigger state +carrying the entity cache, block pointer, and proof-of-indexing +accumulator). + +## Module Compilation + +When a subgraph is deployed, each WASM binary goes through a +multi-stage processing pipeline before any trigger can run. The raw +bytes are first parsed with `parity_wasm`. The start section, if +present, is extracted and re-exported under the name `gn::start` so +that it can be called explicitly after the AssemblyScript heap is set +up rather than during instantiation. Gas metering instructions are then +injected via `wasm_instrument::gas_metering::inject` using the cost +table in `gas_rules.rs`. The instrumented bytes are compiled into +native code by wasmtime. + +All WASM instances in the process share a single global +`wasmtime::Engine`, created once on first use. The engine is configured +with Cranelift, NaN canonicalization for determinism, epoch-based +interruption for timeouts, and a pooling allocator. The pooling +allocator pre-reserves a fixed number of instance slots (controlled by +`GRAPH_WASM_INSTANCE_POOL_SIZE`) so that instantiating a module reuses +a pre-allocated slot rather than performing fresh `mmap`/`munmap` calls +on every trigger. + +After compilation, a `wasmtime::Linker` is built and all host functions +are registered into it. The linker is then used to create an +`InstancePre`, which captures the fully-resolved link between the +compiled module and its imports. This `InstancePre` is stored inside +`ValidModule` and reused for every trigger instantiation, avoiding +repeated symbol resolution. The entire pipeline lives in +`ValidModule::new()` in `runtime/wasm/src/mapping.rs`. + +## Threading Model + +Subgraph runners execute on dedicated OS threads spawned via +`graph::spawn_thread`, not on tokio worker threads. WASM handlers run +directly in the async context of those threads — there is no dedicated +mapping thread and no channel-based handoff between an async task and a +blocking executor. Because the OS threads are separate from the tokio +thread pool, blocking inside a WASM handler does not starve the async +executor that handles networking and database I/O. + +Timeout enforcement uses wasmtime's epoch interruption mechanism. A +single global background task increments the engine's epoch counter at +a fixed interval (the configured handler timeout). Each WASM store sets +its deadline to 2 epochs, so the effective timeout for any handler is +between 1x and 2x the configured interval. When the deadline expires, +wasmtime raises a trap that the runtime classifies as a non-deterministic +timeout. The epoch counter is started lazily by `ensure_epoch_counter()` +in `runtime/wasm/src/mapping.rs`. + +## Trigger Execution Flow + +The path from a blockchain event to a completed handler invocation +proceeds as follows: + +1. The block stream delivers a batch of triggers to the subgraph + runner. +2. The trigger decoder matches each trigger against the data sources + registered for the subgraph and finds the corresponding + `RuntimeHost`. +3. `RuntimeHost::process_mapping_trigger` is called, which delegates to + `run_mapping` in `runtime/wasm/src/host.rs`. +4. `run_mapping` builds a `MappingContext` with a fresh entity cache + snapshot, then calls + `WasmInstance::from_valid_module_with_ctx` to instantiate the module + from the cached `InstancePre`. This is a fast operation because + symbol resolution was already done at compilation time and the + pooling allocator provides a pre-reserved slot. +5. The start function (`gn::start`) is executed if present, followed by + the AssemblyScript `_start` entry point for newer API versions. +6. The trigger data is converted into its AssemblyScript representation + (via the `AscType`/`AscHeap` traits and the `ToAscPtr` + implementation for the trigger type), and the handler function named + in the subgraph manifest is invoked. +7. The handler's entity changes, newly created data sources, and other + side effects are collected in a `BlockState` and returned to the + caller. + +Handler isolation is enforced through the `enter_handler` / +`exit_handler` protocol on the block state. Before a handler runs, +`enter_handler` takes a snapshot of the entity cache. On success, +`exit_handler` commits the handler's changes. On a deterministic error, +`exit_handler_and_discard_changes_due_to_error` reverts the entity +cache to the snapshot and records the error as a `SubgraphError`. This +ensures that a failing handler never leaves partial writes in the +entity cache. The execution logic lives in `WasmInstance::invoke_handler` +in `runtime/wasm/src/module/instance.rs`. + +## Host Functions + +Host functions are the bridge between WASM handlers and the Rust +runtime. They fall into two categories: built-in functions and +chain-specific functions. + +Built-in functions cover entity storage (`store.get`, `store.set`, +`store.remove`, `store.loadRelated`), IPFS access (`ipfs.cat`, +`ipfs.map`, `ipfs.getBlock`), cryptographic primitives +(`crypto.keccak256`), JSON and YAML parsing, big-number arithmetic +(`bigInt.*`, `bigDecimal.*`), type conversions, data source creation, +logging, and ENS resolution. They are registered in `build_linker()` +via the `link!` macro, which wraps each Rust function into a +`wasmtime::Linker` entry, handling gas accounting, error classification, +and the conversion between WASM values and Rust types. + +Chain-specific functions (e.g. `ethereum.call`, `ethereum.encode`) are +provided by the `RuntimeAdapter::host_fns()` trait method. They are +registered through `link_chain_host_fn()`, which looks up the target +function by name at call time from the `MappingContext`'s `host_fns` +map rather than capturing a concrete closure at link time. This allows +different chains to expose different host functions without changing the +linker setup. + +All host functions are registered once at module validation time and +baked into the `InstancePre`. The central registration point is +`build_linker()` in `runtime/wasm/src/module/instance.rs`. + +Rust-to-AssemblyScript type conversion is handled by the `AscType` and +`AscHeap` traits, with proc macros in `runtime/derive` generating the +boilerplate for mapping between Rust structs and their AssemblyScript +memory layout. + +## Gas Metering + +Gas metering limits the computational cost of each handler invocation. +Gas-accounting instructions are injected into the WASM bytecode at +compile time by `wasm_instrument`, so every WASM instruction +contributes to the gas count without requiring per-instruction +interpreter hooks at runtime. + +The cost table in `runtime/wasm/src/gas_rules.rs` assigns a gas value +to each WASM instruction class. Representative costs: loads cost 1573 +gas, stores 2263, arithmetic 25-26, division and remainder 72-82, +direct calls 951, indirect calls 1995, and memory growth 435,000. +These values originate from benchmarks in the Substrate project and are +conservative for wasmtime. + +Host functions also consume gas proportional to their work, tracked via +`HostExports::track_gas_and_ops()`. The special `gas` import — the +function called by the injected metering instructions — is registered +as a synchronous `func_wrap` rather than an async closure because it is +invoked tens of thousands of times per handler and must be as cheap as +possible. + +Each `WasmInstance` carries its own `GasCounter`. When the counter +exceeds the per-handler limit, a deterministic error is raised and the +handler's changes are reverted. + +## Error Handling + +Errors during handler execution are classified by their +`DeterminismLevel`, defined in `runtime/wasm/src/error.rs`: + +- **Deterministic** errors are reproducible regardless of when or where + the handler runs. Examples include integer division by zero, memory + out-of-bounds access, unreachable code, and gas exhaustion. WASM + traps are classified by `is_trap_deterministic()` in + `runtime/wasm/src/module/mod.rs`: traps like `MemoryOutOfBounds`, + `IntegerDivisionByZero`, and `UnreachableCodeReached` are always + deterministic. When a deterministic error occurs, the handler's + entity changes are discarded and the error is recorded as a + `SubgraphError` on the deployment. + +- **Non-deterministic** errors depend on external conditions and may + resolve on retry. Network failures and timeouts (wasmtime's + `Trap::Interrupt`) fall into this category. These errors bubble up to + the subgraph runner, which will retry the trigger. + +- **PossibleReorg** indicates that the block being processed might not + be on the canonical chain. The runtime sets a `possible_reorg` flag + in `WasmInstanceData` when a host function detects this condition. + +- **Unimplemented** is a catch-all for errors that have not yet been + classified. It exists as a transitional category and should be phased + out over time. + +Host functions that encounter errors set the `deterministic_host_trap` +or `possible_reorg` flags on `WasmInstanceData`. After the handler +returns (or traps), `invoke_handler` inspects these flags to determine +how to classify the overall result. Panic safety is provided by +`catch_unwind` around the handler call in `run_mapping`. From 9da838e11a61fd136c68f757ba67657156c11a52 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Sat, 14 Feb 2026 20:48:59 -0800 Subject: [PATCH 24/35] docs: Add WASM ABI memory layout reference Document the binary layout of objects that graph-node serializes into WASM linear memory for subgraph handler invocation. Covers both v0.0.4 (AS v0.6) and v0.0.5+ (AS >= v0.19.2) layouts for core types and Ethereum chain types. --- docs/implementation/abi.md | 956 +++++++++++++++++++++++++++++++++++++ 1 file changed, 956 insertions(+) create mode 100644 docs/implementation/abi.md diff --git a/docs/implementation/abi.md b/docs/implementation/abi.md new file mode 100644 index 00000000000..830d9acc1d0 --- /dev/null +++ b/docs/implementation/abi.md @@ -0,0 +1,956 @@ +# WASM ABI Memory Layout Reference + +This document describes the binary layout of objects that graph-node +serializes into WASM linear memory when invoking subgraph handler functions. +It covers both the legacy **v0.0.4** layout (AssemblyScript v0.6) and the +current **v0.0.5+** layout (AssemblyScript >= v0.19.2), for core types and +Ethereum chain types. + +Source files are referenced relative to the repository root. + +## Table of Contents + +- [1. Overview](#1-overview) +- [2. Object Headers and Allocation](#2-object-headers-and-allocation) +- [3. Pointer Type: AscPtr](#3-pointer-type-ascptr) +- [4. Primitive Types](#4-primitive-types) +- [5. Core Types](#5-core-types) + - [5.1 AscString](#51-ascstring) + - [5.2 ArrayBuffer](#52-arraybuffer) + - [5.3 TypedArray\](#53-typedarrayt) + - [5.4 Array\](#54-arrayt) + - [5.5 AscEnum\](#55-ascenumd) + - [5.6 EnumPayload](#56-enumpayload) + - [5.7 AscTypedMapEntry\](#57-asctypedmapentrykvgt) + - [5.8 AscTypedMap\](#58-asctypedmapkv) + - [5.9 AscEntity](#59-ascentity) + - [5.10 AscBigDecimal](#510-ascbigdecimal) + - [5.11 AscResult\](#511-ascresultve) + - [5.12 AscWrapped\](#512-ascwrappedv) +- [6. Ethereum Types](#6-ethereum-types) + - [6.1 AscEthereumBlock](#61-ascethereumblock) + - [6.2 AscEthereumTransaction](#62-ascethereumtransaction) + - [6.3 AscEthereumEvent](#63-ascethereuevent) + - [6.4 AscLogParam](#64-asclogparam) + - [6.5 AscEthereumLog](#65-ascethereumlog) + - [6.6 AscEthereumTransactionReceipt](#66-ascethereumtransactionreceipt) + - [6.7 AscEthereumCall](#67-ascethereumcall) + - [6.8 AscUnresolvedContractCall](#68-ascunresolvedcontractcall) +- [7. API Version Compatibility Matrix](#7-api-version-compatibility-matrix) +- [8. Allocation Pattern](#8-allocation-pattern) + +--- + +## 1. Overview + +When a subgraph handler is invoked, graph-node must convert Rust-side +trigger data (blocks, transactions, events, etc.) into objects in WASM +linear memory that AssemblyScript code can read. This serialization crosses +the **ABI boundary** between graph-node and subgraph code. + +### Key Traits + +| Trait | Role | Defined in | +|-------|------|------------| +| `AscType` | Defines `to_asc_bytes()` / `from_asc_bytes()` for a type's memory layout | `graph/src/runtime/mod.rs` | +| `AscIndexId` | Provides the `rt_id` used in v0.0.5+ object headers (`INDEX_ASC_TYPE_ID`) | `graph/src/runtime/mod.rs` | +| `ToAscObj` | Converts a Rust value into its Asc representation `C` | `graph/src/runtime/asc_heap.rs` | +| `FromAscObj` | Converts an Asc representation back to Rust | `graph/src/runtime/asc_heap.rs` | +| `AscHeap` | Interface to the WASM linear memory (read/write/allocate) | `graph/src/runtime/asc_heap.rs` | + +### Handler Invocation Flow + +``` +Rust trigger data + | + v + ToAscObj::to_asc_obj() -- recursively converts nested objects + | + v + AscType::to_asc_bytes() -- serializes each object to bytes + | + v + AscPtr::alloc_obj() -- writes bytes into WASM linear memory + | (adds header in v0.0.5+) + v + Single u32 pointer arg -- passed to the WASM handler function +``` + +Allocation is **bottom-up**: leaf objects (strings, byte arrays, BigInts) +are allocated first, producing `AscPtr` values that are then embedded in +parent structs. The final top-level struct pointer is passed to the WASM +handler as a `u32` argument. + +--- + +## 2. Object Headers and Allocation + +### v0.0.4 (AssemblyScript v0.6) + +No object header. Variable-length types (strings, array buffers) carry +**inline length prefixes** as part of their serialized content. + +Allocation writes `to_asc_bytes()` directly via `raw_new()`, and the +returned heap offset is the `AscPtr`. + +### v0.0.5+ (AssemblyScript >= v0.19.2) + +Every object is preceded by a **20-byte header**. The `AscPtr` points past +the header to the start of the content. + +``` + AscPtr points here + | + v ++----------+----------+----------+-------+---------+---------+---------+ +| mm_info | gc_info | gc_info2 | rt_id | rt_size | content | padding | +| 4 bytes | 4 bytes | 4 bytes | 4 B | 4 B | N bytes | 0-15 B | ++----------+----------+----------+-------+---------+---------+---------+ +|<---------- 20-byte header ----------->| +``` + +| Field | Size | Description | +|-------|------|-------------| +| `mm_info` | 4B LE u32 | `16 + full_length` where `full_length = content + alignment_padding` | +| `gc_info` | 4B LE u32 | Always 0 (GC not used by graph-node) | +| `gc_info2` | 4B LE u32 | Always 0 (GC not used by graph-node) | +| `rt_id` | 4B LE u32 | Class identifier from `IndexForAscTypeId` (equivalent to AS `idof`) | +| `rt_size` | 4B LE u32 | Content length in bytes (returned by `content_len()`) | + +**16-byte alignment padding** is appended after the content: + +``` +padding = (16 - (20 + content_length) % 16) % 16 +``` + +The total allocation is `20 + content_length + padding` bytes. + +Reference: `graph/src/runtime/mod.rs` (lines 409-413), +`graph/src/runtime/asc_ptr.rs` (lines 87-123, 145-170) + +--- + +## 3. Pointer Type: AscPtr + +``` ++--------+ +| offset | 4 bytes, little-endian u32 ++--------+ +``` + +- **Size**: 4 bytes +- **Encoding**: Little-endian `u32` +- **Null**: Value `0` represents a null pointer +- **v0.0.4**: Points to the first byte of serialized content +- **v0.0.5+**: Points to the first byte of content (header is at negative + offsets; `rt_size` is at `ptr - 4`) + +An `AscPtr` is itself an `AscValue` and can be embedded in other +`#[repr(C)]` structs. It is also used as the `EnumPayload` representation +for pointer-carrying enum variants. + +Reference: `graph/src/runtime/asc_ptr.rs` + +--- + +## 4. Primitive Types + +All primitives use little-endian encoding. Size equals alignment. + +| Rust Type | Size | Encoding | +|-----------|------|----------| +| `u8` | 1 byte | LE | +| `u16` | 2 bytes | LE | +| `u32` | 4 bytes | LE | +| `u64` | 8 bytes | LE | +| `i8` | 1 byte | LE | +| `i32` | 4 bytes | LE | +| `i64` | 8 bytes | LE | +| `f32` | 4 bytes | LE IEEE 754 | +| `f64` | 8 bytes | LE IEEE 754 | +| `bool` | 1 byte | `0` = false, nonzero = true | + +Reference: `graph/src/runtime/mod.rs` (lines 94-144) + +--- + +## 5. Core Types + +### 5.1 AscString + +UTF-16LE encoded string. + +**v0.0.4** (`runtime/wasm/src/asc_abi/v0_0_4.rs`): + +``` ++--------+-----------------------------------+ +| length | UTF-16LE code units | +| 4 B LE | length * 2 bytes | ++--------+-----------------------------------+ +``` + +| Offset | Field | Size | Description | +|--------|-------|------|-------------| +| 0 | `length` | 4B | Number of UTF-16 code units (u32 LE) | +| 4 | content | `length * 2` B | UTF-16LE encoded characters | + +`asc_size = 4 + length * 2` + +**v0.0.5+** (`runtime/wasm/src/asc_abi/v0_0_5.rs`): + +``` ++-----------------------------------+------------------+ +| UTF-16LE code units | power-of-2 pad | +| rt_size bytes | | ++-----------------------------------+------------------+ +``` + +No inline length prefix. The string length comes from the header's +`rt_size` field (which stores `code_unit_count * 2`, the byte length of +the UTF-16LE data). Extra padding is added to reach the next power of two +of `(byte_count + 20)`. + +**`INDEX_ASC_TYPE_ID`**: `String = 0` + +### 5.2 ArrayBuffer + +Raw binary data buffer. + +**v0.0.4** (`runtime/wasm/src/asc_abi/v0_0_4.rs`): + +``` ++-------------+---------+---------+----------------+ +| byte_length | padding | content | power-of-2 pad | +| 4 B LE | 4 B | N bytes | | ++-------------+---------+---------+----------------+ +``` + +| Offset | Field | Size | Description | +|--------|-------|------|-------------| +| 0 | `byte_length` | 4B | Content length in bytes (u32 LE) | +| 4 | padding | 4B | Zero padding for 8-byte alignment | +| 8 | content | N B | Raw data bytes | +| 8+N | padding | variable | Zeroes to reach next power of two of `(8 + N)` | + +`asc_size = 4 + 4 + byte_length` + +**v0.0.5+** (`runtime/wasm/src/asc_abi/v0_0_5.rs`): + +``` ++---------+----------------+ +| content | power-of-2 pad | +| N bytes | | ++---------+----------------+ +``` + +No inline length. The `rt_size` header field contains the byte length. +Extra padding is added to reach the next power of two of `(N + 20)`. + +**`INDEX_ASC_TYPE_ID`**: `ArrayBuffer = 1` + +### 5.3 TypedArray\ + +A typed view over an `ArrayBuffer`. Subtypes: `Uint8Array` (`TypedArray`), +`Int32Array` (`TypedArray`), etc. + +Type aliases: +- `AscBigInt = Uint8Array = TypedArray` (signed number bytes, LE) +- `AscAddress = Uint8Array` (20 bytes) +- `AscH160 = Uint8Array` (20 bytes) + +**v0.0.4** (`runtime/wasm/src/asc_abi/v0_0_4.rs`): + +``` ++--------+-------------+-------------+ +| buffer | byte_offset | byte_length | +| 4 B | 4 B LE | 4 B LE | ++--------+-------------+-------------+ +``` + +| Offset | Field | Size | Description | +|--------|-------|------|-------------| +| 0 | `buffer` | 4B | `AscPtr` | +| 4 | `byte_offset` | 4B | Byte position in buffer where data starts (u32 LE) | +| 8 | `byte_length` | 4B | Length of the data in bytes (u32 LE) | + +Total: **12 bytes** (PhantomData is zero-sized) + +**v0.0.5+** (`runtime/wasm/src/asc_abi/v0_0_5.rs`): + +``` ++--------+------------+-------------+ +| buffer | data_start | byte_length | +| 4 B | 4 B LE | 4 B LE | ++--------+------------+-------------+ +``` + +| Offset | Field | Size | Description | +|--------|-------|------|-------------| +| 0 | `buffer` | 4B | `AscPtr` | +| 4 | `data_start` | 4B | **Absolute** address in WASM memory where data starts (u32 LE) | +| 8 | `byte_length` | 4B | Length of the data in bytes (u32 LE) | + +Total: **12 bytes** + +The key difference: v0.0.4 uses a relative `byte_offset` within the buffer, +while v0.0.5+ uses an absolute `data_start` address. When reading back, the +offset is computed as `data_start - buffer.wasm_ptr()`. + +**`INDEX_ASC_TYPE_ID`** (selected by element type): + +| Element Type | ID | +|---|---| +| `i8` | `Int8Array = 2` | +| `i16` | `Int16Array = 3` | +| `i32` | `Int32Array = 4` | +| `i64` | `Int64Array = 5` | +| `u8` | `Uint8Array = 6` | +| `u16` | `Uint16Array = 7` | +| `u32` | `Uint32Array = 8` | +| `u64` | `Uint64Array = 9` | +| `f32` | `Float32Array = 10` | +| `f64` | `Float64Array = 11` | + +### 5.4 Array\ + +Growable array backed by an `ArrayBuffer`. + +**v0.0.4** (`runtime/wasm/src/asc_abi/v0_0_4.rs`): + +``` ++--------+--------+ +| buffer | length | +| 4 B | 4 B LE | ++--------+--------+ +``` + +| Offset | Field | Size | Description | +|--------|-------|------|-------------| +| 0 | `buffer` | 4B | `AscPtr` containing the elements | +| 4 | `length` | 4B | Number of elements (u32 LE) | + +Total: **8 bytes** + +**v0.0.5+** (`runtime/wasm/src/asc_abi/v0_0_5.rs`): + +``` ++--------+-------------------+--------------------+--------+ +| buffer | buffer_data_start | buffer_data_length | length | +| 4 B | 4 B LE | 4 B LE | 4 B LE | ++--------+-------------------+--------------------+--------+ +``` + +| Offset | Field | Size | Description | +|--------|-------|------|-------------| +| 0 | `buffer` | 4B | `AscPtr` | +| 4 | `buffer_data_start` | 4B | Absolute address where data starts (u32 LE) | +| 8 | `buffer_data_length` | 4B | Length of backing data in bytes (u32 LE) | +| 12 | `length` | 4B | Number of elements (i32 LE) | + +Total: **16 bytes** + +Note that `length` is `i32` in v0.0.5+ (signed) versus `u32` in v0.0.4. + +**`INDEX_ASC_TYPE_ID`** (selected by element type): + +| Element Type | ID | +|---|---| +| `bool` | `ArrayBool = 13` | +| `Uint8Array` | `ArrayUint8Array = 14` | +| `AscPtr>` | `ArrayEthereumValue = 15` | +| `AscPtr>` | `ArrayStoreValue = 16` | +| `AscPtr>` | `ArrayJsonValue = 17` | +| `AscPtr` | `ArrayString = 18` | +| `AscPtr` | `ArrayEventParam = 19` | +| `u8` | `ArrayU8 = 41` | +| `u16` | `ArrayU16 = 42` | +| `u32` | `ArrayU32 = 43` | +| `u64` | `ArrayU64 = 44` | +| `i8` | `ArrayI8 = 45` | +| `i16` | `ArrayI16 = 46` | +| `i32` | `ArrayI32 = 47` | +| `i64` | `ArrayI64 = 48` | +| `f32` | `ArrayF32 = 49` | +| `f64` | `ArrayF64 = 50` | +| `AscPtr` | `ArrayBigDecimal = 51` | + +### 5.5 AscEnum\ + +Discriminated union used for `EthereumValue`, `StoreValue`, `JsonValue`, etc. + +``` ++------+----------+---------+ +| kind | _padding | payload | +| 4 B | 4 B | 8 B | ++------+----------+---------+ +``` + +| Offset | Field | Size | Description | +|--------|-------|------|-------------| +| 0 | `kind` | 4B | Discriminant enum (u32 LE) | +| 4 | `_padding` | 4B | Explicit padding (always 0) | +| 8 | `payload` | 8B | `EnumPayload` (u64 LE), interpretation depends on `kind` | + +Total: **16 bytes** (same layout for both v0.0.4 and v0.0.5+) + +The `_padding` field is explicit to satisfy `#[repr(C)]` alignment +requirements (the `u64` payload requires 8-byte alignment). + +**Discriminant values for `EthereumValueKind`** (`INDEX_ASC_TYPE_ID = EthereumValue = 30`): + +| Discriminant | Kind | Payload | +|---|---|---| +| 0 | `Address` | `AscPtr` (20 bytes) | +| 1 | `FixedBytes` | `AscPtr` | +| 2 | `Bytes` | `AscPtr` | +| 3 | `Int` | `AscPtr` (signed LE bytes) | +| 4 | `Uint` | `AscPtr` (unsigned LE bytes) | +| 5 | `Bool` | `0` or `1` (direct u64 value) | +| 6 | `String` | `AscPtr` | +| 7 | `FixedArray` | `AscPtr>>>` | +| 8 | `Array` | `AscPtr>>>` | +| 9 | `Tuple` | `AscPtr>>>` | +| 10 | `Function` | `AscPtr` (24 bytes) | + +**Discriminant values for `StoreValueKind`** (`INDEX_ASC_TYPE_ID = StoreValue = 31`): + +| Discriminant | Kind | Payload | +|---|---|---| +| 0 | `String` | `AscPtr` | +| 1 | `Int` | `i32` value (sign-extended in u64) | +| 2 | `BigDecimal` | `AscPtr` | +| 3 | `Bool` | `0` or `1` (direct u64 value) | +| 4 | `Array` | `AscPtr>>>` | +| 5 | `Null` | `0` (unused) | +| 6 | `Bytes` | `AscPtr` | +| 7 | `BigInt` | `AscPtr` (signed LE bytes) | +| 8 | `Int8` | `i64` value (direct in u64) | +| 9 | `Timestamp` | `i64` value (microseconds since epoch, direct in u64) | + +**Discriminant values for `JsonValueKind`** (`INDEX_ASC_TYPE_ID = JsonValue = 32`): + +| Discriminant | Kind | Payload | +|---|---|---| +| 0 | `Null` | `0` (unused) | +| 1 | `Bool` | `0` or `1` (direct u64 value) | +| 2 | `Number` | `AscPtr` (number as string) | +| 3 | `String` | `AscPtr` | +| 4 | `Array` | `AscPtr>>>` | +| 5 | `Object` | `AscPtr>>` | + +Reference: `runtime/wasm/src/asc_abi/class.rs` (lines 499-563), +`runtime/wasm/src/to_from/external.rs` + +### 5.6 EnumPayload + +Untyped 8-byte value that can hold a scalar or a pointer. + +``` ++---------+ +| value | +| 8 B LE | ++---------+ +``` + +- **Size**: 8 bytes (u64 LE) +- Holds either a scalar (`i32`, `i64`, `f64`, `bool` zero-extended to u64) + or an `AscPtr` (lower 32 bits contain the pointer, upper 32 bits are zero) + +Reference: `runtime/wasm/src/asc_abi/class.rs` (lines 416-497) + +### 5.7 AscTypedMapEntry\ + +A key-value pair. Used as the element type of `AscTypedMap` entries arrays. + +``` ++-----+-------+ +| key | value | +| 4 B | 4 B | ++-----+-------+ +``` + +| Offset | Field | Size | Description | +|--------|-------|------|-------------| +| 0 | `key` | 4B | `AscPtr` | +| 4 | `value` | 4B | `AscPtr` | + +Total: **8 bytes** + +**`INDEX_ASC_TYPE_ID`**: + +| Parameterization | ID | +|---|---| +| `>` | `TypedMapEntryStringStoreValue = 34` | +| `>` | `TypedMapEntryStringJsonValue = 35` | + +Reference: `runtime/wasm/src/asc_abi/class.rs` (lines 613-632) + +### 5.8 AscTypedMap\ + +An ordered map backed by an array of entries. + +``` ++---------+ +| entries | +| 4 B | ++---------+ +``` + +| Offset | Field | Size | Description | +|--------|-------|------|-------------| +| 0 | `entries` | 4B | `AscPtr>>>` | + +Total: **4 bytes** + +**`INDEX_ASC_TYPE_ID`**: + +| Parameterization | ID | +|---|---| +| `>` | `TypedMapStringStoreValue = 36` | +| `>` | `TypedMapStringJsonValue = 37` | + +Reference: `runtime/wasm/src/asc_abi/class.rs` (lines 634-659) + +### 5.9 AscEntity + +Type alias: `AscTypedMap>` + +An entity is a typed map where keys are strings and values are store values. +Its full object graph looks like: + +``` +AscEntity (4B) + | + +-> entries: Array>>> + | + +-> ArrayBuffer containing AscPtr elements + | + +-> AscTypedMapEntry (8B each) + | + +-> key: AscString (UTF-16LE) + +-> value: AscEnum (16B) + | + +-> payload may point to: + AscString, AscBigDecimal, + Uint8Array, Array<...>, etc. +``` + +### 5.10 AscBigDecimal + +``` ++--------+-----+ +| digits | exp | +| 4 B | 4 B | ++--------+-----+ +``` + +| Offset | Field | Size | Description | +|--------|-------|------|-------------| +| 0 | `digits` | 4B | `AscPtr` (the significand as signed LE bytes) | +| 4 | `exp` | 4B | `AscPtr` (the exponent as signed LE bytes) | + +Total: **8 bytes** + +**Sign flip on exponent**: The Rust `BigDecimal::as_bigint_and_exponent()` +returns a *negative* exponent (called `scale`). Graph-node **negates** this +before serializing: `exp = -negative_exp`. So a Rust BigDecimal with scale +3 (meaning `digits * 10^-3`) is stored with `exp = 3`. + +**`INDEX_ASC_TYPE_ID`**: `BigDecimal = 12` + +Reference: `runtime/wasm/src/asc_abi/class.rs` (lines 693-704), +`runtime/wasm/src/to_from/external.rs` (lines 91-138) + +### 5.11 AscResult\ + +``` ++-------+-------+ +| value | error | +| 4 B | 4 B | ++-------+-------+ +``` + +| Offset | Field | Size | Description | +|--------|-------|------|-------------| +| 0 | `value` | 4B | `AscPtr>` (null if error) | +| 4 | `error` | 4B | `AscPtr>` (null if success) | + +Total: **8 bytes** + +Exactly one of `value`/`error` is non-null. + +**`INDEX_ASC_TYPE_ID`**: + +| Parameterization | ID | +|---|---| +| `, bool>` | `ResultTypedMapStringJsonValueBool = 39` | +| `>, bool>` | `ResultJsonValueBool = 40` | + +Reference: `runtime/wasm/src/asc_abi/class.rs` (lines 727-745) + +### 5.12 AscWrapped\ + +A simple wrapper holding a single `AscValue`. + +``` ++-------+ +| inner | +| siz V | ++-------+ +``` + +| Offset | Field | Size | Description | +|--------|-------|------|-------------| +| 0 | `inner` | `size_of(V)` | The wrapped value | + +Total: **size_of(V)** (e.g., 4B for `AscPtr`, 1B for `bool`) + +**`INDEX_ASC_TYPE_ID`**: + +| Parameterization | ID | +|---|---| +| `AscPtr` | `WrappedTypedMapStringJsonValue = 27` | +| `bool` | `WrappedBool = 28` | +| `AscPtr>` | `WrappedJsonValue = 29` | + +Reference: `runtime/wasm/src/asc_abi/class.rs` (lines 747-767) + +--- + +## 6. Ethereum Types + +All Ethereum Asc structs are `#[repr(C)]` and consist entirely of `AscPtr` +fields (4 bytes each). Their layouts are identical across v0.0.4 and v0.0.5+ +(only the header/allocation mechanism differs). + +Reference: `chain/ethereum/src/runtime/abi.rs` + +### 6.1 AscEthereumBlock + +**`AscEthereumBlock`** (API versions < 0.0.6): + +``` ++------+-------------+------------+--------+------------+-------------------+ +| hash | parent_hash | uncles_hash| author | state_root | transactions_root | +| 4 B | 4 B | 4 B | 4 B | 4 B | 4 B | ++------+-------------+------------+--------+------------+-------------------+ ++---------------+--------+----------+-----------+-----------+------------+ +| receipts_root | number | gas_used | gas_limit | timestamp | difficulty | +| 4 B | 4 B | 4 B | 4 B | 4 B | 4 B | ++---------------+--------+----------+-----------+-----------+------------+ ++------------------+------+ +| total_difficulty | size | +| 4 B | 4 B | ++------------------+------+ +``` + +| Offset | Field | Size | Type | +|--------|-------|------|------| +| 0 | `hash` | 4B | `AscPtr` (B256, 32 bytes) | +| 4 | `parent_hash` | 4B | `AscPtr` (B256) | +| 8 | `uncles_hash` | 4B | `AscPtr` (B256) | +| 12 | `author` | 4B | `AscPtr` (Address, 20 bytes) | +| 16 | `state_root` | 4B | `AscPtr` (B256) | +| 20 | `transactions_root` | 4B | `AscPtr` (B256) | +| 24 | `receipts_root` | 4B | `AscPtr` (B256) | +| 28 | `number` | 4B | `AscPtr` | +| 32 | `gas_used` | 4B | `AscPtr` | +| 36 | `gas_limit` | 4B | `AscPtr` | +| 40 | `timestamp` | 4B | `AscPtr` | +| 44 | `difficulty` | 4B | `AscPtr` | +| 48 | `total_difficulty` | 4B | `AscPtr` | +| 52 | `size` | 4B | `AscPtr` (nullable) | + +Total: **56 bytes** (14 pointers) + +**`AscEthereumBlock_0_0_6`** (API versions >= 0.0.6): + +Same as above with one additional field: + +| Offset | Field | Size | Type | +|--------|-------|------|------| +| 56 | `base_fee_per_block` | 4B | `AscPtr` (nullable) | + +Total: **60 bytes** (15 pointers) + +**`INDEX_ASC_TYPE_ID`**: `EthereumBlock = 25` + +### 6.2 AscEthereumTransaction + +Three variants exist for different API versions. + +**`AscEthereumTransaction_0_0_1`** (API versions < 0.0.2): + +| Offset | Field | Size | Type | +|--------|-------|------|------| +| 0 | `hash` | 4B | `AscPtr` (B256) | +| 4 | `index` | 4B | `AscPtr` | +| 8 | `from` | 4B | `AscPtr` (Address) | +| 12 | `to` | 4B | `AscPtr` (Address, nullable) | +| 16 | `value` | 4B | `AscPtr` | +| 20 | `gas_limit` | 4B | `AscPtr` | +| 24 | `gas_price` | 4B | `AscPtr` | + +Total: **28 bytes** (7 pointers) + +**`AscEthereumTransaction_0_0_2`** (API versions >= 0.0.2, < 0.0.6): + +| Offset | Field | Size | Type | +|--------|-------|------|------| +| 0-24 | *(same as 0.0.1)* | 28B | | +| 28 | `input` | 4B | `AscPtr` | + +Total: **32 bytes** (8 pointers) + +**`AscEthereumTransaction_0_0_6`** (API versions >= 0.0.6): + +| Offset | Field | Size | Type | +|--------|-------|------|------| +| 0-28 | *(same as 0.0.2)* | 32B | | +| 32 | `nonce` | 4B | `AscPtr` | + +Total: **36 bytes** (9 pointers) + +**`INDEX_ASC_TYPE_ID`**: `EthereumTransaction = 24` + +### 6.3 AscEthereumEvent + +**`AscEthereumEvent`** (API versions < 0.0.7): + +| Offset | Field | Size | Type | +|--------|-------|------|------| +| 0 | `address` | 4B | `AscPtr` | +| 4 | `log_index` | 4B | `AscPtr` | +| 8 | `transaction_log_index` | 4B | `AscPtr` | +| 12 | `log_type` | 4B | `AscPtr` (nullable) | +| 16 | `block` | 4B | `AscPtr` | +| 20 | `transaction` | 4B | `AscPtr` | +| 24 | `params` | 4B | `AscPtr` | + +Total: **28 bytes** (7 pointers) + +The type parameters `T` and `B` are selected based on API version (see +[compatibility matrix](#7-api-version-compatibility-matrix)). + +**`AscEthereumEvent_0_0_7`** (API versions >= 0.0.7): + +| Offset | Field | Size | Type | +|--------|-------|------|------| +| 0-24 | *(same as above)* | 28B | | +| 28 | `receipt` | 4B | `AscPtr` (nullable) | + +Total: **32 bytes** (8 pointers) + +**`INDEX_ASC_TYPE_ID`**: `EthereumEvent = 33` + +### 6.4 AscLogParam + +A single decoded event parameter (name + value). + +``` ++------+-------+ +| name | value | +| 4 B | 4 B | ++------+-------+ +``` + +| Offset | Field | Size | Type | +|--------|-------|------|------| +| 0 | `name` | 4B | `AscPtr` | +| 4 | `value` | 4B | `AscPtr>` | + +Total: **8 bytes** (2 pointers) + +**`INDEX_ASC_TYPE_ID`**: `EventParam = 23` + +### 6.5 AscEthereumLog + +Raw Ethereum log (used inside transaction receipts in API >= 0.0.7). + +| Offset | Field | Size | Type | +|--------|-------|------|------| +| 0 | `address` | 4B | `AscPtr` | +| 4 | `topics` | 4B | `AscPtr` (Array of B256) | +| 8 | `data` | 4B | `AscPtr` | +| 12 | `block_hash` | 4B | `AscPtr` (B256, nullable) | +| 16 | `block_number` | 4B | `AscPtr` (nullable) | +| 20 | `transaction_hash` | 4B | `AscPtr` (B256, nullable) | +| 24 | `transaction_index` | 4B | `AscPtr` (nullable) | +| 28 | `log_index` | 4B | `AscPtr` (nullable) | +| 32 | `transaction_log_index` | 4B | `AscPtr` (always null) | +| 36 | `log_type` | 4B | `AscPtr` (always null) | +| 40 | `removed` | 4B | `AscPtr>` | + +Total: **44 bytes** (11 pointers) + +**`INDEX_ASC_TYPE_ID`**: `Log = 1001` + +### 6.6 AscEthereumTransactionReceipt + +| Offset | Field | Size | Type | +|--------|-------|------|------| +| 0 | `transaction_hash` | 4B | `AscPtr` (B256) | +| 4 | `transaction_index` | 4B | `AscPtr` | +| 8 | `block_hash` | 4B | `AscPtr` (B256, nullable) | +| 12 | `block_number` | 4B | `AscPtr` (nullable) | +| 16 | `cumulative_gas_used` | 4B | `AscPtr` | +| 20 | `gas_used` | 4B | `AscPtr` | +| 24 | `contract_address` | 4B | `AscPtr` (nullable) | +| 28 | `logs` | 4B | `AscPtr` (Array of AscEthereumLog) | +| 32 | `status` | 4B | `AscPtr` (nullable, pre-Byzantium) | +| 36 | `root` | 4B | `AscPtr` (B256, nullable) | +| 40 | `logs_bloom` | 4B | `AscPtr` (256 bytes) | + +Total: **44 bytes** (11 pointers) + +**`INDEX_ASC_TYPE_ID`**: `TransactionReceipt = 1000` + +### 6.7 AscEthereumCall + +**`AscEthereumCall`** (API versions < 0.0.3): + +| Offset | Field | Size | Type | +|--------|-------|------|------| +| 0 | `address` | 4B | `AscPtr` (call target) | +| 4 | `block` | 4B | `AscPtr` | +| 8 | `transaction` | 4B | `AscPtr` | +| 12 | `inputs` | 4B | `AscPtr` | +| 16 | `outputs` | 4B | `AscPtr` | + +Total: **20 bytes** (5 pointers) + +**`AscEthereumCall_0_0_3`** (API versions >= 0.0.3): + +| Offset | Field | Size | Type | +|--------|-------|------|------| +| 0 | `to` | 4B | `AscPtr` | +| 4 | `from` | 4B | `AscPtr` | +| 8 | `block` | 4B | `AscPtr` | +| 12 | `transaction` | 4B | `AscPtr` | +| 16 | `inputs` | 4B | `AscPtr` | +| 20 | `outputs` | 4B | `AscPtr` | + +Total: **24 bytes** (6 pointers) + +Added `from` address and renamed `address` to `to`. + +**`INDEX_ASC_TYPE_ID`**: `EthereumCall = 26` + +### 6.8 AscUnresolvedContractCall + +Used for `eth_call` from subgraph code (read direction: WASM -> Rust). + +**`AscUnresolvedContractCall_0_0_4`** (API versions <= 0.0.4): + +| Offset | Field | Size | Type | +|--------|-------|------|------| +| 0 | `contract_name` | 4B | `AscPtr` | +| 4 | `contract_address` | 4B | `AscPtr` | +| 8 | `function_name` | 4B | `AscPtr` | +| 12 | `function_signature` | 4B | `AscPtr` | +| 16 | `function_args` | 4B | `AscPtr>>>` | + +Total: **20 bytes** (5 pointers) + +**`AscUnresolvedContractCall`** (API versions > 0.0.4): + +| Offset | Field | Size | Type | +|--------|-------|------|------| +| 0 | `contract_name` | 4B | `AscPtr` | +| 4 | `contract_address` | 4B | `AscPtr` | +| 8 | `function_name` | 4B | `AscPtr` | +| 12 | `function_args` | 4B | `AscPtr>>>` | + +Total: **16 bytes** (4 pointers) + +The `function_signature` field was removed; function resolution uses name only. + +**`INDEX_ASC_TYPE_ID`**: `SmartContractCall = 22` (only for the `_0_0_4` variant) + +--- + +## 7. API Version Compatibility Matrix + +The API version determines which struct variants are used when serializing +triggers into WASM memory. + +### Event Handlers + +| API Version | Transaction Struct | Block Struct | Event Struct | +|---|---|---|---| +| < 0.0.2 | `AscEthereumTransaction_0_0_1` (28B) | `AscEthereumBlock` (56B) | `AscEthereumEvent` (28B) | +| >= 0.0.2, < 0.0.6 | `AscEthereumTransaction_0_0_2` (32B) | `AscEthereumBlock` (56B) | `AscEthereumEvent` (28B) | +| >= 0.0.6, < 0.0.7 | `AscEthereumTransaction_0_0_6` (36B) | `AscEthereumBlock_0_0_6` (60B) | `AscEthereumEvent` (28B) | +| >= 0.0.7 | `AscEthereumTransaction_0_0_6` (36B) | `AscEthereumBlock_0_0_6` (60B) | `AscEthereumEvent_0_0_7` (32B) | + +### Call Handlers + +| API Version | Transaction Struct | Block Struct | Call Struct | +|---|---|---|---| +| < 0.0.3 | `AscEthereumTransaction_0_0_1` (28B) | `AscEthereumBlock` (56B) | `AscEthereumCall` (20B) | +| >= 0.0.3, < 0.0.6 | `AscEthereumTransaction_0_0_2` (32B) | `AscEthereumBlock` (56B) | `AscEthereumCall_0_0_3` (24B) | +| >= 0.0.6 | `AscEthereumTransaction_0_0_6` (36B) | `AscEthereumBlock_0_0_6` (60B) | `AscEthereumCall_0_0_3` (24B) | + +### Block Handlers + +| API Version | Block Struct | +|---|---| +| < 0.0.6 | `AscEthereumBlock` (56B) | +| >= 0.0.6 | `AscEthereumBlock_0_0_6` (60B) | + +### Memory Layout Version + +| API Version | Layout | +|---|---| +| <= 0.0.4 | v0.0.4 (no header, inline length prefixes) | +| >= 0.0.5 | v0.0.5+ (20-byte header, 16-byte aligned) | + +Reference: `chain/ethereum/src/trigger.rs` (lines 127-230) + +--- + +## 8. Allocation Pattern + +### Bottom-Up Allocation + +Objects are allocated leaves-first. The `asc_new` function calls +`to_asc_obj` which recursively allocates child objects before constructing +the parent. Each `asc_new` call produces an `AscPtr` that is embedded in +the parent struct. + +### Arena Allocator + +Graph-node uses an arena allocator for WASM heap allocations. The AS +allocator provides a bulk memory region, and graph-node's `raw_new` performs +bump-pointer sub-allocation within it. + +### Example: Ethereum Event Handler Allocation Sequence + +For an event with one `uint256` parameter, the allocation sequence is +(simplified, showing just the types allocated bottom-up): + +``` +1. AscString "paramName" -- log param name +2. AscBigInt -- the uint256 value +3. AscEnum EthereumValueKind -- wraps the BigInt as Uint +4. AscLogParam (name, value) -- references #1 and #3 +5. ArrayBuffer [AscPtr to #4] -- backing buffer for params array +6. Array params -- references #5 +7. Uint8Array (x7) hash, from, to... -- transaction field byte arrays +8. AscBigInt (x5) index, value... -- transaction field big ints +9. AscEthereumTx transaction -- references #7, #8 +10. Uint8Array (x7) hash, parent... -- block field byte arrays +11. AscBigInt (x6) number, gas... -- block field big ints +12. AscEthereumBlock block -- references #10, #11 +13. Uint8Array event address -- 20-byte address +14. AscBigInt log_index -- event log index +15. AscBigInt tx_log_index -- transaction log index +16. AscEthereumEvent event -- references all above + ^ + | + Pointer passed to WASM handler as u32 argument +``` + +Each numbered step is a separate `raw_new` call that bumps the heap pointer. +The final event pointer (#16) is the single `u32` argument to the WASM +handler function. From 1a1808caa7f9b797fdc5f415d9daa5d24f53a596 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Sun, 15 Feb 2026 01:08:36 -0800 Subject: [PATCH 25/35] graph, ethereum: Pre-encode declared calls and skip redundant type checking Declared calls have their types validated at deployment time and their arguments don't change between blocks. Pre-encode the ABI call data in DeclaredCall::as_eth_call() and skip both the type string re-parsing and ABI re-encoding in the hot path (~6-9ms/block saved). --- chain/ethereum/src/ethereum_adapter.rs | 51 ++++++++++--------- chain/ethereum/src/runtime/runtime_adapter.rs | 1 + graph/src/data_source/common.rs | 6 +++ 3 files changed, 35 insertions(+), 23 deletions(-) diff --git a/chain/ethereum/src/ethereum_adapter.rs b/chain/ethereum/src/ethereum_adapter.rs index 5d9b78118a0..968e30d00a8 100644 --- a/chain/ethereum/src/ethereum_adapter.rs +++ b/chain/ethereum/src/ethereum_adapter.rs @@ -1486,32 +1486,37 @@ impl EthereumAdapterTrait for EthereumAdapter { call: &ContractCall, index: u32, ) -> Result { - // Emit custom error for type mismatches. - for (val, kind) in call - .args - .iter() - .zip(call.function.inputs.iter().map(|p| p.selector_type())) - { - let kind: abi::DynSolType = kind.parse().map_err(|err| { - ContractCallError::ABIError(anyhow!( - "failed to parse function input type '{kind}': {err}" - )) - })?; - - if !val.type_check(&kind) { - return Err(ContractCallError::TypeError(val.clone(), kind.clone())); - } - } + let encoded_call = match &call.encoded_call { + // Declared calls are pre-encoded; skip type checking and + // re-encoding + Some(pre_encoded) => pre_encoded.clone(), + None => { + // Emit custom error for type mismatches. + for (val, kind) in call + .args + .iter() + .zip(call.function.inputs.iter().map(|p| p.selector_type())) + { + let kind: abi::DynSolType = kind.parse().map_err(|err| { + ContractCallError::ABIError(anyhow!( + "failed to parse function input type '{kind}': {err}" + )) + })?; - // Encode the call parameters according to the ABI - let req = { - let encoded_call = call - .function - .abi_encode_input(&call.args) - .map_err(ContractCallError::EncodingError)?; - call::Request::new(call.address, encoded_call, index) + if !val.type_check(&kind) { + return Err(ContractCallError::TypeError(val.clone(), kind.clone())); + } + } + + // Encode the call parameters according to the ABI + call.function + .abi_encode_input(&call.args) + .map_err(ContractCallError::EncodingError)? + } }; + let req = call::Request::new(call.address, encoded_call, index); + trace!(logger, "eth_call"; "fn" => &call.function.name, "address" => hex::encode(call.address), diff --git a/chain/ethereum/src/runtime/runtime_adapter.rs b/chain/ethereum/src/runtime/runtime_adapter.rs index a5597efcd4d..9fa51cf63a6 100644 --- a/chain/ethereum/src/runtime/runtime_adapter.rs +++ b/chain/ethereum/src/runtime/runtime_adapter.rs @@ -336,6 +336,7 @@ async fn eth_call( function: function.clone(), args: unresolved_call.function_args.clone(), gas: eth_call_gas, + encoded_call: None, }; // Run Ethereum call in tokio runtime diff --git a/graph/src/data_source/common.rs b/graph/src/data_source/common.rs index bc9b27b5bd6..cecaef9c059 100644 --- a/graph/src/data_source/common.rs +++ b/graph/src/data_source/common.rs @@ -1388,6 +1388,7 @@ impl DeclaredCall { } pub fn as_eth_call(self, block_ptr: BlockPtr, gas: Option) -> (ContractCall, String) { + let encoded_call = self.function.abi_encode_input(&self.args).ok(); ( ContractCall { contract_name: self.contract_name, @@ -1396,6 +1397,7 @@ impl DeclaredCall { function: self.function, args: self.args, gas, + encoded_call, }, self.label, ) @@ -1409,6 +1411,10 @@ pub struct ContractCall { pub function: abi::Function, pub args: Vec, pub gas: Option, + /// Pre-encoded call data. When set, `contract_calls` skips type + /// checking and ABI encoding since the call was already validated and + /// encoded (e.g. for declared calls). + pub encoded_call: Option>, } #[cfg(test)] From 7f3fb34cd4934b15380dabe3d6efab45f4ec8ed3 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Sun, 15 Feb 2026 01:08:41 -0800 Subject: [PATCH 26/35] store: Remove unnecessary call_meta JOIN in batch get_calls The batch get_calls path JOINed call_meta to compute an expired flag that was immediately discarded. Replace with a simpler query that only reads from call_cache (~0.5ms/block saved). --- store/postgres/src/chain_store.rs | 46 ++++++++----------------------- 1 file changed, 12 insertions(+), 34 deletions(-) diff --git a/store/postgres/src/chain_store.rs b/store/postgres/src/chain_store.rs index 41bbb6add18..cb5a8e7656e 100644 --- a/store/postgres/src/chain_store.rs +++ b/store/postgres/src/chain_store.rs @@ -1305,56 +1305,36 @@ mod data { .map(|row| row.map(|(return_value, expired)| (Bytes::from(return_value), expired))) } - pub(super) async fn get_calls_and_access( + /// Batch-fetch cached call results by their IDs. Unlike the + /// singular `get_call_and_access`, this skips the `call_meta` + /// JOIN since the batch path does not need the expired flag. + pub(super) async fn get_calls_batch( &self, conn: &mut AsyncPgConnection, ids: &[&[u8]], - ) -> Result, Bytes, bool)>, Error> { + ) -> Result, Bytes)>, Error> { let rows = match self { Storage::Shared => { use public::eth_call_cache as cache; - use public::eth_call_meta as meta; cache::table - .inner_join(meta::table) .filter(cache::id.eq_any(ids)) - .select(( - cache::id, - cache::return_value, - sql::("CURRENT_DATE > eth_call_meta.accessed_at"), - )) + .select((cache::id, cache::return_value)) .load(conn) .await .map_err(Error::from) } - Storage::Private(Schema { - call_cache, - call_meta, - .. - }) => call_cache + Storage::Private(Schema { call_cache, .. }) => call_cache .table() - .inner_join( - call_meta.table().on(call_meta - .contract_address() - .eq(call_cache.contract_address())), - ) .filter(call_cache.id().eq_any(ids)) - .select(( - call_cache.id(), - call_cache.return_value(), - sql::(&format!( - "CURRENT_DATE > {}.{}", - CallMetaTable::TABLE_NAME, - CallMetaTable::ACCESSED_AT - )), - )) - .load::<(Vec, Vec, bool)>(conn) + .select((call_cache.id(), call_cache.return_value())) + .load::<(Vec, Vec)>(conn) .await .map_err(Error::from), }?; Ok(rows .into_iter() - .map(|(id, return_value, expired)| (id, Bytes::from(return_value), expired)) + .map(|(id, return_value)| (id, Bytes::from(return_value))) .collect()) } @@ -3327,15 +3307,13 @@ impl EthereumCallCache for ChainStore { let conn = &mut self.pool.get_permitted().await?; let rows = conn .transaction::<_, Error, _>(|conn| { - self.storage - .get_calls_and_access(conn, &id_refs) - .scope_boxed() + self.storage.get_calls_batch(conn, &id_refs).scope_boxed() }) .await?; let mut found: Vec = Vec::new(); let mut resps = Vec::new(); - for (id, retval, _) in rows { + for (id, retval) in rows { let idx = ids.iter().position(|i| i.as_ref() == id).ok_or_else(|| { internal_error!( "get_calls returned a call id that was not requested: {}", From 58894eef09f0e36d2a80eb59527595a656a35f48 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Sun, 15 Feb 2026 01:08:46 -0800 Subject: [PATCH 27/35] graph: Skip redundant sort in Entity::sorted_ref for pre-sorted entities Track whether Object entries are in sorted order via a flag set by sort_by_key() and cleared on unsorted insertions. Entity::sorted_ref() and sorted() now skip the sort when the flag is set, which is the common case for entities loaded from the cache (~1-2ms/block saved). --- graph/src/data/store/mod.rs | 9 +++++++-- graph/src/util/intern.rs | 14 ++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/graph/src/data/store/mod.rs b/graph/src/data/store/mod.rs index afdab8512fc..fa0607501f6 100644 --- a/graph/src/data/store/mod.rs +++ b/graph/src/data/store/mod.rs @@ -950,18 +950,23 @@ impl Entity { // This collects the entity into an ordered vector so that it can be iterated deterministically. pub fn sorted(self) -> Vec<(Word, Value)> { + let sorted = self.0.is_sorted(); let mut v: Vec<_> = self .0 .into_iter() .filter(|(k, _)| !k.eq(VID_FIELD)) .collect(); - v.sort_by(|(k1, _), (k2, _)| k1.cmp(k2)); + if !sorted { + v.sort_by(|(k1, _), (k2, _)| k1.cmp(k2)); + } v } pub fn sorted_ref(&self) -> Vec<(&str, &Value)> { let mut v: Vec<_> = self.0.iter().filter(|(k, _)| !k.eq(&VID_FIELD)).collect(); - v.sort_by(|(k1, _), (k2, _)| k1.cmp(k2)); + if !self.0.is_sorted() { + v.sort_by(|(k1, _), (k2, _)| k1.cmp(k2)); + } v } diff --git a/graph/src/util/intern.rs b/graph/src/util/intern.rs index ba47e146f2f..d13e7b8bbab 100644 --- a/graph/src/util/intern.rs +++ b/graph/src/util/intern.rs @@ -190,6 +190,9 @@ pub struct Object { // This could be further improved by using two `Vec`s, one for keys and // one for values. That would avoid losing memory to padding. entries: Vec>, + /// Set by `sort_by_key()`, cleared by mutations. When true, entries + /// are in key-sorted order and callers can skip re-sorting. + sorted: bool, } impl Object { @@ -198,6 +201,7 @@ impl Object { Self { pool, entries: Vec::new(), + sorted: false, } } @@ -266,6 +270,7 @@ impl Object { match self.entries.iter_mut().find(|entry| entry.key == key) { Some(entry) => Some(std::mem::replace(&mut entry.value, value)), None => { + self.sorted = false; self.entries.push(Entry { key, value }); None } @@ -279,6 +284,7 @@ impl Object { } pub fn merge(&mut self, other: Object) { + self.sorted = false; if self.same_pool(&other) { for Entry { key, value } in other.entries { self.insert_atom(key, value); @@ -335,6 +341,14 @@ impl Object { (None, None) => std::cmp::Ordering::Equal, } }); + self.sorted = true; + } + + /// Returns true if entries are known to be in key-sorted order, + /// i.e. `sort_by_key()` was called and no unsorted insertion has + /// happened since. + pub fn is_sorted(&self) -> bool { + self.sorted } } From 41586182b06161d75d72e03b0b6b39274f0f277f Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Sun, 15 Feb 2026 11:38:49 -0800 Subject: [PATCH 28/35] runtime: Cache entity field key AscPtrs via pre-built blob Pre-build a contiguous byte blob containing all entity field key strings as AscString objects, cached on ValidModule and written to each WASM instance's heap once per trigger via a single raw_new call. This eliminates per-key UTF-16 encoding and allocation overhead in store_get. --- graph/src/data/store/mod.rs | 6 + graph/src/schema/input/mod.rs | 10 +- graph/src/util/intern.rs | 38 ++++++ runtime/wasm/src/mapping.rs | 19 +++ runtime/wasm/src/module/context.rs | 189 +++++++++++++++++++++++++++-- 5 files changed, 253 insertions(+), 9 deletions(-) diff --git a/graph/src/data/store/mod.rs b/graph/src/data/store/mod.rs index fa0607501f6..4c496a92ff7 100644 --- a/graph/src/data/store/mod.rs +++ b/graph/src/data/store/mod.rs @@ -970,6 +970,12 @@ impl Entity { v } + /// Iterate entries as `(Atom, &Value)` pairs, including VID. + /// Callers that need to skip VID should filter by the vid atom themselves. + pub fn atom_entries(&self) -> impl Iterator + '_ { + self.0.atom_entries() + } + fn check_id(&self) -> Result<(), EntityValidationError> { match self.get("id") { None => Err(EntityValidationError::missing_id_attribute(format!( diff --git a/graph/src/schema/input/mod.rs b/graph/src/schema/input/mod.rs index 2ec52fe1762..61d66d771fe 100644 --- a/graph/src/schema/input/mod.rs +++ b/graph/src/schema/input/mod.rs @@ -1483,10 +1483,18 @@ impl InputSchema { } // A helper for the `EntityType` constructor - pub(in crate::schema) fn pool(&self) -> &Arc { + pub fn pool(&self) -> &Arc { &self.inner.pool } + /// Return the interned atom for the VID field. + pub fn vid_atom(&self) -> Atom { + self.inner + .pool + .lookup(VID_FIELD) + .expect("vid is always interned") + } + /// Return the entity type for `named`. If the entity type does not /// exist, return an error. Generally, an error should only be possible /// if `named` is based on user input. If `named` is an internal object, diff --git a/graph/src/util/intern.rs b/graph/src/util/intern.rs index d13e7b8bbab..cf69bce2708 100644 --- a/graph/src/util/intern.rs +++ b/graph/src/util/intern.rs @@ -31,6 +31,13 @@ type AtomInt = u16; #[derive(Eq, Hash, PartialEq, PartialOrd, Ord, Clone, Copy, CheapClone, Debug)] pub struct Atom(AtomInt); +impl Atom { + /// The underlying integer value of this atom, suitable for use as a Vec index. + pub fn as_usize(&self) -> usize { + self.0 as usize + } +} + /// An atom and the underlying pool. A `FatAtom` can be used in place of a /// `String` or `Word` #[allow(dead_code)] @@ -125,6 +132,24 @@ impl AtomPool { /// Add `word` to this pool if it is not already in it. Return the atom /// for the word. + /// Total number of atoms across this pool and all ancestors. + pub fn len(&self) -> usize { + self.base_sym as usize + self.atoms.len() + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Iterate all `(Atom, &str)` pairs across this pool and all ancestors. + pub fn iter_atoms(&self) -> impl Iterator + '_ { + (0..self.len()).map(move |i| { + let atom = Atom(i as AtomInt); + let s = self.get(atom).unwrap(); + (atom, s) + }) + } + pub fn intern(&mut self, word: &str) -> Atom { if let Some(atom) = self.lookup(word) { return atom; @@ -317,6 +342,19 @@ impl Object { AtomIter::new(self) } + /// Iterate entries as `(Atom, &V)` pairs, filtering tombstones. + pub fn atom_entries(&self) -> impl Iterator + '_ { + self.entries + .iter() + .filter(|entry| entry.key != TOMBSTONE_KEY) + .map(|entry| (entry.key, &entry.value)) + } + + /// Access the underlying atom pool. + pub fn pool(&self) -> &Arc { + &self.pool + } + /// Sort entries by their string key. Tombstone entries are moved to the /// end. This makes subsequent iteration in key order O(n) instead of /// requiring an O(n log n) sort. diff --git a/runtime/wasm/src/mapping.rs b/runtime/wasm/src/mapping.rs index e2c2aaec3a5..da5d79852a6 100644 --- a/runtime/wasm/src/mapping.rs +++ b/runtime/wasm/src/mapping.rs @@ -6,6 +6,7 @@ use graph::components::subgraph::SharedProofOfIndexing; use graph::parking_lot::RwLock; use graph::prelude::*; use graph::runtime::IndexForAscTypeId; +use graph::util::intern::Atom; use parity_wasm::elements::ExportEntry; use std::collections::{BTreeMap, HashMap}; use std::sync::{Arc, OnceLock}; @@ -108,6 +109,19 @@ fn shared_engine() -> &'static wasmtime::Engine { }) } +/// Pre-built byte blob containing all entity field key strings as AscString +/// objects. Written to each WASM instance's heap once per trigger via a single +/// `raw_new` call, eliminating per-key UTF-16 encoding and allocation overhead. +pub struct KeyBlobData { + /// Concatenated AscString bytes (with headers/padding for API >= 0.0.5). + pub blob: Vec, + /// Indexed by `Atom::as_usize()` → byte offset within blob where the + /// AscPtr should point. `None` for atoms that have no blob entry. + pub offsets: Vec>, + /// Atom for the "vid" field, used to skip during entity iteration. + pub vid_atom: Atom, +} + /// A pre-processed and valid WASM module, ready to be started as a WasmModule. pub struct ValidModule { pub module: wasmtime::Module, @@ -139,6 +153,10 @@ pub struct ValidModule { /// Cache for asc_type_id results. Maps IndexForAscTypeId to their WASM runtime /// type IDs. Populated lazily on first use; deterministic per compiled module. asc_type_id_cache: RwLock>, + + /// Cached key blob for entity field names. Built lazily on first store_get + /// and reused across all triggers for this module. + pub key_blob: RwLock>, } impl ValidModule { @@ -216,6 +234,7 @@ impl ValidModule { start_function, timeout, asc_type_id_cache: RwLock::new(HashMap::new()), + key_blob: RwLock::new(None), }) } diff --git a/runtime/wasm/src/module/context.rs b/runtime/wasm/src/module/context.rs index e5494c95803..f13bc32f81d 100644 --- a/runtime/wasm/src/module/context.rs +++ b/runtime/wasm/src/module/context.rs @@ -1,5 +1,11 @@ +use graph::data::store; use graph::data::value::Word; use graph::runtime::gas; +use graph::runtime::AscHeap; +use graph::runtime::IndexForAscTypeId; +use graph::runtime::{asc_new, gas::GasCounter, DeterministicHostError, HostExportError}; +use graph::runtime::{padding_to_16, AscPtr, AscType, HEADER_SIZE}; +use graph::util::intern::Atom; use graph::util::lfu_cache::LfuCache; use std::collections::HashMap; use wasmtime::AsContext; @@ -11,20 +17,17 @@ use std::time::Instant; use anyhow::Error; use graph::components::store::GetScope; +use graph::data::subgraph::API_VERSION_0_0_4; use never::Never; use crate::asc_abi::class::*; -use crate::HostExports; -use graph::data::store; - -use crate::asc_abi::class::AscEntity; -use crate::asc_abi::class::AscString; +use crate::mapping::KeyBlobData; use crate::mapping::MappingContext; use crate::mapping::ValidModule; +use crate::HostExports; + use crate::ExperimentalFeatures; use graph::prelude::*; -use graph::runtime::AscPtr; -use graph::runtime::{asc_new, gas::GasCounter, DeterministicHostError, HostExportError}; use super::asc_get; use super::AscHeapCtx; @@ -101,6 +104,11 @@ pub struct WasmInstanceData { // This option is needed to break the cyclic dependency between, instance, store, and context. // during execution it should always be populated. asc_heap: Option>, + + /// Per-trigger key blob state. Set lazily on first store_get that returns + /// an entity. Caches the WASM heap base pointer and the offset table so + /// subsequent store_gets don't need to read the ValidModule's RwLock. + key_blob_state: Option, } impl WasmInstanceData { @@ -120,6 +128,7 @@ impl WasmInstanceData { possible_reorg: false, deterministic_host_trap: false, experimental_features, + key_blob_state: None, } } @@ -141,6 +150,170 @@ impl WasmInstanceData { } } +/// Per-trigger snapshot of the key blob data: the WASM heap base pointer +/// plus the offset and vid_atom tables copied from `ValidModule::key_blob`. +struct KeyBlobState { + base_ptr: u32, + offsets: Vec>, + vid_atom: Atom, +} + +impl WasmInstanceContext<'_> { + /// Build the key blob for a schema pool. Creates AscString representations + /// for every interned atom and packs them into a single contiguous blob. + async fn build_key_blob( + &mut self, + pool: &Arc, + vid_atom: Atom, + ) -> Result { + let api_version = self.asc_heap().api_version.clone(); + let is_v0_0_5_plus = api_version > API_VERSION_0_0_4; + + let string_type_id = if is_v0_0_5_plus { + Some(self.asc_type_id(IndexForAscTypeId::String).await?) + } else { + None + }; + + let mut blob = Vec::new(); + let mut offsets = vec![None; pool.len()]; + + for (atom, s) in pool.iter_atoms() { + let utf16: Vec = s.encode_utf16().collect(); + let asc_string = AscString::new(&utf16, &api_version)?; + let mut bytes = asc_string.to_asc_bytes()?; + + if is_v0_0_5_plus { + let aligned_len = padding_to_16(bytes.len()); + bytes.extend(std::iter::repeat_n(0, aligned_len)); + + let content_len = asc_string.content_len(&bytes); + let full_length = bytes.len(); + let type_id = string_type_id.unwrap(); + + // Inline header generation (mirrors AscPtr::generate_header) + let mm_info = (16u32 + full_length as u32).to_le_bytes(); + let gc_info = 0u32.to_le_bytes(); + let gc_info2 = 0u32.to_le_bytes(); + let rt_id = type_id.to_le_bytes(); + let rt_size = (content_len as u32).to_le_bytes(); + + // AscPtr points after the header + offsets[atom.as_usize()] = Some(blob.len() as u32 + HEADER_SIZE as u32); + + blob.extend(mm_info); + blob.extend(gc_info); + blob.extend(gc_info2); + blob.extend(rt_id); + blob.extend(rt_size); + blob.extend(bytes); + } else { + // API <= 0.0.4: no header, AscPtr points to start of bytes + offsets[atom.as_usize()] = Some(blob.len() as u32); + blob.extend(bytes); + } + } + + Ok(KeyBlobData { + blob, + offsets, + vid_atom, + }) + } + + /// Ensure the key blob is built (on ValidModule) and written to this + /// trigger's WASM heap (on WasmInstanceData). Returns a reference to the + /// per-trigger `KeyBlobState`. + async fn ensure_key_blob_state(&mut self, gas: &GasCounter) -> Result<(), HostExportError> { + if self.as_ref().key_blob_state.is_some() { + return Ok(()); + } + + // Ensure blob is built on ValidModule (shared across triggers) + { + let needs_build = self.as_ref().valid_module.key_blob.read().is_none(); + if needs_build { + let schema = &self.as_ref().ctx.state.entity_cache.schema; + let pool = schema.pool().clone(); + let vid_atom = schema.vid_atom(); + let blob_data = self.build_key_blob(&pool, vid_atom).await?; + *self.as_ref().valid_module.key_blob.write() = Some(blob_data); + } + } + + // Clone blob bytes and metadata, then write to WASM heap + let (blob_bytes, offsets, vid_atom) = { + let guard = self.as_ref().valid_module.key_blob.read(); + let data = guard.as_ref().unwrap(); + (data.blob.clone(), data.offsets.clone(), data.vid_atom) + }; + + let base_ptr = self.raw_new(&blob_bytes, gas).await?; + self.as_mut().key_blob_state = Some(KeyBlobState { + base_ptr, + offsets, + vid_atom, + }); + Ok(()) + } + + /// Convert an Entity to AscEntity using cached key AscPtrs from the + /// pre-built blob, avoiding per-key UTF-16 encoding and allocation. + async fn entity_to_asc( + &mut self, + entity: &graph::components::store::Entity, + gas: &GasCounter, + ) -> Result, HostExportError> { + self.ensure_key_blob_state(gas).await?; + + // Read cached state (ensure_key_blob_state guarantees it's Some) + let base_ptr = self.as_ref().key_blob_state.as_ref().unwrap().base_ptr; + let vid_atom = self.as_ref().key_blob_state.as_ref().unwrap().vid_atom; + + // Collect (atom, value) pairs, filtering vid + let entries: Vec<(Atom, &store::Value)> = entity + .atom_entries() + .filter(|(atom, _)| *atom != vid_atom) + .collect(); + + // Build AscTypedMapEntry pointers for each field + let mut entry_ptrs: Vec>>> = + Vec::with_capacity(entries.len()); + + for (atom, value) in entries { + // Look up cached key AscPtr from blob + let offset = self.as_ref().key_blob_state.as_ref().unwrap().offsets[atom.as_usize()] + .unwrap_or_else(|| { + panic!("key blob missing offset for atom {:?}; this is a bug", atom) + }); + let key_ptr: AscPtr = AscPtr::new(base_ptr + offset); + + // Allocate value (always fresh) + let value_ptr: AscPtr> = asc_new(self, value, gas).await?; + + // Allocate entry struct + let entry = AscTypedMapEntry { + key: key_ptr, + value: value_ptr, + }; + let entry_ptr = AscPtr::alloc_obj(entry, self, gas).await?; + entry_ptrs.push(entry_ptr); + } + + // Allocate the entries array + let entries_array = Array::new(&entry_ptrs, self, gas).await?; + let entries_array_ptr = AscPtr::alloc_obj(entries_array, self, gas).await?; + + // Allocate the AscTypedMap (AscEntity) + let typed_map = AscTypedMap { + entries: entries_array_ptr, + }; + let entity_ptr = AscPtr::alloc_obj(typed_map, self, gas).await?; + + Ok(entity_ptr) + } +} + impl WasmInstanceContext<'_> { async fn store_get_scoped( &mut self, @@ -180,7 +353,7 @@ impl WasmInstanceContext<'_> { let ret = match entity_option { Some(entity) => { let _section = host_metrics.stopwatch.start_section("store_get_asc_new"); - asc_new(self, &entity.sorted_ref(), gas).await? + self.entity_to_asc(&entity, gas).await? } None => match &debug_fork { Some(fork) => { From 40302f09db392c5cfe4f59d81a59797af3ff9f2a Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Sun, 15 Feb 2026 17:24:41 -0800 Subject: [PATCH 29/35] graph, core, ethereum: Remove lifetime from HostedTrigger and RunnableTriggers Replace `&'a dyn RuntimeHost` with `Arc>` in HostedTrigger, decoupling trigger prep from execution lifetime-wise. This is a prerequisite for cross-block pipelining where prep for block N+1 runs concurrently with execution of block N. The hosts are already stored as `Arc`, so this change just clones the Arc (one atomic increment per matched trigger) instead of borrowing. All lifetime parameters on HostedTrigger, RunnableTriggers, Decoder, DecoderHook, and TriggerProcessor are removed. --- chain/ethereum/src/data_source.rs | 10 +++---- core/src/subgraph/context/instance/hosts.rs | 18 ++++++------- core/src/subgraph/context/instance/mod.rs | 2 +- core/src/subgraph/runner/mod.rs | 25 +++++++++-------- core/src/subgraph/runner/trigger_runner.rs | 2 +- core/src/subgraph/trigger_processor.rs | 30 ++++++++++----------- graph/src/blockchain/mod.rs | 12 ++++----- graph/src/components/trigger_processor.rs | 30 ++++++++++----------- 8 files changed, 64 insertions(+), 65 deletions(-) diff --git a/chain/ethereum/src/data_source.rs b/chain/ethereum/src/data_source.rs index b2303b6d053..9cff5c1a6ec 100644 --- a/chain/ethereum/src/data_source.rs +++ b/chain/ethereum/src/data_source.rs @@ -1040,9 +1040,9 @@ impl DecoderHook { Ok(labels) } - fn collect_declared_calls<'a>( + fn collect_declared_calls( &self, - runnables: &Vec>, + runnables: &[RunnableTriggers], ) -> Vec<(Arc, DeclaredCall)> { // Extract all hosted triggers from runnables let all_triggers = runnables @@ -1152,13 +1152,13 @@ impl DecoderHook { #[async_trait] impl blockchain::DecoderHook for DecoderHook { - async fn after_decode<'a>( + async fn after_decode( &self, logger: &Logger, block_ptr: &BlockPtr, - runnables: Vec>, + runnables: Vec>, metrics: &Arc, - ) -> Result>, MappingError> { + ) -> Result>, MappingError> { if ENV_VARS.mappings.disable_declared_calls { return Ok(runnables); } diff --git a/core/src/subgraph/context/instance/hosts.rs b/core/src/subgraph/context/instance/hosts.rs index 9c18e12ce1e..83730223c28 100644 --- a/core/src/subgraph/context/instance/hosts.rs +++ b/core/src/subgraph/context/instance/hosts.rs @@ -99,9 +99,9 @@ impl> OnchainHosts { pub fn matches_by_address( &self, address: Option<&[u8]>, - ) -> Box + Send + '_> { + ) -> Box> + Send + '_> { let Some(address) = address else { - return Box::new(self.hosts.iter().map(|host| host.as_ref())); + return Box::new(self.hosts.iter().cloned()); }; let mut matching_hosts: Vec = self @@ -116,7 +116,7 @@ impl> OnchainHosts { Box::new( matching_hosts .into_iter() - .map(move |idx| self.hosts[idx].as_ref()), + .map(move |idx| self.hosts[idx].clone()), ) } } @@ -191,12 +191,12 @@ impl> OffchainHosts { } } - pub fn matches_by_address<'a>( - &'a self, + pub fn matches_by_address( + &self, address: Option<&[u8]>, - ) -> Box + Send + 'a> { + ) -> Box> + Send + '_> { let Some(address) = address else { - return Box::new(self.by_block.values().flatten().map(|host| host.as_ref())); + return Box::new(self.by_block.values().flatten().cloned()); }; Box::new( @@ -204,8 +204,8 @@ impl> OffchainHosts { .get(address) .into_iter() .flatten() // Flatten non-existing `address` into empty. - .map(|host| host.as_ref()) - .chain(self.wildcard_address.iter().map(|host| host.as_ref())), + .cloned() + .chain(self.wildcard_address.iter().cloned()), ) } } diff --git a/core/src/subgraph/context/instance/mod.rs b/core/src/subgraph/context/instance/mod.rs index a6220f105c1..ed6b9ab14b2 100644 --- a/core/src/subgraph/context/instance/mod.rs +++ b/core/src/subgraph/context/instance/mod.rs @@ -234,7 +234,7 @@ where pub fn hosts_for_trigger( &self, trigger: &TriggerData, - ) -> Box + Send + '_> { + ) -> Box> + Send + '_> { match trigger { TriggerData::Onchain(trigger) => self .onchain_hosts diff --git a/core/src/subgraph/runner/mod.rs b/core/src/subgraph/runner/mod.rs index fd758149c3e..14e34476e90 100644 --- a/core/src/subgraph/runner/mod.rs +++ b/core/src/subgraph/runner/mod.rs @@ -807,15 +807,15 @@ where } } - async fn match_and_decode_many<'a, F>( - &'a self, + async fn match_and_decode_many( + &self, logger: &Logger, block: &Arc, triggers: Vec>, hosts_filter: F, - ) -> Result>, MappingError> + ) -> Result>, MappingError> where - F: Fn(&TriggerData) -> Box + Send + 'a>, + F: Fn(&TriggerData) -> Vec>, { let triggers = triggers.into_iter().map(|t| match t { Trigger::Chain(t) => TriggerData::Onchain(t), @@ -853,13 +853,14 @@ where /// /// Takes raw triggers from a block and matches them against all registered /// hosts, returning runnable triggers ready for execution. - async fn match_triggers<'a>( - &'a self, + async fn match_triggers( + &self, logger: &Logger, block: &Arc, triggers: Vec>, - ) -> Result>, MappingError> { - let hosts_filter = |trigger: &TriggerData| self.ctx.instance.hosts_for_trigger(trigger); + ) -> Result>, MappingError> { + let hosts_filter = + |trigger: &TriggerData| self.ctx.instance.hosts_for_trigger(trigger).collect(); self.match_and_decode_many(logger, block, triggers, hosts_filter) .await } @@ -871,7 +872,7 @@ where async fn execute_triggers( &self, block: &Arc, - runnables: Vec>, + runnables: Vec>, block_state: BlockState, proof_of_indexing: &SharedProofOfIndexing, causality_region: &str, @@ -969,9 +970,7 @@ where // Process the triggers in each host in the same order the // corresponding data sources have been created. - let hosts_filter = |_: &'_ TriggerData| -> Box + Send> { - Box::new(runtime_hosts.iter().map(Arc::as_ref)) - }; + let hosts_filter = |_: &TriggerData| -> Vec> { runtime_hosts.clone() }; let runnables = self .match_and_decode_many(logger, &block, triggers, hosts_filter) .await; @@ -1520,7 +1519,7 @@ where let trigger = TriggerData::Offchain(trigger); let process_res = { - let hosts = self.ctx.instance.hosts_for_trigger(&trigger); + let hosts: Vec<_> = self.ctx.instance.hosts_for_trigger(&trigger).collect(); let triggers_res = self.ctx.decoder.match_and_decode( &self.logger, block, diff --git a/core/src/subgraph/runner/trigger_runner.rs b/core/src/subgraph/runner/trigger_runner.rs index 6c89052e5dc..08ec1752cb6 100644 --- a/core/src/subgraph/runner/trigger_runner.rs +++ b/core/src/subgraph/runner/trigger_runner.rs @@ -49,7 +49,7 @@ where pub async fn execute( &self, block: &Arc, - runnables: Vec>, + runnables: Vec>, block_state: BlockState, proof_of_indexing: &SharedProofOfIndexing, causality_region: &str, diff --git a/core/src/subgraph/trigger_processor.rs b/core/src/subgraph/trigger_processor.rs index c3123e87268..71688ddea1f 100644 --- a/core/src/subgraph/trigger_processor.rs +++ b/core/src/subgraph/trigger_processor.rs @@ -21,10 +21,10 @@ where C: Blockchain, T: RuntimeHostBuilder, { - async fn process_trigger<'a>( - &'a self, + async fn process_trigger( + &self, logger: &Logger, - triggers: Vec>, + triggers: Vec>, block: &Arc, mut state: BlockState, proof_of_indexing: &SharedProofOfIndexing, @@ -108,14 +108,14 @@ where } impl> Decoder { - fn match_and_decode_inner<'a>( - &'a self, + fn match_and_decode_inner( + &self, logger: &Logger, block: &Arc, trigger: &TriggerData, - hosts: Box + Send + 'a>, + hosts: Vec>, subgraph_metrics: &Arc, - ) -> Result>, MappingError> { + ) -> Result>, MappingError> { let mut host_mapping = vec![]; { @@ -139,14 +139,14 @@ impl> Decoder { Ok(host_mapping) } - pub(crate) fn match_and_decode<'a>( - &'a self, + pub(crate) fn match_and_decode( + &self, logger: &Logger, block: &Arc, trigger: TriggerData, - hosts: Box + Send + 'a>, + hosts: Vec>, subgraph_metrics: &Arc, - ) -> Result, MappingError> { + ) -> Result, MappingError> { self.match_and_decode_inner(logger, block, &trigger, hosts, subgraph_metrics) .map_err(|e| e.add_trigger_context(&trigger)) .map(|hosted_triggers| RunnableTriggers { @@ -155,16 +155,16 @@ impl> Decoder { }) } - pub(crate) async fn match_and_decode_many<'a, F>( - &'a self, + pub(crate) async fn match_and_decode_many( + &self, logger: &Logger, block: &Arc, triggers: impl Iterator>, hosts_filter: F, metrics: &Arc, - ) -> Result>, MappingError> + ) -> Result>, MappingError> where - F: Fn(&TriggerData) -> Box + Send + 'a>, + F: Fn(&TriggerData) -> Vec>, { let mut runnables = vec![]; for trigger in triggers { diff --git a/graph/src/blockchain/mod.rs b/graph/src/blockchain/mod.rs index f65fcea2e5b..9302b7c45c2 100644 --- a/graph/src/blockchain/mod.rs +++ b/graph/src/blockchain/mod.rs @@ -497,13 +497,13 @@ pub trait MappingTriggerTrait { /// A callback that is called after the triggers have been decoded. #[async_trait] pub trait DecoderHook { - async fn after_decode<'a>( + async fn after_decode( &self, logger: &Logger, block_ptr: &BlockPtr, - triggers: Vec>, + triggers: Vec>, metrics: &Arc, - ) -> Result>, MappingError>; + ) -> Result>, MappingError>; } /// A decoder hook that does nothing and just returns the triggers that were @@ -512,13 +512,13 @@ pub struct NoopDecoderHook; #[async_trait] impl DecoderHook for NoopDecoderHook { - async fn after_decode<'a>( + async fn after_decode( &self, _: &Logger, _: &BlockPtr, - triggers: Vec>, + triggers: Vec>, _: &Arc, - ) -> Result>, MappingError> { + ) -> Result>, MappingError> { Ok(triggers) } } diff --git a/graph/src/components/trigger_processor.rs b/graph/src/components/trigger_processor.rs index f21fe5b7894..499d3b61b20 100644 --- a/graph/src/components/trigger_processor.rs +++ b/graph/src/components/trigger_processor.rs @@ -16,22 +16,22 @@ use super::{ /// A trigger that is almost ready to run: we have a host to run it on, and /// transformed the `TriggerData` into a `MappingTrigger`. -pub struct HostedTrigger<'a, C> +pub struct HostedTrigger where C: Blockchain, { - pub host: &'a dyn RuntimeHost, + pub host: Arc>, pub mapping_trigger: TriggerWithHandler>, } /// The `TriggerData` and the `HostedTriggers` that were derived from it. We /// need to hang on to the `TriggerData` solely for error reporting. -pub struct RunnableTriggers<'a, C> +pub struct RunnableTriggers where C: Blockchain, { pub trigger: TriggerData, - pub hosted_triggers: Vec>, + pub hosted_triggers: Vec>, } #[async_trait] @@ -40,10 +40,10 @@ where C: Blockchain, T: RuntimeHostBuilder, { - async fn process_trigger<'a>( - &'a self, + async fn process_trigger( + &self, logger: &Logger, - triggers: Vec>, + triggers: Vec>, block: &Arc, mut state: BlockState, proof_of_indexing: &SharedProofOfIndexing, @@ -63,25 +63,25 @@ where C: Blockchain, T: RuntimeHostBuilder, { - fn match_and_decode<'a>( - &'a self, + fn match_and_decode( + &self, logger: &Logger, block: &Arc, trigger: TriggerData, - hosts: Box + Send + 'a>, + hosts: Vec>, subgraph_metrics: &Arc, - ) -> Result, MappingError>; + ) -> Result, MappingError>; - fn match_and_decode_many<'a, F>( - &'a self, + fn match_and_decode_many( + &self, logger: &Logger, block: &Arc, triggers: Box>>, hosts_filter: F, subgraph_metrics: &Arc, - ) -> Result>, MappingError> + ) -> Result>, MappingError> where - F: Fn(&TriggerData) -> Box + Send + 'a>, + F: Fn(&TriggerData) -> Vec>, { let mut runnables = vec![]; for trigger in triggers { From 064cf35cf60a73a0bff24bbd3915429cdc88e044 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Sun, 15 Feb 2026 17:31:46 -0800 Subject: [PATCH 30/35] core: Make Decoder cloneable via Arc-wrapped hook Wrap the DecoderHook in Arc and implement Clone for the Decoder struct, enabling it to be cheaply shared across async tasks. This is a prerequisite for spawning a concurrent prep task in cross-block pipelining. Also removes the unnecessary Box indirection from IndexingContext.decoder. --- core/src/subgraph/context/mod.rs | 4 ++-- core/src/subgraph/instance_manager.rs | 2 +- core/src/subgraph/trigger_processor.rs | 13 +++++++++++-- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/core/src/subgraph/context/mod.rs b/core/src/subgraph/context/mod.rs index fa11dff8cf6..cc5ec60ff2c 100644 --- a/core/src/subgraph/context/mod.rs +++ b/core/src/subgraph/context/mod.rs @@ -75,7 +75,7 @@ where pub instances: SubgraphKeepAlive, pub offchain_monitor: OffchainMonitor, pub(crate) trigger_processor: Box>, - pub(crate) decoder: Box>, + pub(crate) decoder: Decoder, } impl> IndexingContext { @@ -87,7 +87,7 @@ impl> IndexingContext { instances: SubgraphKeepAlive, offchain_monitor: OffchainMonitor, trigger_processor: Box>, - decoder: Box>, + decoder: Decoder, ) -> Self { let instance = SubgraphInstance::new( manifest, diff --git a/core/src/subgraph/instance_manager.rs b/core/src/subgraph/instance_manager.rs index 5d0c89ae171..4cc765bf3d0 100644 --- a/core/src/subgraph/instance_manager.rs +++ b/core/src/subgraph/instance_manager.rs @@ -487,7 +487,7 @@ impl SubgraphInstanceManager { let instrument = self.subgraph_store.instrument(&deployment).await?; - let decoder = Box::new(Decoder::new(decoder_hook)); + let decoder = Decoder::new(decoder_hook); let subgraph_data_source_stores = self .get_sourceable_stores::(subgraph_ds_source_deployments, is_runner_test) diff --git a/core/src/subgraph/trigger_processor.rs b/core/src/subgraph/trigger_processor.rs index 71688ddea1f..c95f30fa74b 100644 --- a/core/src/subgraph/trigger_processor.rs +++ b/core/src/subgraph/trigger_processor.rs @@ -90,10 +90,19 @@ where C: Blockchain, T: RuntimeHostBuilder, { - hook: C::DecoderHook, + hook: Arc, _builder: PhantomData, } +impl> Clone for Decoder { + fn clone(&self) -> Self { + Decoder { + hook: self.hook.clone(), + _builder: PhantomData, + } + } +} + impl Decoder where C: Blockchain, @@ -101,7 +110,7 @@ where { pub fn new(hook: C::DecoderHook) -> Self { Decoder { - hook, + hook: Arc::new(hook), _builder: PhantomData, } } From aad1fc88c2305efaf2dc0f9b67790ad619ff89e1 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Sun, 15 Feb 2026 17:39:52 -0800 Subject: [PATCH 31/35] core: Add hosts snapshot capability for cross-block pipelining Add Send + 'static snapshot types (OnchainHostsSnapshot, OffchainHostsSnapshot, HostsSnapshot) that clone Arc host pointers and address indices without mutation methods. This enables spawning a prep task for block N+1 while block N is executing, as the snapshot can be moved to a separate tokio task independently of SubgraphInstance. --- core/src/subgraph/context/instance/hosts.rs | 134 +++++++++++++++++++- core/src/subgraph/context/instance/mod.rs | 15 +++ 2 files changed, 148 insertions(+), 1 deletion(-) diff --git a/core/src/subgraph/context/instance/hosts.rs b/core/src/subgraph/context/instance/hosts.rs index 83730223c28..01564d35095 100644 --- a/core/src/subgraph/context/instance/hosts.rs +++ b/core/src/subgraph/context/instance/hosts.rs @@ -4,12 +4,13 @@ use std::{ }; use graph::{ - blockchain::Blockchain, + blockchain::{Blockchain, TriggerData as _}, cheap_clone::CheapClone, components::{ store::BlockNumber, subgraph::{RuntimeHost, RuntimeHostBuilder}, }, + data_source::TriggerData, }; /// This structure maintains a partition of the hosts by address, for faster trigger matching. This @@ -93,6 +94,15 @@ impl> OnchainHosts { } } + #[allow(dead_code)] + pub fn snapshot(&self) -> OnchainHostsSnapshot { + OnchainHostsSnapshot { + hosts: self.hosts.clone(), + hosts_by_address: self.hosts_by_address.clone(), + hosts_without_address: self.hosts_without_address.clone(), + } + } + /// Returns an iterator over all hosts that match the given address, in the order they were inserted in `hosts`. /// Note that this always includes the hosts without an address, since they match all addresses. /// If no address is provided, returns an iterator over all hosts. @@ -208,4 +218,126 @@ impl> OffchainHosts { .chain(self.wildcard_address.iter().cloned()), ) } + + #[allow(dead_code)] + pub fn snapshot(&self) -> OffchainHostsSnapshot { + OffchainHostsSnapshot { + all_hosts: self.by_block.values().flatten().cloned().collect(), + by_address: self.by_address.clone(), + wildcard_address: self.wildcard_address.clone(), + } + } +} + +/// A `Send + 'static` snapshot of `OnchainHosts` that supports `matches_by_address`. +/// Created by cloning the Arc hosts and the address index; no mutation methods. +#[allow(dead_code)] +pub(super) struct OnchainHostsSnapshot> { + hosts: Vec>, + hosts_by_address: HashMap, Vec>, + hosts_without_address: Vec, +} + +#[allow(dead_code)] +impl> OnchainHostsSnapshot { + pub fn matches_by_address( + &self, + address: Option<&[u8]>, + ) -> Box> + Send + '_> { + let Some(address) = address else { + return Box::new(self.hosts.iter().cloned()); + }; + + let mut matching_hosts: Vec = self + .hosts_by_address + .get(address) + .into_iter() + .flatten() + .copied() + .chain(self.hosts_without_address.iter().copied()) + .collect(); + matching_hosts.sort(); + Box::new( + matching_hosts + .into_iter() + .map(move |idx| self.hosts[idx].clone()), + ) + } +} + +/// A `Send + 'static` snapshot of `OffchainHosts` that supports `matches_by_address`. +#[allow(dead_code)] +pub(super) struct OffchainHostsSnapshot> { + all_hosts: Vec>, + by_address: BTreeMap, Vec>>, + wildcard_address: Vec>, +} + +#[allow(dead_code)] +impl> OffchainHostsSnapshot { + pub fn matches_by_address( + &self, + address: Option<&[u8]>, + ) -> Box> + Send + '_> { + let Some(address) = address else { + return Box::new(self.all_hosts.iter().cloned()); + }; + + Box::new( + self.by_address + .get(address) + .into_iter() + .flatten() + .cloned() + .chain(self.wildcard_address.iter().cloned()), + ) + } +} + +/// A `Send + 'static` snapshot of all host collections, providing the same +/// `hosts_for_trigger` dispatch as `SubgraphInstance`. +#[allow(dead_code)] +pub(crate) struct HostsSnapshot> { + onchain: OnchainHostsSnapshot, + subgraph: OnchainHostsSnapshot, + offchain: OffchainHostsSnapshot, + hosts_len: usize, +} + +#[allow(dead_code)] +impl> HostsSnapshot { + pub(super) fn new( + onchain: OnchainHostsSnapshot, + subgraph: OnchainHostsSnapshot, + offchain: OffchainHostsSnapshot, + hosts_len: usize, + ) -> Self { + Self { + onchain, + subgraph, + offchain, + hosts_len, + } + } + + pub fn hosts_for_trigger(&self, trigger: &TriggerData) -> Vec> { + match trigger { + TriggerData::Onchain(trigger) => self + .onchain + .matches_by_address(trigger.address_match()) + .collect(), + TriggerData::Offchain(trigger) => self + .offchain + .matches_by_address(trigger.source.address().as_deref()) + .collect(), + TriggerData::Subgraph(trigger) => self + .subgraph + .matches_by_address(Some(trigger.source.to_bytes().as_slice())) + .collect(), + } + } + + pub fn hosts_len(&self) -> usize { + self.hosts_len + } } diff --git a/core/src/subgraph/context/instance/mod.rs b/core/src/subgraph/context/instance/mod.rs index ed6b9ab14b2..d69053d50ad 100644 --- a/core/src/subgraph/context/instance/mod.rs +++ b/core/src/subgraph/context/instance/mod.rs @@ -9,6 +9,8 @@ use graph::{ }, prelude::*, }; +#[allow(unused_imports)] +pub(crate) use hosts::HostsSnapshot; use hosts::{OffchainHosts, OnchainHosts}; use std::collections::HashMap; @@ -255,4 +257,17 @@ where pub fn hosts_len(&self) -> usize { self.onchain_hosts.len() + self.offchain_hosts.len() } + + /// Creates a `Send + 'static` snapshot of the current host state for use + /// in a spawned prep task. The snapshot is cheap: it clones Arc pointers + /// and the address indices but not the underlying hosts. + #[allow(dead_code)] + pub fn hosts_snapshot(&self) -> HostsSnapshot { + HostsSnapshot::new( + self.onchain_hosts.snapshot(), + self.subgraph_hosts.snapshot(), + self.offchain_hosts.snapshot(), + self.hosts_len(), + ) + } } From b32baacac5969865578a669c4392513769a0015c Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Sun, 15 Feb 2026 17:53:00 -0800 Subject: [PATCH 32/35] core: Extract prep_block as standalone async function for cross-block pipelining Refactor the trigger matching/decoding stage (Stage 1 of process_block) into a standalone `prep_block` function that takes owned/cloned state instead of borrowing &self from the runner. This enables the same logic to be driven from a spawned task for cross-block pipelining in a subsequent step. - Add PrepResult struct to hold matched triggers and host count - Add prep_block() free function using HostsSnapshot + Decoder - Refactor match_triggers() to delegate to prep_block() - Remove #[allow(dead_code)] from snapshot types now that they're used --- core/src/subgraph/context/instance/hosts.rs | 8 --- core/src/subgraph/context/instance/mod.rs | 2 - core/src/subgraph/context/mod.rs | 1 + core/src/subgraph/runner/mod.rs | 77 +++++++++++++++++---- 4 files changed, 66 insertions(+), 22 deletions(-) diff --git a/core/src/subgraph/context/instance/hosts.rs b/core/src/subgraph/context/instance/hosts.rs index 01564d35095..49e835e3329 100644 --- a/core/src/subgraph/context/instance/hosts.rs +++ b/core/src/subgraph/context/instance/hosts.rs @@ -94,7 +94,6 @@ impl> OnchainHosts { } } - #[allow(dead_code)] pub fn snapshot(&self) -> OnchainHostsSnapshot { OnchainHostsSnapshot { hosts: self.hosts.clone(), @@ -219,7 +218,6 @@ impl> OffchainHosts { ) } - #[allow(dead_code)] pub fn snapshot(&self) -> OffchainHostsSnapshot { OffchainHostsSnapshot { all_hosts: self.by_block.values().flatten().cloned().collect(), @@ -231,14 +229,12 @@ impl> OffchainHosts { /// A `Send + 'static` snapshot of `OnchainHosts` that supports `matches_by_address`. /// Created by cloning the Arc hosts and the address index; no mutation methods. -#[allow(dead_code)] pub(super) struct OnchainHostsSnapshot> { hosts: Vec>, hosts_by_address: HashMap, Vec>, hosts_without_address: Vec, } -#[allow(dead_code)] impl> OnchainHostsSnapshot { pub fn matches_by_address( &self, @@ -266,14 +262,12 @@ impl> OnchainHostsSnapshot { } /// A `Send + 'static` snapshot of `OffchainHosts` that supports `matches_by_address`. -#[allow(dead_code)] pub(super) struct OffchainHostsSnapshot> { all_hosts: Vec>, by_address: BTreeMap, Vec>>, wildcard_address: Vec>, } -#[allow(dead_code)] impl> OffchainHostsSnapshot { pub fn matches_by_address( &self, @@ -296,7 +290,6 @@ impl> OffchainHostsSnapshot { /// A `Send + 'static` snapshot of all host collections, providing the same /// `hosts_for_trigger` dispatch as `SubgraphInstance`. -#[allow(dead_code)] pub(crate) struct HostsSnapshot> { onchain: OnchainHostsSnapshot, subgraph: OnchainHostsSnapshot, @@ -304,7 +297,6 @@ pub(crate) struct HostsSnapshot> { hosts_len: usize, } -#[allow(dead_code)] impl> HostsSnapshot { pub(super) fn new( onchain: OnchainHostsSnapshot, diff --git a/core/src/subgraph/context/instance/mod.rs b/core/src/subgraph/context/instance/mod.rs index d69053d50ad..02b563ee163 100644 --- a/core/src/subgraph/context/instance/mod.rs +++ b/core/src/subgraph/context/instance/mod.rs @@ -9,7 +9,6 @@ use graph::{ }, prelude::*, }; -#[allow(unused_imports)] pub(crate) use hosts::HostsSnapshot; use hosts::{OffchainHosts, OnchainHosts}; use std::collections::HashMap; @@ -261,7 +260,6 @@ where /// Creates a `Send + 'static` snapshot of the current host state for use /// in a spawned prep task. The snapshot is cheap: it clones Arc pointers /// and the address indices but not the underlying hosts. - #[allow(dead_code)] pub fn hosts_snapshot(&self) -> HostsSnapshot { HostsSnapshot::new( self.onchain_hosts.snapshot(), diff --git a/core/src/subgraph/context/mod.rs b/core/src/subgraph/context/mod.rs index cc5ec60ff2c..e173fc229d9 100644 --- a/core/src/subgraph/context/mod.rs +++ b/core/src/subgraph/context/mod.rs @@ -28,6 +28,7 @@ use std::sync::Arc; use graph::parking_lot::RwLock; use tokio::sync::mpsc; +pub(crate) use self::instance::HostsSnapshot; use self::instance::SubgraphInstance; use super::Decoder; diff --git a/core/src/subgraph/runner/mod.rs b/core/src/subgraph/runner/mod.rs index 14e34476e90..66760c4332a 100644 --- a/core/src/subgraph/runner/mod.rs +++ b/core/src/subgraph/runner/mod.rs @@ -1,7 +1,7 @@ mod state; mod trigger_runner; -use crate::subgraph::context::IndexingContext; +use crate::subgraph::context::{HostsSnapshot, IndexingContext}; use crate::subgraph::error::{ ClassifyErrorHelper as _, DetailHelper as _, NonDeterministicErrorHelper as _, ProcessingError, ProcessingErrorKind, @@ -9,6 +9,7 @@ use crate::subgraph::error::{ use crate::subgraph::inputs::IndexingInputs; use crate::subgraph::state::IndexingState; use crate::subgraph::stream::new_block_stream; +use crate::subgraph::trigger_processor::Decoder; use anyhow::Context as _; use graph::blockchain::block_stream::{ BlockStream, BlockStreamEvent, BlockWithTriggers, FirehoseCursor, @@ -36,7 +37,7 @@ use graph::prelude::{ anyhow, hex, retry, thiserror, BlockNumber, BlockPtr, BlockState, CancelGuard, CancelHandle, CancelToken as _, CheapClone as _, EntityCache, EntityModification, Error, InstanceDSTemplateInfo, LogCode, RunnerMetrics, RuntimeHostBuilder, StopwatchMetrics, - StoreError, StreamExtension, UnfailOutcome, Value, ENV_VARS, + StoreError, StreamExtension, SubgraphInstanceMetrics, UnfailOutcome, Value, ENV_VARS, }; use graph::schema::EntityKey; use graph::slog::{debug, error, info, o, trace, warn, Logger}; @@ -57,6 +58,50 @@ const PROCESS_BLOCK_SECTION_NAME: &str = "process_block"; const PROCESS_TRIGGERS_SECTION_NAME: &str = "process_triggers"; const HANDLE_CREATED_DS_SECTION_NAME: &str = "handle_new_data_sources"; +/// The result of the prep stage: matched and decoded triggers ready for +/// execution, plus a snapshot of the host count at prep time (used later +/// by the pipelining logic to detect stale preps after DDS creation). +pub(crate) struct PrepResult { + pub runnables: Vec>, + /// Used by cross-block pipelining (Step 5) to detect stale prep results + /// when DDS have been created between prep and execution. + #[allow(dead_code)] + pub hosts_len: usize, +} + +/// Standalone prep function that matches and decodes triggers without +/// borrowing the runner. Takes owned/cloned state so it can be called +/// inline *or* on a spawned task for cross-block pipelining. +async fn prep_block( + decoder: &Decoder, + hosts: &HostsSnapshot, + logger: &Logger, + block: &Arc, + triggers: Vec>, + metrics: &Arc, +) -> Result, MappingError> +where + C: Blockchain, + T: RuntimeHostBuilder, +{ + let triggers = triggers.into_iter().map(|t| match t { + Trigger::Chain(t) => TriggerData::Onchain(t), + Trigger::Subgraph(t) => TriggerData::Subgraph(t), + }); + + let hosts_len = hosts.hosts_len(); + let hosts_filter = |trigger: &TriggerData| hosts.hosts_for_trigger(trigger); + + let runnables = decoder + .match_and_decode_many(logger, block, triggers, hosts_filter, metrics) + .await?; + + Ok(PrepResult { + runnables, + hosts_len, + }) +} + pub struct SubgraphRunner where C: Blockchain, @@ -852,17 +897,25 @@ where /// Pipeline Stage 1: Match triggers to hosts and decode them. /// /// Takes raw triggers from a block and matches them against all registered - /// hosts, returning runnable triggers ready for execution. + /// hosts, returning runnable triggers ready for execution. Delegates to + /// the standalone `prep_block` function so that the same logic can later + /// be driven from a spawned task for cross-block pipelining. async fn match_triggers( &self, logger: &Logger, block: &Arc, triggers: Vec>, - ) -> Result>, MappingError> { - let hosts_filter = - |trigger: &TriggerData| self.ctx.instance.hosts_for_trigger(trigger).collect(); - self.match_and_decode_many(logger, block, triggers, hosts_filter) - .await + ) -> Result, MappingError> { + let hosts = self.ctx.instance.hosts_snapshot(); + prep_block( + &self.ctx.decoder, + &hosts, + logger, + block, + triggers, + &self.metrics.subgraph, + ) + .await } /// Pipeline Stage 2: Execute matched triggers. @@ -1095,14 +1148,14 @@ where .start_section(PROCESS_TRIGGERS_SECTION_NAME); // Stage 1: Match triggers to hosts and decode - let runnables = self.match_triggers(&logger, &block, triggers).await; + let prep = self.match_triggers(&logger, &block, triggers).await; // Stage 2: Execute triggers - let res = match runnables { - Ok(runnables) => { + let res = match prep { + Ok(prep) => { self.execute_triggers( &block, - runnables, + prep.runnables, block_state, &proof_of_indexing, &causality_region, From 69b163468cc2ac8c522a89cc68c890f80ec7ca33 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Sun, 15 Feb 2026 18:13:40 -0800 Subject: [PATCH 33/35] core: Implement cross-block prep pipelining for block processing After block N finishes processing, eagerly poll the block stream for block N+1. If immediately available, spawn a tokio task that runs trigger matching and decoding (the "prep" phase) concurrently with the state machine's next iteration. When block N+1 enters process_block, the pipelined prep result is awaited instead of prepping inline, hiding ~32ms of prep latency behind the execution + finalization of the previous block. Key design decisions: - Triggers are moved (not cloned) to the prep task. If the task fails (panic), a non-deterministic error is raised and the next retry preps inline. - Host count is checked at consumption time as defense-in-depth against stale preps after DDS creation. - Pending preps are discarded on revert, restart (DDS), and block skip. - Guarded by GRAPH_NODE_PIPELINE_PREP env var (default: enabled). --- core/src/subgraph/runner/mod.rs | 206 +++++++++++++++++++++++++++++--- graph/src/env/mappings.rs | 12 ++ 2 files changed, 204 insertions(+), 14 deletions(-) diff --git a/core/src/subgraph/runner/mod.rs b/core/src/subgraph/runner/mod.rs index 66760c4332a..efe3632d8ad 100644 --- a/core/src/subgraph/runner/mod.rs +++ b/core/src/subgraph/runner/mod.rs @@ -46,6 +46,7 @@ use graph::util::{backoff::ExponentialBackoff, lfu_cache::LfuCache}; use std::sync::Arc; use std::time::{Duration, Instant}; use std::vec; +use tokio::task::JoinHandle; use self::state::{RestartReason, RunnerState, StopReason}; use self::trigger_runner::TriggerRunner; @@ -63,8 +64,8 @@ const HANDLE_CREATED_DS_SECTION_NAME: &str = "handle_new_data_sources"; /// by the pipelining logic to detect stale preps after DDS creation). pub(crate) struct PrepResult { pub runnables: Vec>, - /// Used by cross-block pipelining (Step 5) to detect stale prep results - /// when DDS have been created between prep and execution. + /// Number of hosts at prep time. Planned for use in observability + /// metrics (Step 7) and as an additional staleness check. #[allow(dead_code)] pub hosts_len: usize, } @@ -102,6 +103,21 @@ where }) } +/// A prep result being computed ahead of time for the next block. +/// Stored on the runner between `try_eagerly_prep_next_block` (which +/// spawns the task) and `process_block` (which awaits and consumes it). +struct PendingPrep { + /// Handle to the spawned prep task. + handle: JoinHandle, MappingError>>, + /// Number of hosts at the time the prep was spawned, used to detect + /// stale preps after DDS creation. + hosts_len: usize, + /// Original trigger count (since triggers are moved to the prep task + /// and the `BlockWithTriggers` passed through the state machine has + /// empty `trigger_data`). + trigger_count: usize, +} + pub struct SubgraphRunner where C: Blockchain, @@ -116,6 +132,10 @@ where /// The current state in the runner's state machine. /// This field drives the main loop of the runner. runner_state: RunnerState, + /// A prep result computed for the next block by cross-block pipelining. + /// Populated after a successful block processing when the next block is + /// immediately available from the stream. + pending_prep: Option>, } #[derive(Debug, thiserror::Error)] @@ -157,6 +177,7 @@ where metrics, cancel_handle: None, runner_state: RunnerState::Initializing, + pending_prep: None, } } @@ -595,20 +616,28 @@ where .deployment_head .set(block_ptr.number as f64); - if block.trigger_count() > 0 { + // When pipelining, triggers have been moved to the prep task so + // block.trigger_count() is 0. Use the saved count instead. + let trigger_count = self + .pending_prep + .as_ref() + .map_or_else(|| block.trigger_count(), |p| p.trigger_count); + + if trigger_count > 0 { self.metrics .subgraph .block_trigger_count - .observe(block.trigger_count() as f64); + .observe(trigger_count as f64); } // Check if we should skip this block (optimization for blocks without triggers) - if block.trigger_count() == 0 + if trigger_count == 0 && self.state.skip_ptr_updates_timer.elapsed() <= SKIP_PTR_UPDATES_THRESHOLD && !self.inputs.store.is_deployment_synced() && !close_to_chain_head(&block_ptr, &self.inputs.chain.chain_head_ptr().await?, 1000) { - // Skip this block and continue with the same stream + // Skip this block — also discard pending prep if any. + self.discard_pending_prep(); return Ok(RunnerState::AwaitingBlock { block_stream }); } else { self.state.skip_ptr_updates_timer = Instant::now(); @@ -636,10 +665,17 @@ where // Convert Action to RunnerState match action { - Action::Continue => Ok(RunnerState::AwaitingBlock { block_stream }), - Action::Restart => Ok(RunnerState::Restarting { - reason: RestartReason::DynamicDataSourceCreated, - }), + Action::Continue => { + let next_state = self.try_eagerly_prep_next_block(block_stream); + Ok(next_state) + } + Action::Restart => { + // DDS created — discard any pending prep since the host set changed. + self.discard_pending_prep(); + Ok(RunnerState::Restarting { + reason: RestartReason::DynamicDataSourceCreated, + }) + } Action::Stop => Ok(RunnerState::Stopped { reason: StopReason::MaxEndBlockReached, }), @@ -656,6 +692,9 @@ where revert_to_ptr: BlockPtr, cursor: FirehoseCursor, ) -> Result, SubgraphRunnerError> { + // Discard any pipelined prep — the chain reorganized. + self.discard_pending_prep(); + let stopwatch = &self.metrics.stream.stopwatch; let _section = stopwatch.start_section(HANDLE_REVERT_SECTION_NAME); @@ -918,6 +957,103 @@ where .await } + /// After block N completes successfully, try to eagerly poll the block + /// stream for the next event. If a `ProcessBlock` event is immediately + /// available, spawn a prep task (trigger matching + decoding) for it + /// so the work overlaps with the current block's finalization and the + /// next iteration of the state machine. + /// + /// Returns the next `RunnerState`: + /// - `ProcessingBlock` if we got the next block (prep task is running) + /// - `Reverting` / `AwaitingBlock` / `Stopped` for other events + fn try_eagerly_prep_next_block( + &mut self, + mut block_stream: Cancelable>>, + ) -> RunnerState { + use graph::futures03::Stream as _; + use std::pin::Pin; + use std::task::{Context as TaskContext, Poll, Waker}; + + if !ENV_VARS.mappings.enable_pipeline_prep { + return RunnerState::AwaitingBlock { block_stream }; + } + + // Non-blocking poll: check if the stream already has a next item + // buffered (firehose streams typically buffer at least 1 block). + let waker = Waker::noop(); + let mut cx = TaskContext::from_waker(waker); + let poll_result = Pin::new(&mut block_stream).poll_next(&mut cx); + + match poll_result { + Poll::Ready(Some(Ok(BlockStreamEvent::ProcessBlock(mut block, cursor)))) => { + // Next block is immediately available — spawn a prep task. + let trigger_count = block.trigger_count(); + let triggers = std::mem::take(&mut block.trigger_data); + + let decoder = self.ctx.decoder.clone(); + let hosts = self.ctx.instance.hosts_snapshot(); + let logger = self.logger.cheap_clone(); + let metrics = self.metrics.subgraph.cheap_clone(); + let block_arc: Arc = Arc::new(block.block.clone()); + + let handle = tokio::spawn(async move { + prep_block(&decoder, &hosts, &logger, &block_arc, triggers, &metrics).await + }); + + debug!( + self.logger, + "Spawned pipelined prep for next block"; + "block" => block.ptr().number + ); + + // Store the join handle; the block itself goes into + // ProcessingBlock for the normal state machine flow. + let hosts_len = self.ctx.hosts_len(); + self.pending_prep = Some(PendingPrep { + handle, + hosts_len, + trigger_count, + }); + + // Jump directly to ProcessingBlock, bypassing AwaitingBlock. + RunnerState::ProcessingBlock { + block_stream, + block, + cursor, + } + } + Poll::Ready(Some(Ok(BlockStreamEvent::Revert(to_ptr, cursor)))) => { + // Consumed a revert event — handle it. + RunnerState::Reverting { + block_stream, + to_ptr, + cursor, + } + } + Poll::Ready(Some(Err(e))) => { + debug!( + self.logger, + "Block stream produced a non-fatal error during eager poll"; + "error" => format!("{}", e), + ); + RunnerState::AwaitingBlock { block_stream } + } + Poll::Ready(None) => RunnerState::Stopped { + reason: StopReason::StreamEnded, + }, + Poll::Pending => RunnerState::AwaitingBlock { block_stream }, + } + } + + /// Discard any pending pipelined prep result. + fn discard_pending_prep(&mut self) { + if let Some(pending) = self.pending_prep.take() { + // Aborting the JoinHandle cancels the spawned task. + pending.handle.abort(); + debug!(self.logger, "Discarded pipelined prep result"); + } + } + /// Pipeline Stage 2: Execute matched triggers. /// /// Takes runnable triggers and executes them using the TriggerRunner, @@ -1118,6 +1254,7 @@ where block: BlockWithTriggers, firehose_cursor: FirehoseCursor, ) -> Result { + let pipelined = self.pending_prep.is_some(); let triggers = block.trigger_data; let block = Arc::new(block.block); let block_ptr = block.ptr(); @@ -1127,8 +1264,12 @@ where "block_hash" => format!("{}", block_ptr.hash) )); - info!(logger, "Start processing block"; - "triggers" => triggers.len()); + if pipelined { + info!(logger, "Start processing block (pipelined)"); + } else { + info!(logger, "Start processing block"; + "triggers" => triggers.len()); + } let proof_of_indexing = SharedProofOfIndexing::new(block_ptr.number, self.inputs.poi_version); @@ -1147,8 +1288,45 @@ where .stopwatch .start_section(PROCESS_TRIGGERS_SECTION_NAME); - // Stage 1: Match triggers to hosts and decode - let prep = self.match_triggers(&logger, &block, triggers).await; + // Stage 1: Match triggers to hosts and decode. + // If we have a pipelined prep result from cross-block pipelining, + // await it instead of prepping inline. + let prep = if let Some(pending) = self.pending_prep.take() { + // Validate that the host set hasn't changed (defense in depth). + if pending.hosts_len != self.ctx.hosts_len() { + warn!( + logger, + "Discarding pipelined prep: host count changed"; + "prep_hosts" => pending.hosts_len, + "current_hosts" => self.ctx.hosts_len() + ); + pending.handle.abort(); + // Triggers were moved to the prep task; since we're + // discarding it, we can't fall back to inline prep. + // Return an error that will be retried (without pipelining). + return Err(ProcessingError::Unknown(anyhow!( + "Pipelined prep invalidated by host count change" + ))); + } + match pending.handle.await { + Ok(result) => { + debug!(logger, "Using pipelined prep for block"); + result + } + Err(join_err) => { + // Task panicked or was cancelled. Triggers were moved to + // the prep task so we can't fall back to inline prep. + // Signal a non-deterministic error; the next retry will + // prep inline because pending_prep will be None. + return Err(ProcessingError::Unknown(anyhow!( + "Pipelined prep task failed: {}", + join_err + ))); + } + } + } else { + self.match_triggers(&logger, &block, triggers).await + }; // Stage 2: Execute triggers let res = match prep { diff --git a/graph/src/env/mappings.rs b/graph/src/env/mappings.rs index 152a78e8189..512cd60ca10 100644 --- a/graph/src/env/mappings.rs +++ b/graph/src/env/mappings.rs @@ -96,6 +96,15 @@ pub struct EnvVarsMapping { /// /// Set by `GRAPH_WASM_INSTANCE_POOL_SIZE`. Defaults to 1000. pub wasm_instance_pool_size: u32, + + /// Enable cross-block prep pipelining. When enabled, the runner eagerly + /// polls the block stream for the next block and spawns a prep task + /// (trigger matching + decoding) concurrently with the current block's + /// execution, hiding ~32ms of prep latency behind ~73ms of + /// execution + finalization. + /// + /// Set by the flag `GRAPH_NODE_PIPELINE_PREP`. On by default. + pub enable_pipeline_prep: bool, } /// Cranelift optimization level for WASM compilation. Maps to @@ -172,6 +181,7 @@ impl TryFrom for EnvVarsMapping { fds_max_backoff: Duration::from_secs(x.fds_max_backoff), wasm_opt_level: x.wasm_opt_level, wasm_instance_pool_size: x.wasm_instance_pool_size, + enable_pipeline_prep: x.enable_pipeline_prep.0, }; Ok(vars) } @@ -219,6 +229,8 @@ pub struct InnerMappingHandlers { wasm_opt_level: WasmOptLevel, #[envconfig(from = "GRAPH_WASM_INSTANCE_POOL_SIZE", default = "1000")] wasm_instance_pool_size: u32, + #[envconfig(from = "GRAPH_NODE_PIPELINE_PREP", default = "true")] + enable_pipeline_prep: EnvVarBoolean, } fn validate_ipfs_cache_location(path: PathBuf) -> Result { From 91c1bdc41230dc1129d6378a9cf11327f29e863d Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Sun, 15 Feb 2026 18:23:28 -0800 Subject: [PATCH 34/35] core: Abort pipelined prep task on drop for safe cleanup Add AbortOnDrop wrapper around JoinHandle to ensure spawned prep tasks are always cancelled when the PendingPrep is dropped. This covers edge cases where the runner terminates via error propagation (e.g., failed chain_head_ptr call during skip check) without explicitly discarding the pending prep. Previously, dropping a JoinHandle would detach the task, leaving it running uselessly. --- core/src/subgraph/runner/mod.rs | 46 ++++++++++++++++++++++++++------- 1 file changed, 36 insertions(+), 10 deletions(-) diff --git a/core/src/subgraph/runner/mod.rs b/core/src/subgraph/runner/mod.rs index efe3632d8ad..4be0446f57a 100644 --- a/core/src/subgraph/runner/mod.rs +++ b/core/src/subgraph/runner/mod.rs @@ -103,12 +103,39 @@ where }) } +/// Wrapper around `JoinHandle` that aborts the spawned task on drop. +/// This ensures the prep task is always cleaned up, even when the +/// runner terminates due to error propagation or cancellation. +struct AbortOnDrop(Option>); + +impl AbortOnDrop { + fn new(handle: JoinHandle) -> Self { + Self(Some(handle)) + } + + /// Await the task result, consuming the handle without aborting. + async fn await_result(&mut self) -> Result { + self.0 + .take() + .expect("AbortOnDrop: handle already consumed") + .await + } +} + +impl Drop for AbortOnDrop { + fn drop(&mut self) { + if let Some(handle) = self.0.take() { + handle.abort(); + } + } +} + /// A prep result being computed ahead of time for the next block. /// Stored on the runner between `try_eagerly_prep_next_block` (which /// spawns the task) and `process_block` (which awaits and consumes it). struct PendingPrep { - /// Handle to the spawned prep task. - handle: JoinHandle, MappingError>>, + /// Handle to the spawned prep task; aborted automatically on drop. + handle: AbortOnDrop, MappingError>>, /// Number of hosts at the time the prep was spawned, used to detect /// stale preps after DDS creation. hosts_len: usize, @@ -1010,7 +1037,7 @@ where // ProcessingBlock for the normal state machine flow. let hosts_len = self.ctx.hosts_len(); self.pending_prep = Some(PendingPrep { - handle, + handle: AbortOnDrop::new(handle), hosts_len, trigger_count, }); @@ -1045,11 +1072,10 @@ where } } - /// Discard any pending pipelined prep result. + /// Discard any pending pipelined prep result. The spawned task is + /// aborted automatically via `PendingPrep`'s `Drop` implementation. fn discard_pending_prep(&mut self) { - if let Some(pending) = self.pending_prep.take() { - // Aborting the JoinHandle cancels the spawned task. - pending.handle.abort(); + if self.pending_prep.take().is_some() { debug!(self.logger, "Discarded pipelined prep result"); } } @@ -1291,7 +1317,7 @@ where // Stage 1: Match triggers to hosts and decode. // If we have a pipelined prep result from cross-block pipelining, // await it instead of prepping inline. - let prep = if let Some(pending) = self.pending_prep.take() { + let prep = if let Some(mut pending) = self.pending_prep.take() { // Validate that the host set hasn't changed (defense in depth). if pending.hosts_len != self.ctx.hosts_len() { warn!( @@ -1300,15 +1326,15 @@ where "prep_hosts" => pending.hosts_len, "current_hosts" => self.ctx.hosts_len() ); - pending.handle.abort(); // Triggers were moved to the prep task; since we're // discarding it, we can't fall back to inline prep. + // The spawned task is aborted when `pending` is dropped. // Return an error that will be retried (without pipelining). return Err(ProcessingError::Unknown(anyhow!( "Pipelined prep invalidated by host count change" ))); } - match pending.handle.await { + match pending.handle.await_result().await { Ok(result) => { debug!(logger, "Using pipelined prep for block"); result From 10a30e34b3e15ea5995ad2a59cd09c0b1fd33392 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Sun, 15 Feb 2026 18:34:07 -0800 Subject: [PATCH 35/35] graph, core: Add observability metrics for cross-block prep pipelining Add Prometheus counters and histogram to track pipelining behavior: - deployment_pipeline_prep_hit: pipelined prep result was used - deployment_pipeline_prep_miss: prep done inline (no pipelining) - deployment_pipeline_prep_discard: prep discarded (DDS, revert, skip) - deployment_pipeline_prep_overlap_secs: time the prep task ran concurrently with the previous block's execution --- core/src/subgraph/runner/mod.rs | 23 ++++++++--- graph/src/components/metrics/subgraph.rs | 52 ++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 5 deletions(-) diff --git a/core/src/subgraph/runner/mod.rs b/core/src/subgraph/runner/mod.rs index 4be0446f57a..54135431afe 100644 --- a/core/src/subgraph/runner/mod.rs +++ b/core/src/subgraph/runner/mod.rs @@ -60,12 +60,11 @@ const PROCESS_TRIGGERS_SECTION_NAME: &str = "process_triggers"; const HANDLE_CREATED_DS_SECTION_NAME: &str = "handle_new_data_sources"; /// The result of the prep stage: matched and decoded triggers ready for -/// execution, plus a snapshot of the host count at prep time (used later -/// by the pipelining logic to detect stale preps after DDS creation). +/// execution, plus a snapshot of the host count at prep time (used by the +/// pipelining logic to detect stale preps after DDS creation). pub(crate) struct PrepResult { pub runnables: Vec>, - /// Number of hosts at prep time. Planned for use in observability - /// metrics (Step 7) and as an additional staleness check. + /// Number of hosts at prep time, used as a staleness check. #[allow(dead_code)] pub hosts_len: usize, } @@ -143,6 +142,9 @@ struct PendingPrep { /// and the `BlockWithTriggers` passed through the state machine has /// empty `trigger_data`). trigger_count: usize, + /// When the prep task was spawned, used to compute overlap with the + /// previous block's execution. + spawn_time: Instant, } pub struct SubgraphRunner @@ -1040,6 +1042,7 @@ where handle: AbortOnDrop::new(handle), hosts_len, trigger_count, + spawn_time: Instant::now(), }); // Jump directly to ProcessingBlock, bypassing AwaitingBlock. @@ -1076,6 +1079,7 @@ where /// aborted automatically via `PendingPrep`'s `Drop` implementation. fn discard_pending_prep(&mut self) { if self.pending_prep.take().is_some() { + self.metrics.subgraph.observe_pipeline_prep_discard(); debug!(self.logger, "Discarded pipelined prep result"); } } @@ -1320,6 +1324,7 @@ where let prep = if let Some(mut pending) = self.pending_prep.take() { // Validate that the host set hasn't changed (defense in depth). if pending.hosts_len != self.ctx.hosts_len() { + self.metrics.subgraph.observe_pipeline_prep_discard(); warn!( logger, "Discarding pipelined prep: host count changed"; @@ -1336,10 +1341,17 @@ where } match pending.handle.await_result().await { Ok(result) => { - debug!(logger, "Using pipelined prep for block"); + // Overlap = time from spawn until now (i.e. the time + // the prep task ran concurrently with the previous + // block's execution and finalization). + let overlap = pending.spawn_time.elapsed().as_secs_f64(); + self.metrics.subgraph.observe_pipeline_prep_hit(overlap); + debug!(logger, "Using pipelined prep for block"; + "overlap_secs" => format!("{:.3}", overlap)); result } Err(join_err) => { + self.metrics.subgraph.observe_pipeline_prep_discard(); // Task panicked or was cancelled. Triggers were moved to // the prep task so we can't fall back to inline prep. // Signal a non-deterministic error; the next retry will @@ -1351,6 +1363,7 @@ where } } } else { + self.metrics.subgraph.observe_pipeline_prep_miss(); self.match_triggers(&logger, &block, triggers).await }; diff --git a/graph/src/components/metrics/subgraph.rs b/graph/src/components/metrics/subgraph.rs index abcf2810e6a..43cf5577525 100644 --- a/graph/src/components/metrics/subgraph.rs +++ b/graph/src/components/metrics/subgraph.rs @@ -24,6 +24,11 @@ pub struct SubgraphInstanceMetrics { trigger_processing_duration: Box, blocks_processed_secs: Box, blocks_processed_count: Box, + + pipeline_prep_hit: Counter, + pipeline_prep_miss: Counter, + pipeline_prep_discard: Counter, + pipeline_prep_overlap_secs: Box, } impl SubgraphInstanceMetrics { @@ -96,6 +101,36 @@ impl SubgraphInstanceMetrics { let deployment_synced = DeploymentSyncedMetric::register(®istry, subgraph_hash, &stopwatch.shard()); + let pipeline_prep_hit = registry + .new_deployment_counter( + "deployment_pipeline_prep_hit", + "Number of blocks where a pipelined prep result was used", + subgraph_hash, + ) + .expect("failed to create pipeline_prep_hit counter"); + let pipeline_prep_miss = registry + .new_deployment_counter( + "deployment_pipeline_prep_miss", + "Number of blocks where prep was done inline (no pipelining)", + subgraph_hash, + ) + .expect("failed to create pipeline_prep_miss counter"); + let pipeline_prep_discard = registry + .new_deployment_counter( + "deployment_pipeline_prep_discard", + "Number of pipelined prep results discarded (DDS, revert, skip)", + subgraph_hash, + ) + .expect("failed to create pipeline_prep_discard counter"); + let pipeline_prep_overlap_secs = registry + .new_deployment_histogram( + "deployment_pipeline_prep_overlap_secs", + "Time saved by pipelining: overlap between prep and previous block's execution", + subgraph_hash, + vec![0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0], + ) + .expect("failed to create pipeline_prep_overlap_secs histogram"); + Self { block_trigger_count, block_processing_duration, @@ -107,6 +142,10 @@ impl SubgraphInstanceMetrics { trigger_processing_duration, blocks_processed_secs, blocks_processed_count, + pipeline_prep_hit, + pipeline_prep_miss, + pipeline_prep_discard, + pipeline_prep_overlap_secs, } } @@ -121,6 +160,19 @@ impl SubgraphInstanceMetrics { } } + pub fn observe_pipeline_prep_hit(&self, overlap_secs: f64) { + self.pipeline_prep_hit.inc(); + self.pipeline_prep_overlap_secs.observe(overlap_secs); + } + + pub fn observe_pipeline_prep_miss(&self) { + self.pipeline_prep_miss.inc(); + } + + pub fn observe_pipeline_prep_discard(&self) { + self.pipeline_prep_discard.inc(); + } + pub fn unregister(&self, registry: Arc) { registry.unregister(self.block_processing_duration.clone()); registry.unregister(self.block_trigger_count.clone());