diff --git a/.cargo/config.toml b/.cargo/config.toml index 9d360ec0167..f6795e3c6ec 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,3 +1,5 @@ -[alias] -# Warnings create a lot of noise, we only print errors. -check-clippy = "clippy --no-deps -- --allow warnings" +# Can be safely removed once Cargo's sparse protocol (see +# https://blog.rust-lang.org/2023/03/09/Rust-1.68.0.html#cargos-sparse-protocol) +# becomes the default. +[registries.crates-io] +protocol = "sparse" diff --git a/.config/nextest.toml b/.config/nextest.toml new file mode 100644 index 00000000000..3fa8a93aced --- /dev/null +++ b/.config/nextest.toml @@ -0,0 +1,15 @@ +[[profile.default.overrides]] +filter = 'package(graphman-server)' +priority = -1 +threads-required = 'num-test-threads' # Global mutex + +[[profile.default.overrides]] +filter = 'package(test-store)' +priority = -2 +threads-required = 'num-test-threads' # Global mutex + +[[profile.default.overrides]] +filter = 'package(graph-tests)' +priority = -3 +threads-required = 'num-test-threads' # Global mutex +slow-timeout = { period = "300s", terminate-after = 4 } diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 00000000000..4bd1bc06468 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,42 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the +// README at: https://github.com/devcontainers/templates/tree/main/src/rust +{ + "name": "Rust", + "dockerComposeFile": "docker-compose.yml", + "service": "devcontainer", + "workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}", + "features": { + "ghcr.io/devcontainers/features/rust:1": { + "version": "1.66.0" + } + }, + "customizations": { + "vscode": { + "extensions": [ + "rust-lang.rust-analyzer@prerelease", // rust analyser, pre-release has less bugs + "cschleiden.vscode-github-actions", // github actions + "serayuzgur.crates", // crates + "vadimcn.vscode-lldb" //debug + ], + "settings": { + "editor.formatOnSave": true, + "terminal.integrated.defaultProfile.linux": "zsh" + } + } + }, + + // Use 'mounts' to make the cargo cache persistent in a Docker Volume. + // "mounts": [ + // { + // "source": "devcontainer-cargo-cache-${devcontainerId}", + // "target": "/usr/local/cargo", + // "type": "volume" + // } + // ] + "forwardPorts": [ + 8000, // GraphiQL on node-port + 8020, // create and deploy subgraphs + 5001 //ipfs + ] + +} diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml new file mode 100644 index 00000000000..d26201cc800 --- /dev/null +++ b/.devcontainer/docker-compose.yml @@ -0,0 +1,32 @@ +version: '3' + +services: + devcontainer: + image: mcr.microsoft.com/vscode/devcontainers/rust:bullseye + volumes: + - ../..:/workspaces:cached + network_mode: service:database + command: sleep infinity + ipfs: + image: ipfs/kubo:v0.18.1 + restart: unless-stopped + network_mode: service:database + database: + image: postgres:latest + restart: unless-stopped + command: + [ + "postgres", + "-cshared_preload_libraries=pg_stat_statements" + ] + volumes: + - postgres-data:/var/lib/postgresql/data + environment: + POSTGRES_USER: graph-node + POSTGRES_PASSWORD: let-me-in + POSTGRES_DB: graph-node + + POSTGRES_INITDB_ARGS: "-E UTF8 --locale=C" + +volumes: + postgres-data: \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md deleted file mode 100644 index cb67232349c..00000000000 --- a/.github/ISSUE_TEMPLATE.md +++ /dev/null @@ -1,7 +0,0 @@ -**Do you want to request a *feature* or report a *bug*?** - -**What is the current behavior?** - -**If the current behavior is a bug, please provide the steps to reproduce and if possible a minimal demo of the problem.** - -**What is the expected behavior?** diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml new file mode 100644 index 00000000000..4fe935160de --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug.yml @@ -0,0 +1,54 @@ +name: Bug report +description: Use this issue template if something is not working the way it should be. +title: "[Bug] " +labels: ["bug"] +body: + - type: markdown + attributes: + value: | + Thanks for taking the time to fill out this bug report! + - type: textarea + id: bug-report + attributes: + label: Bug report + description: Please provide a detailed overview of the expected behavior, and what happens instead. The more details, the better. You can use Markdown. + - type: textarea + id: graph-node-logs + attributes: + label: Relevant log output + description: Please copy and paste any relevant log output (either graph-node or hosted service logs). This will be automatically formatted into code, so no need for backticks. Leave blank if it doesn't apply. + render: Shell + - type: markdown + attributes: + value: Does this bug affect a specific subgraph deployment? If not, leave the following blank. + - type: input + attributes: + label: IPFS hash + placeholder: e.g. QmST8VZnjHrwhrW5gTyaiWJDhVcx6TooRv85B49zG7ziLH + validations: + required: false + - type: input + attributes: + label: Subgraph name or link to explorer + placeholder: e.g. https://thegraph.com/explorer/subgraphs/3nXfK3RbFrj6mhkGdoKRowEEti2WvmUdxmz73tben6Mb?view=Overview&chain=mainnet + validations: + required: false + - type: checkboxes + id: checkboxes + attributes: + label: Some information to help us out + options: + - label: Tick this box if this bug is caused by a regression found in the latest release. + - label: Tick this box if this bug is specific to the hosted service. + - label: I have searched the issue tracker to make sure this issue is not a duplicate. + required: true + - type: dropdown + id: operating-system + attributes: + label: OS information + description: What OS are you running? Leave blank if it doesn't apply. + options: + - Windows + - macOS + - Linux + - Other (please specify in your bug report) diff --git a/.github/ISSUE_TEMPLATE/feature.yml b/.github/ISSUE_TEMPLATE/feature.yml new file mode 100644 index 00000000000..47fa2619714 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature.yml @@ -0,0 +1,24 @@ +name: Feature request +description: To request or discuss new features. +title: "[Feature] " +labels: ["enhancement"] +body: + - type: textarea + id: feature-description + attributes: + label: Description + description: Please provide a detailed overview of the desired feature or improvement, along with any examples or useful information. You can use Markdown. + - type: textarea + id: blockers + attributes: + label: Are you aware of any blockers that must be resolved before implementing this feature? If so, which? Link to any relevant GitHub issues. + validations: + required: false + - type: checkboxes + id: checkboxes + attributes: + label: Some information to help us out + options: + - label: Tick this box if you plan on implementing this feature yourself. + - label: I have searched the issue tracker to make sure this issue is not a duplicate. + required: true diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 44043c3072e..977a3b8fc50 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,5 +1,17 @@ version: 2 updates: + +- package-ecosystem: npm + directory: tests/integration-tests + schedule: + interval: weekly + open-pull-requests-limit: 10 + allow: + # We always want to test against the latest Graph CLI tooling: `graph-cli`, + # `graph-ts`. + - dependency-name: "@graphprotocol/graph-*" + versioning-strategy: lockfile-only + - package-ecosystem: cargo directory: "/" schedule: diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml index 102a8b53e78..96fa5ba1cb8 100644 --- a/.github/workflows/audit.yml +++ b/.github/workflows/audit.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/checkout@v2 - - uses: actions-rs/audit-check@v1 + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 + - uses: rustsec/audit-check@69366f33c96575abad1ee0dba8212993eecbe998 #v2.0.0 with: token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bd8dead4c92..59bbd6ce598 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,21 +1,30 @@ name: Continuous Integration - on: push: branches: [master] pull_request: - types: [opened, synchronize, reopened] + branches: [master] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.ref }} env: CARGO_TERM_COLOR: always RUST_BACKTRACE: full - THEGRAPH_STORE_POSTGRES_DIESEL_URL: "postgresql://postgres:postgres@localhost:5432/graph_node_test" + RUSTFLAGS: "-C link-arg=-fuse-ld=lld -D warnings" + THEGRAPH_STORE_POSTGRES_DIESEL_URL: "postgresql://graph:graph@localhost:5432/graph-test" + GRAPH_STORE_CONNECTION_TIMEOUT: "30000" jobs: unit-tests: name: Run unit tests - runs-on: ubuntu-latest - timeout-minutes: 60 + runs-on: nscloud-ubuntu-22.04-amd64-16x32 + timeout-minutes: 20 services: ipfs: image: ipfs/go-ipfs:v0.10.0 @@ -24,36 +33,51 @@ jobs: postgres: image: postgres env: - POSTGRES_PASSWORD: postgres - POSTGRES_DB: graph_node_test - POSTGRES_INITDB_ARGS: "-E UTF8 --locale=C" + POSTGRES_USER: graph + POSTGRES_PASSWORD: graph + POSTGRES_DB: graph-test + POSTGRES_INITDB_ARGS: "-E UTF8 --locale=C -c max_connections=1000 -c shared_buffers=2GB" options: >- - --health-cmd pg_isready + --health-cmd "pg_isready -U graph" --health-interval 10s --health-timeout 5s --health-retries 5 + --name postgres ports: - 5432:5432 - env: - RUSTFLAGS: "-C link-arg=-fuse-ld=lld -D warnings" steps: - - name: Checkout sources - uses: actions/checkout@v2 - - uses: Swatinem/rust-cache@v2 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5 - - name: Install lld - run: sudo apt-get install -y lld protobuf-compiler + - name: Setup dependencies + run: | + sudo apt-get update + sudo apt-get install -y lld protobuf-compiler - - name: Run unit tests - uses: actions-rs/cargo@v1 + - name: Setup rust toolchain + uses: actions-rust-lang/setup-rust-toolchain@fb51252c7ba57d633bc668f941da052e410add48 # v1 + + - name: Setup just + uses: extractions/setup-just@e33e0265a09d6d736e2ee1e0eb685ef1de4669ff # v3 + + - name: Install pnpm + uses: pnpm/action-setup@a7487c7e89a18df4991f7f222e4898a00d66ddda # v4 + + - name: Install Node.js + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 with: - command: test - args: --verbose --workspace --exclude graph-tests -- --nocapture + node-version: 20 + cache: pnpm + + - name: Install Node.js dependencies + run: pnpm install + + - name: Run unit tests + run: just test-unit runner-tests: name: Subgraph Runner integration tests - runs-on: ubuntu-latest - timeout-minutes: 60 + runs-on: nscloud-ubuntu-22.04-amd64-16x32 + timeout-minutes: 20 services: ipfs: image: ipfs/go-ipfs:v0.10.0 @@ -62,84 +86,116 @@ jobs: postgres: image: postgres env: - POSTGRES_PASSWORD: postgres - POSTGRES_DB: graph_node_test - POSTGRES_INITDB_ARGS: "-E UTF8 --locale=C" + POSTGRES_USER: graph + POSTGRES_PASSWORD: graph + POSTGRES_DB: graph-test + POSTGRES_INITDB_ARGS: "-E UTF8 --locale=C -c max_connections=1000 -c shared_buffers=2GB" options: >- - --health-cmd pg_isready + --health-cmd "pg_isready -U graph" --health-interval 10s --health-timeout 5s --health-retries 5 + --name postgres ports: - 5432:5432 - env: - RUSTFLAGS: "-C link-arg=-fuse-ld=lld -D warnings" steps: - - name: Checkout sources - uses: actions/checkout@v2 - - uses: Swatinem/rust-cache@v2 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5 - - name: Install lld - run: sudo apt-get install -y lld protobuf-compiler + - name: Setup dependencies + run: | + sudo apt-get update + sudo apt-get install -y lld protobuf-compiler - - name: Run runner tests - id: runner-tests-1 - uses: actions-rs/cargo@v1 - env: - TESTS_GANACHE_HARD_WAIT_SECONDS: "60" + - name: Setup rust toolchain + uses: actions-rust-lang/setup-rust-toolchain@fb51252c7ba57d633bc668f941da052e410add48 # v1 + + - name: Setup just + uses: extractions/setup-just@e33e0265a09d6d736e2ee1e0eb685ef1de4669ff # v3 + + - name: Install pnpm + uses: pnpm/action-setup@a7487c7e89a18df4991f7f222e4898a00d66ddda # v4 + + - name: Install Node.js + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 with: - command: test - args: --verbose --package graph-tests -- --skip parallel_integration_tests + node-version: 20 + cache: pnpm + + - name: Install Node.js dependencies + run: pnpm install + + - name: Run runner tests + run: just test-runner integration-tests: name: Run integration tests - runs-on: ubuntu-latest - timeout-minutes: 60 - env: - RUSTFLAGS: "-C link-arg=-fuse-ld=lld -D warnings" + runs-on: nscloud-ubuntu-22.04-amd64-16x32 + timeout-minutes: 20 + services: + ipfs: + image: ipfs/go-ipfs:v0.10.0 + ports: + - 3001:5001 + postgres: + image: postgres + env: + POSTGRES_USER: graph-node + POSTGRES_PASSWORD: let-me-in + POSTGRES_DB: graph-node + POSTGRES_INITDB_ARGS: "-E UTF8 --locale=C -c max_connections=1000 -c shared_buffers=2GB" + options: >- + --health-cmd "pg_isready -U graph-node" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + --name postgres + ports: + - 3011:5432 steps: - - name: Checkout sources - uses: actions/checkout@v2 - - uses: Swatinem/rust-cache@v2 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5 - - name: Install Node 14 - uses: actions/setup-node@v3 - with: - node-version: "14" - cache: yarn - cache-dependency-path: "tests/integration-tests/yarn.lock" + - name: Setup dependencies + run: | + sudo apt-get update + sudo apt-get install -y lld protobuf-compiler - - name: Install lld and jq - run: sudo apt-get install -y lld jq protobuf-compiler + - name: Setup rust toolchain + uses: actions-rust-lang/setup-rust-toolchain@fb51252c7ba57d633bc668f941da052e410add48 # v1 - - name: Build graph-node - uses: actions-rs/cargo@v1 - with: - command: build - args: --bin graph-node - - # Integration tests are a bit flaky, running them twice increases the - # chances of one run succeeding. - - name: Run integration tests (round 1) - id: integration-tests-1 - uses: actions-rs/cargo@v1 - env: - N_CONCURRENT_TESTS: "4" - TESTS_GANACHE_HARD_WAIT_SECONDS: "30" + - name: Setup just + uses: extractions/setup-just@e33e0265a09d6d736e2ee1e0eb685ef1de4669ff # v3 + + - name: Install Foundry + uses: foundry-rs/foundry-toolchain@82dee4ba654bd2146511f85f0d013af94670c4de # v1 with: - command: test - args: --verbose --package graph-tests parallel_integration_tests -- --nocapture - continue-on-error: true - - name: Run integration tests (round 2) - id: integration-tests-2 - uses: actions-rs/cargo@v1 - if: ${{ steps.integration-tests-1.outcome == 'failure' }} - env: - N_CONCURRENT_TESTS: "4" - TESTS_GANACHE_HARD_WAIT_SECONDS: "30" + version: v1.4.0 + + - name: Install pnpm + uses: pnpm/action-setup@a7487c7e89a18df4991f7f222e4898a00d66ddda # v4 + + - name: Install Node.js + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 with: - command: test - args: --verbose --package graph-tests parallel_integration_tests -- --nocapture + node-version: 20 + cache: pnpm + + - name: Install Node.js dependencies + run: pnpm install + + - name: Start anvil + run: anvil --gas-limit 100000000000 --base-fee 1 --timestamp 1743944919 --port 3021 & + + - name: Build graph-node + run: just build --test integration_tests + + - name: Run integration tests + run: | + export PATH="${{ github.workspace }}/node_modules/.bin:$PATH" + just test-integration + + - name: Cat graph-node.log + if: always() + run: cat tests/integration-tests/graph-node.log || echo "No graph-node.log" rustfmt: name: Check rustfmt style @@ -148,45 +204,55 @@ jobs: env: RUSTFLAGS: "-D warnings" steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5 + - uses: actions-rust-lang/setup-rust-toolchain@fb51252c7ba57d633bc668f941da052e410add48 # v1 + + - name: Setup just + uses: extractions/setup-just@e33e0265a09d6d736e2ee1e0eb685ef1de4669ff # v3 - name: Check formatting - uses: actions-rs/cargo@v1 - with: - command: fmt - args: --all -- --check + run: just format --check clippy: name: Clippy linting runs-on: ubuntu-latest - timeout-minutes: 60 + timeout-minutes: 10 + env: + RUSTFLAGS: "-D warnings" steps: - - uses: actions/checkout@v2 - # Unlike rustfmt, Clippy actually compiles stuff so it benefits from - # caching. - - uses: Swatinem/rust-cache@v2 - - name: Install deps - run: sudo apt-get install -y protobuf-compiler - - name: Run Clippy - uses: actions-rs/cargo@v1 - with: - command: check-clippy + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5 + - name: Setup dependencies + run: | + sudo apt-get update + sudo apt-get install -y protobuf-compiler + + - name: Setup rust toolchain + uses: actions-rust-lang/setup-rust-toolchain@fb51252c7ba57d633bc668f941da052e410add48 # v1 + + - name: Setup just + uses: extractions/setup-just@e33e0265a09d6d736e2ee1e0eb685ef1de4669ff # v3 + + - name: Run linting + run: just lint release-check: name: Build in release mode runs-on: ubuntu-latest - timeout-minutes: 60 + timeout-minutes: 10 env: RUSTFLAGS: "-D warnings" steps: - - uses: actions/checkout@v2 - - uses: Swatinem/rust-cache@v2 - - name: Install dependencies + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5 + - name: Setup dependencies run: | sudo apt-get update - sudo apt-get -y install libpq-dev protobuf-compiler + sudo apt-get install -y protobuf-compiler + + - name: Setup rust toolchain + uses: actions-rust-lang/setup-rust-toolchain@fb51252c7ba57d633bc668f941da052e410add48 # v1 + + - name: Setup just + uses: extractions/setup-just@e33e0265a09d6d736e2ee1e0eb685ef1de4669ff # v3 + - name: Cargo check (release) - uses: actions-rs/cargo@v1 - with: - command: check - args: --release + run: just check --release diff --git a/.github/workflows/code-coverage.yml b/.github/workflows/code-coverage.yml deleted file mode 100644 index 2f476c797cb..00000000000 --- a/.github/workflows/code-coverage.yml +++ /dev/null @@ -1,74 +0,0 @@ -name: Code coverage - -on: - workflow_dispatch: - schedule: - # Run it every 3 days. - - cron: "0 3 * * *" - -env: - CARGO_TERM_COLOR: always - RUST_BACKTRACE: full - THEGRAPH_STORE_POSTGRES_DIESEL_URL: "postgresql://postgres:postgres@localhost:5432/graph_node_test" - RUSTFLAGS: "-C link-arg=-fuse-ld=lld -D warnings" - N_CONCURRENT_TESTS: "4" - TESTS_GANACHE_HARD_WAIT_SECONDS: "30" - -jobs: - # Heavily inspired from . - coverage: - name: Code coverage of integration tests - runs-on: ubuntu-latest - timeout-minutes: 60 - services: - ipfs: - image: ipfs/go-ipfs:v0.10.0 - ports: - - 5001:5001 - postgres: - image: postgres - env: - POSTGRES_PASSWORD: postgres - POSTGRES_DB: graph_node_test - POSTGRES_INITDB_ARGS: "-E UTF8 --locale=C" - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - - 5432:5432 - steps: - - uses: actions/checkout@v3 - - uses: Swatinem/rust-cache@v2 - - name: Install Node 14 - uses: actions/setup-node@v3 - with: - node-version: "14" - cache: yarn - cache-dependency-path: "tests/integration-tests/yarn.lock" - - name: Install lld - run: sudo apt-get install -y lld jq - - uses: actions-rs/cargo@v1 - with: - command: install - args: cargo-llvm-cov - - - name: Build graph-node - uses: actions-rs/cargo@v1 - with: - command: build - args: --bin graph-node - - - name: Generate code coverage - run: cargo llvm-cov --package graph-tests --lcov --output-path lcov.info -- --nocapture - - uses: actions/upload-artifact@v3 - with: - name: code-coverage-info - path: lcov.info - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 - with: - # No token needed, because the repo is public. - files: lcov.info - fail_ci_if_error: true diff --git a/.github/workflows/gnd-binary-build.yml b/.github/workflows/gnd-binary-build.yml new file mode 100644 index 00000000000..237de3342b5 --- /dev/null +++ b/.github/workflows/gnd-binary-build.yml @@ -0,0 +1,355 @@ +name: Build gnd Binaries + +on: + release: + types: [published] + workflow_dispatch: + inputs: + release_tag: + description: 'Existing release tag to upload binaries to and publish as on npm (e.g. v0.44.0). Leave empty to only build artifacts.' + type: string + default: '' + dry_run: + description: 'Dry-run npm publish (no actual publish)' + type: boolean + default: false + +jobs: + build: + name: Build gnd for ${{ matrix.target }} + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + include: + - target: x86_64-unknown-linux-gnu + runner: ubuntu-22.04 + asset_name: gnd-linux-x86_64 + - target: aarch64-unknown-linux-gnu + runner: ubuntu-22.04 + asset_name: gnd-linux-aarch64 + - target: x86_64-apple-darwin + runner: macos-14 + asset_name: gnd-macos-x86_64 + - target: aarch64-apple-darwin + runner: macos-latest + asset_name: gnd-macos-aarch64 + - target: x86_64-pc-windows-msvc + runner: windows-latest + asset_name: gnd-windows-x86_64.exe + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Cache built binary + id: bin-cache + uses: actions/cache@v5 + with: + path: | + ${{ matrix.asset_name }}.gz + ${{ matrix.asset_name }}.zip + key: gnd-${{ matrix.target }}-${{ hashFiles('Cargo.lock', '**/Cargo.toml', '**/*.rs') }} + + - name: Install Rust toolchain + if: steps.bin-cache.outputs.cache-hit != 'true' + run: | + rustup toolchain install stable + rustup target add ${{ matrix.target }} + rustup default stable + + - name: Rust Cache + if: steps.bin-cache.outputs.cache-hit != 'true' + uses: Swatinem/rust-cache@v2 + with: + key: ${{ matrix.target }} + + - name: Install dependencies (Ubuntu) + if: steps.bin-cache.outputs.cache-hit != 'true' && startsWith(matrix.runner, 'ubuntu') + run: | + sudo apt-get update + sudo apt-get install -y protobuf-compiler musl-tools + if [ "${{ matrix.target }}" = "aarch64-unknown-linux-gnu" ]; then + sudo apt-get install -y gcc-aarch64-linux-gnu + fi + + - name: Install dependencies (macOS) + if: steps.bin-cache.outputs.cache-hit != 'true' && startsWith(matrix.runner, 'macos') + run: | + brew install protobuf + + - name: Install protobuf (Windows) + if: steps.bin-cache.outputs.cache-hit != 'true' && startsWith(matrix.runner, 'windows') + run: choco install protoc + + - name: Build gnd binary (Unix/Mac) + if: steps.bin-cache.outputs.cache-hit != 'true' && !startsWith(matrix.runner, 'windows') + run: cargo build --bin gnd --release --target ${{ matrix.target }} + env: + CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER: aarch64-linux-gnu-gcc + + - name: Build gnd binary (Windows) + if: steps.bin-cache.outputs.cache-hit != 'true' && startsWith(matrix.runner, 'windows') + run: cargo build --bin gnd --release --target ${{ matrix.target }} + + - name: Sign macOS binary + if: steps.bin-cache.outputs.cache-hit != 'true' && startsWith(matrix.runner, 'macos') + uses: lando/code-sign-action@v3 + with: + file: target/${{ matrix.target }}/release/gnd + certificate-data: ${{ secrets.APPLE_CERT_DATA }} + certificate-password: ${{ secrets.APPLE_CERT_PASSWORD }} + certificate-id: ${{ secrets.APPLE_TEAM_ID }} + options: --options runtime --entitlements entitlements.plist + + - name: Notarize macOS binary + if: steps.bin-cache.outputs.cache-hit != 'true' && startsWith(matrix.runner, 'macos') + uses: lando/notarize-action@v2 + with: + product-path: target/${{ matrix.target }}/release/gnd + appstore-connect-username: ${{ secrets.NOTARIZATION_USERNAME }} + appstore-connect-password: ${{ secrets.NOTARIZATION_PASSWORD }} + appstore-connect-team-id: ${{ secrets.APPLE_TEAM_ID }} + + - name: Prepare binary (Unix) + if: steps.bin-cache.outputs.cache-hit != 'true' && !startsWith(matrix.runner, 'windows') + run: | + cp target/${{ matrix.target }}/release/gnd ${{ matrix.asset_name }} + chmod +x ${{ matrix.asset_name }} + gzip ${{ matrix.asset_name }} + + - name: Prepare binary (Windows) + if: steps.bin-cache.outputs.cache-hit != 'true' && startsWith(matrix.runner, 'windows') + run: | + copy target\${{ matrix.target }}\release\gnd.exe ${{ matrix.asset_name }} + 7z a -tzip ${{ matrix.asset_name }}.zip ${{ matrix.asset_name }} + + - name: Upload artifact + uses: actions/upload-artifact@v7 + with: + name: ${{ matrix.asset_name }} + path: | + ${{ matrix.asset_name }}.gz + ${{ matrix.asset_name }}.zip + if-no-files-found: error + + release: + name: Create Release + needs: build + if: startsWith(github.ref, 'refs/tags/') || inputs.release_tag != '' + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - name: Setup GitHub CLI + run: | + # GitHub CLI is pre-installed on GitHub-hosted runners + gh --version + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Download all artifacts + uses: actions/download-artifact@v8 + with: + path: artifacts + + - name: Display structure of downloaded artifacts + run: ls -R artifacts + + - name: Upload Assets to Release + run: | + VERSION="${{ inputs.release_tag != '' && inputs.release_tag || github.ref_name }}" + + gh release upload $VERSION --clobber --repo $GITHUB_REPOSITORY \ + artifacts/gnd-linux-x86_64/gnd-linux-x86_64.gz \ + artifacts/gnd-linux-aarch64/gnd-linux-aarch64.gz \ + artifacts/gnd-macos-x86_64/gnd-macos-x86_64.gz \ + artifacts/gnd-macos-aarch64/gnd-macos-aarch64.gz \ + artifacts/gnd-windows-x86_64.exe/gnd-windows-x86_64.exe.zip + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + publish-npm: + name: Publish npm package for ${{ matrix.platform }} + needs: release + if: startsWith(github.ref, 'refs/tags/') || inputs.release_tag != '' + runs-on: ubuntu-latest + permissions: + id-token: write + contents: read + strategy: + fail-fast: false + matrix: + include: + - platform: linux-x64 + asset: gnd-linux-x86_64.gz + os_field: linux + cpu_field: x64 + extract: gunzip + - platform: linux-arm64 + asset: gnd-linux-aarch64.gz + os_field: linux + cpu_field: arm64 + extract: gunzip + - platform: darwin-x64 + asset: gnd-macos-x86_64.gz + os_field: darwin + cpu_field: x64 + extract: gunzip + - platform: darwin-arm64 + asset: gnd-macos-aarch64.gz + os_field: darwin + cpu_field: arm64 + extract: gunzip + - platform: win32-x64 + asset: gnd-windows-x86_64.exe.zip + os_field: win32 + cpu_field: x64 + extract: unzip + steps: + - uses: actions/setup-node@v6 + with: + node-version: 24 + + - name: Download gnd binary + env: + GH_TOKEN: ${{ github.token }} + run: | + gh release download "${{ inputs.release_tag != '' && inputs.release_tag || github.ref_name }}" \ + --repo "${{ github.repository }}" \ + --pattern "${{ matrix.asset }}" \ + --output ./binary-archive + + - name: Extract binary + run: | + mkdir -p pkg/bin + if [ "${{ matrix.extract }}" = "gunzip" ]; then + gunzip -c ./binary-archive > pkg/bin/gnd + chmod +x pkg/bin/gnd + else + unzip ./binary-archive -d pkg/bin + mv pkg/bin/*.exe pkg/bin/gnd.exe + fi + + - name: Determine version and npm tag + id: version + shell: bash + run: | + VERSION="${{ inputs.release_tag != '' && inputs.release_tag || github.ref_name }}" + VERSION="${VERSION#v}" + echo "version=${VERSION}" >> $GITHUB_OUTPUT + # Prerelease versions (e.g. 0.42.2-dev.1) need an explicit --tag + if [[ "$VERSION" == *-* ]]; then + # Extract prerelease identifier (e.g. "dev" from "0.42.2-dev.1") + PRE="${VERSION#*-}" + TAG="${PRE%%.*}" + echo "tag=${TAG}" >> $GITHUB_OUTPUT + else + echo "tag=latest" >> $GITHUB_OUTPUT + fi + + - name: Create package.json + shell: bash + run: | + if [ "${{ matrix.os_field }}" = "win32" ]; then + BIN_PATH="./bin/gnd.exe" + else + BIN_PATH="./bin/gnd" + fi + + cat > pkg/package.json << EOF + { + "name": "@graphprotocol/gnd-${{ matrix.platform }}", + "version": "${{ steps.version.outputs.version }}", + "description": "gnd binary for ${{ matrix.platform }}", + "os": ["${{ matrix.os_field }}"], + "cpu": ["${{ matrix.cpu_field }}"], + "bin": { + "gnd": "${BIN_PATH}" + }, + "publishConfig": { + "access": "public", + "provenance": true + }, + "license": "(Apache-2.0 OR MIT)", + "repository": { + "type": "git", + "url": "https://github.com/graphprotocol/graph-node.git" + } + } + EOF + + - name: Publish + run: npm publish --provenance --access public --tag ${{ steps.version.outputs.tag }} ${{ inputs.dry_run && '--dry-run' || '' }} + working-directory: pkg + + publish-npm-wrapper: + name: Publish @graphprotocol/gnd wrapper + needs: publish-npm + if: startsWith(github.ref, 'refs/tags/') || inputs.release_tag != '' + runs-on: ubuntu-latest + permissions: + id-token: write + contents: read + steps: + - uses: actions/checkout@v6 + + - uses: actions/setup-node@v6 + with: + node-version: 24 + + - name: Determine version and npm tag + id: version + shell: bash + run: | + VERSION="${{ inputs.release_tag != '' && inputs.release_tag || github.ref_name }}" + VERSION="${VERSION#v}" + echo "version=${VERSION}" >> $GITHUB_OUTPUT + if [[ "$VERSION" == *-* ]]; then + PRE="${VERSION#*-}" + TAG="${PRE%%.*}" + echo "tag=${TAG}" >> $GITHUB_OUTPUT + else + echo "tag=latest" >> $GITHUB_OUTPUT + fi + + - name: Create wrapper package + shell: bash + run: | + VERSION="${{ steps.version.outputs.version }}" + + mkdir -p pkg/bin + cp gnd/npm/bin/gnd.js pkg/bin/gnd.js + cp gnd/npm/README.md pkg/README.md + + cat > pkg/package.json << EOF + { + "name": "@graphprotocol/gnd", + "version": "${VERSION}", + "description": "Graph Node Development CLI", + "bin": { + "gnd": "./bin/gnd.js" + }, + "optionalDependencies": { + "@graphprotocol/gnd-darwin-arm64": "${VERSION}", + "@graphprotocol/gnd-darwin-x64": "${VERSION}", + "@graphprotocol/gnd-linux-arm64": "${VERSION}", + "@graphprotocol/gnd-linux-x64": "${VERSION}", + "@graphprotocol/gnd-win32-x64": "${VERSION}" + }, + "publishConfig": { + "access": "public", + "provenance": true + }, + "license": "(Apache-2.0 OR MIT)", + "repository": { + "type": "git", + "url": "https://github.com/graphprotocol/graph-node.git" + } + } + EOF + + - name: Publish + run: npm publish --provenance --access public --tag ${{ steps.version.outputs.tag }} ${{ inputs.dry_run && '--dry-run' || '' }} + working-directory: pkg \ No newline at end of file diff --git a/.gitignore b/.gitignore index bf084b6c226..0480ebcf19b 100644 --- a/.gitignore +++ b/.gitignore @@ -22,9 +22,20 @@ lcov.info /tests/**/build /tests/**/generated -/tests/**/node_modules -/tests/**/yarn-error.log -# Built solidity contracts. -/tests/**/bin -/tests/**/truffle_output +# Node dependencies +node_modules/ +.pnpm-store/ + +# Docker volumes and debug logs +.postgres +logfile + +# Nix related files +.direnv +.envrc +.data + +# Local claude settings +.claude/settings.local.json +.claude/ralph-loop.local.md diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index dcaabb18e56..00000000000 --- a/.travis.yml +++ /dev/null @@ -1,87 +0,0 @@ -dist: bionic -language: rust -# This line would cache cargo-audit once installed, -# but instead will fail from the 10 minute timeout after -# printing the line "creating directory /home/travis/.cache/sccache" -#cache: cargo -rust: - - stable - - beta - -# Select pre-installed services -addons: - postgresql: "10" - apt: - packages: - - postgresql-10 - - postgresql-client-10 -services: - - postgresql - - docker - -before_install: - # Install Node.js 11.x - - nvm install 11 && nvm use 11 - # Install IPFS - - wget "https://dist.ipfs.io/go-ipfs/v0.10.0/go-ipfs_v0.10.0_linux-amd64.tar.gz" -O /tmp/ipfs.tar.gz - - pushd . && cd $HOME/bin && tar -xzvf /tmp/ipfs.tar.gz && popd - - export PATH="$HOME/bin/go-ipfs:$PATH" - - ipfs init - -matrix: - fast_finish: true - include: - # Some env var is always necessary to differentiate included builds - # Check coding style - - env: CHECK_FORMATTING=true - rust: stable - script: - - rustup component add rustfmt - - cargo fmt --all -- --check - - # Make sure release builds compile - - env: CHECK_RELEASE=true - rust: stable - script: - - cargo check --release - - # Check for warnings - - env: RUSTFLAGS="-D warnings" - rust: stable - script: - - cargo check --tests - - # Build tagged commits in release mode - - env: RELEASE=true - if: tag IS present - script: - - cargo build -p graph-node --release - - mv target/release/graph-node target/release/graph-node-$TRAVIS_OS_NAME - -env: - global: - - PGPORT=5432 - - THEGRAPH_STORE_POSTGRES_DIESEL_URL=postgresql://travis:travis@localhost:5432/graph_node_test - # Added because https://nodejs.org/dist/ had issues - - NVM_NODEJS_ORG_MIRROR=https://cnpmjs.org/mirrors/node/ - -# Test pipeline -before_script: - - psql -c "ALTER USER travis WITH PASSWORD 'travis';" - - psql -c 'create database graph_node_test;' -U travis - -script: - # Run tests - - ipfs daemon &> /dev/null & - - RUST_BACKTRACE=1 cargo test --verbose --all -- --nocapture - - killall ipfs - -deploy: - provider: releases - api_key: - secure: ygpZedRG+/Qg/lPhifyNQ+4rExjZ4nGyJjB4DYT1fuePMyKXfiCPGicaWRGR3ZnZGNRjdKaIkF97vBsZ0aHwW+AykwOxlXrkAFvCKA0Tb82vaYqCLrBs/Y5AEhuCWLFDz5cXDPMkptf+uLX/s3JCF0Mxo5EBN2JfBQ8vS6ScKEwqn2TiLLBQKTQ4658TFM4H5KiXktpyVVdlRvpoS3pRIPMqNU/QpGPQigaiKyYD5+azCrAXeaKT9bBS1njVbxI69Go4nraWZn7wIhZCrwJ+MxGNTOxwasypsWm/u1umhRVLM1rL2i7RRqkIvzwn22YMaU7FZKCx8huXcj0cB8NtHZSw7GhJDDDv3e7puZxl3m/c/7ks76UF95syLzoM/9FWEFew8Ti+5MApzKQj5YWHOCIEzBWPeqAcA8Y+Az7w2h1ZgNbjDgSvjGAFSpE8m+SM0A2TOOZ1g/t/yfbEl8CWO6Y8v2x1EONkp7X0CqJgASMp+h8kzKCbuYyRnghlToY+5wYuh4M9Qg9UeJCt9dOblRBVJwW5CFr62kgE/gso8F9tXXHkRTv3hfk5madZR1Vn5A7KadEO8epfV4IQNsd+VHfoxoJSprx5f77Q2bLMBD1GT/qMqECgSznoTkU5ajkKJRqUw4AwLTohrYir76j61eQfxOhXExY/EM8xvlxpd1w= - file: target/release/graph-node-$TRAVIS_OS_NAME - repo: graphprotocol/graph-node - on: - tags: true - skip_cleanup: true diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000000..b8ec983428b --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,288 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Project Overview + +Graph Node is a Rust-based decentralized blockchain indexing protocol that enables efficient querying of blockchain data through GraphQL. It's the core component of The Graph protocol, written as a Cargo workspace with multiple crates organized by functionality. + +## Essential Development Commands + +### Testing Workflow + +⚠️ **Only run integration tests when explicitly requested or when changes require full system testing** + +Use unit tests for regular development and only run integration tests when: + +- Explicitly asked to do so +- Making changes to integration/end-to-end functionality +- Debugging issues that require full system testing +- Preparing releases or major changes + +### Unit Tests + +Unit tests are inlined with source code. + +**Prerequisites:** + +1. PostgreSQL running on localhost:5432 (with initialised `graph-test` database) +2. IPFS running on localhost:5001 +3. PNPM +4. Foundry (for smart contract compilation) +5. Environment variable `THEGRAPH_STORE_POSTGRES_DIESEL_URL` set to `postgresql://graph:graph@127.0.0.1:5432/graph-test` + +The environment dependencies and environment setup are operated by the human. + +**Running Unit Tests:** + +```bash +# Run unit tests +just test-unit + +# Run specific tests (e.g. `data_source::common::tests`) +just test-unit data_source::common::tests +``` + +**⚠️ Test Verification Requirements:** +When filtering for specific tests, ensure the intended test name(s) appear in the output. + +### Runner Tests (Integration Tests) + +**Prerequisites:** + +1. PostgreSQL running on localhost:5432 (with initialised `graph-test` database) +2. IPFS running on localhost:5001 +3. PNPM +4. Foundry (for smart contract compilation) +5. Environment variable `THEGRAPH_STORE_POSTGRES_DIESEL_URL` set to `postgresql://graph:graph@127.0.0.1:5432/graph-test` + +**Running Runner Tests:** + +```bash +# Run runner tests. +just test-runner + +# Run specific tests (e.g. `block_handlers`) +just test-runner block_handlers +``` + +**⚠️ Test Verification Requirements:** +When filtering for specific tests, ensure the intended test name(s) appear in the output. + +**Important Notes:** + +- Runner tests take moderate time (10-20 seconds) +- Tests automatically reset the database between runs +- Some tests can pass without IPFS, but tests involving file data sources require it + +### Integration Tests + +**Prerequisites:** + +1. PostgreSQL running on localhost:3011 (with initialised `graph-node` database) +2. IPFS running on localhost:3001 +3. Anvil running on localhost:3021 +4. PNPM +5. Foundry (for smart contract compilation) + +The environment dependencies and environment setup are operated by the human. + +**Running Integration Tests:** + +```bash +# Run all integration tests (automatically builds graph-node and gnd) +just test-integration + +# Run a specific integration test case (e.g., "grafted" test case) +TEST_CASE=grafted just test-integration + +# (Optional) Use graph-cli instead of gnd for compatibility testing +GRAPH_CLI=node_modules/.bin/graph just test-integration + +# Override ports if using different service ports (e.g., for local development) +POSTGRES_TEST_PORT=5432 ETHEREUM_TEST_PORT=8545 IPFS_TEST_PORT=5001 just test-integration +``` + +**⚠️ Test Verification Requirements:** + +- **ALWAYS verify tests actually ran** - Check the output for "test result: ok. X passed" where X > 0 +- **If output shows "0 passed" or "0 tests run"**, the TEST_CASE variable or filter was wrong - fix and re-run +- **Never trust exit code 0 alone** - Cargo can exit successfully even when no tests matched your filter + +**Important Notes:** + +- Integration tests take significant time (several minutes) +- Tests automatically reset the database between runs +- Logs are written to `tests/integration-tests/graph-node.log` +- **If a test hangs for >10 minutes**, it's likely stuck - kill with `pkill -9 integration_tests` and check logs +- CI uses the default ports (3011, 3021, 3001) - local development can override with environment variables + +### Code Quality + +```bash +# 🚨 MANDATORY: Format all code IMMEDIATELY after any .rs file edit +just format + +# 🚨 MANDATORY: Check code for warnings and errors - MUST have zero warnings +just lint + +# 🚨 MANDATORY: Check in release mode to catch linking/optimization issues that cargo check misses +just check --release +``` + +🚨 **CRITICAL REQUIREMENTS for ANY implementation**: + +- **🚨 MANDATORY**: `cargo fmt --all` MUST be run before any commit +- **🚨 MANDATORY**: `just lint` MUST show zero warnings before any commit +- **🚨 MANDATORY**: `cargo check --release` MUST complete successfully before any commit +- **🚨 MANDATORY**: The unit test suite MUST pass before any commit + +Forgetting any of these means you failed to follow instructions. Before any commit or PR, ALL of the above MUST be satisfied! No exceptions! + +## High-Level Architecture + +### Core Components + +- **`graph/`**: Core abstractions, traits, and shared types +- **`node/`**: Main executable and CLI (graphman) +- **`chain/`**: Blockchain-specific adapters (ethereum, near, substreams) +- **`runtime/`**: WebAssembly runtime for subgraph execution +- **`store/`**: PostgreSQL-based storage layer +- **`graphql/`**: GraphQL query execution engine +- **`server/`**: HTTP/WebSocket APIs + +### Data Flow + +``` +Blockchain → Chain Adapter → Block Stream → Trigger Processing → Runtime → Store → GraphQL API +``` + +1. **Chain Adapters** connect to blockchain nodes and convert data to standardized formats +2. **Block Streams** provide event-driven streaming of blockchain blocks +3. **Trigger Processing** matches blockchain events to subgraph handlers +4. **Runtime** executes subgraph code in WebAssembly sandbox +5. **Store** persists entities with block-level granularity +6. **GraphQL** processes queries and returns results + +### Key Abstractions + +- **`Blockchain`** trait: Core blockchain interface +- **`Store`** trait: Storage abstraction with read/write variants +- **`RuntimeHost`**: WASM execution environment +- **`TriggerData`**: Standardized blockchain events +- **`EventConsumer`/`EventProducer`**: Component communication + +### Architecture Patterns + +- **Event-driven**: Components communicate through async streams and channels +- **Trait-based**: Extensive use of traits for abstraction and modularity +- **Async/await**: Tokio-based async runtime throughout +- **Multi-shard**: Database sharding for scalability +- **Sandboxed execution**: WASM runtime with gas metering + +## Development Guidelines + +### Commit Convention + +Use format: `{crate-name}: {description}` + +- Single crate: `store: Support 'Or' filters` +- Multiple crates: `core, graphql: Add event source to store` +- All crates: `all: {description}` + +### Git Workflow + +- Rebase on master (don't merge master into feature branch) +- Keep commits logical and atomic +- Squash commits to clean up history before merging + +## Crate Structure + +### Core Crates + +- **`graph`**: Shared types, traits, and utilities +- **`node`**: Main binary and component wiring +- **`core`**: Business logic and subgraph management + +### Blockchain Integration + +- **`chain/ethereum`**: Ethereum chain support +- **`chain/near`**: NEAR protocol support +- **`chain/substreams`**: Substreams data source support + +### Infrastructure + +- **`store/postgres`**: PostgreSQL storage implementation +- **`runtime/wasm`**: WebAssembly runtime and host functions +- **`graphql`**: Query processing and execution +- **`server/`**: HTTP/WebSocket servers + +### Key Dependencies + +- **`diesel`**: PostgreSQL ORM +- **`tokio`**: Async runtime +- **`tonic`**: gRPC framework +- **`wasmtime`**: WebAssembly runtime +- **`web3`**: Ethereum interaction + +## Test Environment Requirements + +### Process Compose Setup (Recommended) + +The repository includes a process-compose-flake setup that provides native, declarative service management. + +Currently, the human is required to operate the service dependencies as illustrated below. + +**Unit Tests:** + +```bash +# Human: Start PostgreSQL + IPFS for unit tests in a separate terminal +# PostgreSQL: localhost:5432, IPFS: localhost:5001 +nix run .#unit + +# Claude: Run unit tests +just test-unit +``` + +**Runner Tests:** + +```bash +# Human: Start PostgreSQL + IPFS for runner tests in a separate terminal +# PostgreSQL: localhost:5432, IPFS: localhost:5001 +nix run .#unit # NOTE: Runner tests are using the same nix services stack as the unit test + +# Claude: Run runner tests +just test-runner +``` + +**Integration Tests:** + +```bash +# Human: Start all services for integration tests in a separate terminal +# PostgreSQL: localhost:3011, IPFS: localhost:3001, Anvil: localhost:3021 +nix run .#integration + +# Claude: Run integration tests (automatically builds graph-node and gnd) +just test-integration +``` + +**Services Configuration:** +The services are configured to use the test suite default ports for unit- and integration tests respectively. + +| Service | Unit Tests Port | Integration Tests Port | Database/Config | +| ---------------- | --------------- | ---------------------- | ----------------------------------------------- | +| PostgreSQL | 5432 | 3011 | `graph-test` / `graph-node` | +| IPFS | 5001 | 3001 | Data in `./.data/unit` or `./.data/integration` | +| Anvil (Ethereum) | - | 3021 | Deterministic test chain | + +**Service Configuration:** +The setup combines built-in services-flake services with custom multiService modules: + +**Built-in Services:** + +- **PostgreSQL**: Uses services-flake's postgres service with a helper function (`mkPostgresConfig`) that provides graph-specific defaults including required extensions. + +**Custom Services** (located in `./nix`): + +- `ipfs.nix`: IPFS (kubo) with automatic initialization and configurable ports +- `anvil.nix`: Ethereum test chain with deterministic configuration diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index aa67ac5d73d..7992c32c49f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -5,7 +5,7 @@ Welcome to the Graph Protocol! Thanks a ton for your interest in contributing. If you run into any problems feel free to create an issue. PRs are much appreciated for simple things. Here's [a list of good first issues](https://github.com/graphprotocol/graph-node/labels/good%20first%20issue). If it's something more complex we'd appreciate having a quick chat in GitHub Issues or Discord. -Join the conversation on our [Discord](https://discord.gg/9a5VCua). +Join the conversation on our [Discord](https://discord.gg/graphprotocol). Please follow the [Code of Conduct](https://github.com/graphprotocol/graph-node/blob/master/CODE_OF_CONDUCT.md) for all the communications and at events. Thank you! @@ -15,7 +15,7 @@ Install development helpers: ```sh cargo install cargo-watch -rustup component add rustfmt-preview +rustup component add rustfmt ``` Set environment variables: @@ -79,7 +79,7 @@ Please do not merge master into your branch as you develop your pull request; instead, rebase your branch on top of the latest master if your pull request branch is long-lived. -We try to keep the hostory of the `master` branch linear, and avoid merge +We try to keep the history of the `master` branch linear, and avoid merge commits. Once your pull request is approved, merge it following these steps: ``` diff --git a/Cargo.lock b/Cargo.lock index 13a40c7430c..5e204e6ad4d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "Inflector" @@ -14,5630 +14,10318 @@ dependencies = [ [[package]] name = "addr2line" -version = "0.15.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7a2e47a1fbe209ee101dd6d61285226744c6c8d3c21c8dc878ba6cb9f467f3a" -dependencies = [ - "gimli 0.24.0", -] - -[[package]] -name = "addr2line" -version = "0.16.0" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e61f2b7f93d2c7d2b08263acaa4a363b3e276806c68af6134c44f523bf1aacd" +checksum = "59317f77929f0e679d39364702289274de2f0f0b22cbf50b2b8cff2169a0b27a" dependencies = [ - "gimli 0.25.0", + "gimli", ] [[package]] -name = "adler" -version = "1.0.2" +name = "adler2" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" [[package]] -name = "aho-corasick" -version = "0.7.18" +name = "ahash" +version = "0.8.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ - "memchr", + "cfg-if", + "const-random", + "getrandom 0.3.1", + "once_cell", + "version_check", + "zerocopy", ] [[package]] -name = "android_system_properties" -version = "0.1.5" +name = "aho-corasick" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" dependencies = [ - "libc", + "memchr", ] [[package]] -name = "anyhow" -version = "1.0.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "224afbd727c3d6e4b90103ece64b8d1b67fbb1973b1046c2281eed3f3803f800" - -[[package]] -name = "arc-swap" -version = "1.3.0" +name = "alloc-no-stdlib" +version = "2.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e906254e445520903e7fc9da4f709886c84ae4bc4ddaf0e093188d66df4dc820" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" [[package]] -name = "arrayref" -version = "0.3.6" +name = "alloc-stdlib" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544" +checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" +dependencies = [ + "alloc-no-stdlib", +] [[package]] -name = "arrayvec" -version = "0.5.2" +name = "allocator-api2" +version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" [[package]] -name = "arrayvec" -version = "0.7.2" +name = "alloy" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" +checksum = "5e1e915a830ea2ee123c9d3886bfec3302a7ddcd73a973ab165ed45aca2e42c3" +dependencies = [ + "alloy-consensus", + "alloy-contract", + "alloy-core", + "alloy-eips", + "alloy-genesis", + "alloy-json-rpc", + "alloy-network", + "alloy-provider", + "alloy-pubsub", + "alloy-rpc-client", + "alloy-rpc-types", + "alloy-serde", + "alloy-signer", + "alloy-signer-local", + "alloy-transport", + "alloy-transport-http", + "alloy-transport-ipc", + "alloy-transport-ws", + "alloy-trie", +] [[package]] -name = "ascii" -version = "0.9.3" +name = "alloy-chains" +version = "0.2.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eab1c04a571841102f5345a8fc0f6bb3d31c315dec879b5c6e42e40ce7ffa34e" +checksum = "ef3a72a2247c34a8545ee99e562b1b9b69168e5000567257ae51e91b4e6b1193" +dependencies = [ + "alloy-primitives", + "num_enum", + "strum 0.27.2", +] [[package]] -name = "assert-json-diff" -version = "2.0.2" +name = "alloy-consensus" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47e4f2b81832e72834d7518d8487a0396a28cc408186a2e8854c0f98011faf12" +checksum = "83447eeb17816e172f1dfc0db1f9dc0b7c5d069bd1f7cecbecceb382bf931015" dependencies = [ + "alloy-eips", + "alloy-primitives", + "alloy-rlp", + "alloy-serde", + "alloy-trie", + "alloy-tx-macros", + "arbitrary", + "auto_impl", + "borsh", + "c-kzg", + "derive_more", + "either", + "k256", + "once_cell", + "rand 0.8.6", + "secp256k1 0.30.0", "serde", "serde_json", + "serde_with", + "thiserror 2.0.18", ] [[package]] -name = "async-recursion" -version = "1.0.0" +name = "alloy-consensus-any" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cda8f4bcc10624c4e85bc66b3f452cca98cfa5ca002dc83a16aad2367641bea" +checksum = "5406343e306856dc2be762700e98a16904de45dee14a07f233e742ce68daff2f" dependencies = [ - "proc-macro2", - "quote", - "syn", + "alloy-consensus", + "alloy-eips", + "alloy-primitives", + "alloy-rlp", + "alloy-serde", + "arbitrary", + "serde", ] [[package]] -name = "async-stream" -version = "0.3.4" +name = "alloy-contract" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad445822218ce64be7a341abfb0b1ea43b5c23aa83902542a4542e78309d8e5e" +checksum = "3ae8b60d71b92824e095b4003ff01fd2bc923017b7568997c5f459240e83499c" dependencies = [ - "async-stream-impl", - "futures-core", - "pin-project-lite", + "alloy-consensus", + "alloy-dyn-abi", + "alloy-json-abi", + "alloy-network", + "alloy-network-primitives", + "alloy-primitives", + "alloy-provider", + "alloy-pubsub", + "alloy-rpc-types-eth", + "alloy-sol-types", + "alloy-transport", + "futures 0.3.31", + "futures-util", + "serde_json", + "thiserror 2.0.18", + "tracing", ] [[package]] -name = "async-stream-impl" -version = "0.3.4" +name = "alloy-core" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4655ae1a7b0cdf149156f780c5bf3f1352bc53cbd9e0a361a7ef7b22947e965" +checksum = "62ddde5968de6044d67af107ad835bc0069a7ca245870b94c5958a7d8712b184" dependencies = [ - "proc-macro2", - "quote", - "syn", + "alloy-dyn-abi", + "alloy-json-abi", + "alloy-primitives", + "alloy-rlp", + "alloy-sol-types", ] [[package]] -name = "async-trait" -version = "0.1.51" +name = "alloy-dyn-abi" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44318e776df68115a881de9a8fd1b9e53368d7a4a5ce4cc48517da3393233a5e" +checksum = "a475bb02d9cef2dbb99065c1664ab3fe1f9352e21d6d5ed3f02cdbfc06ed1abc" dependencies = [ - "proc-macro2", - "quote", - "syn", + "alloy-json-abi", + "alloy-primitives", + "alloy-sol-type-parser", + "alloy-sol-types", + "arbitrary", + "itoa", + "proptest", + "serde", + "serde_json", + "winnow 1.0.0", ] [[package]] -name = "atomic_refcell" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "857253367827bd9d0fd973f0ef15506a96e79e41b0ad7aa691203a4e3214f6c8" - -[[package]] -name = "atty" -version = "0.2.14" +name = "alloy-eip2124" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +checksum = "741bdd7499908b3aa0b159bba11e71c8cddd009a2c2eb7a06e825f1ec87900a5" dependencies = [ - "hermit-abi 0.1.19", - "libc", - "winapi", + "alloy-primitives", + "alloy-rlp", + "arbitrary", + "crc", + "rand 0.8.6", + "serde", + "thiserror 2.0.18", ] [[package]] -name = "autocfg" -version = "1.0.1" +name = "alloy-eip2930" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" +checksum = "9441120fa82df73e8959ae0e4ab8ade03de2aaae61be313fbf5746277847ce25" +dependencies = [ + "alloy-primitives", + "alloy-rlp", + "arbitrary", + "borsh", + "rand 0.8.6", + "serde", +] [[package]] -name = "axum" -version = "0.6.1" +name = "alloy-eip7702" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08b108ad2665fa3f6e6a517c3d80ec3e77d224c47d605167aefaa5d7ef97fa48" +checksum = "2919c5a56a1007492da313e7a3b6d45ef5edc5d33416fdec63c0d7a2702a0d20" dependencies = [ - "async-trait", - "axum-core", - "bitflags", - "bytes", - "futures-util", - "http", - "http-body", - "hyper", - "itoa 1.0.1", - "matchit", - "memchr", - "mime", - "percent-encoding", - "pin-project-lite", - "rustversion", + "alloy-primitives", + "alloy-rlp", + "arbitrary", + "borsh", + "k256", + "rand 0.8.6", "serde", - "sync_wrapper", - "tower 0.4.13", - "tower-http", - "tower-layer 0.3.2", - "tower-service 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "thiserror 2.0.18", ] [[package]] -name = "axum-core" -version = "0.3.0" +name = "alloy-eip7928" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79b8558f5a0581152dc94dcd289132a1d377494bdeafcd41869b3258e3e2ad92" +checksum = "d3231de68d5d6e75332b7489cfcc7f4dfabeba94d990a10e4b923af0e6623540" dependencies = [ - "async-trait", - "bytes", - "futures-util", - "http", - "http-body", - "mime", - "rustversion", - "tower-layer 0.3.2", - "tower-service 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "alloy-primitives", + "alloy-rlp", + "arbitrary", + "borsh", + "serde", ] [[package]] -name = "backtrace" -version = "0.3.61" +name = "alloy-eips" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7a905d892734eea339e896738c14b9afce22b5318f64b951e70bf3844419b01" +checksum = "0dca4c89ace90684b4b77366d00631ed498c9af962079af2a5dbc593a0618a77" dependencies = [ - "addr2line 0.16.0", - "cc", - "cfg-if 1.0.0", - "libc", - "miniz_oxide 0.4.4", - "object 0.26.0", - "rustc-demangle", + "alloy-eip2124", + "alloy-eip2930", + "alloy-eip7702", + "alloy-eip7928", + "alloy-primitives", + "alloy-rlp", + "alloy-serde", + "arbitrary", + "auto_impl", + "borsh", + "c-kzg", + "derive_more", + "either", + "serde", + "serde_with", + "sha2 0.10.9", ] [[package]] -name = "base-x" -version = "0.2.11" +name = "alloy-genesis" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cbbc9d0964165b47557570cce6c952866c2678457aca742aafc9fb771d30270" +checksum = "ab0e0fe9e6d1120ad7bb9254c3fc2b9bc80a8df42a033fb626be6559c13d5153" +dependencies = [ + "alloy-eips", + "alloy-primitives", + "alloy-serde", + "alloy-trie", + "borsh", + "serde", + "serde_with", +] [[package]] -name = "base64" -version = "0.13.1" +name = "alloy-json-abi" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" +checksum = "7c36c9d7f9021601b04bfef14a4b64849f6d73116a4e91e071d7fbfe10247901" +dependencies = [ + "alloy-primitives", + "alloy-sol-type-parser", + "serde", + "serde_json", +] [[package]] -name = "base64" -version = "0.20.0" +name = "alloy-json-rpc" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ea22880d78093b0cbe17c89f64a7d457941e65759157ec6cb31a31d652b05e5" +checksum = "ec0a82e56b1843bce483942d54fcadea92e676f1bde162e93c7d3b621fabc4e1" +dependencies = [ + "alloy-primitives", + "alloy-sol-types", + "http 1.4.2", + "serde", + "serde_json", + "thiserror 2.0.18", + "tracing", +] [[package]] -name = "base64-url" -version = "1.4.13" +name = "alloy-network" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67a99c239d0c7e77c85dddfa9cebce48704b3c49550fcd3b84dd637e4484899f" +checksum = "a7db7b095b0b1db1d18ce7e91dcd2e82007f2d52bfb8125e6b64633a74a06bc3" dependencies = [ - "base64 0.13.1", + "alloy-consensus", + "alloy-consensus-any", + "alloy-eips", + "alloy-json-rpc", + "alloy-network-primitives", + "alloy-primitives", + "alloy-rpc-types-any", + "alloy-rpc-types-eth", + "alloy-serde", + "alloy-signer", + "alloy-sol-types", + "async-trait", + "auto_impl", + "derive_more", + "futures-utils-wasm", + "serde", + "serde_json", + "thiserror 2.0.18", ] [[package]] -name = "beef" -version = "0.5.2" +name = "alloy-network-primitives" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a8241f3ebb85c056b509d4327ad0358fbbba6ffb340bf388f26350aeda225b1" +checksum = "cd28d9bfd11729037d194f2b1d43db8642eb3f342032691f4ca96bb745479c3c" dependencies = [ + "alloy-consensus", + "alloy-eips", + "alloy-primitives", + "alloy-serde", "serde", ] [[package]] -name = "bigdecimal" -version = "0.1.2" +name = "alloy-primitives" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1374191e2dd25f9ae02e3aa95041ed5d747fc77b3c102b49fe2dd9a8117a6244" +checksum = "4885c1409b6936c4898e646ef58baf6ec54edaf6d8179f79df805a7b85b7cf3e" dependencies = [ - "num-bigint", - "num-integer", - "num-traits", + "alloy-rlp", + "arbitrary", + "bytes", + "cfg-if", + "const-hex", + "derive_more", + "foldhash 0.2.0", + "hashbrown 0.17.0", + "indexmap 2.14.0", + "itoa", + "k256", + "keccak-asm", + "paste", + "proptest", + "proptest-derive 0.8.0", + "rand 0.9.3", + "rapidhash", + "ruint", + "rustc-hash", + "secp256k1 0.31.1", "serde", + "sha3", ] [[package]] -name = "bincode" -version = "1.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" -dependencies = [ +name = "alloy-provider" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8955ab30418343de57b356de2ea60200f9fb8016a7ea3bc7f5c6176f01a8b1cf" +dependencies = [ + "alloy-chains", + "alloy-consensus", + "alloy-eips", + "alloy-json-rpc", + "alloy-network", + "alloy-network-primitives", + "alloy-primitives", + "alloy-pubsub", + "alloy-rpc-client", + "alloy-rpc-types-anvil", + "alloy-rpc-types-debug", + "alloy-rpc-types-eth", + "alloy-rpc-types-trace", + "alloy-rpc-types-txpool", + "alloy-signer", + "alloy-sol-types", + "alloy-transport", + "alloy-transport-http", + "alloy-transport-ipc", + "alloy-transport-ws", + "async-stream", + "async-trait", + "auto_impl", + "dashmap", + "either", + "futures 0.3.31", + "futures-utils-wasm", + "lru", + "parking_lot", + "pin-project", + "reqwest 0.13.2", "serde", + "serde_json", + "thiserror 2.0.18", + "tokio", + "tracing", + "url", + "wasmtimer", ] [[package]] -name = "bitflags" -version = "1.3.1" +name = "alloy-pubsub" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2da1976d75adbe5fbc88130ecd119529cf1cc6a93ae1546d8696ee66f0d21af1" +checksum = "7cd85cfea1fa8ebd20d3475e961fe3a3624c0eb4659ea137715c0c83c8aeaff0" +dependencies = [ + "alloy-json-rpc", + "alloy-primitives", + "alloy-transport", + "auto_impl", + "bimap", + "futures 0.3.31", + "parking_lot", + "serde", + "serde_json", + "tokio", + "tokio-stream", + "tower 0.5.2", + "tracing", + "wasmtimer", +] [[package]] -name = "bitvec" -version = "1.0.0" +name = "alloy-rlp" +version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1489fcb93a5bb47da0462ca93ad252ad6af2145cce58d10d46a83931ba9f016b" +checksum = "5f70d83b765fdc080dbcd4f4db70d8d23fe4761f2f02ebfa9146b833900634b4" dependencies = [ - "funty", - "radium", - "tap", - "wyz", + "alloy-rlp-derive", + "arrayvec", + "bytes", ] [[package]] -name = "blake2b_simd" -version = "1.0.0" +name = "alloy-rlp-derive" +version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72936ee4afc7f8f736d1c38383b56480b5497b4617b4a77bdbf1d2ababc76127" +checksum = "64b728d511962dda67c1bc7ea7c03736ec275ed2cf4c35d9585298ac9ccf3b73" dependencies = [ - "arrayref", - "arrayvec 0.7.2", - "constant_time_eq 0.1.5", + "proc-macro2", + "quote", + "syn 2.0.118", ] [[package]] -name = "blake2s_simd" -version = "1.0.0" +name = "alloy-rpc-client" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db539cc2b5f6003621f1cd9ef92d7ded8ea5232c7de0f9faa2de251cd98730d4" +checksum = "24f461f091dc8f657e73b5dea18fd63d5c7049720cd252f1eade4a7ebed6a7e1" dependencies = [ - "arrayref", - "arrayvec 0.7.2", - "constant_time_eq 0.1.5", + "alloy-json-rpc", + "alloy-primitives", + "alloy-pubsub", + "alloy-transport", + "alloy-transport-http", + "alloy-transport-ipc", + "alloy-transport-ws", + "futures 0.3.31", + "pin-project", + "reqwest 0.13.2", + "serde", + "serde_json", + "tokio", + "tokio-stream", + "tower 0.5.2", + "tracing", + "url", + "wasmtimer", ] [[package]] -name = "blake3" -version = "0.3.8" +name = "alloy-rpc-types" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b64485778c4f16a6a5a9d335e80d449ac6c70cdd6a06d2af18a6f6f775a125b3" +checksum = "052c031d1f7c5611997056bbcb8814e5cbf20f7efeee8c3de690555172038cf2" dependencies = [ - "arrayref", - "arrayvec 0.5.2", - "cc", - "cfg-if 0.1.10", - "constant_time_eq 0.1.5", - "crypto-mac 0.8.0", - "digest 0.9.0", + "alloy-primitives", + "alloy-rpc-types-anvil", + "alloy-rpc-types-debug", + "alloy-rpc-types-engine", + "alloy-rpc-types-eth", + "alloy-rpc-types-trace", + "alloy-rpc-types-txpool", + "alloy-serde", + "serde", ] [[package]] -name = "blake3" -version = "1.3.3" +name = "alloy-rpc-types-anvil" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42ae2468a89544a466886840aa467a25b766499f4f04bf7d9fcd10ecee9fccef" +checksum = "2ff111a54268dc0bbd3b17f98571a7e27cc661dc081ad2999d91888647eb2e11" dependencies = [ - "arrayref", - "arrayvec 0.7.2", - "cc", - "cfg-if 1.0.0", - "constant_time_eq 0.2.4", - "digest 0.10.5", + "alloy-primitives", + "alloy-rpc-types-eth", + "alloy-serde", + "serde", ] [[package]] -name = "block-buffer" -version = "0.9.0" +name = "alloy-rpc-types-any" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" +checksum = "0a6561ed4759c974d9c144500a59e3fb8c1d87327a12900d5ce455c0cae6dcb6" dependencies = [ - "generic-array", + "alloy-consensus-any", + "alloy-network-primitives", + "alloy-primitives", + "alloy-rpc-types-eth", + "alloy-serde", + "serde", + "serde_json", ] [[package]] -name = "block-buffer" -version = "0.10.2" +name = "alloy-rpc-types-debug" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bf7fe51849ea569fd452f37822f606a5cabb684dc918707a0193fd4664ff324" +checksum = "48b9ad6eee93dd35a9ec0a6c1c6b180892a900ee17a6ed6500921552dd71e846" dependencies = [ - "generic-array", + "alloy-primitives", + "alloy-rlp", + "derive_more", + "serde", + "serde_with", ] [[package]] -name = "bollard" -version = "0.10.1" +name = "alloy-rpc-types-engine" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "699194c00f3a2effd3358d47f880646818e3d483190b17ebcdf598c654fb77e9" +checksum = "7eba59e1c069f168a01982f42a57797736923b76aa854194df4930be17867e1c" dependencies = [ - "base64 0.13.1", - "bollard-stubs", - "bytes", - "chrono", - "ct-logs", - "dirs-next", - "futures-core", - "futures-util", - "hex", - "http", - "hyper", - "hyper-unix-connector", - "log", - "pin-project", + "alloy-consensus", + "alloy-eips", + "alloy-primitives", + "alloy-rlp", + "alloy-serde", + "arbitrary", + "derive_more", + "rand 0.8.6", "serde", - "serde_derive", - "serde_json", - "serde_urlencoded", - "thiserror", - "tokio", - "tokio-util 0.6.7", - "url", - "winapi", + "strum 0.27.2", ] [[package]] -name = "bollard-stubs" -version = "1.41.0" +name = "alloy-rpc-types-eth" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed2f2e73fffe9455141e170fb9c1feb0ac521ec7e7dcd47a7cab72a658490fb8" +checksum = "175a2a5b6017d7f61b5e4b800d21215fe8e94fe729d00828e13bb6d93dcf3492" dependencies = [ - "chrono", + "alloy-consensus", + "alloy-consensus-any", + "alloy-eips", + "alloy-network-primitives", + "alloy-primitives", + "alloy-rlp", + "alloy-serde", + "alloy-sol-types", + "arbitrary", + "itertools 0.13.0", "serde", + "serde_json", "serde_with", + "thiserror 2.0.18", ] [[package]] -name = "bs58" -version = "0.4.0" +name = "alloy-rpc-types-trace" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "771fe0050b883fcc3ea2359b1a96bcfbc090b7116eae7c3c512c7a083fdf23d3" +checksum = "514b4b1ce3354f65067b4fc7eb75358e0f2ec8be3340c96dea65d6894f9ca435" +dependencies = [ + "alloy-primitives", + "alloy-rpc-types-eth", + "alloy-serde", + "serde", + "serde_json", + "thiserror 2.0.18", +] [[package]] -name = "bstr" -version = "0.2.16" +name = "alloy-rpc-types-txpool" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90682c8d613ad3373e66de8c6411e0ae2ab2571e879d2efbf73558cc66f21279" +checksum = "76e34a42ebb4a71ab0bfdebc6d2f3c7bf809f01edf154d08fed159d10d1ef1d4" dependencies = [ - "memchr", + "alloy-primitives", + "alloy-rpc-types-eth", + "alloy-serde", + "serde", ] [[package]] -name = "bumpalo" -version = "3.12.0" +name = "alloy-serde" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" +checksum = "cc21a8772af7d78bba286726aa245bd2ff81cd9abe230afea2e91578996831c9" +dependencies = [ + "alloy-primitives", + "arbitrary", + "serde", + "serde_json", +] [[package]] -name = "byte-slice-cast" -version = "1.2.0" +name = "alloy-signer" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d30c751592b77c499e7bce34d99d67c2c11bdc0574e9a488ddade14150a4698" +checksum = "8ffbce94c50dd9d4d1f83e044c5595bbd3ada981bd3057ce28b3a5470e77385d" +dependencies = [ + "alloy-primitives", + "async-trait", + "auto_impl", + "either", + "elliptic-curve", + "k256", + "thiserror 2.0.18", +] [[package]] -name = "byteorder" -version = "1.4.3" +name = "alloy-signer-local" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" +checksum = "e48366d2c42b8d95ef951101bafa28486590f21b7a1e68b6b2d069746557bbe3" +dependencies = [ + "alloy-consensus", + "alloy-network", + "alloy-primitives", + "alloy-signer", + "async-trait", + "k256", + "rand 0.8.6", + "thiserror 2.0.18", +] [[package]] -name = "bytes" -version = "1.2.1" +name = "alloy-sol-macro" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec8a7b6a70fde80372154c65702f00a0f56f3e1c36abbc6c440484be248856db" +checksum = "840128ed2b2971d6d4668a553fe403a82683d3acc646c73e75887e7157408033" +dependencies = [ + "alloy-sol-macro-expander", + "alloy-sol-macro-input", + "proc-macro-error2", + "proc-macro2", + "quote", + "syn 2.0.118", +] [[package]] -name = "cc" -version = "1.0.69" +name = "alloy-sol-macro-expander" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e70cc2f62c6ce1868963827bd677764c62d07c3d9a3e1fb1177ee1a9ab199eb2" +checksum = "63ec265e5d65d725175f6ca7711c970824c90ef9c0d1f1973711d4150ee612dd" dependencies = [ - "jobserver", + "alloy-json-abi", + "alloy-sol-macro-input", + "const-hex", + "heck 0.5.0", + "indexmap 2.14.0", + "proc-macro-error2", + "proc-macro2", + "quote", + "sha3", + "syn 2.0.118", + "syn-solidity", ] [[package]] -name = "cfg-if" -version = "0.1.10" +name = "alloy-sol-macro-input" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" +checksum = "89bf01077f18650876cfa682eb1f949967b5cde03f1a51c955c469d2c9b4aa67" +dependencies = [ + "alloy-json-abi", + "const-hex", + "dunce", + "heck 0.5.0", + "macro-string", + "proc-macro2", + "quote", + "serde_json", + "syn 2.0.118", + "syn-solidity", +] [[package]] -name = "cfg-if" -version = "1.0.0" +name = "alloy-sol-type-parser" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +checksum = "857b470ecdd2ed38beaf82ad1a38c516a8ff75266750f38b9eeed001d575241b" +dependencies = [ + "serde", + "winnow 1.0.0", +] [[package]] -name = "chrono" -version = "0.4.23" +name = "alloy-sol-types" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16b0a3d9ed01224b22057780a37bb8c5dbfe1be8ba48678e7bf57ec4b385411f" +checksum = "384cf252de0db2dec52821eac037a7f57e2aa33fe5b900ce6fe39973402341f1" dependencies = [ - "iana-time-zone", - "js-sys", - "num-integer", - "num-traits", + "alloy-json-abi", + "alloy-primitives", + "alloy-sol-macro", "serde", - "time 0.1.44", - "wasm-bindgen", - "winapi", ] [[package]] -name = "cid" -version = "0.10.1" +name = "alloy-transport" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd94671561e36e4e7de75f753f577edafb0e7c05d6e4547229fdf7938fbcd2c3" +checksum = "86052fdcec72d37ca4aa4b66254601e7453c45a6e1c70aa4561033d002fb80cc" dependencies = [ - "core2", - "multibase", - "multihash 0.18.0", + "alloy-json-rpc", + "auto_impl", + "base64", + "derive_more", + "futures 0.3.31", + "futures-utils-wasm", + "parking_lot", "serde", - "unsigned-varint", + "serde_json", + "thiserror 2.0.18", + "tokio", + "tower 0.5.2", + "tracing", + "url", + "wasmtimer", ] [[package]] -name = "clap" -version = "3.2.23" +name = "alloy-transport-http" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71655c45cb9845d3270c9d6df84ebe72b4dad3c2ba3f7023ad47c144e4e473a5" +checksum = "b273587487921274f4f5d0ef2c7ef36944dcbb75a4e2318e69eae822bd263f91" dependencies = [ - "atty", - "bitflags", - "clap_derive", - "clap_lex", - "indexmap", - "once_cell", - "strsim", - "termcolor", - "textwrap", + "alloy-json-rpc", + "alloy-transport", + "itertools 0.13.0", + "reqwest 0.13.2", + "serde_json", + "tower 0.5.2", + "tracing", + "url", ] [[package]] -name = "clap_derive" -version = "3.2.18" +name = "alloy-transport-ipc" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea0c8bce528c4be4da13ea6fead8965e95b6073585a2f05204bd8f4119f82a65" +checksum = "bfb89df168b24773ef603af14f2449c05a7d3f27d05d3eceaea6bf96cccae168" dependencies = [ - "heck 0.4.1", - "proc-macro-error", - "proc-macro2", - "quote", - "syn", + "alloy-json-rpc", + "alloy-pubsub", + "alloy-transport", + "bytes", + "futures 0.3.31", + "interprocess", + "pin-project", + "serde", + "serde_json", + "tokio", + "tokio-util", + "tracing", ] [[package]] -name = "clap_lex" -version = "0.2.2" +name = "alloy-transport-ws" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5538cd660450ebeb4234cfecf8f2284b844ffc4c50531e66d584ad5b91293613" +checksum = "33e32e0b47d3b3bf5770b7c132090c614b008d307c5e1544f1925f5b7e9e9af6" dependencies = [ - "os_str_bytes", + "alloy-pubsub", + "alloy-transport", + "futures 0.3.31", + "http 1.4.2", + "rustls", + "serde_json", + "tokio", + "tokio-tungstenite 0.28.0", + "tracing", + "url", + "ws_stream_wasm", ] [[package]] -name = "combine" -version = "3.8.1" +name = "alloy-trie" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da3da6baa321ec19e1cc41d31bf599f00c783d0517095cdaf0332e3fe8d20680" +checksum = "428aa0f0e0658ff091f8f667c406e034b431cb10abd39de4f507520968acc499" dependencies = [ - "ascii", - "byteorder", - "either", - "memchr", - "unreachable", + "alloy-primitives", + "alloy-rlp", + "arbitrary", + "arrayvec", + "derive_arbitrary", + "derive_more", + "nybbles", + "proptest", + "proptest-derive 0.5.1", + "serde", + "smallvec", + "tracing", ] [[package]] -name = "common-multipart-rfc7578" -version = "0.6.0" +name = "alloy-tx-macros" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baee326bc603965b0f26583e1ecd7c111c41b49bd92a344897476a352798869" +checksum = "01a0035943b75fe1e249f52e688492d7a1b1826bc2d19b8e1d5d3c24a2ad8f50" dependencies = [ - "bytes", - "futures-core", - "futures-util", - "http", - "mime", - "mime_guess", - "rand", - "thiserror", + "darling 0.23.0", + "proc-macro2", + "quote", + "syn 2.0.118", ] [[package]] -name = "console" -version = "0.13.0" +name = "android_system_properties" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a50aab2529019abfabfa93f1e6c41ef392f91fbf179b347a7e96abb524884a08" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" dependencies = [ - "encode_unicode", - "lazy_static", "libc", - "regex", - "terminal_size", - "unicode-width", - "winapi", - "winapi-util", ] [[package]] -name = "const_fn_assert" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27d614f23f34f7b5165a77dc1591f497e2518f9cec4b4f4b92bfc4dc6cf7a190" - -[[package]] -name = "constant_time_eq" -version = "0.1.5" +name = "anstream" +version = "0.6.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" +checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" +dependencies = [ + "anstyle", + "anstyle-parse 0.2.4", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] [[package]] -name = "constant_time_eq" -version = "0.2.4" +name = "anstream" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3ad85c1f65dc7b37604eb0e89748faf0b9653065f2a8ef69f96a687ec1e9279" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse 1.0.0", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] [[package]] -name = "convert_case" -version = "0.4.0" +name = "anstyle" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" [[package]] -name = "core-foundation" -version = "0.9.1" +name = "anstyle-parse" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a89e2ae426ea83155dccf10c0fa6b1463ef6d5fcb44cee0b224a408fa640a62" +checksum = "c03a11a9034d92058ceb6ee011ce58af4a9bf61491aa7e1e59ecd24bd40d22d4" dependencies = [ - "core-foundation-sys", - "libc", + "utf8parse", ] [[package]] -name = "core-foundation-sys" -version = "0.8.3" +name = "anstyle-parse" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] [[package]] -name = "core2" -version = "0.4.0" +name = "anstyle-query" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b49ba7ef1ad6107f8824dbe97de947cbaac53c44e7f9756a1fba0d37c1eec505" +checksum = "ad186efb764318d35165f1758e7dcef3b10628e26d41a44bc5550652e6804391" dependencies = [ - "memchr", + "windows-sys 0.52.0", ] [[package]] -name = "cpp_demangle" -version = "0.3.3" +name = "anstyle-wincon" +version = "3.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ea47428dc9d2237f3c6bc134472edfd63ebba0af932e783506dcfd66f10d18a" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ - "cfg-if 1.0.0", + "anstyle", + "once_cell_polyfill", + "windows-sys 0.60.2", ] [[package]] -name = "cpufeatures" -version = "0.1.5" +name = "anyhow" +version = "1.0.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66c99696f6c9dd7f35d486b9d04d7e6e202aa3e8c40d553f2fdf5e7e0c6a71ef" -dependencies = [ - "libc", -] +checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" [[package]] -name = "cpufeatures" -version = "0.2.2" +name = "arbitrary" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59a6001667ab124aebae2a495118e11d30984c3a653e99d86d58971708cf5e4b" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" dependencies = [ - "libc", + "derive_arbitrary", ] [[package]] -name = "cranelift-bforest" -version = "0.74.0" +name = "arc-swap" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8ca3560686e7c9c7ed7e0fe77469f2410ba5d7781b1acaa9adc8d8deea28e3e" +checksum = "6a3a1fd6f75306b68087b831f025c712524bcb19aad54e557b1129cfa0a2b207" dependencies = [ - "cranelift-entity", + "rustversion", ] [[package]] -name = "cranelift-codegen" -version = "0.74.0" +name = "arcstr" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf9bf1ffffb6ce3d2e5ebc83549bd2436426c99b31cc550d521364cbe35d276" -dependencies = [ - "cranelift-bforest", - "cranelift-codegen-meta", - "cranelift-codegen-shared", - "cranelift-entity", - "gimli 0.24.0", - "log", - "regalloc", - "serde", - "smallvec", - "target-lexicon", -] +checksum = "03918c3dbd7701a85c6b9887732e2921175f26c350b4563841d0958c21d57e6d" [[package]] -name = "cranelift-codegen-meta" -version = "0.74.0" +name = "ark-ff" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cc21936a5a6d07e23849ffe83e5c1f6f50305c074f4b2970ca50c13bf55b821" +checksum = "6b3235cc41ee7a12aaaf2c575a2ad7b46713a8a50bda2fc3b003a04845c05dd6" dependencies = [ - "cranelift-codegen-shared", - "cranelift-entity", + "ark-ff-asm 0.3.0", + "ark-ff-macros 0.3.0", + "ark-serialize 0.3.0", + "ark-std 0.3.0", + "derivative", + "num-bigint 0.4.6", + "num-traits", + "paste", + "rustc_version 0.3.3", + "zeroize", ] [[package]] -name = "cranelift-codegen-shared" -version = "0.74.0" +name = "ark-ff" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca5b6ffaa87560bebe69a5446449da18090b126037920b0c1c6d5945f72faf6b" +checksum = "ec847af850f44ad29048935519032c33da8aa03340876d351dfab5660d2966ba" dependencies = [ - "serde", + "ark-ff-asm 0.4.2", + "ark-ff-macros 0.4.2", + "ark-serialize 0.4.2", + "ark-std 0.4.0", + "derivative", + "digest 0.10.7", + "itertools 0.10.5", + "num-bigint 0.4.6", + "num-traits", + "paste", + "rustc_version 0.4.0", + "zeroize", ] [[package]] -name = "cranelift-entity" -version = "0.74.0" +name = "ark-ff" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d6b4a8bef04f82e4296782646f733c641d09497df2fabf791323fefaa44c64c" -dependencies = [ - "serde", +checksum = "a177aba0ed1e0fbb62aa9f6d0502e9b46dad8c2eab04c14258a1212d2557ea70" +dependencies = [ + "ark-ff-asm 0.5.0", + "ark-ff-macros 0.5.0", + "ark-serialize 0.5.0", + "ark-std 0.5.0", + "arrayvec", + "digest 0.10.7", + "educe", + "itertools 0.13.0", + "num-bigint 0.4.6", + "num-traits", + "paste", + "zeroize", ] [[package]] -name = "cranelift-frontend" -version = "0.74.0" +name = "ark-ff-asm" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c31b783b351f966fce33e3c03498cb116d16d97a8f9978164a60920bd0d3a99c" +checksum = "db02d390bf6643fb404d3d22d31aee1c4bc4459600aef9113833d17e786c6e44" dependencies = [ - "cranelift-codegen", - "log", - "smallvec", - "target-lexicon", + "quote", + "syn 1.0.109", ] [[package]] -name = "cranelift-native" -version = "0.74.0" +name = "ark-ff-asm" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a77c88d3dd48021ff1e37e978a00098524abd3513444ae252c08d37b310b3d2a" +checksum = "3ed4aa4fe255d0bc6d79373f7e31d2ea147bcf486cba1be5ba7ea85abdb92348" dependencies = [ - "cranelift-codegen", - "target-lexicon", + "quote", + "syn 1.0.109", ] [[package]] -name = "cranelift-wasm" -version = "0.74.0" +name = "ark-ff-asm" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edb6d408e2da77cdbbd65466298d44c86ae71c1785d2ab0d8657753cdb4d9d89" +checksum = "62945a2f7e6de02a31fe400aa489f0e0f5b2502e69f95f853adb82a96c7a6b60" dependencies = [ - "cranelift-codegen", - "cranelift-entity", - "cranelift-frontend", - "itertools", - "log", - "serde", - "smallvec", - "thiserror", - "wasmparser", + "quote", + "syn 2.0.118", ] [[package]] -name = "crc32fast" -version = "1.2.1" +name = "ark-ff-macros" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81156fece84ab6a9f2afdb109ce3ae577e42b1228441eded99bd77f627953b1a" +checksum = "db2fd794a08ccb318058009eefdf15bcaaaaf6f8161eb3345f907222bac38b20" dependencies = [ - "cfg-if 1.0.0", + "num-bigint 0.4.6", + "num-traits", + "quote", + "syn 1.0.109", ] [[package]] -name = "crossbeam" -version = "0.8.2" +name = "ark-ff-macros" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2801af0d36612ae591caa9568261fddce32ce6e08a7275ea334a06a4ad021a2c" +checksum = "7abe79b0e4288889c4574159ab790824d0033b9fdcb2a112a3182fac2e514565" dependencies = [ - "cfg-if 1.0.0", - "crossbeam-channel", - "crossbeam-deque", - "crossbeam-epoch", - "crossbeam-queue", - "crossbeam-utils", + "num-bigint 0.4.6", + "num-traits", + "proc-macro2", + "quote", + "syn 1.0.109", ] [[package]] -name = "crossbeam-channel" -version = "0.5.5" +name = "ark-ff-macros" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c02a4d71819009c192cf4872265391563fd6a84c81ff2c0f2a7026ca4c1d85c" +checksum = "09be120733ee33f7693ceaa202ca41accd5653b779563608f1234f78ae07c4b3" dependencies = [ - "cfg-if 1.0.0", - "crossbeam-utils", + "num-bigint 0.4.6", + "num-traits", + "proc-macro2", + "quote", + "syn 2.0.118", ] [[package]] -name = "crossbeam-deque" -version = "0.8.1" +name = "ark-serialize" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e" +checksum = "1d6c2b318ee6e10f8c2853e73a83adc0ccb88995aa978d8a3408d492ab2ee671" dependencies = [ - "cfg-if 1.0.0", - "crossbeam-epoch", - "crossbeam-utils", + "ark-std 0.3.0", + "digest 0.9.0", ] [[package]] -name = "crossbeam-epoch" -version = "0.9.5" +name = "ark-serialize" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ec02e091aa634e2c3ada4a392989e7c3116673ef0ac5b72232439094d73b7fd" +checksum = "adb7b85a02b83d2f22f89bd5cac66c9c89474240cb6207cb1efc16d098e822a5" dependencies = [ - "cfg-if 1.0.0", - "crossbeam-utils", - "lazy_static", - "memoffset", - "scopeguard", + "ark-std 0.4.0", + "digest 0.10.7", + "num-bigint 0.4.6", ] [[package]] -name = "crossbeam-queue" -version = "0.3.2" +name = "ark-serialize" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b10ddc024425c88c2ad148c1b0fd53f4c6d38db9697c9f1588381212fa657c9" +checksum = "3f4d068aaf107ebcd7dfb52bc748f8030e0fc930ac8e360146ca54c1203088f7" dependencies = [ - "cfg-if 1.0.0", - "crossbeam-utils", + "ark-std 0.5.0", + "arrayvec", + "digest 0.10.7", + "num-bigint 0.4.6", ] [[package]] -name = "crossbeam-utils" -version = "0.8.8" +name = "ark-std" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bf124c720b7686e3c2663cf54062ab0f68a88af2fb6a030e87e30bf721fcb38" +checksum = "1df2c09229cbc5a028b1d70e00fdb2acee28b1055dfb5ca73eea49c5a25c4e7c" dependencies = [ - "cfg-if 1.0.0", - "lazy_static", + "num-traits", + "rand 0.8.6", ] [[package]] -name = "crunchy" -version = "0.2.2" +name = "ark-std" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" +checksum = "94893f1e0c6eeab764ade8dc4c0db24caf4fe7cbbaafc0eba0a9030f447b5185" +dependencies = [ + "num-traits", + "rand 0.8.6", +] [[package]] -name = "crypto-common" -version = "0.1.3" +name = "ark-std" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57952ca27b5e3606ff4dd79b0020231aaf9d6aa76dc05fd30137538c50bd3ce8" +checksum = "246a225cc6131e9ee4f24619af0f19d67761fff15d7ccc22e42b80846e69449a" dependencies = [ - "generic-array", - "typenum", + "num-traits", + "rand 0.8.6", ] [[package]] -name = "crypto-mac" -version = "0.8.0" +name = "arrayvec" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b584a330336237c1eecd3e94266efb216c56ed91225d634cb2991c5f3fd1aeab" +checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" dependencies = [ - "generic-array", - "subtle", + "serde", ] [[package]] -name = "crypto-mac" -version = "0.10.1" +name = "arrow" +version = "59.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bff07008ec701e8028e2ceb8f83f0e4274ee62bd2dbdc4fefff2e9a91824081a" +checksum = "ffaaa3e009861fd829d0a24dd6f115aa8e4634324bb092147d43baafe69ca4a7" dependencies = [ - "generic-array", - "subtle", + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-csv", + "arrow-data", + "arrow-ipc", + "arrow-json", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", + "arrow-string", ] [[package]] -name = "ct-logs" -version = "0.8.0" +name = "arrow-arith" +version = "59.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1a816186fa68d9e426e3cb4ae4dff1fcd8e4a2c34b781bf7a822574a0d0aac8" +checksum = "3ac95125e1d71c4a252b5a9c729aef111e80418f08aaa6dbabd1ba66918247fc" dependencies = [ - "sct 0.6.1", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "num-traits", ] [[package]] -name = "ctor" -version = "0.1.20" +name = "arrow-array" +version = "59.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e98e2ad1a782e33928b96fc3948e7c355e5af34ba4de7670fe8bac2a3b2006d" +checksum = "0c60c79628e9a97cb90d7a0dc3e944f216a902f837d4ecabc14d524bddbbc137" dependencies = [ - "quote", - "syn", + "ahash", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "half", + "hashbrown 0.17.0", + "num-complex", + "num-integer", + "num-traits", ] [[package]] -name = "darling" -version = "0.13.0" +name = "arrow-buffer" +version = "59.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "757c0ded2af11d8e739c4daea1ac623dd1624b06c844cf3f5a39f1bdbd99bb12" +checksum = "6026f638c400e9878c1b1cc05c3cfd46fbf381285916ab408678701c1df46c1a" dependencies = [ - "darling_core", - "darling_macro", + "bytes", + "half", + "num-bigint 0.4.6", + "num-traits", ] [[package]] -name = "darling_core" -version = "0.13.0" +name = "arrow-cast" +version = "59.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c34d8efb62d0c2d7f60ece80f75e5c63c1588ba68032740494b0b9a996466e3" +checksum = "c82c236c3caf8df5664284f3f1fbe89938852163998c3fdbf37e84ac220445e9" dependencies = [ - "fnv", - "ident_case", - "proc-macro2", - "quote", - "strsim", - "syn", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-ord", + "arrow-schema", + "arrow-select", + "atoi", + "base64", + "chrono", + "half", + "lexical-core", + "num-traits", + "ryu", ] [[package]] -name = "darling_macro" -version = "0.13.0" +name = "arrow-csv" +version = "59.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ade7bff147130fe5e6d39f089c6bd49ec0250f35d70b2eebf72afdfc919f15cc" +checksum = "12714e5fb7954159af1e26d4e0d37108bcf1a2ad5ee5c5bf02a944d564d588b7" dependencies = [ - "darling_core", - "quote", - "syn", + "arrow-array", + "arrow-cast", + "arrow-schema", + "chrono", + "csv", + "csv-core", + "regex", ] [[package]] -name = "data-encoding" -version = "2.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ee2393c4a91429dffb4bedf19f4d6abf27d8a732c8ce4980305d782e5426d57" - -[[package]] -name = "data-encoding-macro" -version = "0.1.12" +name = "arrow-data" +version = "59.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86927b7cd2fe88fa698b87404b287ab98d1a0063a34071d92e575b72d3029aca" +checksum = "7bd568aa70c4ec5947027b0d5caee94877433b661a0bb9e8ddceeeb5f0c9b1ab" dependencies = [ - "data-encoding", - "data-encoding-macro-internal", + "arrow-buffer", + "arrow-schema", + "half", + "num-integer", + "num-traits", ] [[package]] -name = "data-encoding-macro-internal" -version = "0.1.10" +name = "arrow-flight" +version = "59.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5bbed42daaa95e780b60a50546aa345b8413a1e46f9a40a12907d3598f038db" +checksum = "68365401e834743d708094927e2ca727a32d639fe900df04b936e07a36701b74" dependencies = [ - "data-encoding", - "syn", + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-ipc", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", + "arrow-string", + "base64", + "bytes", + "futures 0.3.31", + "once_cell", + "paste", + "prost", + "prost-types", + "tonic", + "tonic-prost", ] [[package]] -name = "defer" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "647605a6345d5e89c3950a36a638c56478af9b414c55c6f2477c73b115f9acde" - -[[package]] -name = "derive_more" -version = "0.99.17" +name = "arrow-ipc" +version = "59.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321" +checksum = "e57ee4d470eab1a021bc4b63fa2b2c15d572892bf227b0a982d3b755a6c662b5" dependencies = [ - "convert_case", - "proc-macro2", - "quote", - "rustc_version", - "syn", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "flatbuffers", ] [[package]] -name = "diesel" -version = "1.4.8" +name = "arrow-json" +version = "59.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b28135ecf6b7d446b43e27e225622a038cc4e2930a1022f51cdb97ada19b8e4d" +checksum = "38f47e0e7a284e1f3707a780dc8cd5451b1614e9e398ea2d9ca03c7a2fe9a9ed" dependencies = [ - "bigdecimal", - "bitflags", - "byteorder", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-ord", + "arrow-schema", + "arrow-select", "chrono", - "diesel_derives", - "num-bigint", - "num-integer", + "half", + "indexmap 2.14.0", + "itoa", + "lexical-core", + "memchr", "num-traits", - "pq-sys", - "r2d2", + "ryu", + "serde_core", "serde_json", + "simdutf8", ] [[package]] -name = "diesel-derive-enum" -version = "1.1.2" +name = "arrow-ord" +version = "59.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8910921b014e2af16298f006de12aa08af894b71f0f49a486ab6d74b17bbed" +checksum = "a79cf73ad2eba8686ec2aa9bbf8671208e509025f166afc040cedbd94ffe4983" dependencies = [ - "heck 0.4.1", - "proc-macro2", - "quote", - "syn", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", ] [[package]] -name = "diesel-dynamic-schema" -version = "1.0.0" -source = "git+https://github.com/diesel-rs/diesel-dynamic-schema?rev=a8ec4fb1#a8ec4fb11de6242488ba3698d74406f4b5073dc4" +name = "arrow-row" +version = "59.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cea0f7d8ed6182f14952761e2c0f989852d5aa334fcbc49f73a9f2247c25b879" dependencies = [ - "diesel", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "half", ] [[package]] -name = "diesel_derives" -version = "1.4.1" +name = "arrow-schema" +version = "59.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45f5098f628d02a7a0f68ddba586fb61e80edec3bdc1be3b921f4ceec60858d3" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] +checksum = "80b3e786a0dd9103acd583a6fb486dbf2f3268466cc0bd571dcf34cef231c1f1" [[package]] -name = "diesel_migrations" -version = "1.4.0" +name = "arrow-select" +version = "59.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf3cde8413353dc7f5d72fa8ce0b99a560a359d2c5ef1e5817ca731cd9008f4c" +checksum = "067a67e0361f6c31f4a7248759f36ca4ca71b187a941ed4d49da1c7d3d4db624" dependencies = [ - "migrations_internals", - "migrations_macros", + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "num-traits", ] [[package]] -name = "diff" -version = "0.1.12" +name = "arrow-string" +version = "59.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e25ea47919b1560c4e3b7fe0aaab9becf5b84a10325ddf7db0f0ba5e1026499" +checksum = "99bc95847f3ff62a2b03d6f8ce2e3e78f01362060549a2a311898dd442f6256d" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "memchr", + "num-traits", + "regex", + "regex-syntax", +] [[package]] -name = "difflib" -version = "0.4.0" +name = "ascii_utils" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" +checksum = "71938f30533e4d95a6d17aa530939da3842c2ab6f4f84b9dae68447e4129f74a" [[package]] -name = "digest" -version = "0.9.0" +name = "assert-json-diff" +version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" +checksum = "47e4f2b81832e72834d7518d8487a0396a28cc408186a2e8854c0f98011faf12" dependencies = [ - "generic-array", + "serde", + "serde_json", ] [[package]] -name = "digest" -version = "0.10.5" +name = "async-compression" +version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adfbc57365a37acbd2ebf2b64d7e69bb766e2fea813521ed536f5d0520dcf86c" +checksum = "ddb939d66e4ae03cee6091612804ba446b12878410cfa17f785f4dd67d4014e8" dependencies = [ - "block-buffer 0.10.2", - "crypto-common", - "subtle", + "brotli", + "flate2", + "futures-core", + "memchr", + "pin-project-lite", + "tokio", ] [[package]] -name = "directories-next" -version = "2.0.0" +name = "async-graphql" +version = "7.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "339ee130d97a610ea5a5872d2bbb130fdf68884ff09d3028b81bec8a1ac23bbc" +checksum = "1057a9f7ccf2404d94571dec3451ade1cb524790df6f1ada0d19c2a49f6b0f40" dependencies = [ - "cfg-if 1.0.0", - "dirs-sys-next", + "async-graphql-derive", + "async-graphql-parser", + "async-graphql-value", + "async-io", + "async-trait", + "asynk-strim", + "base64", + "bytes", + "chrono", + "fast_chemail", + "fnv", + "futures-util", + "handlebars", + "http 1.4.2", + "indexmap 2.14.0", + "mime", + "multer", + "num-traits", + "pin-project-lite", + "regex", + "serde", + "serde_json", + "serde_urlencoded", + "static_assertions_next", + "tempfile", + "thiserror 2.0.18", ] [[package]] -name = "dirs" -version = "4.0.0" +name = "async-graphql-axum" +version = "7.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059" +checksum = "a1e37c5532e4b686acf45e7162bc93da91fc2c702fb0d465efc2c20c8f973795" dependencies = [ - "dirs-sys", + "async-graphql", + "axum", + "bytes", + "futures-util", + "serde_json", + "tokio", + "tokio-stream", + "tokio-util", + "tower-service 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] -name = "dirs-next" -version = "2.0.0" +name = "async-graphql-derive" +version = "7.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" +checksum = "2e6cbeadc8515e66450fba0985ce722192e28443697799988265d86304d7cc68" dependencies = [ - "cfg-if 1.0.0", - "dirs-sys-next", + "Inflector", + "async-graphql-parser", + "darling 0.23.0", + "proc-macro-crate", + "proc-macro2", + "quote", + "strum 0.27.2", + "syn 2.0.118", + "thiserror 2.0.18", ] [[package]] -name = "dirs-sys" -version = "0.3.7" +name = "async-graphql-parser" +version = "7.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" +checksum = "e64ef70f77a1c689111e52076da1cd18f91834bcb847de0a9171f83624b07fbf" dependencies = [ - "libc", - "redox_users", - "winapi", + "async-graphql-value", + "pest", + "serde", + "serde_json", ] [[package]] -name = "dirs-sys-next" -version = "0.1.2" +name = "async-graphql-value" +version = "7.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" +checksum = "3e3ef112905abea9dea592fc868a6873b10ebd3f983e83308f995d6284e9ba41" dependencies = [ - "libc", - "redox_users", - "winapi", + "bytes", + "indexmap 2.14.0", + "serde", + "serde_json", ] [[package]] -name = "either" -version = "1.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90e5c1c8368803113bf0c9584fc495a58b86dc8a29edbf8fe877d21d9507e797" - -[[package]] -name = "encode_unicode" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" - -[[package]] -name = "encoding_rs" -version = "0.8.28" +name = "async-io" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80df024fbc5ac80f87dfef0d9f5209a252f2a497f7f42944cff24d8253cac065" +checksum = "456b8a8feb6f42d237746d4b3e9a178494627745c3c56c6ea55d92ba50d026fc" dependencies = [ - "cfg-if 1.0.0", + "autocfg", + "cfg-if", + "concurrent-queue", + "futures-io", + "futures-lite", + "parking", + "polling", + "rustix 1.1.4", + "slab", + "windows-sys 0.61.2", ] [[package]] -name = "env_logger" -version = "0.7.1" +name = "async-lock" +version = "3.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36" +checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311" dependencies = [ - "atty", - "humantime 1.3.0", - "log", - "regex", - "termcolor", + "event-listener", + "event-listener-strategy", + "pin-project-lite", ] [[package]] -name = "env_logger" -version = "0.9.3" +name = "async-recursion" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a12e6657c4c97ebab115a42dcee77225f7f482cdd841cf7088c657a42e9e00e7" +checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ - "atty", - "humantime 2.1.0", - "log", - "regex", - "termcolor", + "proc-macro2", + "quote", + "syn 2.0.118", ] [[package]] -name = "envconfig" -version = "0.10.0" +name = "async-stream" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea81cc7e21f55a9d9b1efb6816904978d0bfbe31a50347cb24b2e75564bcac9b" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" dependencies = [ - "envconfig_derive", + "async-stream-impl", + "futures-core", + "pin-project-lite", ] [[package]] -name = "envconfig_derive" -version = "0.10.0" +name = "async-stream-impl" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dfca278e5f84b45519acaaff758ebfa01f18e96998bc24b8f1b722dd804b9bf" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] -name = "errno" -version = "0.2.7" +name = "async-trait" +version = "0.1.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa68f2fb9cae9d37c9b2b3584aba698a2e97f72d7aef7b9f7aa71d8b54ce46fe" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ - "errno-dragonfly", - "libc", - "winapi", + "proc-macro2", + "quote", + "syn 2.0.118", ] [[package]] -name = "errno-dragonfly" -version = "0.1.1" +name = "async_io_stream" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14ca354e36190500e1e1fb267c647932382b54053c50b14970856c0b00a35067" +checksum = "b6d7b9decdf35d8908a7e3ef02f64c5e9b1695e230154c0e8de3969142d9b94c" dependencies = [ - "gcc", - "libc", + "futures 0.3.31", + "pharos", + "rustc_version 0.4.0", ] [[package]] -name = "ethabi" -version = "17.2.0" +name = "asynk-strim" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4966fba78396ff92db3b817ee71143eccd98acf0f876b8d600e585a670c5d1b" +checksum = "52697735bdaac441a29391a9e97102c74c6ef0f9b60a40cf109b1b404e29d2f6" dependencies = [ - "ethereum-types", - "hex", - "once_cell", - "regex", - "serde", - "serde_json", - "sha3", - "thiserror", - "uint", + "futures-core", + "pin-project-lite", ] [[package]] -name = "ethbloom" -version = "0.12.1" +name = "atoi" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11da94e443c60508eb62cf256243a64da87304c2802ac2528847f79d750007ef" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" dependencies = [ - "crunchy", - "fixed-hash", - "impl-rlp", - "impl-serde", - "tiny-keccak 2.0.2", + "num-traits", ] [[package]] -name = "ethereum-types" -version = "0.13.1" +name = "atomic-waker" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2827b94c556145446fcce834ca86b7abf0c39a805883fe20e72c5bfdb5a0dc6" -dependencies = [ - "ethbloom", - "fixed-hash", - "impl-rlp", - "impl-serde", - "primitive-types", - "uint", -] +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] -name = "fallible-iterator" -version = "0.2.0" +name = "atomic_refcell" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" +checksum = "21e4227379beff4205943696e6c3e0cd809bacdf3f0edd6e3dd153e2269571a4" [[package]] -name = "file-per-thread-logger" -version = "0.1.4" +name = "auto_impl" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fdbe0d94371f9ce939b555dd342d0686cc4c0cadbcd4b61d70af5ff97eb4126" +checksum = "ffdcb70bdbc4d478427380519163274ac86e52916e10f0a8889adf0f96d3fee7" dependencies = [ - "env_logger 0.7.1", - "log", + "proc-macro2", + "quote", + "syn 2.0.118", ] [[package]] -name = "firestorm" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31586bda1b136406162e381a3185a506cdfc1631708dd40cba2f6628d8634499" - -[[package]] -name = "firestorm" -version = "0.5.0" +name = "autocfg" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d3d6188b8804df28032815ea256b6955c9625c24da7525f387a7af02fbb8f01" +checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" [[package]] -name = "fixed-hash" -version = "0.7.0" +name = "aws-lc-rs" +version = "1.16.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfcf0ed7fe52a17a03854ec54a9f76d6d84508d1c0e66bc1793301c73fc8493c" +checksum = "0ec6fb3fe69024a75fa7e1bfb48aa6cf59706a101658ea01bfd33b2b248a038f" dependencies = [ - "byteorder", - "rand", - "rustc-hex", - "static_assertions", + "aws-lc-sys", + "zeroize", ] [[package]] -name = "fixedbitset" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "398ea4fabe40b9b0d885340a2a991a44c8a645624075ad966d21f88688e2b69e" - -[[package]] -name = "flate2" -version = "1.0.25" +name = "aws-lc-sys" +version = "0.40.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8a2db397cb1c8772f31494cb8917e48cd1e64f0fa7efac59fbd741a0a8ce841" +checksum = "f50037ee5e1e41e7b8f9d161680a725bd1626cb6f8c7e901f91f942850852fe7" dependencies = [ - "crc32fast", - "miniz_oxide 0.6.2", + "cc", + "cmake", + "dunce", + "fs_extra", ] [[package]] -name = "fnv" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" - -[[package]] -name = "foreign-types" -version = "0.3.2" +name = "axum" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" dependencies = [ - "foreign-types-shared", + "axum-core", + "base64", + "bytes", + "form_urlencoded", + "futures-util", + "http 1.4.2", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sha1 0.10.6", + "sync_wrapper", + "tokio", + "tokio-tungstenite 0.29.0", + "tower 0.5.2", + "tower-layer 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "tower-service 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "tracing", ] [[package]] -name = "foreign-types-shared" -version = "0.1.1" +name = "axum-core" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" +checksum = "59446ce19cd142f8833f856eb31f3eb097812d1479ab224f54d72428ca21ea22" +dependencies = [ + "bytes", + "futures-core", + "http 1.4.2", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "tower-service 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "tracing", +] [[package]] -name = "form_urlencoded" -version = "1.1.0" +name = "backon" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8" +checksum = "cffb0e931875b666fc4fcb20fee52e9bbd1ef836fd9e9e04ec21555f9f85f7ef" dependencies = [ - "percent-encoding", + "fastrand", ] [[package]] -name = "funty" -version = "2.0.0" +name = "base-x" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" +checksum = "4cbbc9d0964165b47557570cce6c952866c2678457aca742aafc9fb771d30270" [[package]] -name = "futures" -version = "0.1.31" +name = "base16ct" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a471a38ef8ed83cd6e40aa59c1ffe17db6855c18e3604d9c4ed8c08ebc28678" +checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" [[package]] -name = "futures" -version = "0.3.16" +name = "base64" +version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1adc00f486adfc9ce99f77d717836f0c5aa84965eb0b4f051f4e83f7cab53f8b" -dependencies = [ - "futures-channel", - "futures-core", - "futures-executor", - "futures-io", - "futures-sink", - "futures-task", - "futures-util", -] +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] -name = "futures-channel" -version = "0.3.16" +name = "base64ct" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d809780667f4410e7c41b07f52439b94d2bdf8528eeedc287fa38d3b7f95d82" + +[[package]] +name = "bigdecimal" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74ed2411805f6e4e3d9bc904c95d5d423b89b3b25dc0250aa74729de20629ff9" +checksum = "1374191e2dd25f9ae02e3aa95041ed5d747fc77b3c102b49fe2dd9a8117a6244" dependencies = [ - "futures-core", - "futures-sink", + "num-bigint 0.2.6", + "num-integer", + "num-traits", + "serde", ] [[package]] -name = "futures-core" -version = "0.3.16" +name = "bimap" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af51b1b4a7fdff033703db39de8802c673eb91855f2e0d47dcf3bf2c0ef01f99" +checksum = "230c5f1ca6a325a32553f8640d31ac9b49f2411e901e427570154868b46da4f7" [[package]] -name = "futures-executor" -version = "0.3.16" +name = "bit-set" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d0d535a57b87e1ae31437b892713aee90cd2d7b0ee48727cd11fc72ef54761c" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" dependencies = [ - "futures-core", - "futures-task", - "futures-util", + "bit-vec", ] [[package]] -name = "futures-io" -version = "0.3.16" +name = "bit-vec" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b0e06c393068f3a6ef246c75cdca793d6a46347e75286933e5e75fd2fd11582" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" [[package]] -name = "futures-macro" -version = "0.3.16" +name = "bitcoin-io" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c54913bae956fb8df7f4dc6fc90362aa72e69148e3f39041fbe8742d21e0ac57" -dependencies = [ - "autocfg", - "proc-macro-hack", - "proc-macro2", - "quote", - "syn", -] +checksum = "2dee39a0ee5b4095224a0cfc6bf4cc1baf0f9624b96b367e53b66d974e51d953" [[package]] -name = "futures-sink" -version = "0.3.16" +name = "bitcoin_hashes" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0f30aaa67363d119812743aa5f33c201a7a66329f97d1a887022971feea4b53" +checksum = "26ec84b80c482df901772e931a9a681e26a1b9ee2302edeff23cb30328745c8b" +dependencies = [ + "bitcoin-io", + "hex-conservative", +] [[package]] -name = "futures-task" -version = "0.3.16" +name = "bitflags" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbe54a98670017f3be909561f6ad13e810d9a51f3f061b902062ca3da80799f2" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] -name = "futures-timer" -version = "3.0.2" +name = "bitflags" +version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c" +checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" [[package]] -name = "futures-util" -version = "0.3.16" +name = "bitvec" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67eb846bfd58e44a8481a00049e82c43e0ccb5d61f8dc071057cb19249dd4d78" +checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" dependencies = [ - "autocfg", - "futures 0.1.31", - "futures-channel", - "futures-core", - "futures-io", - "futures-macro", - "futures-sink", - "futures-task", - "memchr", - "pin-project-lite", - "pin-utils", - "proc-macro-hack", - "proc-macro-nested", - "slab", + "funty", + "radium", + "tap", + "wyz", ] [[package]] -name = "gcc" -version = "0.3.55" +name = "blake3" +version = "1.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f5f3913fa0bfe7ee1fd8248b6b9f42a5af4b9d65ec2dd2c3c26132b950ecfc2" +checksum = "6d9e454fc11f76977dc803893aff6304ed33d6a26efae8696573bea74baa27ae" +dependencies = [ + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "cpufeatures 0.3.0", +] [[package]] -name = "generic-array" -version = "0.14.4" +name = "block-buffer" +version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "501466ecc8a30d1d3b7fc9229b122b2ce8ed6e9d9223f1138d4babb253e51817" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" dependencies = [ - "typenum", - "version_check", + "generic-array", ] [[package]] -name = "getrandom" -version = "0.2.3" +name = "block-buffer" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcd999463524c52659517fe2cea98493cfe485d10565e7b0fb07dbba7ad2753" +checksum = "cdd35008169921d80bc60d3d0ab416eecb028c4cd653352907921d95084790be" dependencies = [ - "cfg-if 1.0.0", - "libc", - "wasi", + "hybrid-array", ] [[package]] -name = "gimli" -version = "0.24.0" +name = "blst" +version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e4075386626662786ddb0ec9081e7c7eeb1ba31951f447ca780ef9f5d568189" +checksum = "dcdb4c7013139a150f9fc55d123186dbfaba0d912817466282c73ac49e71fb45" dependencies = [ - "fallible-iterator", - "indexmap", - "stable_deref_trait", + "cc", + "glob", + "threadpool", + "zeroize", ] [[package]] -name = "gimli" -version = "0.25.0" +name = "borsh" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0a01e0497841a3b2db4f8afa483cce65f7e96a3498bd6c541734792aeac8fe7" +checksum = "d1da5ab77c1437701eeff7c88d968729e7766172279eab0676857b3d63af7a6f" +dependencies = [ + "borsh-derive", + "cfg_aliases", +] [[package]] -name = "git-testament" -version = "0.2.4" +name = "borsh-derive" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "986bf57c808270f3a0a0652c3bfce0f5d667aa5f5b465616dc697c7f390834b1" +checksum = "0686c856aa6aac0c4498f936d7d6a02df690f614c03e4d906d1018062b5c5e2c" dependencies = [ - "git-testament-derive", - "no-std-compat", + "once_cell", + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.118", ] [[package]] -name = "git-testament-derive" -version = "0.1.14" +name = "brotli" +version = "8.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a782db5866c7ab75f3552dda4cbf34e3e257cc64c963c6ed5af1e12818e8ae6" +checksum = "4bd8b9603c7aa97359dbd97ecf258968c95f3adddd6db2f7e7a5bef101c84560" dependencies = [ - "log", - "proc-macro2", - "quote", - "syn", - "time 0.3.17", + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", ] [[package]] -name = "globset" -version = "0.4.8" +name = "brotli-decompressor" +version = "5.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10463d9ff00a2a068db14231982f5132edebad0d7660cd956a1c30292dbcbfbd" +checksum = "874bb8112abecc98cbd6d81ea4fa7e94fb9449648c93cc89aa40c81c24d7de03" dependencies = [ - "aho-corasick", - "bstr", - "fnv", - "log", - "regex", + "alloc-no-stdlib", + "alloc-stdlib", ] [[package]] -name = "graph" -version = "0.30.0" +name = "bs58" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "771fe0050b883fcc3ea2359b1a96bcfbc090b7116eae7c3c512c7a083fdf23d3" + +[[package]] +name = "bs58" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" dependencies = [ - "Inflector", - "anyhow", - "async-stream", - "async-trait", - "atomic_refcell", - "bigdecimal", - "bytes", - "chrono", - "cid", - "clap", - "diesel", - "diesel_derives", - "envconfig", - "ethabi", - "futures 0.1.31", - "futures 0.3.16", - "graphql-parser", - "hex", - "http", - "isatty", - "itertools", - "lazy_static", - "maplit", - "num-bigint", - "num-traits", - "num_cpus", - "parking_lot 0.12.1", - "petgraph", - "priority-queue", - "prometheus", - "prost", - "prost-types", - "rand", - "regex", - "reqwest", - "semver", - "serde", - "serde_derive", - "serde_json", - "serde_plain", - "serde_yaml", - "slog", - "slog-async", - "slog-envlogger", - "slog-term", - "stable-hash 0.3.3", - "stable-hash 0.4.2", - "strum", - "strum_macros", - "test-store", - "thiserror", - "tiny-keccak 1.5.0", - "tokio", - "tokio-retry", - "tokio-stream", - "tonic", - "tonic-build", - "url", - "wasmparser", - "web3", + "tinyvec", ] [[package]] -name = "graph-chain-arweave" -version = "0.30.0" +name = "bstr" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05efc5cfd9110c8416e471df0e96702d58690178e206e61b7173706673c93706" dependencies = [ - "base64-url", - "diesel", - "graph", - "graph-runtime-derive", - "graph-runtime-wasm", - "prost", - "prost-types", + "memchr", "serde", - "sha2 0.10.6", - "tonic-build", ] [[package]] -name = "graph-chain-common" -version = "0.30.0" +name = "bumpalo" +version = "3.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" dependencies = [ - "anyhow", - "heck 0.4.1", - "protobuf 3.2.0", - "protobuf-parse", + "allocator-api2", ] [[package]] -name = "graph-chain-cosmos" -version = "0.30.0" +name = "byte-slice-cast" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3ac9f8b63eca6fd385229b3675f6cc0dc5c8a5c8a54a59d4f52ffd670d87b0c" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" dependencies = [ - "anyhow", - "graph", - "graph-chain-common", - "graph-runtime-derive", - "graph-runtime-wasm", - "prost", - "prost-types", - "semver", "serde", - "tonic-build", ] [[package]] -name = "graph-chain-ethereum" -version = "0.30.0" +name = "c-kzg" +version = "2.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e00bf4b112b07b505472dbefd19e37e53307e2bfed5a79e0cc161d58ccd0e687" dependencies = [ - "anyhow", - "base64 0.20.0", - "dirs-next", - "envconfig", - "futures 0.1.31", - "graph", - "graph-mock", - "graph-runtime-derive", - "graph-runtime-wasm", + "arbitrary", + "blst", + "cc", + "glob", "hex", - "http", - "itertools", - "jsonrpc-core", - "lazy_static", - "prost", - "prost-types", - "semver", + "libc", + "once_cell", "serde", - "test-store", - "tiny-keccak 1.5.0", - "tonic-build", ] [[package]] -name = "graph-chain-near" -version = "0.30.0" +name = "cc" +version = "1.2.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43c5703da9466b66a946814e1adf53ea2c90f10063b86290cc9eb67ce3478a20" dependencies = [ - "base64 0.20.0", - "diesel", - "graph", - "graph-runtime-derive", - "graph-runtime-wasm", - "prost", - "prost-types", - "serde", - "tonic-build", + "find-msvc-tools", + "jobserver", + "libc", + "shlex", ] [[package]] -name = "graph-chain-substreams" -version = "0.30.0" +name = "cesu8" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "chacha20" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" dependencies = [ - "anyhow", - "async-stream", - "base64 0.20.0", - "dirs-next", - "envconfig", - "futures 0.1.31", - "graph", - "graph-core", - "graph-runtime-wasm", - "hex", - "http", - "itertools", - "jsonrpc-core", - "lazy_static", - "prost", - "prost-types", - "semver", - "serde", - "tiny-keccak 1.5.0", - "tokio", - "tonic-build", + "cfg-if", + "cpufeatures 0.3.0", + "rand_core 0.10.0", ] [[package]] -name = "graph-core" -version = "0.30.0" +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" dependencies = [ - "anyhow", - "async-stream", - "async-trait", - "atomic_refcell", - "bytes", - "cid", - "futures 0.1.31", - "futures 0.3.16", - "graph", - "graph-chain-arweave", - "graph-chain-cosmos", - "graph-chain-ethereum", - "graph-chain-near", - "graph-chain-substreams", - "graph-mock", - "graph-runtime-wasm", - "graphql-parser", - "hex", - "ipfs-api", - "ipfs-api-backend-hyper", - "lazy_static", - "lru_time_cache", - "pretty_assertions", - "semver", + "iana-time-zone", + "js-sys", + "num-traits", "serde", - "serde_json", - "serde_yaml", - "test-store", - "tower 0.4.12", - "tower-test", - "uuid", + "wasm-bindgen", + "windows-link 0.2.1", ] [[package]] -name = "graph-graphql" -version = "0.30.0" +name = "cid" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21a304f95f84d169a6f31c4d0a30d784643aaa0bbc9c1e449a2c23e963ec4971" dependencies = [ - "Inflector", - "anyhow", - "async-recursion", - "crossbeam", - "defer", - "graph", - "graph-chain-ethereum", - "graphql-parser", - "graphql-tools", - "indexmap", - "lazy_static", - "parking_lot 0.12.1", - "pretty_assertions", - "stable-hash 0.3.3", - "stable-hash 0.4.2", - "test-store", + "multibase", + "multihash", + "unsigned-varint", ] [[package]] -name = "graph-mock" -version = "0.30.0" +name = "clap" +version = "4.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84b3edb18336f4df585bc9aa31dd99c036dfa5dc5e9a2939a722a188f3a8970d" dependencies = [ - "graph", + "clap_builder", + "clap_derive", ] [[package]] -name = "graph-node" -version = "0.30.0" +name = "clap_builder" +version = "4.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1c09dd5ada6c6c78075d6fd0da3f90d8080651e2d6cc8eb2f1aaa4034ced708" +dependencies = [ + "anstream 0.6.21", + "anstyle", + "clap_lex", + "strsim", + "terminal_size", +] + +[[package]] +name = "clap_complete" +version = "4.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b4be9c4c4b1f30b78d8a750e0822b6a6102d97e62061c583a6c1dea2dfb33ae" dependencies = [ "clap", - "crossbeam-channel", - "diesel", - "env_logger 0.9.3", - "futures 0.3.16", - "git-testament", - "graph", - "graph-chain-arweave", - "graph-chain-cosmos", - "graph-chain-ethereum", - "graph-chain-near", - "graph-chain-substreams", - "graph-core", - "graph-graphql", - "graph-runtime-wasm", - "graph-server-http", - "graph-server-index-node", - "graph-server-json-rpc", - "graph-server-metrics", - "graph-server-websocket", - "graph-store-postgres", - "graphql-parser", - "http", - "json-structural-diff", - "lazy_static", - "prometheus", - "serde", - "serde_regex", - "shellexpand", - "termcolor", - "toml 0.7.1", - "url", ] [[package]] -name = "graph-runtime-derive" -version = "0.30.0" +name = "clap_derive" +version = "4.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bac35c6dafb060fd4d275d9a4ffae97917c13a6327903a8be2153cd964f7085" dependencies = [ - "heck 0.4.1", + "heck 0.5.0", "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] -name = "graph-runtime-test" -version = "0.30.0" +name = "clap_lex" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b82cf0babdbd58558212896d1a4272303a57bdb245c2bf1147185fb45640e70" + +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" dependencies = [ - "graph", - "graph-chain-ethereum", - "graph-core", - "graph-mock", - "graph-runtime-derive", - "graph-runtime-wasm", - "rand", - "semver", - "test-store", - "wasmtime", + "cc", ] [[package]] -name = "graph-runtime-wasm" -version = "0.30.0" +name = "cobs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa961b519f0b462e3a3b4a34b64d119eeaca1d59af726fe450bbba07a9fc0a1" +dependencies = [ + "thiserror 2.0.18", +] + +[[package]] +name = "colorchoice" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422" + +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" dependencies = [ - "anyhow", - "async-trait", - "atomic_refcell", - "bs58", "bytes", - "defer", - "ethabi", - "futures 0.1.31", - "graph", - "graph-runtime-derive", - "hex", - "lazy_static", - "never", - "parity-wasm", - "semver", - "strum", - "strum_macros", - "uuid", - "wasm-instrument", - "wasmtime", + "futures-core", + "memchr", + "pin-project-lite", + "tokio", + "tokio-util", ] [[package]] -name = "graph-server-http" -version = "0.30.0" +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" dependencies = [ - "futures 0.1.31", - "graph", - "graph-graphql", - "graph-mock", - "graphql-parser", - "http", - "hyper", - "serde", + "crossbeam-utils", ] [[package]] -name = "graph-server-index-node" -version = "0.30.0" +name = "console" +version = "0.15.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8" dependencies = [ - "blake3 1.3.3", - "either", - "futures 0.3.16", - "graph", - "graph-chain-arweave", - "graph-chain-cosmos", - "graph-chain-ethereum", - "graph-chain-near", - "graph-graphql", - "graphql-parser", - "http", - "hyper", - "lazy_static", - "serde", + "encode_unicode", + "libc", + "once_cell", + "unicode-width", + "windows-sys 0.59.0", +] + +[[package]] +name = "console" +version = "0.16.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fe5f465a4f6fee88fad41b85d990f84c835335e85b5d9e6e63e0d06d28cba7c" +dependencies = [ + "encode_unicode", + "libc", + "unicode-width", + "windows-sys 0.61.2", +] + +[[package]] +name = "const-hex" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bb320cac8a0750d7f25280aa97b09c26edfe161164238ecbbb31092b079e735" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.12", + "proptest", + "serde_core", +] + +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom 0.2.15", + "once_cell", + "tiny-keccak", +] + +[[package]] +name = "constant_time_eq" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + +[[package]] +name = "convert_case" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "633458d4ef8c78b72454de2d54fd6ab2e60f9e02be22f3c6104cdc8a4e0fceb9" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b55271e5c8c478ad3f38ad24ef34923091e0548492a266d19b3c0b4d82574c63" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpp_demangle" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96e58d342ad113c2b878f16d5d034c03be492ae460cdbc02b7f0f2284d310c7d" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "cpufeatures" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" +dependencies = [ + "libc", +] + +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + +[[package]] +name = "cranelift-assembler-x64" +version = "0.133.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e06aeba2c965fc446d13c56a6ccb2631b78445d7544543dd9a25289977630914" +dependencies = [ + "cranelift-assembler-x64-meta", +] + +[[package]] +name = "cranelift-assembler-x64-meta" +version = "0.133.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee2d2dde4ec1352715595b5cfa6fe2e5b8ebb9da3457b3ee8db0aa2808c069aa" +dependencies = [ + "cranelift-srcgen", +] + +[[package]] +name = "cranelift-bforest" +version = "0.133.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03b4982ef9fa54ec9eee841e891e7ddc5434be1250e88de31572e000c888f30b" +dependencies = [ + "cranelift-entity", + "wasmtime-internal-core", +] + +[[package]] +name = "cranelift-bitset" +version = "0.133.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "529143118c4eeb58c39ecb02319557d512be6c61348486422974ab8e3906b8a8" +dependencies = [ + "serde", + "serde_derive", + "wasmtime-internal-core", +] + +[[package]] +name = "cranelift-codegen" +version = "0.133.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7780677247ad3577e3a6a3ebf43f39b325a11d6393db72b2c9968a910d4d13d" +dependencies = [ + "bumpalo", + "cranelift-assembler-x64", + "cranelift-bforest", + "cranelift-bitset", + "cranelift-codegen-meta", + "cranelift-codegen-shared", + "cranelift-control", + "cranelift-entity", + "cranelift-isle", + "gimli", + "hashbrown 0.17.0", + "libm", + "log", + "postcard", + "pulley-interpreter", + "regalloc2", + "rustc-hash", + "serde", + "serde_derive", + "sha2 0.10.9", + "smallvec", + "target-lexicon", + "wasmtime-internal-core", +] + +[[package]] +name = "cranelift-codegen-meta" +version = "0.133.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac9645250416cbf92454fe61160e17e026e0ce405906a54500b114f923ddffc9" +dependencies = [ + "cranelift-assembler-x64-meta", + "cranelift-codegen-shared", + "cranelift-srcgen", + "heck 0.5.0", + "pulley-interpreter", +] + +[[package]] +name = "cranelift-codegen-shared" +version = "0.133.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20ee8d222ff0fd3681791979afbf88586ac9f49010d3db96b3cbe4c96759aee3" + +[[package]] +name = "cranelift-control" +version = "0.133.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "591abe6f5312bd2c4220f1b3bead56c2ad00257c52668015ba013b85dcf2a17a" +dependencies = [ + "arbitrary", +] + +[[package]] +name = "cranelift-entity" +version = "0.133.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5300c49cf940526fe771517b3b3eabd5d0ff164ee61698579cf403fe8d3af3c" +dependencies = [ + "cranelift-bitset", + "serde", + "serde_derive", + "wasmtime-internal-core", +] + +[[package]] +name = "cranelift-frontend" +version = "0.133.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da4adbf760207fdbbe130f1191cce01cdef66831a9f648b1f39ff2800d126d45" +dependencies = [ + "cranelift-codegen", + "hashbrown 0.17.0", + "log", + "smallvec", + "target-lexicon", +] + +[[package]] +name = "cranelift-isle" +version = "0.133.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8315b21ff018226a42a60a4702c2dd75f6447cac26e9bca622e14c22088c2ff5" + +[[package]] +name = "cranelift-native" +version = "0.133.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d506ef23a60715bde451b06620b14402166ded3b648454fccbf04f3e46a4aa70" +dependencies = [ + "cranelift-codegen", + "libc", + "target-lexicon", +] + +[[package]] +name = "cranelift-srcgen" +version = "0.133.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48ed47e602652e3410f9387fc0db70fefadcee4d78a78881421aabcab4e26b89" + +[[package]] +name = "crc" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + +[[package]] +name = "crc32fast" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1137cd7e7fc0fb5d3c5a8678be38ec56e819125d8d7907411fe24ccb943faca8" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-epoch", + "crossbeam-queue", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-queue" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" + +[[package]] +name = "crossterm" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8b9f2e4c67f833b660cdb0a3523065869fb35570177239812ed4c905aeff87b" +dependencies = [ + "bitflags 2.11.1", + "crossterm_winapi", + "derive_more", + "document-features", + "mio", + "parking_lot", + "rustix 1.1.4", + "signal-hook", + "signal-hook-mio", + "winapi", +] + +[[package]] +name = "crossterm_winapi" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b" +dependencies = [ + "winapi", +] + +[[package]] +name = "crunchy" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" + +[[package]] +name = "crypto-bigint" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" +dependencies = [ + "generic-array", + "rand_core 0.6.4", + "subtle", + "zeroize", +] + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "crypto-common" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77727bb15fa921304124b128af125e7e3b968275d1b108b379190264f4423710" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "csv" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde_core", +] + +[[package]] +name = "csv-core" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70" +dependencies = [ + "memchr", +] + +[[package]] +name = "darling" +version = "0.20.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989" +dependencies = [ + "darling_core 0.20.10", + "darling_macro 0.20.10", +] + +[[package]] +name = "darling" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cdf337090841a411e2a7f3deb9187445851f91b309c0c0a29e05f74a00a48c0" +dependencies = [ + "darling_core 0.21.3", + "darling_macro 0.21.3", +] + +[[package]] +name = "darling" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" +dependencies = [ + "darling_core 0.23.0", + "darling_macro 0.23.0", +] + +[[package]] +name = "darling_core" +version = "0.20.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.118", +] + +[[package]] +name = "darling_core" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1247195ecd7e3c85f83c8d2a366e4210d588e802133e1e355180a9870b517ea4" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.118", +] + +[[package]] +name = "darling_core" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "serde", + "strsim", + "syn 2.0.118", +] + +[[package]] +name = "darling_macro" +version = "0.20.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" +dependencies = [ + "darling_core 0.20.10", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "darling_macro" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" +dependencies = [ + "darling_core 0.21.3", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "darling_macro" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" +dependencies = [ + "darling_core 0.23.0", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "dashmap" +version = "6.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + +[[package]] +name = "data-encoding" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2" + +[[package]] +name = "data-encoding-macro" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1559b6cba622276d6d63706db152618eeb15b89b3e4041446b05876e352e639" +dependencies = [ + "data-encoding", + "data-encoding-macro-internal", +] + +[[package]] +name = "data-encoding-macro-internal" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "332d754c0af53bc87c108fed664d121ecf59207ec4196041f04d6ab9002ad33f" +dependencies = [ + "data-encoding", + "syn 1.0.109", +] + +[[package]] +name = "deadpool" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0be2b1d1d6ec8d846f05e137292d0b89133caf95ef33695424c09568bdd39b1b" +dependencies = [ + "deadpool-runtime 0.1.4", + "lazy_static", + "num_cpus", + "tokio", +] + +[[package]] +name = "deadpool" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "883466cb8db62725aee5f4a6011e8a5d42912b42632df32aad57fc91127c6e04" +dependencies = [ + "deadpool-runtime 0.3.1", + "num_cpus", + "tokio", +] + +[[package]] +name = "deadpool-runtime" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "092966b41edc516079bdf31ec78a2e0588d1d0c08f78b91d8307215928642b2b" + +[[package]] +name = "deadpool-runtime" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2657f61fb1dd8bf37a8d51093cc7cee4e77125b22f7753f49b289f831bec2bae" +dependencies = [ + "tokio", +] + +[[package]] +name = "debugid" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef552e6f588e446098f6ba40d89ac146c8c7b64aade83c051ee00bb5d2bc18d" +dependencies = [ + "uuid", +] + +[[package]] +name = "defer" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "930c7171c8df9fb1782bdf9b918ed9ed2d33d1d22300abb754f9085bc48bf8e8" + +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid 0.9.6", + "zeroize", +] + +[[package]] +name = "deranged" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ececcb659e7ba858fb4f10388c250a7252eb0a27373f1a72b8748afdd248e587" +dependencies = [ + "powerfmt", + "serde_core", +] + +[[package]] +name = "derivative" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "derive_arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "derive_builder" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +dependencies = [ + "darling 0.20.10", + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core", + "syn 2.0.118", +] + +[[package]] +name = "derive_more" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" +dependencies = [ + "convert_case", + "proc-macro2", + "quote", + "rustc_version 0.4.0", + "syn 2.0.118", + "unicode-xid", +] + +[[package]] +name = "diesel" +version = "2.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9940fb8467a0a06312218ed384185cb8536aa10d8ec017d0ce7fad2c1bd882d5" +dependencies = [ + "bigdecimal", + "bitflags 2.11.1", + "byteorder", + "chrono", + "diesel_derives", + "downcast-rs", + "itoa", + "num-bigint 0.2.6", + "num-integer", + "num-traits", + "pq-sys", + "r2d2", + "serde_json", +] + +[[package]] +name = "diesel-async" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd39af30158d444884f166fe4c58f35dc40ad71ad017bb59408a3448526ff4bd" +dependencies = [ + "deadpool 0.13.0", + "diesel", + "futures-core", + "futures-util", + "pin-project-lite", + "tokio", + "tokio-postgres", +] + +[[package]] +name = "diesel-derive-enum" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81c5131a2895ef64741dad1d483f358c2a229a3a2d1b256778cdc5e146db64d4" +dependencies = [ + "heck 0.4.1", + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "diesel-dynamic-schema" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "030a2287b125235908614c5f32f9b3bdc43c4d639846853d66e8a68c75a02756" +dependencies = [ + "diesel", +] + +[[package]] +name = "diesel_derives" +version = "2.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1817b7f4279b947fc4cafddec12b0e5f8727141706561ce3ac94a60bddd1cf5" +dependencies = [ + "diesel_table_macro_syntax", + "dsl_auto_type", + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "diesel_migrations" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d0f4a98124ba6d4ca75da535f65984badec16a003b6e2f94a01e31a79490b8" +dependencies = [ + "diesel", + "migrations_internals", + "migrations_macros", +] + +[[package]] +name = "diesel_table_macro_syntax" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe2444076b48641147115697648dc743c2c00b61adade0f01ce67133c7babe8c" +dependencies = [ + "syn 2.0.118", +] + +[[package]] +name = "diff" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" + +[[package]] +name = "difflib" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" + +[[package]] +name = "digest" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" +dependencies = [ + "generic-array", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer 0.10.4", + "const-oid 0.9.6", + "crypto-common 0.1.6", + "subtle", +] + +[[package]] +name = "digest" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4850db49bf08e663084f7fb5c87d202ef91a3907271aff24a94eb97ff039153c" +dependencies = [ + "block-buffer 0.12.0", + "const-oid 0.10.2", + "crypto-common 0.2.1", +] + +[[package]] +name = "directories-next" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339ee130d97a610ea5a5872d2bbb130fdf68884ff09d3028b81bec8a1ac23bbc" +dependencies = [ + "cfg-if", + "dirs-sys-next", +] + +[[package]] +name = "dirs" +version = "5.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-next" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" +dependencies = [ + "cfg-if", + "dirs-sys-next", +] + +[[package]] +name = "dirs-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys 0.48.0", +] + +[[package]] +name = "dirs-sys-next" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "doctest-file" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aac81fa3e28d21450aa4d2ac065992ba96a1d7303efbce51a95f4fd175b67562" + +[[package]] +name = "document-features" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4b8a88685455ed29a21542a33abd9cb6510b6b129abadabdcef0f4c55bc8f61" +dependencies = [ + "litrs", +] + +[[package]] +name = "downcast-rs" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "117240f60069e65410b3ae1bb213295bd828f707b5bec6596a1afc8793ce0cbc" + +[[package]] +name = "dsl_auto_type" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd122633e4bef06db27737f21d3738fb89c8f6d5360d6d9d7635dda142a7757e" +dependencies = [ + "darling 0.21.3", + "either", + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + +[[package]] +name = "ecdsa" +version = "0.16.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" +dependencies = [ + "der", + "digest 0.10.7", + "elliptic-curve", + "rfc6979", + "serdect", + "signature", + "spki", +] + +[[package]] +name = "educe" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7bc049e1bd8cdeb31b68bbd586a9464ecf9f3944af3958a7a9d0f8b9799417" +dependencies = [ + "enum-ordinalize", + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +dependencies = [ + "serde", +] + +[[package]] +name = "elliptic-curve" +version = "0.13.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" +dependencies = [ + "base16ct", + "crypto-bigint", + "digest 0.10.7", + "ff", + "generic-array", + "group", + "pkcs8", + "rand_core 0.6.4", + "sec1", + "serdect", + "subtle", + "zeroize", +] + +[[package]] +name = "embedded-io" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced" + +[[package]] +name = "embedded-io" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" + +[[package]] +name = "encode_unicode" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" + +[[package]] +name = "encoding_rs" +version = "0.8.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "enum-ordinalize" +version = "4.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a1091a7bb1f8f2c4b28f1fe2cef4980ca2d410a3d727d67ecc3178c9b0800f0" +dependencies = [ + "enum-ordinalize-derive", +] + +[[package]] +name = "enum-ordinalize-derive" +version = "4.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ca9601fb2d62598ee17836250842873a413586e5d7ed88b356e38ddbb0ec631" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "env_filter" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900d271a03799a1ee8d1ca9b19893b48ca674a9284fefcfb85f05e74ed314217" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "env_logger" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de671bd27a75a797dc9ae289ba1e77276e75e2026408aab65185384e2d5cd3f6" +dependencies = [ + "anstream 1.0.0", + "anstyle", + "env_filter", + "jiff", + "log", +] + +[[package]] +name = "envconfig" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2934d78aeb06e54961db5b96cfca2b01f5cb01cf11c34f7b088932fd48233ac6" +dependencies = [ + "envconfig_derive", +] + +[[package]] +name = "envconfig_derive" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "452c458dfb890a2fea64e4c894f83daad61689fb9bbe84c382e1ce6d7536710b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "erased-serde" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c138974f9d5e7fe373eb04df7cae98833802ae4b11c24ac7039a21d5af4b26c" +dependencies = [ + "serde", +] + +[[package]] +name = "errno" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cea14ef9355e3beab063703aa9dab15afd25f0667c341310c1e5274bb1d0da18" +dependencies = [ + "libc", + "windows-sys 0.59.0", +] + +[[package]] +name = "event-listener" +version = "5.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + +[[package]] +name = "fallible-iterator" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" + +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fast_chemail" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "495a39d30d624c2caabe6312bfead73e7717692b44e0b32df168c275a2e8e9e4" +dependencies = [ + "ascii_utils", +] + +[[package]] +name = "fastrand" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + +[[package]] +name = "fastrlp" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139834ddba373bbdd213dffe02c8d110508dcf1726c2be27e8d1f7d7e1856418" +dependencies = [ + "arrayvec", + "auto_impl", + "bytes", +] + +[[package]] +name = "fastrlp" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce8dba4714ef14b8274c371879b175aa55b16b30f269663f19d576f380018dc4" +dependencies = [ + "arrayvec", + "auto_impl", + "bytes", +] + +[[package]] +name = "ff" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393" +dependencies = [ + "rand_core 0.6.4", + "subtle", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "firestorm" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31586bda1b136406162e381a3185a506cdfc1631708dd40cba2f6628d8634499" + +[[package]] +name = "firestorm" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c5f6c2c942da57e2aaaa84b8a521489486f14e75e7fa91dab70aba913975f98" + +[[package]] +name = "fixed-hash" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "835c052cb0c08c1acf6ffd71c022172e18723949c8282f2b9f27efbc51e64534" +dependencies = [ + "byteorder", + "rand 0.8.6", + "rustc-hex", + "static_assertions", +] + +[[package]] +name = "fixedbitset" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" + +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + +[[package]] +name = "flatbuffers" +version = "25.12.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" +dependencies = [ + "bitflags 2.11.1", + "rustc_version 0.4.0", +] + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", + "zlib-rs", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + +[[package]] +name = "fsevent-sys" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76ee7a02da4d231650c7cea31349b889be2f45ddb3ef3032d2ec8185f6313fd2" +dependencies = [ + "libc", +] + +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + +[[package]] +name = "futures" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a471a38ef8ed83cd6e40aa59c1ffe17db6855c18e3604d9c4ed8c08ebc28678" + +[[package]] +name = "futures" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" + +[[package]] +name = "futures-executor" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" + +[[package]] +name = "futures-lite" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f78e10609fe0e0b3f4157ffab1876319b5b0db102a2c60dc4626306dc46b44ad" +dependencies = [ + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "futures-macro" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "futures-sink" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" + +[[package]] +name = "futures-task" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" + +[[package]] +name = "futures-util" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +dependencies = [ + "futures 0.1.31", + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "futures-utils-wasm" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42012b0f064e01aa58b545fe3727f90f7dd4020f4a3ea735b50344965f5a57e9" + +[[package]] +name = "fuzzy-matcher" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54614a3312934d066701a80f20f15fa3b56d67ac7722b39eea5b4c9dd1d66c94" +dependencies = [ + "thread_local", +] + +[[package]] +name = "fxprof-processed-profile" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25234f20a3ec0a962a61770cfe39ecf03cb529a6e474ad8cff025ed497eda557" +dependencies = [ + "bitflags 2.11.1", + "debugid", + "rustc-hash", + "serde", + "serde_derive", + "serde_json", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", + "zeroize", +] + +[[package]] +name = "getrandom" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi 0.11.0+wasi-snapshot-preview1", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43a49c392881ce6d5c3b8cb70f98717b7c07aabbdff06687b9030dbfbe2725f8" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi 0.13.3+wasi-0.2.2", + "wasm-bindgen", + "windows-targets 0.52.6", +] + +[[package]] +name = "getrandom" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "rand_core 0.10.0", + "wasip2", + "wasip3", +] + +[[package]] +name = "gimli" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf7f043f89559805f8c7cacc432749b2fa0d0a0a9ee46ce47164ed5ba7f126c" +dependencies = [ + "fnv", + "hashbrown 0.16.1", + "indexmap 2.14.0", + "stable_deref_trait", +] + +[[package]] +name = "git-testament" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a74999c921479f919c87a9d2e6922a79a18683f18105344df8e067149232e51" +dependencies = [ + "git-testament-derive", +] + +[[package]] +name = "git-testament-derive" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbeac967e71eb3dc1656742fc7521ec7cd3b6b88738face65bf1fddf702bc4c0" +dependencies = [ + "log", + "proc-macro2", + "quote", + "syn 2.0.118", + "time", +] + +[[package]] +name = "glob" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" + +[[package]] +name = "globset" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52dfc19153a48bde0cbd630453615c8151bce3a5adfac7a0aebfbf0a1e1f57e3" +dependencies = [ + "aho-corasick", + "bstr", + "log", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "gnd" +version = "0.45.0" +dependencies = [ + "Inflector", + "anyhow", + "async-trait", + "clap", + "clap_complete", + "console 0.16.4", + "env_logger", + "git-testament", + "globset", + "graph", + "graph-chain-ethereum", + "graph-core", + "graph-graphql", + "graph-node", + "graph-store-postgres", + "graphql-tools", + "hex", + "indicatif", + "inquire", + "lazy_static", + "notify", + "open", + "pgtemp", + "pq-sys", + "regex", + "reqwest 0.12.23", + "semver 1.0.28", + "serde", + "serde_json", + "serde_yaml", + "sha1 0.11.0", + "similar", + "tempfile", + "thiserror 2.0.18", + "tokio", + "tokio-util", + "tower 0.5.2", + "url", + "walkdir", + "wasmparser 0.118.2", +] + +[[package]] +name = "graph" +version = "0.45.0" +dependencies = [ + "Inflector", + "ahash", + "alloy", + "anyhow", + "arrow", + "arrow-flight", + "async-stream", + "async-trait", + "atomic_refcell", + "base64", + "bigdecimal", + "bs58 0.5.1", + "bytes", + "chrono", + "cid", + "clap", + "csv", + "defer", + "derive_more", + "diesel", + "diesel_derives", + "envconfig", + "futures 0.1.31", + "futures 0.3.31", + "graph_derive", + "graphql-tools", + "half", + "hex", + "hex-literal", + "http 0.2.12", + "http 1.4.2", + "http-body-util", + "hyper", + "hyper-util", + "indoc", + "itertools 0.15.0", + "lazy-regex", + "lazy_static", + "lru_time_cache", + "maplit", + "num-bigint 0.2.6", + "num-integer", + "num-traits", + "object_store", + "parking_lot", + "petgraph 0.8.3", + "portable-atomic", + "priority-queue", + "prometheus", + "prost", + "prost-types", + "rand 0.9.3", + "redis", + "regex", + "reqwest 0.12.23", + "rustls", + "semver 1.0.28", + "serde", + "serde_derive", + "serde_json", + "serde_plain", + "serde_regex", + "serde_yaml", + "sha2 0.11.0", + "slog", + "slog-async", + "slog-envlogger", + "slog-term", + "sqlparser 0.57.0", + "sqlparser 0.62.0", + "stable-hash 0.3.4", + "stable-hash 0.4.4", + "strum_macros 0.28.0", + "tempfile", + "thiserror 2.0.18", + "tokio", + "tokio-retry", + "tokio-stream", + "tokio-util", + "toml 1.1.2+spec-1.1.0", + "tonic", + "tonic-prost", + "tonic-prost-build", + "url", + "wasmparser 0.118.2", + "wiremock", +] + +[[package]] +name = "graph-chain-common" +version = "0.45.0" +dependencies = [ + "anyhow", + "heck 0.5.0", + "protobuf", + "protobuf-parse", +] + +[[package]] +name = "graph-chain-ethereum" +version = "0.45.0" +dependencies = [ + "anyhow", + "async-trait", + "base64", + "envconfig", + "futures 0.3.31", + "graph", + "graph-runtime-derive", + "graph-runtime-wasm", + "hex", + "itertools 0.15.0", + "jsonrpc-core", + "prost", + "prost-types", + "semver 1.0.28", + "serde", + "thiserror 2.0.18", + "tokio", + "tokio-stream", + "tonic-prost-build", + "tower 0.5.2", +] + +[[package]] +name = "graph-chain-near" +version = "0.45.0" +dependencies = [ + "anyhow", + "async-trait", + "diesel", + "graph", + "graph-runtime-derive", + "graph-runtime-wasm", + "prost", + "prost-types", + "serde", + "tokio", + "tonic-prost-build", +] + +[[package]] +name = "graph-core" +version = "0.45.0" +dependencies = [ + "alloy", + "anyhow", + "arrow", + "async-trait", + "bytes", + "chrono", + "futures 0.3.31", + "graph", + "graph-chain-ethereum", + "graph-chain-near", + "graph-runtime-wasm", + "indoc", + "itertools 0.15.0", + "parking_lot", + "prometheus", + "serde_yaml", + "slog", + "strum 0.28.0", + "thiserror 2.0.18", + "tokio", + "tokio-retry", + "tokio-util", + "tower 0.5.3", + "tower-test", + "wiremock", +] + +[[package]] +name = "graph-graphql" +version = "0.45.0" +dependencies = [ + "anyhow", + "async-recursion", + "async-trait", + "crossbeam", + "graph", + "graphql-tools", + "lazy_static", + "parking_lot", + "stable-hash 0.3.4", + "stable-hash 0.4.4", +] + +[[package]] +name = "graph-node" +version = "0.45.0" +dependencies = [ + "anyhow", + "clap", + "console 0.16.4", + "diesel", + "diesel-async", + "env_logger", + "git-testament", + "graph", + "graph-chain-ethereum", + "graph-chain-near", + "graph-core", + "graph-graphql", + "graph-server-http", + "graph-server-index-node", + "graph-server-json-rpc", + "graph-server-metrics", + "graph-store-postgres", + "graphman", + "graphman-server", + "indicatif", + "itertools 0.15.0", + "json-structural-diff", + "lazy_static", + "prometheus", + "serde", + "shellexpand", + "termcolor", + "tokio", + "tokio-util", + "url", +] + +[[package]] +name = "graph-runtime-derive" +version = "0.45.0" +dependencies = [ + "quote", + "syn 2.0.118", +] + +[[package]] +name = "graph-runtime-test" +version = "0.45.0" +dependencies = [ + "async-trait", + "graph", + "graph-chain-ethereum", + "graph-runtime-wasm", + "rand 0.9.3", + "semver 1.0.28", + "test-store", + "wasmtime", +] + +[[package]] +name = "graph-runtime-wasm" +version = "0.45.0" +dependencies = [ + "anyhow", + "async-trait", + "bs58 0.4.0", + "graph", + "graph-runtime-derive", + "hex", + "never", + "parity-wasm", + "semver 1.0.28", + "serde_yaml", + "wasm-instrument", + "wasmtime", +] + +[[package]] +name = "graph-server-http" +version = "0.45.0" +dependencies = [ + "async-trait", + "graph", + "graph-core", + "graph-graphql", + "serde", +] + +[[package]] +name = "graph-server-index-node" +version = "0.45.0" +dependencies = [ + "async-trait", + "blake3", + "git-testament", + "graph", + "graph-chain-ethereum", + "graph-chain-near", + "graph-graphql", +] + +[[package]] +name = "graph-server-json-rpc" +version = "0.45.0" +dependencies = [ + "axum", + "graph", + "serde", + "serde_json", + "slog", + "thiserror 2.0.18", + "tokio", +] + +[[package]] +name = "graph-server-metrics" +version = "0.45.0" +dependencies = [ + "graph", +] + +[[package]] +name = "graph-store-postgres" +version = "0.45.0" +dependencies = [ + "Inflector", + "anyhow", + "arrow", + "async-trait", + "blake3", + "chrono", + "clap", + "deadpool 0.13.0", + "derive_more", + "diesel", + "diesel-async", + "diesel-derive-enum", + "diesel-dynamic-schema", + "diesel_derives", + "diesel_migrations", + "fallible-iterator 0.3.0", + "git-testament", + "graph", + "graphman-store", + "hex", + "itertools 0.15.0", + "lazy_static", + "lru_time_cache", + "openssl", + "parquet", + "postgres", + "postgres-openssl", + "pretty_assertions", + "rand 0.9.3", + "serde", + "serde_json", + "serde_yaml", + "sqlparser 0.62.0", + "stable-hash 0.3.4", + "thiserror 2.0.18", + "tokio", + "tokio-stream", +] + +[[package]] +name = "graph-tests" +version = "0.45.0" +dependencies = [ + "anyhow", + "assert-json-diff", + "async-stream", + "async-trait", + "graph", + "graph-chain-ethereum", + "graph-core", + "graph-graphql", + "graph-node", + "graph-runtime-wasm", + "graph-server-index-node", + "graph-store-postgres", + "serde", + "serde_yaml", + "slog", + "slog-async", + "slog-term", + "tokio", + "tokio-stream", + "tokio-util", +] + +[[package]] +name = "graph_derive" +version = "0.45.0" +dependencies = [ + "proc-macro-utils", + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "graphman" +version = "0.45.0" +dependencies = [ + "anyhow", + "diesel", + "diesel-async", + "graph", + "graph-store-postgres", + "graphman-store", + "itertools 0.15.0", + "thiserror 2.0.18", + "tokio", +] + +[[package]] +name = "graphman-server" +version = "0.45.0" +dependencies = [ + "anyhow", + "async-graphql", + "async-graphql-axum", + "axum", + "chrono", + "diesel", + "diesel-async", + "graph", + "graph-store-postgres", + "graphman", + "graphman-store", + "lazy_static", + "reqwest 0.12.23", + "serde", + "serde_json", + "slog", + "test-store", + "thiserror 2.0.18", + "tokio", + "tower-http 0.7.0", +] + +[[package]] +name = "graphman-store" +version = "0.45.0" +dependencies = [ + "anyhow", + "async-trait", + "chrono", + "diesel", + "strum 0.28.0", +] + +[[package]] +name = "graphql-tools" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b1ae57fa544e67a661c97805be3e36b97e7d15d65d67bb34beac24d940f987e" +dependencies = [ + "combine", + "itoa", + "lazy_static", + "ryu", + "serde", + "serde_json", + "serde_with", + "thiserror 2.0.18", +] + +[[package]] +name = "group" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" +dependencies = [ + "ff", + "rand_core 0.6.4", + "subtle", +] + +[[package]] +name = "h2" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http 1.4.2", + "indexmap 2.14.0", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "num-traits", + "zerocopy", +] + +[[package]] +name = "handlebars" +version = "6.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b3f9296c208515b87bd915a2f5d1163d4b3f863ba83337d7713cf478055948e" +dependencies = [ + "derive_builder", + "log", + "num-order", + "pest", + "pest_derive", + "serde", + "serde_json", + "thiserror 2.0.18", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + +[[package]] +name = "hashbrown" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" +dependencies = [ + "foldhash 0.1.5", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] + +[[package]] +name = "hashbrown" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" +dependencies = [ + "foldhash 0.2.0", + "serde", + "serde_core", +] + +[[package]] +name = "hdrhistogram" +version = "7.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "765c9198f173dd59ce26ff9f95ef0aafd0a0fe01fb9d72841bc5066a4c06511d" +dependencies = [ + "byteorder", + "num-traits", +] + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hermit-abi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" + +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hex-conservative" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fda06d18ac606267c40c04e41b9947729bf8b9efe74bd4e82b61a5f26a510b9f" +dependencies = [ + "arrayvec", +] + +[[package]] +name = "hex-literal" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e712f64ec3850b98572bffac52e2c6f282b29fe6c5fa6d42334b30be438d95c1" + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest 0.10.7", +] + +[[package]] +name = "home" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "http" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" +dependencies = [ + "bytes", + "http 1.4.2", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http 1.4.2", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fcc0b4a115bf80b728eb8ea024ad5bd707b615bfed49e0665b6e0f86fd082d9" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "humantime" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" + +[[package]] +name = "hybrid-array" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a79f2aff40c18ab8615ddc5caa9eb5b96314aef18fe5823090f204ad988e813" +dependencies = [ + "typenum", +] + +[[package]] +name = "hyper" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http 1.4.2", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ee4be2c948921a1a5320b629c4193916ed787a7f7f293fd3f7f5a6c9de74155" +dependencies = [ + "futures-util", + "http 1.4.2", + "hyper", + "hyper-util", + "rustls", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower-service 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "hyper-timeout" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" +dependencies = [ + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http 1.4.2", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "system-configuration", + "tokio", + "tower-layer 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "tower-service 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "tracing", + "windows-registry", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "ibig" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1fcc7f316b2c079dde77564a1360639c1a956a23fa96122732e416cb10717bb" +dependencies = [ + "cfg-if", + "num-traits", + "rand 0.8.6", + "static_assertions", +] + +[[package]] +name = "icu_collections" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locid" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_locid_transform" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_locid_transform_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locid_transform_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" + +[[package]] +name = "icu_normalizer" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "utf16_iter", + "utf8_iter", + "write16", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" + +[[package]] +name = "icu_properties" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locid_transform", + "icu_properties_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" + +[[package]] +name = "icu_provider" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_provider_macros", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_provider_macros" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "impl-codec" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba6a270039626615617f3f36d15fc827041df3b78c439da2cadfa47455a77f2f" +dependencies = [ + "parity-scale-codec", +] + +[[package]] +name = "impl-trait-for-tuples" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11d7a9f6330b71fea57921c9b61c47ee6e84f72d394754eff6163ae67e7395eb" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", + "serde", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "arbitrary", + "equivalent", + "hashbrown 0.17.0", + "serde", + "serde_core", +] + +[[package]] +name = "indicatif" +version = "0.18.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "993f007684f2e9727160da8b960ec161264703bfd1af084fd2e34d040c9a0dd4" +dependencies = [ + "console 0.16.4", + "portable-atomic", + "unicode-width", + "unit-prefix", + "web-time", +] + +[[package]] +name = "indoc" +version = "2.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706" +dependencies = [ + "rustversion", +] + +[[package]] +name = "inotify" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f37dccff2791ab604f9babef0ba14fbe0be30bd368dc541e2b08d07c8aa908f3" +dependencies = [ + "bitflags 2.11.1", + "inotify-sys", + "libc", +] + +[[package]] +name = "inotify-sys" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e05c02b5e89bff3b946cedeca278abc628fe811e604f027c45a8aa3cf793d0eb" +dependencies = [ + "libc", +] + +[[package]] +name = "inquire" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6654738b8024300cf062d04a1c13c10c8e2cea598ec1c47dc9b6641159429756" +dependencies = [ + "bitflags 2.11.1", + "crossterm", + "dyn-clone", + "fuzzy-matcher", + "unicode-segmentation", + "unicode-width", +] + +[[package]] +name = "interprocess" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d941b405bd2322993887859a8ee6ac9134945a24ec5ec763a8a962fc64dfec2d" +dependencies = [ + "doctest-file", + "futures-core", + "libc", + "recvmsg", + "tokio", + "widestring", + "windows-sys 0.52.0", +] + +[[package]] +name = "ipnet" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" + +[[package]] +name = "is-docker" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "928bae27f42bc99b60d9ac7334e3a21d10ad8f1835a4e12ec3ec0464765ed1b3" +dependencies = [ + "once_cell", +] + +[[package]] +name = "is-terminal" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f23ff5ef2b80d608d61efee834934d862cd92461afc0560dedf493e4c033738b" +dependencies = [ + "hermit-abi 0.3.9", + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "is-wsl" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "173609498df190136aa7dea1a91db051746d339e18476eed5ca40521f02d7aa5" +dependencies = [ + "is-docker", + "once_cell", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8478577c03552c21db0e2724ffb8986a5ce7af88107e6be5d2ee6e158c12800" + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" + +[[package]] +name = "ittapi" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b996fe614c41395cdaedf3cf408a9534851090959d90d54a535f675550b64b1" +dependencies = [ + "anyhow", + "ittapi-sys", + "log", +] + +[[package]] +name = "ittapi-sys" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52f5385394064fa2c886205dba02598013ce83d3e92d33dbdc0c52fe0e7bf4fc" +dependencies = [ + "cc", +] + +[[package]] +name = "jiff" +version = "0.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a3546dc96b6d42c5f24902af9e2538e82e39ad350b0c766eb3fbf2d8f3d8359" +dependencies = [ + "jiff-static", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", +] + +[[package]] +name = "jiff-static" +version = "0.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a8c8b344124222efd714b73bb41f8b5120b27a7cc1c75593a6ff768d9d05aa4" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "jni" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" +dependencies = [ + "cesu8", + "cfg-if", + "combine", + "jni-sys 0.3.1", + "log", + "thiserror 1.0.61", + "walkdir", + "windows-sys 0.45.0", +] + +[[package]] +name = "jni-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41a652e1f9b6e0275df1f15b32661cf0d4b78d4d87ddec5e0c3c20f097433258" +dependencies = [ + "jni-sys 0.4.1", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.118", +] + +[[package]] +name = "jobserver" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2b099aaa34a9751c5bf0878add70444e1ed2dd73f347be99003d4577277de6e" +dependencies = [ + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03d04c30968dffe80775bd4d7fb676131cd04a1fb46d2686dbffbaec2d9dfd31" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "json-structural-diff" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e878e36a8a44c158505c2c818abdc1350413ad83dcb774a0459f6a7ef2b65cbf" +dependencies = [ + "console 0.15.11", + "difflib", + "regex", + "serde_json", +] + +[[package]] +name = "jsonrpc-core" +version = "18.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14f7f76aef2d054868398427f6c54943cf3d1caa9a7ec7d0c38d69df97a965eb" +dependencies = [ + "futures 0.3.31", + "futures-executor", + "futures-util", + "log", + "serde", + "serde_derive", + "serde_json", +] + +[[package]] +name = "k256" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6e3919bbaa2945715f0bb6d3934a173d1e9a59ac23767fbaaef277265a7411b" +dependencies = [ + "cfg-if", + "ecdsa", + "elliptic-curve", + "once_cell", + "serdect", + "sha2 0.10.9", +] + +[[package]] +name = "keccak" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e24a010dd405bd7ed803e5253182815b41bf2e6a80cc3bfc066658e03a198aa" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", +] + +[[package]] +name = "keccak-asm" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1766b89733097006f3a1388a02849865d6bc98c89273cb622e29fdd209922183" +dependencies = [ + "digest 0.10.7", + "sha3-asm", +] + +[[package]] +name = "kqueue" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eac30106d7dce88daf4a3fcb4879ea939476d5074a9b7ddd0fb97fa4bed5596a" +dependencies = [ + "kqueue-sys", + "libc", +] + +[[package]] +name = "kqueue-sys" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed9625ffda8729b85e45cf04090035ac368927b8cebc34898e7c120f52e4838b" +dependencies = [ + "bitflags 1.3.2", + "libc", +] + +[[package]] +name = "lazy-regex" +version = "3.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bae91019476d3ec7147de9aa291cadb6d870abf2f3015d2da73a90325ac1496" +dependencies = [ + "lazy-regex-proc_macros", + "once_cell", + "regex", +] + +[[package]] +name = "lazy-regex-proc_macros" +version = "3.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4de9c1e1439d8b7b3061b2d209809f447ca33241733d9a3c01eabf2dc8d94358" +dependencies = [ + "proc-macro2", + "quote", + "regex", + "syn 2.0.118", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "leb128" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" + +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + +[[package]] +name = "lexical-core" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d8d125a277f807e55a77304455eb7b1cb52f2b18c143b60e766c120bd64a594" +dependencies = [ + "lexical-parse-float", + "lexical-parse-integer", + "lexical-util", + "lexical-write-float", + "lexical-write-integer", +] + +[[package]] +name = "lexical-parse-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52a9f232fbd6f550bc0137dcb5f99ab674071ac2d690ac69704593cb4abbea56" +dependencies = [ + "lexical-parse-integer", + "lexical-util", +] + +[[package]] +name = "lexical-parse-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a7a039f8fb9c19c996cd7b2fcce303c1b2874fe1aca544edc85c4a5f8489b34" +dependencies = [ + "lexical-util", +] + +[[package]] +name = "lexical-util" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2604dd126bb14f13fb5d1bd6a66155079cb9fa655b37f875b3a742c705dbed17" + +[[package]] +name = "lexical-write-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50c438c87c013188d415fbabbb1dceb44249ab81664efbd31b14ae55dabb6361" +dependencies = [ + "lexical-util", + "lexical-write-integer", +] + +[[package]] +name = "lexical-write-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "409851a618475d2d5796377cad353802345cba92c867d9fbcde9cf4eac4e14df" +dependencies = [ + "lexical-util", +] + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "libredox" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" +dependencies = [ + "bitflags 2.11.1", + "libc", +] + +[[package]] +name = "linux-raw-sys" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23fb14cb19457329c82206317a5663005a4d404783dc74f4252769b0d5f42856" + +[[package]] +name = "litrs" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "lru" +version = "0.16.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1dc47f592c06f33f8e3aea9591776ec7c9f9e4124778ff8a3c3b87159f7e593" +dependencies = [ + "hashbrown 0.16.1", +] + +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + +[[package]] +name = "lru_time_cache" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9106e1d747ffd48e6be5bb2d97fa706ed25b144fbee4d5c02eae110cd8d6badd" + +[[package]] +name = "lz4_flex" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db9a0d582c2874f68138a16ce1867e0ffde6c0bb0a0df85e1f36d04146db488a" +dependencies = [ + "twox-hash", +] + +[[package]] +name = "mach2" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dae608c151f68243f2b000364e1f7b186d9c29845f7d2d85bd31b9ad77ad552b" + +[[package]] +name = "macro-string" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59a9dbbfc75d2688ed057456ce8a3ee3f48d12eec09229f560f3643b9f275653" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "maplit" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" + +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest 0.10.7", +] + +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" + +[[package]] +name = "memfd" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad38eb12aea514a0466ea40a80fd8cc83637065948eb4a426e4aa46261175227" +dependencies = [ + "rustix 1.1.4", +] + +[[package]] +name = "migrations_internals" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36c791ecdf977c99f45f23280405d7723727470f6689a5e6dbf513ac547ae10d" +dependencies = [ + "serde", + "toml 0.9.12+spec-1.1.0", +] + +[[package]] +name = "migrations_macros" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36fc5ac76be324cfd2d3f2cf0fdf5d5d3c4f14ed8aaebadb09e304ba42282703" +dependencies = [ + "migrations_internals", + "proc-macro2", + "quote", +] + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "mime_guess" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" +dependencies = [ + "mime", + "unicase", +] + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd" +dependencies = [ + "libc", + "log", + "wasi 0.11.0+wasi-snapshot-preview1", + "windows-sys 0.52.0", +] + +[[package]] +name = "multer" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83e87776546dc87511aa5ee218730c92b666d7264ab6ed41f9d215af9cd5224b" +dependencies = [ + "bytes", + "encoding_rs", + "futures-util", + "http 1.4.2", + "httparse", + "memchr", + "mime", + "spin", + "version_check", +] + +[[package]] +name = "multibase" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b3539ec3c1f04ac9748a260728e855f261b4977f5c3406612c884564f329404" +dependencies = [ + "base-x", + "data-encoding", + "data-encoding-macro", +] + +[[package]] +name = "multihash" +version = "0.19.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "577c63b00ad74d57e8c9aa870b5fccebf2fd64a308a5aee9f1bb88e4aea19447" +dependencies = [ + "unsigned-varint", +] + +[[package]] +name = "multimap" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "defc4c55412d89136f966bbb339008b474350e5e6e78d2714439c386b3137a03" + +[[package]] +name = "native-tls" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8614eb2c83d59d1c8cc974dd3f920198647674a0a035e1af1fa58707e317466" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework 2.11.0", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "never" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96aba5aa877601bb3f6dd6a63a969e1f82e60646e81e71b14496995e9853c91" + +[[package]] +name = "nix" +version = "0.31.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" +dependencies = [ + "bitflags 2.11.1", + "cfg-if", + "cfg_aliases", + "libc", +] + +[[package]] +name = "notify" +version = "8.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d3d07927151ff8575b7087f245456e549fea62edf0ec4e565a5ee50c8402bc3" +dependencies = [ + "bitflags 2.11.1", + "fsevent-sys", + "inotify", + "kqueue", + "libc", + "log", + "mio", + "notify-types", + "walkdir", + "windows-sys 0.60.2", +] + +[[package]] +name = "notify-types" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e0826a989adedc2a244799e823aece04662b66609d96af8dff7ac6df9a8925d" + +[[package]] +name = "num-bigint" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", + "serde", +] + +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-conv" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf97ec579c3c42f953ef76dbf8d55ac91fb219dde70e49aa4a6b7d74e9919050" + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-modular" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17bb261bf36fa7d83f4c294f834e91256769097b3cb505d44831e0a179ac647f" + +[[package]] +name = "num-order" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "537b596b97c40fcf8056d153049eb22f481c17ebce72a513ec9286e4986d1bb6" +dependencies = [ + "num-modular", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "num_cpus" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +dependencies = [ + "hermit-abi 0.3.9", + "libc", +] + +[[package]] +name = "num_enum" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1207a7e20ad57b847bbddc6776b968420d38292bbfe2089accff5e19e82454c" +dependencies = [ + "num_enum_derive", + "rustversion", +] + +[[package]] +name = "num_enum_derive" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff32365de1b6743cb203b710788263c44a03de03802daf96092f2da4fe6ba4d7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "nybbles" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5676b5c379cf5b03da1df2b3061c4a4e2aa691086a56ac923e08c143f53f59" +dependencies = [ + "alloy-rlp", + "arbitrary", + "cfg-if", + "proptest", + "ruint", + "serde", + "smallvec", +] + +[[package]] +name = "object" +version = "0.39.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e5a6c098c7a3b6547378093f5cc30bc54fd361ce711e05293a5cc589562739b" +dependencies = [ + "crc32fast", + "hashbrown 0.17.0", + "indexmap 2.14.0", + "memchr", +] + +[[package]] +name = "object_store" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "765784b4390c6bcf80316e5a22f4e3661b639c9d8c83246856643c27d8ce9dbe" +dependencies = [ + "async-trait", + "aws-lc-rs", + "base64", + "bytes", + "chrono", + "form_urlencoded", + "futures-channel", + "futures-core", + "futures-util", + "http 1.4.2", + "http-body-util", + "humantime", + "hyper", + "itertools 0.15.0", + "nix", + "parking_lot", + "percent-encoding", + "quick-xml", + "rand 0.10.1", + "reqwest 0.13.2", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "thiserror 2.0.18", + "tokio", + "tracing", + "url", + "walkdir", + "wasm-bindgen-futures", + "web-time", + "windows-sys 0.61.2", +] + +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "open" +version = "5.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd8d3b65c44123a56e0133d2cd06ce4361bd3ca99d41198b2f25e3c3db9b8b4a" +dependencies = [ + "is-wsl", + "libc", +] + +[[package]] +name = "openssl" +version = "0.10.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77823a27f0babb03091cb9ed9ef80af3b39dbc82f97e8fa530374b7dafd87a45" +dependencies = [ + "bitflags 2.11.1", + "cfg-if", + "foreign-types", + "libc", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-src" +version = "300.5.0+3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8ce546f549326b0e6052b649198487d91320875da901e7bd11a06d1ee3f9c2f" +dependencies = [ + "cc", +] + +[[package]] +name = "openssl-sys" +version = "0.9.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b47e7e6bb2c38cd930d25a23b40fa52e068c10e85f3e03a7f5ba5aaca5713695" +dependencies = [ + "cc", + "libc", + "openssl-src", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + +[[package]] +name = "parity-scale-codec" +version = "3.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "306800abfa29c7f16596b5970a588435e3d5b3149683d00c12b699cc19f895ee" +dependencies = [ + "arrayvec", + "bitvec", + "byte-slice-cast", + "impl-trait-for-tuples", + "parity-scale-codec-derive", + "serde", +] + +[[package]] +name = "parity-scale-codec-derive" +version = "3.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d830939c76d294956402033aee57a6da7b438f2294eb94864c37b0569053a42c" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "parity-wasm" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1ad0aff30c1da14b1254fcb2af73e1fa9a28670e584a626f53a369d0e157304" + +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall 0.5.2", + "smallvec", + "windows-link 0.2.1", +] + +[[package]] +name = "parquet" +version = "59.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "970dff83e97d953c827ae8176f6bf4e9f77bf62daacc01ec5df348ec5eacd913" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-ipc", + "arrow-schema", + "arrow-select", + "base64", + "brotli", + "bytes", + "chrono", + "flate2", + "half", + "hashbrown 0.17.0", + "lz4_flex", + "num-bigint 0.4.6", + "num-integer", + "num-traits", + "paste", + "seq-macro", + "simdutf8", + "snap", + "twox-hash", + "zstd", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pest" +version = "2.7.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd53dff83f26735fdc1ca837098ccf133605d794cdae66acfc2bfac3ec809d95" +dependencies = [ + "memchr", + "thiserror 1.0.61", + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.7.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a548d2beca6773b1c244554d36fcf8548a8a58e74156968211567250e48e49a" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.7.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c93a82e8d145725dcbaf44e5ea887c8a869efdcc28706df2d08c69e17077183" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "pest_meta" +version = "2.7.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a941429fea7e08bedec25e4f6785b6ffaacc6b755da98df5ef3e7dcf4a124c4f" +dependencies = [ + "once_cell", + "pest", + "sha2 0.10.9", +] + +[[package]] +name = "petgraph" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" +dependencies = [ + "fixedbitset 0.4.2", + "indexmap 2.14.0", +] + +[[package]] +name = "petgraph" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" +dependencies = [ + "fixedbitset 0.5.7", + "hashbrown 0.15.2", + "indexmap 2.14.0", + "serde", +] + +[[package]] +name = "pgtemp" +version = "0.6.0" +source = "git+https://github.com/graphprotocol/pgtemp?branch=initdb-args#08a95d441d74ce0a50b6e0a55dbf96d8362d8fb7" +dependencies = [ + "libc", + "tempfile", + "tokio", + "url", +] + +[[package]] +name = "pharos" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9567389417feee6ce15dd6527a8a1ecac205ef62c2932bcf3d9f6fc5b78b414" +dependencies = [ + "futures 0.3.31", + "rustc_version 0.4.0", +] + +[[package]] +name = "phf" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" +dependencies = [ + "phf_shared", + "serde", +] + +[[package]] +name = "phf_shared" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + +[[package]] +name = "pkg-config" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" + +[[package]] +name = "polling" +version = "3.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0e4f59085d47d8241c88ead0f274e8a0cb551f3625263c05eb8dd897c34218" +dependencies = [ + "cfg-if", + "concurrent-queue", + "hermit-abi 0.5.2", + "pin-project-lite", + "rustix 1.1.4", + "windows-sys 0.61.2", +] + +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + +[[package]] +name = "portable-atomic-util" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "postcard" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6764c3b5dd454e283a30e6dfe78e9b31096d9e32036b5d1eaac7a6119ccb9a24" +dependencies = [ + "cobs", + "embedded-io 0.4.0", + "embedded-io 0.6.1", + "serde", +] + +[[package]] +name = "postgres" +version = "0.19.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c48ece1c6cda0db61b058c1721378da76855140e9214339fa1317decacb176" +dependencies = [ + "bytes", + "fallible-iterator 0.2.0", + "futures-util", + "log", + "tokio", + "tokio-postgres", +] + +[[package]] +name = "postgres-openssl" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06743eefaa1a5c0ef2ccb6d9abf6528790a229eabd62ddcabf9b2a3aeff09fa4" +dependencies = [ + "openssl", + "tokio", + "tokio-openssl", + "tokio-postgres", +] + +[[package]] +name = "postgres-protocol" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbef655056b916eb868048276cfd5d6a7dea4f81560dfd047f97c8c6fe3fcfd4" +dependencies = [ + "base64", + "byteorder", + "bytes", + "fallible-iterator 0.2.0", + "hmac", + "md-5", + "memchr", + "rand 0.9.3", + "sha2 0.10.9", + "stringprep", +] + +[[package]] +name = "postgres-types" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef4605b7c057056dd35baeb6ac0c0338e4975b1f2bef0f65da953285eb007095" +dependencies = [ + "bytes", + "fallible-iterator 0.2.0", + "postgres-protocol", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + +[[package]] +name = "pq-src" +version = "0.3.10+libpq-18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56ef39ce621f4993d6084fdcd4cbf1e01c84bdba53109cfad095d2cf441b85b9" +dependencies = [ + "cc", + "openssl-sys", +] + +[[package]] +name = "pq-sys" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "574ddd6a267294433f140b02a726b0640c43cf7c6f717084684aaa3b285aba61" +dependencies = [ + "libc", + "pkg-config", + "pq-src", + "vcpkg", +] + +[[package]] +name = "pretty_assertions" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ae130e2f271fbc2ac3a40fb1d07180839cdbbe443c7a27e1e3c13c5cac0116d" +dependencies = [ + "diff", + "yansi", +] + +[[package]] +name = "prettyplease" +version = "0.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f12335488a2f3b0a83b14edad48dca9879ce89b2edd10e80237e4e852dd645e" +dependencies = [ + "proc-macro2", + "syn 2.0.118", +] + +[[package]] +name = "primitive-types" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b34d9fd68ae0b74a41b21c03c2f62847aa0ffea044eee893b4c140b37e244e2" +dependencies = [ + "fixed-hash", + "impl-codec", + "uint 0.9.5", +] + +[[package]] +name = "priority-queue" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93980406f12d9f8140ed5abe7155acb10bb1e69ea55c88960b9c2f117445ef96" +dependencies = [ + "equivalent", + "indexmap 2.14.0", + "serde", +] + +[[package]] +name = "proc-macro-crate" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d37c51ca738a55da99dc0c4a34860fd675453b8b36209178c2249bb13651284" +dependencies = [ + "toml_edit", +] + +[[package]] +name = "proc-macro-error-attr2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "proc-macro-error2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802" +dependencies = [ + "proc-macro-error-attr2", + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "proc-macro-utils" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eeaf08a13de400bc215877b5bdc088f241b12eb42f0a548d3390dc1c56bb7071" +dependencies = [ + "proc-macro2", + "quote", + "smallvec", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prometheus" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ca5326d8d0b950a9acd87e6a3f94745394f62e4dae1b1ee22b2bc0c394af43a" +dependencies = [ + "cfg-if", + "fnv", + "lazy_static", + "libc", + "memchr", + "parking_lot", + "protobuf", + "reqwest 0.12.23", + "thiserror 2.0.18", +] + +[[package]] +name = "proptest" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee689443a2bd0a16ab0348b52ee43e3b2d1b1f931c8aa5c9f8de4c86fbe8c40" +dependencies = [ + "bit-set", + "bit-vec", + "bitflags 2.11.1", + "num-traits", + "rand 0.9.3", + "rand_chacha 0.9.0", + "rand_xorshift", + "regex-syntax", + "rusty-fork", + "tempfile", + "unarray", +] + +[[package]] +name = "proptest-derive" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ee1c9ac207483d5e7db4940700de86a9aae46ef90c48b57f99fe7edb8345e49" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "proptest-derive" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c57924a81864dddafba92e1bf92f9bf82f97096c44489548a60e888e1547549b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", ] [[package]] -name = "graph-server-json-rpc" -version = "0.30.0" +name = "prost" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1" dependencies = [ - "graph", - "jsonrpsee", - "serde", + "bytes", + "prost-derive", ] [[package]] -name = "graph-server-metrics" -version = "0.30.0" +name = "prost-build" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7" dependencies = [ - "graph", - "hyper", + "heck 0.5.0", + "itertools 0.13.0", + "log", + "multimap", + "petgraph 0.8.3", + "prettyplease", + "prost", + "prost-types", + "pulldown-cmark", + "pulldown-cmark-to-cmark", + "regex", + "syn 2.0.118", + "tempfile", ] [[package]] -name = "graph-server-websocket" -version = "0.30.0" +name = "prost-derive" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "futures 0.1.31", - "graph", - "graphql-parser", - "http", - "lazy_static", - "serde", - "serde_derive", - "tokio-tungstenite", - "uuid", + "itertools 0.13.0", + "proc-macro2", + "quote", + "syn 2.0.118", ] [[package]] -name = "graph-store-postgres" -version = "0.30.0" +name = "prost-types" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f94967dc7688f3054c7fac87473ffae4cc4c3904800e2d9f5b857246d8963b0a" dependencies = [ - "Inflector", - "anyhow", - "async-trait", - "blake3 1.3.3", - "clap", - "derive_more", - "diesel", - "diesel-derive-enum", - "diesel-dynamic-schema", - "diesel_derives", - "diesel_migrations", - "fallible-iterator", - "futures 0.3.16", - "git-testament", - "graph", - "graph-chain-ethereum", - "graph-graphql", - "graph-mock", - "graphql-parser", - "hex", - "hex-literal", - "itertools", - "lazy_static", - "lru_time_cache", - "maybe-owned", - "openssl", - "pin-utils", - "postgres", - "postgres-openssl", - "pretty_assertions", - "rand", - "serde", - "stable-hash 0.3.3", - "test-store", - "uuid", + "prost", ] [[package]] -name = "graph-tests" -version = "0.30.0" +name = "protobuf" +version = "3.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d65a1d4ddae7d8b5de68153b48f6aa3bba8cb002b243dbdbc55a5afbc98f99f4" dependencies = [ - "anyhow", - "assert-json-diff", - "async-stream", - "bollard", - "cid", - "futures 0.3.16", - "graph", - "graph-chain-ethereum", - "graph-chain-near", - "graph-core", - "graph-graphql", - "graph-mock", - "graph-node", - "graph-runtime-wasm", - "graph-server-index-node", - "graph-store-postgres", - "graphql-parser", - "hyper", - "lazy_static", - "serde", - "serde_yaml", - "slog", - "tokio", - "tokio-stream", - "uuid", + "once_cell", + "protobuf-support", + "thiserror 1.0.61", ] [[package]] -name = "graphql-parser" -version = "0.4.0" +name = "protobuf-parse" +version = "3.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2ebc8013b4426d5b81a4364c419a95ed0b404af2b82e2457de52d9348f0e474" +checksum = "b4aeaa1f2460f1d348eeaeed86aea999ce98c1bded6f089ff8514c9d9dbdc973" dependencies = [ - "combine", - "thiserror", + "anyhow", + "indexmap 2.14.0", + "log", + "protobuf", + "protobuf-support", + "tempfile", + "thiserror 1.0.61", + "which", ] [[package]] -name = "graphql-tools" -version = "0.2.1" +name = "protobuf-support" +version = "3.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bc3a979aca9d796ff03ff71f4013e203a1f69bf1f37899ae4a8e676bb236608" +checksum = "3e36c2f31e0a47f9280fb347ef5e461ffcd2c52dd520d8e216b52f93b0b0d7d6" dependencies = [ - "graphql-parser", - "lazy_static", - "serde", - "serde_json", + "thiserror 1.0.61", ] [[package]] -name = "h2" -version = "0.3.13" +name = "psm" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37a82c6d637fc9515a4694bbf1cb2457b79d81ce52b3108bdeea58b07dd34a57" +checksum = "5787f7cda34e3033a72192c018bc5883100330f362ef279a8cbccfce8bb4e874" dependencies = [ - "bytes", - "fnv", - "futures-core", - "futures-sink", - "futures-util", - "http", - "indexmap", - "slab", - "tokio", - "tokio-util 0.7.1", - "tracing", + "cc", ] [[package]] -name = "hashbrown" -version = "0.12.1" +name = "pulldown-cmark" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db0d4cf898abf0081f964436dc980e96670a0f36863e4b83aaacdb65c9d7ccc3" +checksum = "1e8bbe1a966bd2f362681a44f6edce3c2310ac21e4d5067a6e7ec396297a6ea0" +dependencies = [ + "bitflags 2.11.1", + "memchr", + "unicase", +] [[package]] -name = "hdrhistogram" -version = "7.5.2" +name = "pulldown-cmark-to-cmark" +version = "22.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f19b9f54f7c7f55e31401bb647626ce0cf0f67b0004982ce815b3ee72a02aa8" +checksum = "50793def1b900256624a709439404384204a5dc3a6ec580281bfaac35e882e90" dependencies = [ - "byteorder", - "num-traits", + "pulldown-cmark", ] [[package]] -name = "headers" -version = "0.3.5" +name = "pulley-interpreter" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4c4eb0471fcb85846d8b0690695ef354f9afb11cb03cac2e1d7c9253351afb0" +checksum = "38b92604caae1a1899b6a5b54967289dd538177c626004c91accf9d0ec7e4a12" dependencies = [ - "base64 0.13.1", - "bitflags", - "bytes", - "headers-core", - "http", - "httpdate", - "mime", - "sha-1 0.9.7", + "cranelift-bitset", + "log", + "pulley-macros", + "wasmtime-internal-core", ] [[package]] -name = "headers-core" -version = "0.2.0" +name = "pulley-macros" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7f66481bfee273957b1f20485a4ff3362987f85b2c236580d81b4eb7a326429" +checksum = "5a7ac85c0bb3fb351f10d531230aaa5e366b46d7c4e5328e5f02801d6dac1165" dependencies = [ - "http", + "proc-macro2", + "quote", + "syn 2.0.118", ] [[package]] -name = "heck" -version = "0.3.3" +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + +[[package]] +name = "quick-xml" +version = "0.40.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" +checksum = "2474bd2e5029e7ccb6abb2ba48cf2383a333851dedf495901544281590c7da7f" dependencies = [ - "unicode-segmentation", + "memchr", + "serde", ] [[package]] -name = "heck" -version = "0.4.1" +name = "quinn" +version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.18", + "tokio", + "tracing", + "web-time", +] [[package]] -name = "hermit-abi" -version = "0.1.19" +name = "quinn-proto" +version = "0.11.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" dependencies = [ - "libc", + "aws-lc-rs", + "bytes", + "getrandom 0.3.1", + "lru-slab", + "rand 0.9.3", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.18", + "tinyvec", + "tracing", + "web-time", ] [[package]] -name = "hermit-abi" -version = "0.2.6" +name = "quinn-udp" +version = "0.5.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" dependencies = [ + "cfg_aliases", "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.60.2", ] [[package]] -name = "hex" -version = "0.4.3" +name = "quote" +version = "1.0.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" +dependencies = [ + "proc-macro2", +] [[package]] -name = "hex-literal" -version = "0.3.4" +name = "r-efi" +version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ebdb29d2ea9ed0083cd8cece49bbd968021bd99b0849edb4a9a7ee0fdf6a4e0" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] -name = "hmac" -version = "0.10.1" +name = "r2d2" +version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1441c6b1e930e2817404b5046f1f989899143a12bf92de603b69f4e0aee1e15" +checksum = "51de85fb3fb6524929c8a2eb85e6b6d363de4e8c48f9e2c2eac4944abc181c93" dependencies = [ - "crypto-mac 0.10.1", - "digest 0.9.0", + "log", + "parking_lot", + "scheduled-thread-pool", ] [[package]] -name = "http" -version = "0.2.8" +name = "radium" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399" -dependencies = [ - "bytes", - "fnv", - "itoa 1.0.1", -] +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" [[package]] -name = "http-body" -version = "0.4.5" +name = "rand" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" dependencies = [ - "bytes", - "http", - "pin-project-lite", + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", + "serde", ] [[package]] -name = "http-range-header" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bfe8eed0a9285ef776bb792479ea3834e8b94e13d615c2f66d03dd50a435a29" - -[[package]] -name = "httparse" -version = "1.7.1" +name = "rand" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "496ce29bb5a52785b44e0f7ca2847ae0bb839c9bd28f69acac9b99d461c0c04c" +checksum = "7ec095654a25171c2124e9e3393a930bddbffdc939556c914957a4c3e0a87166" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.3", + "serde", +] [[package]] -name = "httpdate" -version = "1.0.1" +name = "rand" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6456b8a6c8f33fee7d958fcd1b60d55b11940a79e63ae87013e6d22e26034440" +checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" +dependencies = [ + "chacha20", + "getrandom 0.4.2", + "rand_core 0.10.0", +] [[package]] -name = "humantime" -version = "1.3.0" +name = "rand_chacha" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ - "quick-error", + "ppv-lite86", + "rand_core 0.6.4", ] [[package]] -name = "humantime" -version = "2.1.0" +name = "rand_chacha" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.3", +] [[package]] -name = "hyper" -version = "0.14.18" +name = "rand_core" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b26ae0a80afebe130861d90abf98e3814a4f28a4c6ffeb5ab8ebb2be311e0ef2" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "bytes", - "futures-channel", - "futures-core", - "futures-util", - "h2", - "http", - "http-body", - "httparse", - "httpdate", - "itoa 1.0.1", - "pin-project-lite", - "socket2", - "tokio", - "tower-service 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "tracing", - "want", + "getrandom 0.2.15", ] [[package]] -name = "hyper-multipart-rfc7578" -version = "0.8.0" +name = "rand_core" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0eb2cf73e96e9925f4bed948e763aa2901c2f1a3a5f713ee41917433ced6671" +checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" dependencies = [ - "bytes", - "common-multipart-rfc7578", - "futures-core", - "http", - "hyper", + "getrandom 0.3.1", + "serde", ] [[package]] -name = "hyper-rustls" -version = "0.23.0" +name = "rand_core" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c8d0fd677905edcbeedbf2edb6494d676f0e98d54d5cf9bda0b061cb8fb8aba" + +[[package]] +name = "rand_xorshift" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d87c48c02e0dc5e3b849a2041db3029fd066650f8f717c07bf8ed78ccb895cac" +checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a" dependencies = [ - "http", - "hyper", - "log", - "rustls", - "rustls-native-certs", - "tokio", - "tokio-rustls", + "rand_core 0.9.3", ] [[package]] -name = "hyper-timeout" -version = "0.4.1" +name = "rapidhash" +version = "4.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" +checksum = "5d8b5b858a440a0bc02625b62dd95131b9201aa9f69f411195dd4a7cfb1de3d7" dependencies = [ - "hyper", - "pin-project-lite", - "tokio", - "tokio-io-timeout", + "rustversion", ] [[package]] -name = "hyper-tls" -version = "0.5.0" +name = "rayon" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" dependencies = [ - "bytes", - "hyper", - "native-tls", - "tokio", - "tokio-native-tls", + "either", + "rayon-core", ] [[package]] -name = "hyper-unix-connector" -version = "0.2.2" +name = "rayon-core" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24ef1fd95d34b4ff007d3f0590727b5cf33572cace09b42032fc817dc8b16557" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" dependencies = [ - "anyhow", - "hex", - "hyper", - "pin-project", - "tokio", + "crossbeam-deque", + "crossbeam-utils", ] [[package]] -name = "iana-time-zone" -version = "0.1.47" +name = "recursive" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c495f162af0bf17656d0014a0eded5f3cd2f365fdd204548c2869db89359dc7" +checksum = "0786a43debb760f491b1bc0269fe5e84155353c67482b9e60d0cfb596054b43e" dependencies = [ - "android_system_properties", - "core-foundation-sys", - "js-sys", - "once_cell", - "wasm-bindgen", - "winapi", + "recursive-proc-macro-impl", + "stacker", ] [[package]] -name = "ibig" -version = "0.3.2" +name = "recursive-proc-macro-impl" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61c5022ee7f7a2feb0bd2fdc4b8ec882cd14903cebf33e7c1847e3f3a282f8b7" +checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b" dependencies = [ - "cfg-if 1.0.0", - "const_fn_assert", - "num-traits", - "rand", - "static_assertions", + "quote", + "syn 2.0.118", ] [[package]] -name = "ident_case" -version = "1.0.1" +name = "recvmsg" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" +checksum = "d3edd4d5d42c92f0a659926464d4cce56b562761267ecf0f469d85b7de384175" [[package]] -name = "idna" -version = "0.2.3" +name = "redis" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8" +checksum = "a12e6b5f4d8ef33944e833e2b1859ad478deab6e431d7337b30ee2efe21f7543" dependencies = [ - "matches", - "unicode-bidi", - "unicode-normalization", + "arc-swap", + "arcstr", + "async-lock", + "backon", + "bytes", + "cfg-if", + "combine", + "futures-channel", + "futures-util", + "itoa", + "num-bigint 0.4.6", + "percent-encoding", + "pin-project-lite", + "ryu", + "sha1_smol", + "socket2", + "tokio", + "tokio-util", + "url", + "xxhash-rust", ] [[package]] -name = "idna" -version = "0.3.0" +name = "redox_syscall" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" dependencies = [ - "unicode-bidi", - "unicode-normalization", + "bitflags 1.3.2", ] [[package]] -name = "impl-codec" -version = "0.6.0" +name = "redox_syscall" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba6a270039626615617f3f36d15fc827041df3b78c439da2cadfa47455a77f2f" +checksum = "c82cf8cff14456045f55ec4241383baeff27af886adb72ffb2162f99911de0fd" dependencies = [ - "parity-scale-codec", + "bitflags 2.11.1", ] [[package]] -name = "impl-rlp" -version = "0.3.0" +name = "redox_users" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f28220f89297a075ddc7245cd538076ee98b01f2a9c23a53a4f1105d5a322808" +checksum = "bd283d9651eeda4b2a83a43c1c91b266c40fd76ecd39a50a8c630ae69dc72891" dependencies = [ - "rlp", + "getrandom 0.2.15", + "libredox", + "thiserror 1.0.61", ] [[package]] -name = "impl-serde" -version = "0.3.2" +name = "ref-cast" +version = "1.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4551f042f3438e64dbd6226b20527fc84a6e1fe65688b58746a2f53623f25f5c" +checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" dependencies = [ - "serde", + "ref-cast-impl", ] [[package]] -name = "impl-trait-for-tuples" -version = "0.2.1" +name = "ref-cast-impl" +version = "1.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5dacb10c5b3bb92d46ba347505a9041e676bb20ad220101326bffb0c93031ee" +checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] -name = "indexmap" -version = "1.9.2" +name = "regalloc2" +version = "0.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399" +checksum = "de2c52737737f8609e94f975dee22854a2d5c125772d4b1cf292120f4d45c186" dependencies = [ - "autocfg", - "hashbrown", + "allocator-api2", + "bumpalo", + "hashbrown 0.17.0", + "log", + "rustc-hash", "serde", + "smallvec", ] [[package]] -name = "instant" -version = "0.1.10" +name = "regex" +version = "1.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bee0328b1209d157ef001c94dd85b4f8f64139adb0eac2659f4b08382b2f474d" +checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" dependencies = [ - "cfg-if 1.0.0", + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", ] [[package]] -name = "ipfs-api" -version = "0.17.0" +name = "regex-automata" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d8cc57cf12ae4af611e53dd04053e1cfb815917c51c410aa30399bf377046ab" +checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" dependencies = [ - "ipfs-api-backend-hyper", + "aho-corasick", + "memchr", + "regex-syntax", ] [[package]] -name = "ipfs-api-backend-hyper" -version = "0.6.0" +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "reqwest" +version = "0.12.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9d131b408b4caafe1e7c00d410a09ad3eb7e3ab68690cf668e86904b2176b4" +checksum = "d429f34c8092b2d42c7c93cec323bb4adeb7c67698f70839adec842ec10c7ceb" dependencies = [ - "async-trait", - "base64 0.13.1", + "async-compression", + "base64", "bytes", - "futures 0.3.16", - "http", + "encoding_rs", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http 1.4.2", + "http-body", + "http-body-util", "hyper", - "hyper-multipart-rfc7578", "hyper-rustls", - "ipfs-api-prelude", - "thiserror", + "hyper-tls", + "hyper-util", + "js-sys", + "log", + "mime", + "mime_guess", + "native-tls", + "percent-encoding", + "pin-project-lite", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-native-tls", + "tokio-util", + "tower 0.5.2", + "tower-http 0.6.11", + "tower-service 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams 0.4.0", + "web-sys", ] [[package]] -name = "ipfs-api-prelude" -version = "0.6.0" +name = "reqwest" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b74065805db266ba2c6edbd670b23c4714824a955628472b2e46cc9f3a869cb" +checksum = "ab3f43e3283ab1488b624b44b0e988d0acea0b3214e694730a055cb6b2efa801" dependencies = [ - "async-trait", + "base64", "bytes", - "cfg-if 1.0.0", - "common-multipart-rfc7578", - "dirs", - "futures 0.3.16", - "http", - "multiaddr", - "multibase", + "futures-core", + "futures-util", + "h2", + "http 1.4.2", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "rustls-platform-verifier", "serde", "serde_json", - "serde_urlencoded", - "thiserror", + "sync_wrapper", "tokio", - "tokio-util 0.7.1", - "tracing", - "typed-builder", - "walkdir", + "tokio-rustls", + "tokio-util", + "tower 0.5.2", + "tower-http 0.6.11", + "tower-service 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams 0.5.0", + "web-sys", ] [[package]] -name = "ipnet" -version = "2.3.1" +name = "rfc6979" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68f2d64f2edebec4ce84ad108148e67e1064789bee435edc5b60ad398714a3a9" +checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" +dependencies = [ + "hmac", + "subtle", +] [[package]] -name = "isatty" -version = "0.1.9" +name = "ring" +version = "0.17.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e31a8281fc93ec9693494da65fbf28c0c2aa60a2eaec25dc58e2f31952e95edc" +checksum = "70ac5d832aa16abd7d1def883a8545280c20a60f523a370aa3a9617c2b8550ee" dependencies = [ - "cfg-if 0.1.10", + "cc", + "cfg-if", + "getrandom 0.2.15", "libc", - "redox_syscall 0.1.57", - "winapi", + "untrusted", + "windows-sys 0.52.0", ] [[package]] -name = "itertools" -version = "0.10.5" +name = "rlp" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +checksum = "bb919243f34364b6bd2fc10ef797edbfa75f33c252e7998527479c6d6b47e1ec" dependencies = [ - "either", + "bytes", + "rustc-hex", ] [[package]] -name = "itoa" -version = "0.4.7" +name = "ruint" +version = "1.17.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c141e807189ad38a07276942c6623032d3753c8859c146104ac2e4d68865945a" +dependencies = [ + "alloy-rlp", + "arbitrary", + "ark-ff 0.3.0", + "ark-ff 0.4.2", + "ark-ff 0.5.0", + "bytes", + "fastrlp 0.3.1", + "fastrlp 0.4.0", + "num-bigint 0.4.6", + "num-integer", + "num-traits", + "parity-scale-codec", + "primitive-types", + "proptest", + "rand 0.8.6", + "rand 0.9.3", + "rlp", + "ruint-macro", + "serde_core", + "valuable", + "zeroize", +] + +[[package]] +name = "ruint-macro" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736" +checksum = "48fd7bd8a6377e15ad9d42a8ec25371b94ddc67abe7c8b9127bec79bebaaae18" [[package]] -name = "itoa" -version = "1.0.1" +name = "rustc-demangle" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35" +checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" [[package]] -name = "jobserver" -version = "0.1.23" +name = "rustc-hash" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5ca711fd837261e14ec9e674f092cbb931d3fa1482b017ae59328ddc6f3212b" -dependencies = [ - "libc", -] +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" [[package]] -name = "js-sys" -version = "0.3.59" +name = "rustc-hex" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "258451ab10b34f8af53416d1fdab72c22e805f0c92a1136d59470ec0b11138b2" -dependencies = [ - "wasm-bindgen", -] +checksum = "3e75f6a532d0fd9f7f13144f392b6ad56a32696bfcd9c78f797f16bbb6f072d6" [[package]] -name = "json-structural-diff" -version = "0.1.0" +name = "rustc_version" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25c7940d3c84d2079306c176c7b2b37622b6bc5e43fbd1541b1e4a4e1fd02045" +checksum = "f0dfe2087c51c460008730de8b57e6a320782fbfb312e1f4d520e6c6fae155ee" dependencies = [ - "console", - "difflib", - "regex", - "serde_json", + "semver 0.11.0", ] [[package]] -name = "jsonrpc-core" -version = "18.0.0" +name = "rustc_version" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14f7f76aef2d054868398427f6c54943cf3d1caa9a7ec7d0c38d69df97a965eb" +checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" dependencies = [ - "futures 0.3.16", - "futures-executor", - "futures-util", - "log", - "serde", - "serde_derive", - "serde_json", + "semver 1.0.28", ] [[package]] -name = "jsonrpsee" -version = "0.15.1" +name = "rustix" +version = "0.38.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bd0d559d5e679b1ab2f869b486a11182923863b1b3ee8b421763cdd707b783a" +checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" dependencies = [ - "jsonrpsee-core", - "jsonrpsee-http-server", - "jsonrpsee-types", + "bitflags 2.11.1", + "errno", + "libc", + "linux-raw-sys 0.4.14", + "windows-sys 0.52.0", ] [[package]] -name = "jsonrpsee-core" -version = "0.15.1" +name = "rustix" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3dc3e9cf2ba50b7b1d7d76a667619f82846caa39e8e8daa8a4962d74acaddca" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "anyhow", - "arrayvec 0.7.2", - "async-trait", - "beef", - "futures-channel", - "futures-util", - "globset", - "http", - "hyper", - "jsonrpsee-types", - "lazy_static", - "parking_lot 0.12.1", - "rand", - "rustc-hash", - "serde", - "serde_json", - "thiserror", - "tokio", - "tracing", - "unicase", + "bitflags 2.11.1", + "errno", + "libc", + "linux-raw-sys 0.12.1", + "windows-sys 0.52.0", ] [[package]] -name = "jsonrpsee-http-server" -version = "0.15.1" +name = "rustls" +version = "0.23.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03802f0373a38c2420c70b5144742d800b509e2937edc4afb116434f07120117" +checksum = "6b92b125634d9b795e7beca796cc790df15a7fb38323bf3196fda83292d06b1f" dependencies = [ - "futures-channel", - "futures-util", - "hyper", - "jsonrpsee-core", - "jsonrpsee-types", - "serde", - "serde_json", - "tokio", - "tracing", - "tracing-futures", + "aws-lc-rs", + "log", + "once_cell", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", ] [[package]] -name = "jsonrpsee-types" -version = "0.15.1" +name = "rustls-native-certs" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e290bba767401b646812f608c099b922d8142603c9e73a50fb192d3ac86f4a0d" +checksum = "7fcff2dd52b58a8d98a70243663a0d234c4e2b79235637849d15913394a247d3" dependencies = [ - "anyhow", - "beef", - "serde", - "serde_json", - "thiserror", - "tracing", + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework 3.7.0", ] [[package]] -name = "keccak" -version = "0.1.0" +name = "rustls-pki-types" +version = "1.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67c21572b4949434e4fc1e1978b99c5f77064153c59d998bf13ecd96fb5ecba7" +checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" +dependencies = [ + "web-time", + "zeroize", +] [[package]] -name = "lazy_static" -version = "1.4.0" +name = "rustls-platform-verifier" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +checksum = "1d99feebc72bae7ab76ba994bb5e121b8d83d910ca40b36e0921f53becc41784" +dependencies = [ + "core-foundation 0.10.0", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework 3.7.0", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.52.0", +] [[package]] -name = "leb128" -version = "0.2.4" +name = "rustls-platform-verifier-android" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3576a87f2ba00f6f106fdfcd16db1d698d648a26ad8e0573cad8537c3c362d2a" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" [[package]] -name = "libc" -version = "0.2.131" +name = "rustls-webpki" +version = "0.103.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04c3b4822ccebfa39c02fc03d1534441b22ead323fa0f48bb7ddd8e6ba076a40" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +dependencies = [ + "aws-lc-rs", + "ring", + "rustls-pki-types", + "untrusted", +] [[package]] -name = "linked-hash-map" -version = "0.5.4" +name = "rustversion" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3" +checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6" [[package]] -name = "lock_api" -version = "0.4.6" +name = "rusty-fork" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88943dd7ef4a2e5a4bfa2753aaab3013e34ce2533d1996fb18ef591e315e2b3b" +checksum = "cc6bf79ff24e648f6da1f8d1f011e9cac26491b619e6b9280f2b47f1774e6ee2" dependencies = [ - "scopeguard", + "fnv", + "quick-error", + "tempfile", + "wait-timeout", ] [[package]] -name = "log" -version = "0.4.17" +name = "ryu" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" -dependencies = [ - "cfg-if 1.0.0", -] +checksum = "a50f4cf475b65d88e057964e0e9bb1f0aa9bbb2036dc65c64596b42932536984" [[package]] -name = "lru_time_cache" -version = "0.11.11" +name = "same-file" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9106e1d747ffd48e6be5bb2d97fa706ed25b144fbee4d5c02eae110cd8d6badd" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] [[package]] -name = "mach" -version = "0.3.2" +name = "schannel" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b823e83b2affd8f40a9ee8c29dbc56404c1e34cd2710921f2801e2cf29527afa" +checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" dependencies = [ - "libc", + "windows-sys 0.52.0", ] [[package]] -name = "maplit" -version = "1.0.2" +name = "scheduled-thread-pool" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" +checksum = "3cbc66816425a074528352f5789333ecff06ca41b36b0b0efdfbb29edc391a19" +dependencies = [ + "parking_lot", +] [[package]] -name = "matches" -version = "0.1.8" +name = "schemars" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" +checksum = "4cd191f9397d57d581cddd31014772520aa448f65ef991055d7f61582c65165f" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] [[package]] -name = "matchit" -version = "0.7.0" +name = "schemars" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b87248edafb776e59e6ee64a79086f65890d3510f2c656c000bf2a7e8a0aea40" +checksum = "a2b42f36aa1cd011945615b92222f6bf73c599a102a300334cd7f8dbeec726cc" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] [[package]] -name = "maybe-owned" -version = "0.3.4" +name = "scopeguard" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4facc753ae494aeb6e3c22f839b158aebd4f9270f55cd3c79906c45476c47ab4" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] -name = "md-5" -version = "0.9.1" +name = "sec1" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b5a279bb9607f9f53c22d496eade00d138d1bdcccd07d74650387cf94942a15" +checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" dependencies = [ - "block-buffer 0.9.0", - "digest 0.9.0", - "opaque-debug", + "base16ct", + "der", + "generic-array", + "pkcs8", + "serdect", + "subtle", + "zeroize", ] [[package]] -name = "memchr" -version = "2.5.0" +name = "secp256k1" +version = "0.30.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +checksum = "b50c5943d326858130af85e049f2661ba3c78b26589b8ab98e65e80ae44a1252" +dependencies = [ + "bitcoin_hashes", + "rand 0.8.6", + "secp256k1-sys 0.10.1", + "serde", +] [[package]] -name = "memoffset" -version = "0.6.4" +name = "secp256k1" +version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59accc507f1338036a0477ef61afdae33cde60840f4dfe481319ce3ad116ddf9" +checksum = "2c3c81b43dc2d8877c216a3fccf76677ee1ebccd429566d3e67447290d0c42b2" dependencies = [ - "autocfg", + "bitcoin_hashes", + "rand 0.9.3", + "secp256k1-sys 0.11.0", ] [[package]] -name = "migrations_internals" -version = "1.4.1" +name = "secp256k1-sys" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b4fc84e4af020b837029e017966f86a1c2d5e83e64b589963d5047525995860" +checksum = "d4387882333d3aa8cb20530a17c69a3752e97837832f34f6dccc760e715001d9" dependencies = [ - "diesel", + "cc", ] [[package]] -name = "migrations_macros" -version = "1.4.2" +name = "secp256k1-sys" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9753f12909fd8d923f75ae5c3258cae1ed3c8ec052e1b38c93c21a6d157f789c" +checksum = "dcb913707158fadaf0d8702c2db0e857de66eb003ccfdda5924b5f5ac98efb38" dependencies = [ - "migrations_internals", - "proc-macro2", - "quote", - "syn", + "cc", ] [[package]] -name = "mime" -version = "0.3.16" +name = "security-framework" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" +checksum = "c627723fd09706bacdb5cf41499e95098555af3c3c29d014dc3c458ef6be11c0" +dependencies = [ + "bitflags 2.11.1", + "core-foundation 0.9.4", + "core-foundation-sys", + "libc", + "security-framework-sys", +] [[package]] -name = "mime_guess" -version = "2.0.3" +name = "security-framework" +version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2684d4c2e97d99848d30b324b00c8fcc7e5c897b7cbb5819b09e7c90e8baf212" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ - "mime", - "unicase", + "bitflags 2.11.1", + "core-foundation 0.10.0", + "core-foundation-sys", + "libc", + "security-framework-sys", ] [[package]] -name = "miniz_oxide" -version = "0.4.4" +name = "security-framework-sys" +version = "2.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" dependencies = [ - "adler", - "autocfg", + "core-foundation-sys", + "libc", ] [[package]] -name = "miniz_oxide" -version = "0.6.2" +name = "semver" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b275950c28b37e794e8c55d88aeb5e139d0ce23fdbbeda68f8d7174abdf9e8fa" +checksum = "f301af10236f6df4160f7c3f04eec6dbc70ace82d23326abad5edee88801c6b6" dependencies = [ - "adler", + "semver-parser", ] [[package]] -name = "mio" -version = "0.7.13" +name = "semver" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c2bdb6314ec10835cd3293dd268473a835c02b7b352e788be788b3c6ca6bb16" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" dependencies = [ - "libc", - "log", - "miow", - "ntapi", - "winapi", + "serde", + "serde_core", ] [[package]] -name = "miow" -version = "0.3.7" +name = "semver-parser" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9f1c5b025cda876f66ef43a113f91ebc9f4ccef34843000e0adf6ebbab84e21" +checksum = "9900206b54a3527fdc7b8a938bffd94a568bac4f4aa8113b209df75a09c0dec2" dependencies = [ - "winapi", + "pest", ] [[package]] -name = "more-asserts" -version = "0.2.1" +name = "send_wrapper" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0debeb9fcf88823ea64d64e4a815ab1643f33127d995978e099942ce38f25238" +checksum = "cd0b0ec5f1c1ca621c432a25813d8d60c88abe6d3e08a3eb9cf37d97a0fe3d73" [[package]] -name = "multiaddr" -version = "0.17.0" +name = "seq-macro" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b53e0cc5907a5c216ba6584bf74be8ab47d6d6289f72793b2dddbf15dc3bf8c" -dependencies = [ - "arrayref", - "byteorder", - "data-encoding", - "multibase", - "multihash 0.17.0", - "percent-encoding", - "serde", - "static_assertions", - "unsigned-varint", - "url", -] +checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" [[package]] -name = "multibase" -version = "0.9.1" +name = "serde" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b3539ec3c1f04ac9748a260728e855f261b4977f5c3406612c884564f329404" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" dependencies = [ - "base-x", - "data-encoding", - "data-encoding-macro", + "serde_core", + "serde_derive", ] [[package]] -name = "multihash" -version = "0.17.0" +name = "serde_core" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "835d6ff01d610179fbce3de1694d007e500bf33a7f29689838941d6bf783ae40" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" dependencies = [ - "core2", - "multihash-derive", - "unsigned-varint", + "serde_derive", ] [[package]] -name = "multihash" -version = "0.18.0" +name = "serde_derive" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15e5d911412e631e1de11eb313e4dd71f73fd964401102aab23d6c8327c431ba" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ - "blake2b_simd", - "blake2s_simd", - "blake3 1.3.3", - "core2", - "digest 0.10.5", - "multihash-derive", - "sha2 0.10.6", - "sha3", - "unsigned-varint", + "proc-macro2", + "quote", + "syn 2.0.118", ] [[package]] -name = "multihash-derive" -version = "0.8.0" +name = "serde_json" +version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc076939022111618a5026d3be019fd8b366e76314538ff9a1b59ffbcbf98bcd" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" dependencies = [ - "proc-macro-crate", - "proc-macro-error", - "proc-macro2", - "quote", - "syn", - "synstructure", + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", ] [[package]] -name = "multimap" -version = "0.8.3" +name = "serde_path_to_error" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" +checksum = "af99884400da37c88f5e9146b7f1fd0fbcae8f6eec4e9da38b67d05486f814a6" +dependencies = [ + "itoa", + "serde", +] [[package]] -name = "native-tls" -version = "0.2.8" +name = "serde_plain" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48ba9f7719b5a0f42f338907614285fb5fd70e53858141f69898a1fb7203b24d" +checksum = "9ce1fc6db65a611022b23a0dec6975d63fb80a302cb3388835ff02c097258d50" dependencies = [ - "lazy_static", - "libc", - "log", - "openssl", - "openssl-probe", - "openssl-sys", - "schannel", - "security-framework", - "security-framework-sys", - "tempfile", + "serde", ] [[package]] -name = "never" -version = "0.1.0" +name = "serde_regex" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c96aba5aa877601bb3f6dd6a63a969e1f82e60646e81e71b14496995e9853c91" +checksum = "bafc8d0c5330cecff10f16b459b479fd9acaa5b4acd7167301414e21b0057012" +dependencies = [ + "regex", + "serde", +] [[package]] -name = "no-std-compat" -version = "0.4.1" +name = "serde_spanned" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b93853da6d84c2e3c7d730d6473e8817692dd89be387eb01b94d7f108ecb5b8c" +checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" +dependencies = [ + "serde_core", +] [[package]] -name = "nom8" -version = "0.2.0" +name = "serde_urlencoded" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae01545c9c7fc4486ab7debaf2aad7003ac19431791868fb2e8066df97fad2f8" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" dependencies = [ - "memchr", + "form_urlencoded", + "itoa", + "ryu", + "serde", ] [[package]] -name = "ntapi" -version = "0.3.6" +name = "serde_with" +version = "3.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f6bb902e437b6d86e03cce10a7e2af662292c5dfef23b65899ea3ac9354ad44" +checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c" dependencies = [ - "winapi", + "base64", + "bs58 0.5.1", + "chrono", + "hex", + "indexmap 1.9.3", + "indexmap 2.14.0", + "schemars 0.9.0", + "schemars 1.2.1", + "serde_core", + "serde_json", + "serde_with_macros", + "time", ] [[package]] -name = "num-bigint" -version = "0.2.6" +name = "serde_with_macros" +version = "3.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304" +checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660" dependencies = [ - "autocfg", - "num-integer", - "num-traits", - "serde", + "darling 0.23.0", + "proc-macro2", + "quote", + "syn 2.0.118", ] [[package]] -name = "num-integer" -version = "0.1.44" +name = "serde_yaml" +version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "autocfg", - "num-traits", + "indexmap 2.14.0", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", ] [[package]] -name = "num-traits" -version = "0.2.15" +name = "serdect" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +checksum = "a84f14a19e9a014bb9f4512488d9829a68e04ecabffb0f9904cd1ace94598177" dependencies = [ - "autocfg", + "base16ct", + "serde", ] [[package]] -name = "num_cpus" -version = "1.15.0" +name = "sha1" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ - "hermit-abi 0.2.6", - "libc", + "cfg-if", + "cpufeatures 0.2.12", + "digest 0.10.7", ] [[package]] -name = "object" -version = "0.24.0" +name = "sha1" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a5b3dd1c072ee7963717671d1ca129f1048fda25edea6b752bfc71ac8854170" +checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" dependencies = [ - "crc32fast", - "indexmap", + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.2", ] [[package]] -name = "object" -version = "0.26.0" +name = "sha1_smol" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d" + +[[package]] +name = "sha2" +version = "0.10.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c55827317fb4c08822499848a14237d2874d6f139828893017237e7ab93eb386" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ - "memchr", + "cfg-if", + "cpufeatures 0.2.12", + "digest 0.10.7", ] [[package]] -name = "once_cell" -version = "1.13.1" +name = "sha2" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "074864da206b4973b84eb91683020dbefd6a8c3f0f38e054d93954e891935e4e" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.2", +] [[package]] -name = "opaque-debug" -version = "0.3.0" +name = "sha3" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" +checksum = "be176f1a57ce4e3d31c1a166222d9768de5954f811601fb7ca06fc8203905ce1" +dependencies = [ + "digest 0.11.2", + "keccak", +] [[package]] -name = "openssl" -version = "0.10.45" +name = "sha3-asm" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b102428fd03bc5edf97f62620f7298614c45cedf287c271e7ed450bbaf83f2e1" +checksum = "9f3f15d4e239ebe08413eed880e0f9b5af4b40ee0472543320efa91d488e96a7" dependencies = [ - "bitflags", - "cfg-if 1.0.0", - "foreign-types", - "libc", - "once_cell", - "openssl-macros", - "openssl-sys", + "cc", + "cfg-if", ] [[package]] -name = "openssl-macros" -version = "0.1.0" +name = "shellexpand" +version = "3.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" +checksum = "32824fab5e16e6c4d86dc1ba84489390419a39f97699852b66480bb87d297ed8" dependencies = [ - "proc-macro2", - "quote", - "syn", + "dirs", ] [[package]] -name = "openssl-probe" -version = "0.1.4" +name = "shlex" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28988d872ab76095a6e6ac88d99b54fd267702734fd7ffe610ca27f533ddb95a" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] -name = "openssl-sys" -version = "0.9.80" +name = "signal-hook" +version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23bbbf7854cd45b83958ebe919f0e8e516793727652e27fda10a8384cfc790b7" +checksum = "d881a16cf4426aa584979d30bd82cb33429027e42122b169753d6ef1085ed6e2" dependencies = [ - "autocfg", - "cc", "libc", - "pkg-config", - "vcpkg", + "signal-hook-registry", ] [[package]] -name = "os_str_bytes" -version = "6.0.0" +name = "signal-hook-mio" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e22443d1643a904602595ba1cd8f7d896afe56d26712531c5ff73a15b2fbf64" +checksum = "b75a19a7a740b25bc7944bdee6172368f988763b744e3d4dfe753f6b4ece40cc" +dependencies = [ + "libc", + "mio", + "signal-hook", +] [[package]] -name = "output_vt100" -version = "0.1.2" +name = "signal-hook-registry" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53cdc5b785b7a58c5aad8216b3dfa114df64b0b06ae6e1501cef91df2fbdf8f9" +checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" dependencies = [ - "winapi", + "libc", ] [[package]] -name = "parity-scale-codec" -version = "3.0.0" +name = "signature" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a7f3fcf5e45fc28b84dcdab6b983e77f197ec01f325a33f404ba6855afd1070" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" dependencies = [ - "arrayvec 0.7.2", - "bitvec", - "byte-slice-cast", - "impl-trait-for-tuples", - "parity-scale-codec-derive", - "serde", + "digest 0.10.7", + "rand_core 0.6.4", ] [[package]] -name = "parity-scale-codec-derive" -version = "3.0.0" +name = "simd-adler32" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c6e626dc84025ff56bf1476ed0e30d10c84d7f89a475ef46ebabee1095a8fba" -dependencies = [ - "proc-macro-crate", - "proc-macro2", - "quote", - "syn", -] +checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" [[package]] -name = "parity-wasm" -version = "0.45.0" +name = "simdutf8" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1ad0aff30c1da14b1254fcb2af73e1fa9a28670e584a626f53a369d0e157304" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" [[package]] -name = "parking_lot" -version = "0.11.2" +name = "similar" +version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +checksum = "e6505efef05804732ed8a3f2d4f279429eb485bd69d5b0cc6b19cc02005cda16" dependencies = [ - "instant", - "lock_api", - "parking_lot_core 0.8.5", + "bstr", ] [[package]] -name = "parking_lot" -version = "0.12.1" +name = "siphasher" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" -dependencies = [ - "lock_api", - "parking_lot_core 0.9.1", -] +checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" [[package]] -name = "parking_lot_core" -version = "0.8.5" +name = "slab" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d76e8e1493bcac0d2766c42737f34458f1c8c50c0d23bcb24ea953affb273216" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" dependencies = [ - "cfg-if 1.0.0", - "instant", - "libc", - "redox_syscall 0.2.10", - "smallvec", - "winapi", + "autocfg", ] [[package]] -name = "parking_lot_core" -version = "0.9.1" +name = "slog" +version = "2.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28141e0cc4143da2443301914478dc976a61ffdb3f043058310c70df2fed8954" +checksum = "9b3b8565691b22d2bdfc066426ed48f837fc0c5f2c8cad8d9718f7f99d6995c1" dependencies = [ - "cfg-if 1.0.0", - "libc", - "redox_syscall 0.2.10", - "smallvec", - "windows-sys", + "anyhow", + "erased-serde", + "rustversion", + "serde_core", ] [[package]] -name = "paste" -version = "1.0.5" +name = "slog-async" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acbf547ad0c65e31259204bd90935776d1c693cec2f4ff7abb7a1bbbd40dfe58" +checksum = "72c8038f898a2c79507940990f05386455b3a317d8f18d4caea7cbc3d5096b84" +dependencies = [ + "crossbeam-channel", + "slog", + "take_mut", + "thread_local", +] [[package]] -name = "percent-encoding" +name = "slog-envlogger" version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" +checksum = "906a1a0bc43fed692df4b82a5e2fbfc3733db8dad8bb514ab27a4f23ad04f5c0" +dependencies = [ + "log", + "regex", + "slog", + "slog-async", + "slog-scope", + "slog-stdlog", + "slog-term", +] [[package]] -name = "petgraph" -version = "0.6.3" +name = "slog-scope" +version = "4.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dd7d28ee937e54fe3080c91faa1c3a46c06de6252988a7f4592ba2310ef22a4" +checksum = "2f95a4b4c3274cd2869549da82b57ccc930859bdbf5bcea0424bc5f140b3c786" dependencies = [ - "fixedbitset", - "indexmap", + "arc-swap", + "lazy_static", + "slog", ] [[package]] -name = "phf" -version = "0.8.0" +name = "slog-stdlog" +version = "4.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dfb61232e34fcb633f43d12c58f83c1df82962dcdfa565a4e866ffc17dafe12" +checksum = "6706b2ace5bbae7291d3f8d2473e2bfab073ccd7d03670946197aec98471fa3e" dependencies = [ - "phf_shared", + "log", + "slog", + "slog-scope", ] [[package]] -name = "phf_shared" -version = "0.8.0" +name = "slog-term" +version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c00cf8b9eafe68dde5e9eaa2cef8ee84a9336a47d566ec55ca16589633b65af7" +checksum = "b6e022d0b998abfe5c3782c1f03551a596269450ccd677ea51c56f8b214610e8" dependencies = [ - "siphasher", + "is-terminal", + "slog", + "term", + "thread_local", + "time", ] [[package]] -name = "pin-project" -version = "1.0.12" +name = "smallvec" +version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad29a609b6bcd67fee905812e544992d216af9d755757c05ed2d0e15a74c6ecc" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" dependencies = [ - "pin-project-internal", + "arbitrary", + "serde", ] [[package]] -name = "pin-project-internal" -version = "1.0.12" +name = "snap" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" + +[[package]] +name = "socket2" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" +checksum = "233504af464074f9d066d7b5416c5f9b894a5862a6506e306f7b816cdd6f1807" dependencies = [ - "proc-macro2", - "quote", - "syn", + "libc", + "windows-sys 0.59.0", ] [[package]] -name = "pin-project-lite" -version = "0.2.9" +name = "spin" +version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" [[package]] -name = "pin-utils" -version = "0.1.0" +name = "spki" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] [[package]] -name = "pkg-config" -version = "0.3.19" +name = "sqlparser" +version = "0.57.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c" +checksum = "07c5f081b292a3d19637f0b32a79e28ff14a9fd23ef47bd7fce08ff5de221eca" +dependencies = [ + "log", + "recursive", + "sqlparser_derive 0.3.0", +] [[package]] -name = "postgres" -version = "0.19.1" +name = "sqlparser" +version = "0.62.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7871ee579860d8183f542e387b176a25f2656b9fb5211e045397f745a68d1c2" +checksum = "13c6d1b651dc4edf07eead2a0c6c78016ce971bc2c10da5266861b13f25e7cec" dependencies = [ - "bytes", - "fallible-iterator", - "futures 0.3.16", "log", - "tokio", - "tokio-postgres", + "recursive", + "sqlparser_derive 0.5.0", ] [[package]] -name = "postgres-openssl" -version = "0.5.0" +name = "sqlparser_derive" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1de0ea6504e07ca78355a6fb88ad0f36cafe9e696cbc6717f16a207f3a60be72" +checksum = "da5fc6819faabb412da764b99d3b713bb55083c11e7e0c00144d386cd6a1939c" dependencies = [ - "futures 0.3.16", - "openssl", - "tokio", - "tokio-openssl", - "tokio-postgres", + "proc-macro2", + "quote", + "syn 2.0.118", ] [[package]] -name = "postgres-protocol" -version = "0.6.1" +name = "sqlparser_derive" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff3e0f70d32e20923cabf2df02913be7c1842d4c772db8065c00fcfdd1d1bff3" +checksum = "a6dd45d8fc1c79299bfbb7190e42ccbbdf6a5f52e4a6ad98d92357ea965bd289" dependencies = [ - "base64 0.13.1", - "byteorder", - "bytes", - "fallible-iterator", - "hmac", - "md-5", - "memchr", - "rand", - "sha2 0.9.5", - "stringprep", + "proc-macro2", + "quote", + "syn 2.0.118", ] [[package]] -name = "postgres-types" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "430f4131e1b7657b0cd9a2b0c3408d77c9a43a042d300b8c77f981dffcc43a2f" +name = "stable-hash" +version = "0.3.4" +source = "git+https://github.com/graphprotocol/stable-hash?rev=42b85d936c6b9f1b7a4b8385d2f081d5564daa69#42b85d936c6b9f1b7a4b8385d2f081d5564daa69" dependencies = [ - "bytes", - "fallible-iterator", - "postgres-protocol", + "blake3", + "firestorm 0.4.6", + "ibig", + "lazy_static", + "leb128", + "num-traits", ] [[package]] -name = "ppv-lite86" -version = "0.2.10" +name = "stable-hash" +version = "0.4.4" +source = "git+https://github.com/graphprotocol/stable-hash?rev=70300f974e61529675aef3f20803f26a23f0e0ea#70300f974e61529675aef3f20803f26a23f0e0ea" +dependencies = [ + "blake3", + "firestorm 0.5.1", + "ibig", + "lazy_static", + "leb128", + "num-traits", + "uint 0.10.1", + "xxhash-rust", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" [[package]] -name = "pq-sys" -version = "0.4.6" +name = "stacker" +version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ac25eee5a0582f45a67e837e350d784e7003bd29a5f460796772061ca49ffda" +checksum = "e1f8b29fb42aafcea4edeeb6b2f2d7ecd0d969c48b4cf0d2e64aafc471dd6e59" dependencies = [ - "vcpkg", + "cc", + "cfg-if", + "libc", + "psm", + "windows-sys 0.59.0", ] [[package]] -name = "pretty_assertions" -version = "1.3.0" +name = "static_assertions" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a25e9bcb20aa780fd0bb16b72403a9064d6b3f22f026946029acb941a50af755" -dependencies = [ - "ctor", - "diff", - "output_vt100", - "yansi", -] +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] -name = "prettyplease" -version = "0.1.10" +name = "static_assertions_next" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7beae5182595e9a8b683fa98c4317f956c9a2dec3b9716990d20023cc60c766" + +[[package]] +name = "stringprep" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9e07e3a46d0771a8a06b5f4441527802830b43e679ba12f44960f48dd4c6803" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" dependencies = [ - "proc-macro2", - "syn", + "unicode-bidi", + "unicode-normalization", + "unicode-properties", ] [[package]] -name = "primitive-types" +name = "strsim" version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e28720988bff275df1f51b171e1b2a18c30d194c4d2b61defdacecd625a5d94a" -dependencies = [ - "fixed-hash", - "impl-codec", - "impl-rlp", - "impl-serde", - "uint", -] +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] -name = "priority-queue" -version = "0.7.0" +name = "strum" +version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a03dfae8e64d4aa651415e2a4321f9f09f2e388a2f8bec36bed03bc22c0b687" +checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" dependencies = [ - "indexmap", - "take_mut", + "strum_macros 0.27.2", ] [[package]] -name = "proc-macro-crate" -version = "1.1.0" +name = "strum" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ebace6889caf889b4d3f76becee12e90353f2b8c7d875534a71e5742f8f6f83" +checksum = "9628de9b8791db39ceda2b119bbe13134770b56c138ec1d3af810d045c04f9bd" dependencies = [ - "thiserror", - "toml 0.5.11", + "strum_macros 0.28.0", ] [[package]] -name = "proc-macro-error" -version = "1.0.4" +name = "strum_macros" +version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" dependencies = [ - "proc-macro-error-attr", + "heck 0.5.0", "proc-macro2", "quote", - "syn", - "version_check", + "syn 2.0.118", ] [[package]] -name = "proc-macro-error-attr" -version = "1.0.4" +name = "strum_macros" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +checksum = "ab85eea0270ee17587ed4156089e10b9e6880ee688791d45a905f5b1ca36f664" dependencies = [ + "heck 0.5.0", "proc-macro2", "quote", - "version_check", + "syn 2.0.118", ] [[package]] -name = "proc-macro-hack" -version = "0.5.19" +name = "subtle" +version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] -name = "proc-macro-nested" -version = "0.1.7" +name = "syn" +version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc881b2c22681370c6a780e47af9840ef841837bc98118431d4e1868bd0c1086" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] [[package]] -name = "proc-macro2" -version = "1.0.51" +name = "syn" +version = "2.0.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d727cae5b39d21da60fa540906919ad737832fe0b1c165da3a34d6548c849d6" +checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" dependencies = [ + "proc-macro2", + "quote", "unicode-ident", ] [[package]] -name = "prometheus" -version = "0.13.3" +name = "syn-solidity" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "449811d15fbdf5ceb5c1144416066429cf82316e2ec8ce0c1f6f8a02e7bbcf8c" +checksum = "ec005042c7d952febc1a3ef5b0f6674e9054aa836877a31c90b20e25b3d31744" dependencies = [ - "cfg-if 1.0.0", - "fnv", - "lazy_static", - "libc", - "memchr", - "parking_lot 0.12.1", - "protobuf 2.25.0", - "reqwest", - "thiserror", + "paste", + "proc-macro2", + "quote", + "syn 2.0.118", ] [[package]] -name = "prost" -version = "0.11.6" +name = "sync_wrapper" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21dc42e00223fc37204bd4aa177e69420c604ca4a183209a8f9de30c6d934698" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" dependencies = [ - "bytes", - "prost-derive", + "futures-core", ] [[package]] -name = "prost-build" -version = "0.11.5" +name = "synstructure" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb5320c680de74ba083512704acb90fe00f28f79207286a848e730c45dd73ed6" +checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ - "bytes", - "heck 0.4.1", - "itertools", - "lazy_static", - "log", - "multimap", - "petgraph", - "prettyplease", - "prost", - "prost-types", - "regex", - "syn", - "tempfile", - "which", + "proc-macro2", + "quote", + "syn 2.0.118", ] [[package]] -name = "prost-derive" -version = "0.11.6" +name = "system-configuration" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bda8c0881ea9f722eb9629376db3d0b903b462477c1aafcb0566610ac28ac5d" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" dependencies = [ - "anyhow", - "itertools", - "proc-macro2", - "quote", - "syn", + "bitflags 2.11.1", + "core-foundation 0.9.4", + "system-configuration-sys", ] [[package]] -name = "prost-types" -version = "0.11.6" +name = "system-configuration-sys" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5e0526209433e96d83d750dd81a99118edbc55739e7e61a46764fd2ad537788" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" dependencies = [ - "bytes", - "prost", + "core-foundation-sys", + "libc", ] [[package]] -name = "protobuf" -version = "2.25.0" +name = "take_mut" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "020f86b07722c5c4291f7c723eac4676b3892d47d9a7708dc2779696407f039b" +checksum = "f764005d11ee5f36500a149ace24e00e3da98b0158b3e2d53a7495660d3f4d60" [[package]] -name = "protobuf" -version = "3.2.0" +name = "tap" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b55bad9126f378a853655831eb7363b7b01b81d19f8cb1218861086ca4a1a61e" -dependencies = [ - "once_cell", - "protobuf-support", - "thiserror", -] +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] -name = "protobuf-parse" -version = "3.2.0" +name = "target-lexicon" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" + +[[package]] +name = "tempfile" +version = "3.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d39b14605eaa1f6a340aec7f320b34064feb26c93aec35d6a9a2272a8ddfa49" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ - "anyhow", - "indexmap", - "log", - "protobuf 3.2.0", - "protobuf-support", - "tempfile", - "thiserror", - "which", + "fastrand", + "getrandom 0.4.2", + "once_cell", + "rustix 1.1.4", + "windows-sys 0.52.0", ] [[package]] -name = "protobuf-support" -version = "3.2.0" +name = "term" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5d4d7b8601c814cfb36bcebb79f0e61e45e1e93640cf778837833bbed05c372" +checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f" dependencies = [ - "thiserror", + "dirs-next", + "rustversion", + "winapi", ] [[package]] -name = "psm" -version = "0.1.14" +name = "termcolor" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14ce37fa8c0428a37307d163292add09b3aedc003472e6b3622486878404191d" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" dependencies = [ - "cc", + "winapi-util", ] [[package]] -name = "quick-error" -version = "1.2.3" +name = "terminal_size" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" +checksum = "21bebf2b7c9e0a515f6e0f8c51dc0f8e4696391e6f1ff30379559f8365fb0df7" +dependencies = [ + "rustix 0.38.34", + "windows-sys 0.48.0", +] [[package]] -name = "quote" -version = "1.0.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b" +name = "test-store" +version = "0.45.0" dependencies = [ - "proc-macro2", + "async-trait", + "diesel", + "diesel-async", + "graph", + "graph-chain-ethereum", + "graph-graphql", + "graph-node", + "graph-store-postgres", + "hex", + "hex-literal", + "lazy_static", + "pretty_assertions", + "prost-types", + "serde_json", + "tokio", ] [[package]] -name = "r2d2" -version = "0.8.9" +name = "thiserror" +version = "1.0.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "545c5bc2b880973c9c10e4067418407a0ccaa3091781d1671d46eb35107cb26f" +checksum = "c546c80d6be4bc6a00c0f01730c08df82eaa7a7a61f11d656526506112cc1709" dependencies = [ - "log", - "parking_lot 0.11.2", - "scheduled-thread-pool", + "thiserror-impl 1.0.61", ] [[package]] -name = "radium" -version = "0.7.0" +name = "thiserror" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl 2.0.18", +] [[package]] -name = "rand" -version = "0.8.5" +name = "thiserror-impl" +version = "1.0.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" dependencies = [ - "libc", - "rand_chacha", - "rand_core", + "proc-macro2", + "quote", + "syn 2.0.118", ] [[package]] -name = "rand_chacha" -version = "0.3.1" +name = "thiserror-impl" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ - "ppv-lite86", - "rand_core", + "proc-macro2", + "quote", + "syn 2.0.118", ] [[package]] -name = "rand_core" -version = "0.6.3" +name = "thread_local" +version = "1.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7" +checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" dependencies = [ - "getrandom", + "cfg-if", + "once_cell", ] [[package]] -name = "rayon" -version = "1.5.1" +name = "threadpool" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06aca804d41dbc8ba42dfd964f0d01334eceb64314b9ecf7c5fad5188a06d90" +checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa" dependencies = [ - "autocfg", - "crossbeam-deque", - "either", - "rayon-core", + "num_cpus", ] [[package]] -name = "rayon-core" -version = "1.9.1" +name = "time" +version = "0.3.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d78120e2c850279833f1dd3582f730c4ab53ed95aeaaaa862a2a5c71b1656d8e" +checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" dependencies = [ - "crossbeam-channel", - "crossbeam-deque", - "crossbeam-utils", - "lazy_static", - "num_cpus", + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", ] [[package]] -name = "redox_syscall" -version = "0.1.57" +name = "time-core" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" +checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" [[package]] -name = "redox_syscall" -version = "0.2.10" +name = "time-macros" +version = "0.2.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8383f39639269cde97d255a32bdb68c047337295414940c68bdd30c2e13203ff" +checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" dependencies = [ - "bitflags", + "num-conv", + "time-core", ] [[package]] -name = "redox_users" -version = "0.4.0" +name = "tiny-keccak" +version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "528532f3d801c87aec9def2add9ca802fe569e44a544afe633765267840abe64" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" dependencies = [ - "getrandom", - "redox_syscall 0.2.10", + "crunchy", ] [[package]] -name = "regalloc" -version = "0.0.31" +name = "tinystr" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "571f7f397d61c4755285cd37853fe8e03271c243424a907415909379659381c5" +checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" dependencies = [ - "log", - "rustc-hash", - "serde", - "smallvec", + "displaydoc", + "zerovec", ] [[package]] -name = "regex" -version = "1.5.5" +name = "tinyvec" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a11647b6b25ff05a515cb92c365cec08801e83423a235b51e231e1808747286" +checksum = "ce6b6a2fb3a985e99cebfaefa9faa3024743da73304ca1c683a36429613d3d22" dependencies = [ - "aho-corasick", - "memchr", - "regex-syntax", + "tinyvec_macros", ] [[package]] -name = "regex-syntax" -version = "0.6.25" +name = "tinyvec_macros" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] -name = "region" -version = "2.2.0" +name = "tokio" +version = "1.50.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877e54ea2adcd70d80e9179344c97f93ef0dffd6b03e1f4529e6e83ab2fa9ae0" +checksum = "27ad5e34374e03cfffefc301becb44e9dc3c17584f414349ebe29ed26661822d" dependencies = [ - "bitflags", + "bytes", "libc", - "mach", - "winapi", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", ] [[package]] -name = "remove_dir_all" -version = "0.5.3" +name = "tokio-macros" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" dependencies = [ - "winapi", + "proc-macro2", + "quote", + "syn 2.0.118", ] [[package]] -name = "reqwest" -version = "0.11.4" +name = "tokio-native-tls" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "246e9f61b9bb77df069a947682be06e31ac43ea37862e244a69f177694ea6d22" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-openssl" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ffab79df67727f6acf57f1ff743091873c24c579b1e2ce4d8f53e47ded4d63d" dependencies = [ - "base64 0.13.1", - "bytes", - "encoding_rs", - "futures-core", "futures-util", - "http", - "http-body", - "hyper", - "hyper-tls", - "ipnet", - "js-sys", - "lazy_static", + "openssl", + "openssl-sys", + "tokio", +] + +[[package]] +name = "tokio-postgres" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b40d66d9b2cfe04b628173409368e58247e8eddbbd3b0e6c6ba1d09f20f6c9e" +dependencies = [ + "async-trait", + "byteorder", + "bytes", + "fallible-iterator 0.2.0", + "futures-channel", + "futures-util", "log", - "mime", - "mime_guess", - "native-tls", + "parking_lot", "percent-encoding", + "phf", "pin-project-lite", - "serde", - "serde_json", - "serde_urlencoded", + "postgres-protocol", + "postgres-types", + "rand 0.9.3", + "socket2", "tokio", - "tokio-native-tls", - "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", - "winreg", + "tokio-util", + "whoami", ] [[package]] -name = "ring" -version = "0.16.20" +name = "tokio-retry" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +checksum = "4a129d95275ebf4c493ec53bf0f8cd95f5ac161bc4f381700809a54f595d4470" dependencies = [ - "cc", - "libc", - "once_cell", - "spin", - "untrusted", - "web-sys", - "winapi", + "pin-project-lite", + "rand 0.10.1", + "tokio", ] [[package]] -name = "rlp" -version = "0.5.1" +name = "tokio-rustls" +version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "999508abb0ae792aabed2460c45b89106d97fe4adac593bdaef433c2605847b5" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" dependencies = [ - "bytes", - "rustc-hex", + "rustls", + "tokio", ] [[package]] -name = "rustc-demangle" -version = "0.1.20" +name = "tokio-stream" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dead70b0b5e03e9c814bcb6b01e03e68f7c57a80aa48c72ec92152ab3e818d49" +checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", + "tokio-util", +] [[package]] -name = "rustc-hash" -version = "1.1.0" +name = "tokio-test" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +checksum = "2468baabc3311435b55dd935f702f42cd1b8abb7e754fb7dfb16bd36aa88f9f7" +dependencies = [ + "async-stream", + "bytes", + "futures-core", + "tokio", + "tokio-stream", +] [[package]] -name = "rustc-hex" -version = "2.1.0" +name = "tokio-tungstenite" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e75f6a532d0fd9f7f13144f392b6ad56a32696bfcd9c78f797f16bbb6f072d6" +checksum = "d25a406cddcc431a75d3d9afc6a7c0f7428d4891dd973e4d54c56b46127bf857" +dependencies = [ + "futures-util", + "log", + "rustls", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tungstenite 0.28.0", + "webpki-roots 0.26.11", +] [[package]] -name = "rustc_version" -version = "0.4.0" +name = "tokio-tungstenite" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +checksum = "8f72a05e828585856dacd553fba484c242c46e391fb0e58917c942ee9202915c" dependencies = [ - "semver", + "futures-util", + "log", + "tokio", + "tungstenite 0.29.0", ] [[package]] -name = "rustls" -version = "0.20.4" +name = "tokio-util" +version = "0.7.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fbfeb8d0ddb84706bc597a5574ab8912817c52a397f819e5b614e2265206921" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" dependencies = [ - "log", - "ring", - "sct 0.7.0", - "webpki", + "bytes", + "futures-core", + "futures-io", + "futures-sink", + "pin-project-lite", + "tokio", ] [[package]] -name = "rustls-native-certs" -version = "0.6.2" +name = "toml" +version = "0.9.12+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0167bac7a9f490495f3c33013e7722b53cb087ecbe082fb0c6387c96f634ea50" +checksum = "cf92845e79fc2e2def6a5d828f0801e29a2f8acc037becc5ab08595c7d5e9863" dependencies = [ - "openssl-probe", - "rustls-pemfile", - "schannel", - "security-framework", + "indexmap 2.14.0", + "serde_core", + "serde_spanned", + "toml_datetime 0.7.5+spec-1.1.0", + "toml_parser", + "toml_writer", + "winnow 0.7.13", ] [[package]] -name = "rustls-pemfile" -version = "1.0.0" +name = "toml" +version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7522c9de787ff061458fe9a829dc790a3f5b22dc571694fc5883f448b94d9a9" +checksum = "81f3d15e84cbcd896376e6730314d59fb5a87f31e4b038454184435cd57defee" dependencies = [ - "base64 0.13.1", + "indexmap 2.14.0", + "serde_core", + "serde_spanned", + "toml_datetime 1.1.1+spec-1.1.0", + "toml_parser", + "toml_writer", + "winnow 1.0.0", ] [[package]] -name = "rustversion" -version = "1.0.11" +name = "toml_datetime" +version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5583e89e108996506031660fe09baa5011b9dd0341b89029313006d1fb508d70" +checksum = "4badfd56924ae69bcc9039335b2e017639ce3f9b001c393c1b2d1ef846ce2cbf" [[package]] -name = "ryu" -version = "1.0.5" +name = "toml_datetime" +version = "0.7.5+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" +checksum = "92e1cfed4a3038bc5a127e35a2d360f145e1f4b971b551a2ba5fd7aedf7e1347" +dependencies = [ + "serde_core", +] [[package]] -name = "same-file" -version = "1.0.6" +name = "toml_datetime" +version = "1.1.1+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" dependencies = [ - "winapi-util", + "serde_core", ] [[package]] -name = "schannel" -version = "0.1.19" +name = "toml_edit" +version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75" +checksum = "6a8534fd7f78b5405e860340ad6575217ce99f38d4d5c8f2442cb5ecb50090e1" dependencies = [ - "lazy_static", - "winapi", + "indexmap 2.14.0", + "toml_datetime 0.6.6", + "winnow 0.5.40", ] [[package]] -name = "scheduled-thread-pool" -version = "0.2.5" +name = "toml_parser" +version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc6f74fd1204073fa02d5d5d68bec8021be4c38690b61264b2fdb48083d0e7d7" +checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" dependencies = [ - "parking_lot 0.11.2", + "winnow 1.0.0", ] [[package]] -name = "scopeguard" -version = "1.1.0" +name = "toml_writer" +version = "1.1.1+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +checksum = "756daf9b1013ebe47a8776667b466417e2d4c5679d441c26230efd9ef78692db" [[package]] -name = "scroll" -version = "0.10.2" +name = "tonic" +version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fda28d4b4830b807a8b43f7b0e6b5df875311b3e7621d84577188c175b6ec1ec" +checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef" dependencies = [ - "scroll_derive", + "async-trait", + "axum", + "base64", + "bytes", + "flate2", + "h2", + "http 1.4.2", + "http-body", + "http-body-util", + "hyper", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "rustls-native-certs", + "socket2", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-stream", + "tower 0.5.2", + "tower-layer 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "tower-service 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "tracing", ] [[package]] -name = "scroll_derive" -version = "0.10.5" +name = "tonic-build" +version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aaaae8f38bb311444cfb7f1979af0bc9240d95795f75f9ceddf6a59b79ceffa0" +checksum = "c68f61875ac5293cf72e6c8cf0158086428c82c37229e98c840878f1706b0322" dependencies = [ + "prettyplease", "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] -name = "sct" -version = "0.6.1" +name = "tonic-prost" +version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b362b83898e0e69f38515b82ee15aa80636befe47c3b6d3d89a911e78fc228ce" +checksum = "50849f68853be452acf590cde0b146665b8d507b3b8af17261df47e02c209ea0" dependencies = [ - "ring", - "untrusted", + "bytes", + "prost", + "tonic", ] [[package]] -name = "sct" -version = "0.7.0" +name = "tonic-prost-build" +version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" +checksum = "654e5643eff75d7f8c99197ce1440ed19a3474eada74c12bbac488b2cafdae27" dependencies = [ - "ring", - "untrusted", + "prettyplease", + "proc-macro2", + "prost-build", + "prost-types", + "quote", + "syn 2.0.118", + "tempfile", + "tonic-build", ] [[package]] -name = "secp256k1" -version = "0.21.3" +name = "tower" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c42e6f1735c5f00f51e43e28d6634141f2bcad10931b2609ddd74a86d751260" +checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" dependencies = [ - "secp256k1-sys", + "futures-core", + "futures-util", + "hdrhistogram", + "indexmap 2.14.0", + "pin-project-lite", + "slab", + "sync_wrapper", + "tokio", + "tokio-util", + "tower-layer 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "tower-service 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "tracing", ] [[package]] -name = "secp256k1-sys" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "957da2573cde917463ece3570eab4a0b3f19de6f1646cde62e6fd3868f566036" +name = "tower" +version = "0.5.3" +source = "git+https://github.com/tower-rs/tower.git#df06d70dbea345facbffb5881fe8647f53bf424d" dependencies = [ - "cc", + "futures-core", + "futures-util", + "hdrhistogram", + "indexmap 2.14.0", + "pin-project-lite", + "slab", + "sync_wrapper", + "tokio", + "tokio-util", + "tower-layer 0.3.3 (git+https://github.com/tower-rs/tower.git)", + "tower-service 0.3.3 (git+https://github.com/tower-rs/tower.git)", + "tracing", ] [[package]] -name = "security-framework" -version = "2.3.1" +name = "tower-http" +version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23a2ac85147a3a11d77ecf1bc7166ec0b92febfa4461c37944e180f319ece467" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ - "bitflags", - "core-foundation", - "core-foundation-sys", - "libc", - "security-framework-sys", + "bitflags 2.11.1", + "bytes", + "futures-util", + "http 1.4.2", + "http-body", + "pin-project-lite", + "tower 0.5.2", + "tower-layer 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "tower-service 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "url", ] [[package]] -name = "security-framework-sys" -version = "2.3.0" +name = "tower-http" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e4effb91b4b8b6fb7732e670b6cee160278ff8e6bf485c7805d9e319d76e284" +checksum = "b11f75e912b0c2be01b63d8cf8057b8c3f97cf34abb3d431a3a4c8675498e233" dependencies = [ - "core-foundation-sys", - "libc", + "bitflags 2.11.1", + "bytes", + "http 1.4.2", + "percent-encoding", + "pin-project-lite", + "tower-layer 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "tower-service 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] -name = "semver" -version = "1.0.16" +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "git+https://github.com/tower-rs/tower.git#df06d70dbea345facbffb5881fe8647f53bf424d" + +[[package]] +name = "tower-service" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58bc9567378fc7690d6b2addae4e60ac2eeea07becb2c64b9f218b53865cba2a" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "git+https://github.com/tower-rs/tower.git#df06d70dbea345facbffb5881fe8647f53bf424d" + +[[package]] +name = "tower-test" +version = "0.4.1" +source = "git+https://github.com/tower-rs/tower.git#df06d70dbea345facbffb5881fe8647f53bf424d" dependencies = [ - "serde", + "pin-project-lite", + "tokio", + "tokio-test", + "tower-layer 0.3.3 (git+https://github.com/tower-rs/tower.git)", + "tower-service 0.3.3 (git+https://github.com/tower-rs/tower.git)", ] [[package]] -name = "serde" -version = "1.0.152" +name = "tracing" +version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ - "serde_derive", + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", ] [[package]] -name = "serde_derive" -version = "1.0.152" +name = "tracing-attributes" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" +checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] -name = "serde_json" -version = "1.0.66" +name = "tracing-core" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "336b10da19a12ad094b59d870ebde26a45402e5b470add4b5fd03c5048a32127" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" dependencies = [ - "itoa 0.4.7", - "ryu", - "serde", + "once_cell", ] [[package]] -name = "serde_plain" -version = "1.0.1" +name = "try-lock" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6018081315db179d0ce57b1fe4b62a12a0028c9cf9bbef868c9cf477b3c34ae" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "tungstenite" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8628dcc84e5a09eb3d8423d6cb682965dea9133204e8fb3efee74c2a0c259442" dependencies = [ - "serde", + "bytes", + "data-encoding", + "http 1.4.2", + "httparse", + "log", + "rand 0.9.3", + "rustls", + "rustls-pki-types", + "sha1 0.10.6", + "thiserror 2.0.18", + "utf-8", ] [[package]] -name = "serde_regex" -version = "1.1.0" +name = "tungstenite" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8136f1a4ea815d7eac4101cfd0b16dc0cb5e1fe1b8609dfd728058656b7badf" +checksum = "6c01152af293afb9c7c2a57e4b559c5620b421f6d133261c60dd2d0cdb38e6b8" dependencies = [ - "regex", - "serde", + "bytes", + "data-encoding", + "http 1.4.2", + "httparse", + "log", + "rand 0.9.3", + "sha1 0.10.6", + "thiserror 2.0.18", ] [[package]] -name = "serde_spanned" -version = "0.6.1" +name = "twox-hash" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0efd8caf556a6cebd3b285caf480045fcc1ac04f6bd786b09a6f11af30c4fcf4" -dependencies = [ - "serde", -] +checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" [[package]] -name = "serde_urlencoded" -version = "0.7.0" +name = "typenum" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edfa57a7f8d9c1d260a549e7224100f6c43d43f9103e06dd8b4095a9b2b43ce9" -dependencies = [ - "form_urlencoded", - "itoa 0.4.7", - "ryu", - "serde", -] +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] -name = "serde_with" -version = "1.9.4" +name = "ucd-trie" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ad9fdbb69badc8916db738c25efd04f0a65297d26c2f8de4b62e57b8c12bc72" -dependencies = [ - "rustversion", - "serde", - "serde_with_macros", -] +checksum = "ed646292ffc8188ef8ea4d1e0e0150fb15a5c2e12ad9b8fc191ae7a8a7f3c4b9" [[package]] -name = "serde_with_macros" -version = "1.4.2" +name = "uint" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1569374bd54623ec8bd592cf22ba6e03c0f177ff55fbc8c29a49e296e7adecf" +checksum = "76f64bba2c53b04fcab63c01a7d7427eadc821e3bc48c34dc9ba29c501164b52" dependencies = [ - "darling", - "proc-macro2", - "quote", - "syn", + "byteorder", + "crunchy", + "hex", + "static_assertions", ] [[package]] -name = "serde_yaml" -version = "0.8.26" +name = "uint" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "578a7433b776b56a35785ed5ce9a7e777ac0598aac5a6dd1b4b18a307c7fc71b" +checksum = "6f9227a75a5a540a464c832ad4a4195dbdbecd8787610a56262721fde6f04f90" dependencies = [ - "indexmap", - "ryu", - "serde", - "yaml-rust", + "byteorder", + "crunchy", + "hex", + "static_assertions", ] [[package]] -name = "sha-1" -version = "0.9.7" +name = "unarray" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a0c8611594e2ab4ebbf06ec7cbbf0a99450b8570e96cbf5188b5d5f6ef18d81" -dependencies = [ - "block-buffer 0.9.0", - "cfg-if 1.0.0", - "cpufeatures 0.1.5", - "digest 0.9.0", - "opaque-debug", -] +checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" [[package]] -name = "sha-1" -version = "0.10.0" +name = "unicase" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "028f48d513f9678cda28f6e4064755b3fbb2af6acd672f2c209b62323f7aea0f" +checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89" dependencies = [ - "cfg-if 1.0.0", - "cpufeatures 0.2.2", - "digest 0.10.5", + "version_check", ] [[package]] -name = "sha2" -version = "0.9.5" +name = "unicode-bidi" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b362ae5752fd2137731f9fa25fd4d9058af34666ca1966fb969119cc35719f12" -dependencies = [ - "block-buffer 0.9.0", - "cfg-if 1.0.0", - "cpufeatures 0.1.5", - "digest 0.9.0", - "opaque-debug", -] +checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" [[package]] -name = "sha2" -version = "0.10.6" +name = "unicode-ident" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" -dependencies = [ - "cfg-if 1.0.0", - "cpufeatures 0.2.2", - "digest 0.10.5", -] +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" [[package]] -name = "sha3" -version = "0.10.1" +name = "unicode-normalization" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "881bf8156c87b6301fc5ca6b27f11eeb2761224c7081e69b409d5a1951a70c86" +checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" dependencies = [ - "digest 0.10.5", - "keccak", + "tinyvec", ] [[package]] -name = "shellexpand" -version = "2.1.0" +name = "unicode-properties" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83bdb7831b2d85ddf4a7b148aa19d0587eddbe8671a436b7bd1182eaad0f2829" -dependencies = [ - "dirs-next", -] +checksum = "e4259d9d4425d9f0661581b804cb85fe66a4c631cadd8f490d1c13a35d5d9291" [[package]] -name = "signal-hook-registry" -version = "1.4.0" +name = "unicode-segmentation" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51e73328dc4ac0c7ccbda3a494dfa03df1de2f46018127f60c693f2648455b0" -dependencies = [ - "libc", -] +checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" [[package]] -name = "siphasher" -version = "0.3.6" +name = "unicode-width" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "729a25c17d72b06c68cb47955d44fda88ad2d3e7d77e025663fdd69b93dd71a1" +checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" [[package]] -name = "slab" -version = "0.4.4" +name = "unicode-xid" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c307a32c1c5c437f38c7fd45d753050587732ba8628319fbdf12a7e289ccc590" +checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" [[package]] -name = "slog" -version = "2.7.0" +name = "unit-prefix" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8347046d4ebd943127157b94d63abb990fcf729dc4e9978927fdf4ac3c998d06" +checksum = "81e544489bf3d8ef66c953931f56617f423cd4b5494be343d9b9d3dda037b9a3" [[package]] -name = "slog-async" -version = "2.7.0" +name = "unsafe-libyaml" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "766c59b252e62a34651412870ff55d8c4e6d04df19b43eecb2703e417b097ffe" -dependencies = [ - "crossbeam-channel", - "slog", - "take_mut", - "thread_local", -] +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" [[package]] -name = "slog-envlogger" -version = "2.2.0" +name = "unsigned-varint" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "906a1a0bc43fed692df4b82a5e2fbfc3733db8dad8bb514ab27a4f23ad04f5c0" -dependencies = [ - "log", - "regex", - "slog", - "slog-async", - "slog-scope", - "slog-stdlog", - "slog-term", -] +checksum = "eb066959b24b5196ae73cb057f45598450d2c5f71460e98c49b738086eff9c06" [[package]] -name = "slog-scope" -version = "4.4.0" +name = "untrusted" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f95a4b4c3274cd2869549da82b57ccc930859bdbf5bcea0424bc5f140b3c786" -dependencies = [ - "arc-swap", - "lazy_static", - "slog", -] +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] -name = "slog-stdlog" -version = "4.1.1" +name = "url" +version = "2.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6706b2ace5bbae7291d3f8d2473e2bfab073ccd7d03670946197aec98471fa3e" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" dependencies = [ - "log", - "slog", - "slog-scope", + "form_urlencoded", + "idna", + "percent-encoding", + "serde", + "serde_derive", ] [[package]] -name = "slog-term" -version = "2.8.0" +name = "utf-8" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95c1e7e5aab61ced6006149ea772770b84a0d16ce0f7885def313e4829946d76" -dependencies = [ - "atty", - "chrono", - "slog", - "term", - "thread_local", -] +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" [[package]] -name = "smallvec" -version = "1.6.1" +name = "utf16_iter" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e" +checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" [[package]] -name = "socket2" -version = "0.4.1" +name = "utf8_iter" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "765f090f0e423d2b55843402a07915add955e7d60657db13707a159727326cad" -dependencies = [ - "libc", - "winapi", -] +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" [[package]] -name = "soketto" -version = "0.7.1" +name = "utf8parse" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d1c5305e39e09653383c2c7244f2f78b3bcae37cf50c64cb4789c9f5096ec2" -dependencies = [ - "base64 0.13.1", - "bytes", - "futures 0.3.16", - "httparse", - "log", - "rand", - "sha-1 0.9.7", -] +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] -name = "spin" -version = "0.5.2" +name = "uuid" +version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +checksum = "e0f540e3240398cce6128b64ba83fdbdd86129c16a3aa1a3a252efd66eb3d587" [[package]] -name = "stable-hash" -version = "0.3.3" +name = "valuable" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10196e68950ed99c0d2db7a30ffaf4dfe0bbf2f9af2ae0457ee8ad396e0a2dd7" -dependencies = [ - "blake3 0.3.8", - "firestorm 0.4.6", - "ibig", - "lazy_static", - "leb128", - "num-traits", -] +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" [[package]] -name = "stable-hash" -version = "0.4.2" +name = "vcpkg" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af75bd21beb162eab69de76abbb803d4111735ead00d5086dcc6f4ddb3b53cc9" -dependencies = [ - "blake3 0.3.8", - "firestorm 0.5.0", - "ibig", - "lazy_static", - "leb128", - "num-traits", - "xxhash-rust", -] +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" [[package]] -name = "stable_deref_trait" -version = "1.2.0" +name = "version_check" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] -name = "static_assertions" -version = "1.1.0" +name = "wait-timeout" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" +dependencies = [ + "libc", +] [[package]] -name = "stringprep" -version = "0.1.2" +name = "walkdir" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ee348cb74b87454fff4b551cbf727025810a004f88aeacae7f85b87f4e9a1c1" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" dependencies = [ - "unicode-bidi", - "unicode-normalization", + "same-file", + "winapi-util", ] [[package]] -name = "strsim" -version = "0.10.0" +name = "want" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] [[package]] -name = "strum" -version = "0.21.0" +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aaf86bbcfd1fa9670b7a129f64fc0c9fcbbfe4f1bc4210e9e98fe71ffc12cde2" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] -name = "strum_macros" -version = "0.21.1" +name = "wasi" +version = "0.13.3+wasi-0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d06aaeeee809dbc59eb4556183dd927df67db1540de5be8d3ec0b6636358a5ec" +checksum = "26816d2e1a4a36a2940b96c5296ce403917633dff8f3440e9b236ed6f6bacad2" dependencies = [ - "heck 0.3.3", - "proc-macro2", - "quote", - "syn", + "wit-bindgen-rt", ] [[package]] -name = "subtle" -version = "2.4.1" +name = "wasip2" +version = "1.0.2+wasi-0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" +checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" +dependencies = [ + "wit-bindgen", +] [[package]] -name = "syn" -version = "1.0.107" +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", + "wit-bindgen", ] [[package]] -name = "sync_wrapper" -version = "0.1.1" +name = "wasite" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20518fe4a4c9acf048008599e464deb21beeae3d3578418951a189c235a7a9a8" +checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" [[package]] -name = "synstructure" -version = "0.12.5" +name = "wasm-bindgen" +version = "0.2.125" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "474aaa926faa1603c40b7885a9eaea29b444d1cb2850cb7c0e37bb1a4182f4fa" +checksum = "8ddb3f79143bced6de84270411622a2699cee572fc0875aeaf1e7867cf9fca1a" dependencies = [ - "proc-macro2", - "quote", - "syn", - "unicode-xid", + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", ] [[package]] -name = "take_mut" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f764005d11ee5f36500a149ace24e00e3da98b0158b3e2d53a7495660d3f4d60" - -[[package]] -name = "tap" -version = "1.0.1" +name = "wasm-bindgen-futures" +version = "0.4.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" +checksum = "503b14d284f2c8dac03b819967e155ea753f573586193b2b2c95990cb5d69280" +dependencies = [ + "js-sys", + "wasm-bindgen", +] [[package]] -name = "target-lexicon" -version = "0.12.1" +name = "wasm-bindgen-macro" +version = "0.2.125" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0652da4c4121005e9ed22b79f6c5f2d9e2752906b53a33e9490489ba421a6fb" +checksum = "4e21a184b13fb19e157296e2c46056aec9092264fab83e4ba59e68c61b323c3d" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] [[package]] -name = "tempfile" -version = "3.2.0" +name = "wasm-bindgen-macro-support" +version = "0.2.125" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22" +checksum = "fecefd9c35bd935a20fc3fc344b5f29138961e4f47fb03297d88f2587afb5ebd" dependencies = [ - "cfg-if 1.0.0", - "libc", - "rand", - "redox_syscall 0.2.10", - "remove_dir_all", - "winapi", + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.118", + "wasm-bindgen-shared", ] [[package]] -name = "term" -version = "0.7.0" +name = "wasm-bindgen-shared" +version = "0.2.125" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f" +checksum = "23939e44bb9a5d7576fa2b563dc2e136628f1224e88a8deed09e04858b77871f" dependencies = [ - "dirs-next", - "rustversion", - "winapi", + "unicode-ident", ] [[package]] -name = "termcolor" -version = "1.2.0" +name = "wasm-compose" +version = "0.251.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" +checksum = "b089037d7eb453ed57b560fe7833de0707411c8b9fdc429745ced77e2a1bacb9" dependencies = [ - "winapi-util", + "anyhow", + "heck 0.5.0", + "indexmap 2.14.0", + "log", + "petgraph 0.6.5", + "smallvec", + "wasm-encoder 0.251.0", + "wasmparser 0.251.0", + "wat", ] [[package]] -name = "terminal_size" -version = "0.1.17" +name = "wasm-encoder" +version = "0.244.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "633c1a546cee861a1a6d0dc69ebeca693bf4296661ba7852b9d21d159e0506df" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" dependencies = [ - "libc", - "winapi", + "leb128fmt", + "wasmparser 0.244.0", ] [[package]] -name = "test-store" -version = "0.30.0" +name = "wasm-encoder" +version = "0.251.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a879a421bd17c528b74721b2abf4c62e8f1d1889c2ba8c3c50d02deaf2ce395" dependencies = [ - "diesel", - "graph", - "graph-chain-ethereum", - "graph-graphql", - "graph-mock", - "graph-node", - "graph-store-postgres", - "graphql-parser", - "hex-literal", - "lazy_static", - "prost-types", - "serde", + "leb128fmt", + "wasmparser 0.251.0", ] [[package]] -name = "textwrap" -version = "0.16.0" +name = "wasm-encoder" +version = "0.252.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" +checksum = "8185ae345fa5687c054626ff9a50e7089797a343d9904d1dc9820eb4c4d3196f" +dependencies = [ + "leb128fmt", + "wasmparser 0.252.0", +] [[package]] -name = "thiserror" -version = "1.0.31" +name = "wasm-instrument" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd829fe32373d27f76265620b5309d0340cb8550f523c1dda251d6298069069a" +checksum = "2a47ecb37b9734d1085eaa5ae1a81e60801fd8c28d4cabdd8aedb982021918bc" dependencies = [ - "thiserror-impl", + "parity-wasm", ] [[package]] -name = "thiserror-impl" -version = "1.0.31" +name = "wasm-metadata" +version = "0.244.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0396bc89e626244658bef819e22d0cc459e795a5ebe878e6ec336d1674a8d79a" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" dependencies = [ - "proc-macro2", - "quote", - "syn", + "anyhow", + "indexmap 2.14.0", + "wasm-encoder 0.244.0", + "wasmparser 0.244.0", ] [[package]] -name = "thread_local" -version = "1.1.4" +name = "wasm-streams" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5516c27b78311c50bf42c071425c560ac799b11c30b31f87e3081965fe5e0180" +checksum = "b65dc4c90b63b118468cf747d8bf3566c1913ef60be765b5730ead9e0a3ba129" dependencies = [ - "once_cell", + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", ] [[package]] -name = "time" -version = "0.1.44" +name = "wasm-streams" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" +checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb" dependencies = [ - "libc", - "wasi", - "winapi", + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", ] [[package]] -name = "time" -version = "0.3.17" +name = "wasmparser" +version = "0.118.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a561bf4617eebd33bca6434b988f39ed798e527f51a1e797d0ee4f61c0a38376" +checksum = "77f1154f1ab868e2a01d9834a805faca7bf8b50d041b4ca714d005d0dab1c50c" dependencies = [ - "itoa 1.0.1", - "serde", - "time-core", - "time-macros", + "indexmap 2.14.0", + "semver 1.0.28", ] [[package]] -name = "time-core" -version = "0.1.0" +name = "wasmparser" +version = "0.244.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags 2.11.1", + "hashbrown 0.15.2", + "indexmap 2.14.0", + "semver 1.0.28", +] [[package]] -name = "time-macros" -version = "0.2.6" +name = "wasmparser" +version = "0.251.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d967f99f534ca7e495c575c62638eebc2898a8c84c119b89e250477bc4ba16b2" +checksum = "437970b35b1a85cfde9c74b2398352d8d653f3bd8e3a3db0c063ea8f5b4b36ff" dependencies = [ - "time-core", + "bitflags 2.11.1", + "hashbrown 0.17.0", + "indexmap 2.14.0", + "semver 1.0.28", + "serde", ] [[package]] -name = "tiny-keccak" -version = "1.5.0" +name = "wasmparser" +version = "0.252.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d8a021c69bb74a44ccedb824a046447e2c84a01df9e5c20779750acb38e11b2" +checksum = "d3eb099dcadcde5be9eef55e3a337128efd4e44b4c93122487e4d2e4e1c6627c" dependencies = [ - "crunchy", + "bitflags 2.11.1", + "indexmap 2.14.0", + "semver 1.0.28", ] [[package]] -name = "tiny-keccak" -version = "2.0.2" +name = "wasmprinter" +version = "0.251.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +checksum = "8798c1a699bd25648b6708eefe94d97c6f9891febb94b42cca1f7a4b086ea64e" dependencies = [ - "crunchy", + "anyhow", + "termcolor", + "wasmparser 0.251.0", ] [[package]] -name = "tinyvec" -version = "1.3.1" +name = "wasmtime" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "848a1e1181b9f6753b5e96a092749e29b11d19ede67dfbbd6c7dc7e0f49b5338" +checksum = "c4213d2f019a5e44aa8a61d8826dd33a505bff79f749b14a8bafd67321cb9351" dependencies = [ - "tinyvec_macros", + "addr2line", + "async-trait", + "bitflags 2.11.1", + "bumpalo", + "cc", + "cfg-if", + "encoding_rs", + "futures 0.3.31", + "fxprof-processed-profile", + "gimli", + "ittapi", + "libc", + "log", + "mach2", + "memfd", + "object", + "once_cell", + "postcard", + "pulley-interpreter", + "rayon", + "rustix 1.1.4", + "semver 1.0.28", + "serde", + "serde_derive", + "serde_json", + "smallvec", + "target-lexicon", + "tempfile", + "wasm-compose", + "wasm-encoder 0.251.0", + "wasmparser 0.251.0", + "wasmtime-environ", + "wasmtime-internal-cache", + "wasmtime-internal-component-macro", + "wasmtime-internal-component-util", + "wasmtime-internal-core", + "wasmtime-internal-cranelift", + "wasmtime-internal-fiber", + "wasmtime-internal-jit-debug", + "wasmtime-internal-jit-icache-coherence", + "wasmtime-internal-unwinder", + "wasmtime-internal-versioned-export-macros", + "wat", + "windows-sys 0.61.2", + "wit-parser 0.251.0", ] [[package]] -name = "tinyvec_macros" -version = "0.1.0" +name = "wasmtime-environ" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" +checksum = "d45863de41977ec6453e859cf843d456fa3fcb45a659b66d16e794f90ec4f5b7" +dependencies = [ + "anyhow", + "cpp_demangle", + "cranelift-bforest", + "cranelift-bitset", + "cranelift-entity", + "gimli", + "hashbrown 0.17.0", + "indexmap 2.14.0", + "log", + "object", + "postcard", + "rustc-demangle", + "semver 1.0.28", + "serde", + "serde_derive", + "sha2 0.10.9", + "smallvec", + "target-lexicon", + "wasm-encoder 0.251.0", + "wasmparser 0.251.0", + "wasmprinter", + "wasmtime-internal-component-util", + "wasmtime-internal-core", +] [[package]] -name = "tokio" -version = "1.16.1" +name = "wasmtime-internal-cache" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c27a64b625de6d309e8c57716ba93021dccf1b3b5c97edd6d3dd2d2135afc0a" +checksum = "438bc7dc45fb75297d75f79a9a0ce852345d13ebc6a6863f6f688f013836a9dd" dependencies = [ - "bytes", - "libc", - "memchr", - "mio", - "num_cpus", - "once_cell", - "parking_lot 0.11.2", - "pin-project-lite", - "signal-hook-registry", - "tokio-macros", - "winapi", + "base64", + "directories-next", + "log", + "postcard", + "rustix 1.1.4", + "serde", + "serde_derive", + "sha2 0.10.9", + "toml 0.9.12+spec-1.1.0", + "wasmtime-environ", + "windows-sys 0.61.2", + "zstd", ] [[package]] -name = "tokio-io-timeout" -version = "1.1.1" +name = "wasmtime-internal-component-macro" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90c49f106be240de154571dd31fbe48acb10ba6c6dd6f6517ad603abffa42de9" +checksum = "f1e48f8d4966d62a10b6d70722bc432c1e163890be2801d3b5784589ad36ffc3" dependencies = [ - "pin-project-lite", - "tokio", + "anyhow", + "proc-macro2", + "quote", + "syn 2.0.118", + "wasmtime-internal-component-util", + "wasmtime-internal-wit-bindgen", + "wit-parser 0.251.0", ] [[package]] -name = "tokio-macros" -version = "1.7.0" +name = "wasmtime-internal-component-util" +version = "46.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819ad5abd5822a22dbf4014475cdfd1fe790707761cd732d74aaa3ba4d5ba489" + +[[package]] +name = "wasmtime-internal-core" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b557f72f448c511a979e2564e55d74e6c4432fc96ff4f6241bc6bded342643b7" +checksum = "3fc28372e36eaf8cf70faa83b5779137f7e99c8d18569a125d1580e735cc9e4d" dependencies = [ - "proc-macro2", - "quote", - "syn", + "anyhow", + "hashbrown 0.17.0", + "libm", + "serde", ] [[package]] -name = "tokio-native-tls" -version = "0.3.0" +name = "wasmtime-internal-cranelift" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" +checksum = "a433efc6e35112a5457e1dc8bc4d8d39820ac7722267e89bc04e5df641f32124" dependencies = [ - "native-tls", - "tokio", + "cfg-if", + "cranelift-codegen", + "cranelift-control", + "cranelift-entity", + "cranelift-frontend", + "cranelift-native", + "gimli", + "itertools 0.14.0", + "log", + "object", + "pulley-interpreter", + "smallvec", + "target-lexicon", + "thiserror 2.0.18", + "wasmparser 0.251.0", + "wasmtime-environ", + "wasmtime-internal-core", + "wasmtime-internal-unwinder", + "wasmtime-internal-versioned-export-macros", ] [[package]] -name = "tokio-openssl" -version = "0.6.3" +name = "wasmtime-internal-fiber" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08f9ffb7809f1b20c1b398d92acf4cc719874b3b2b2d9ea2f09b4a80350878a" +checksum = "18a1d3a39d0d210f6b8574ee96a4315e0a14c67f3a1fc3cd5372cb10d2fb4422" dependencies = [ - "futures-util", - "openssl", - "openssl-sys", - "tokio", + "cc", + "cfg-if", + "libc", + "rustix 1.1.4", + "wasmtime-environ", + "wasmtime-internal-versioned-export-macros", + "windows-sys 0.61.2", ] [[package]] -name = "tokio-postgres" -version = "0.7.2" +name = "wasmtime-internal-jit-debug" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d2b1383c7e4fb9a09e292c7c6afb7da54418d53b045f1c1fac7a911411a2b8b" +checksum = "9f667288cb4dfa68a4639ffac4d5628535dda64ebdc2b990526efb12b30ba803" dependencies = [ - "async-trait", - "byteorder", - "bytes", - "fallible-iterator", - "futures 0.3.16", - "log", - "parking_lot 0.11.2", - "percent-encoding", - "phf", - "pin-project-lite", - "postgres-protocol", - "postgres-types", - "socket2", - "tokio", - "tokio-util 0.6.7", + "cc", + "object", + "rustix 1.1.4", + "wasmtime-internal-versioned-export-macros", ] [[package]] -name = "tokio-retry" -version = "0.3.0" +name = "wasmtime-internal-jit-icache-coherence" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f57eb36ecbe0fc510036adff84824dd3c24bb781e21bfa67b69d556aa85214f" +checksum = "eba651d44ab0faad4c58106b3adb45068189fb65ef50f0c404b6d9e3bf81a357" dependencies = [ - "pin-project", - "rand", - "tokio", + "cfg-if", + "libc", + "wasmtime-internal-core", + "windows-sys 0.61.2", ] [[package]] -name = "tokio-rustls" -version = "0.23.3" +name = "wasmtime-internal-unwinder" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4151fda0cf2798550ad0b34bcfc9b9dcc2a9d2471c895c68f3a8818e54f2389e" +checksum = "2ecc52563b0558af2a7487eb710de07cc4532564b55528876129238e83118cb1" dependencies = [ - "rustls", - "tokio", - "webpki", + "cfg-if", + "cranelift-codegen", + "log", + "object", + "wasmtime-environ", ] [[package]] -name = "tokio-stream" -version = "0.1.12" +name = "wasmtime-internal-versioned-export-macros" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fb52b74f05dbf495a8fba459fdc331812b96aa086d9eb78101fa0d4569c3313" +checksum = "e747f4a074699ba1b4e4d841fb263f9b7df5bd1555181c4752bf5990d21ba676" dependencies = [ - "futures-core", - "pin-project-lite", - "tokio", - "tokio-util 0.7.1", + "proc-macro2", + "quote", + "syn 2.0.118", ] [[package]] -name = "tokio-test" -version = "0.4.2" +name = "wasmtime-internal-wit-bindgen" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53474327ae5e166530d17f2d956afcb4f8a004de581b3cae10f12006bc8163e3" +checksum = "80009f46991622814196d96fac6fc0a938f46b5cba737a8f4e21e24e5a03856f" dependencies = [ - "async-stream", - "bytes", - "futures-core", - "tokio", - "tokio-stream", + "anyhow", + "bitflags 2.11.1", + "heck 0.5.0", + "indexmap 2.14.0", + "wit-parser 0.251.0", ] [[package]] -name = "tokio-tungstenite" -version = "0.17.2" +name = "wasmtimer" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f714dd15bead90401d77e04243611caec13726c2408afd5b31901dfcdcb3b181" +checksum = "1c598d6b99ea013e35844697fc4670d08339d5cda15588f193c6beedd12f644b" dependencies = [ - "futures-util", - "log", - "tokio", - "tungstenite", + "futures 0.3.31", + "js-sys", + "parking_lot", + "pin-utils", + "slab", + "wasm-bindgen", ] [[package]] -name = "tokio-util" -version = "0.6.7" +name = "wast" +version = "252.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1caa0b0c8d94a049db56b5acf8cba99dc0623aab1b26d5b5f5e2d945846b3592" +checksum = "942a3449d6a593fccc111a6241c8df52bda168af30e40bf9580d4394d7374c65" dependencies = [ - "bytes", - "futures-core", - "futures-io", - "futures-sink", - "log", - "pin-project-lite", - "tokio", + "bumpalo", + "leb128fmt", + "memchr", + "unicode-width", + "wasm-encoder 0.252.0", ] [[package]] -name = "tokio-util" -version = "0.7.1" +name = "wat" +version = "1.252.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0edfdeb067411dba2044da6d1cb2df793dd35add7888d73c16e3381ded401764" +checksum = "c72a4ba7088f7bac94cf516e49882bdf97068904a563768cf249efc839ec42cb" dependencies = [ - "bytes", - "futures-core", - "futures-sink", - "pin-project-lite", - "tokio", - "tracing", + "wast", ] [[package]] -name = "toml" -version = "0.5.11" +name = "web-sys" +version = "0.3.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" +checksum = "a6430a72df5eb332242960fe84b3002a241163998241eb596d4f739b9757061d" dependencies = [ - "serde", + "js-sys", + "wasm-bindgen", ] [[package]] -name = "toml" -version = "0.7.1" +name = "web-time" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "772c1426ab886e7362aedf4abc9c0d1348a979517efedfc25862944d10137af0" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" dependencies = [ - "serde", - "serde_spanned", - "toml_datetime", - "toml_edit", + "js-sys", + "wasm-bindgen", ] [[package]] -name = "toml_datetime" -version = "0.6.1" +name = "webpki-root-certs" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ab8ed2edee10b50132aed5f331333428b011c99402b5a534154ed15746f9622" +checksum = "f31141ce3fc3e300ae89b78c0dd67f9708061d1d2eda54b8209346fd6be9a92c" dependencies = [ - "serde", + "rustls-pki-types", ] [[package]] -name = "toml_edit" -version = "0.19.1" +name = "webpki-roots" +version = "0.26.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90a238ee2e6ede22fb95350acc78e21dc40da00bb66c0334bde83de4ed89424e" +checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" dependencies = [ - "indexmap", - "nom8", - "serde", - "serde_spanned", - "toml_datetime", + "webpki-roots 1.0.5", ] [[package]] -name = "tonic" -version = "0.8.3" +name = "webpki-roots" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f219fad3b929bef19b1f86fbc0358d35daed8f2cac972037ac0dc10bbb8d5fb" +checksum = "12bed680863276c63889429bfd6cab3b99943659923822de1c8a39c49e4d722c" dependencies = [ - "async-stream", - "async-trait", - "axum", - "base64 0.13.1", - "bytes", - "flate2", - "futures-core", - "futures-util", - "h2", - "http", - "http-body", - "hyper", - "hyper-timeout", - "percent-encoding", - "pin-project", - "prost", - "prost-derive", - "rustls-native-certs", - "rustls-pemfile", - "tokio", - "tokio-rustls", - "tokio-stream", - "tokio-util 0.7.1", - "tower 0.4.13", - "tower-layer 0.3.2", - "tower-service 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "tracing", - "tracing-futures", + "rustls-pki-types", ] [[package]] -name = "tonic-build" -version = "0.8.4" +name = "which" +version = "4.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bf5e9b9c0f7e0a7c027dcfaba7b2c60816c7049171f679d99ee2ff65d0de8c4" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" dependencies = [ - "prettyplease", - "proc-macro2", - "prost-build", - "quote", - "syn", + "either", + "home", + "once_cell", + "rustix 0.38.34", ] [[package]] -name = "tower" -version = "0.4.12" -source = "git+https://github.com/tower-rs/tower.git#74881d531141ba0f07b7f58e2a72e3594e5a665c" +name = "whoami" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a44ab49fad634e88f55bf8f9bb3abd2f27d7204172a112c7c9987e01c1c94ea9" dependencies = [ - "futures-core", - "futures-util", - "hdrhistogram", - "indexmap", - "pin-project-lite", - "slab", - "sync_wrapper", - "tokio", - "tokio-util 0.7.1", - "tower-layer 0.3.1", - "tower-service 0.3.1 (git+https://github.com/tower-rs/tower.git)", - "tracing", + "redox_syscall 0.4.1", + "wasite", + "web-sys", ] [[package]] -name = "tower" -version = "0.4.13" +name = "widestring" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72069c3113ab32ab29e5584db3c6ec55d416895e60715417b5b883a357c3e471" + +[[package]] +name = "winapi" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" dependencies = [ - "futures-core", - "futures-util", - "indexmap", - "pin-project", - "pin-project-lite", - "rand", - "slab", - "tokio", - "tokio-util 0.7.1", - "tower-layer 0.3.2", - "tower-service 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "tracing", + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", ] [[package]] -name = "tower-http" -version = "0.3.2" +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e980386f06883cf4d0578d6c9178c81f68b45d77d00f2c2c1bc034b3439c2c56" +checksum = "4d4cc384e1e73b93bafa6fb4f1df8c41695c8a91cf9c4c64358067d15a7b6c6b" dependencies = [ - "bitflags", - "bytes", - "futures-core", - "futures-util", - "http", - "http-body", - "http-range-header", - "pin-project-lite", - "tower 0.4.13", - "tower-layer 0.3.2", - "tower-service 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "windows-sys 0.52.0", ] [[package]] -name = "tower-layer" -version = "0.3.1" -source = "git+https://github.com/tower-rs/tower.git#74881d531141ba0f07b7f58e2a72e3594e5a665c" +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] -name = "tower-layer" -version = "0.3.2" +name = "windows-core" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +dependencies = [ + "windows-targets 0.52.6", +] [[package]] -name = "tower-service" -version = "0.3.1" +name = "windows-link" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" +checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" [[package]] -name = "tower-service" -version = "0.3.1" -source = "git+https://github.com/tower-rs/tower.git#74881d531141ba0f07b7f58e2a72e3594e5a665c" +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" [[package]] -name = "tower-test" -version = "0.4.0" -source = "git+https://github.com/tower-rs/tower.git#74881d531141ba0f07b7f58e2a72e3594e5a665c" +name = "windows-registry" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3bab093bdd303a1240bb99b8aba8ea8a69ee19d34c9e2ef9594e708a4878820" dependencies = [ - "futures-util", - "pin-project-lite", - "tokio", - "tokio-test", - "tower-layer 0.3.1", - "tower-service 0.3.1 (git+https://github.com/tower-rs/tower.git)", + "windows-link 0.1.3", + "windows-result", + "windows-strings", ] [[package]] -name = "tracing" -version = "0.1.36" +name = "windows-result" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fce9567bd60a67d08a16488756721ba392f24f29006402881e43b19aac64307" +checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" dependencies = [ - "cfg-if 1.0.0", - "log", - "pin-project-lite", - "tracing-attributes", - "tracing-core", + "windows-link 0.1.3", ] [[package]] -name = "tracing-attributes" -version = "0.1.22" +name = "windows-strings" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11c75893af559bc8e10716548bdef5cb2b983f8e637db9d0e15126b61b484ee2" +checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57" dependencies = [ - "proc-macro2", - "quote", - "syn", + "windows-link 0.1.3", ] [[package]] -name = "tracing-core" -version = "0.1.29" +name = "windows-sys" +version = "0.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aeea4303076558a00714b823f9ad67d58a3bbda1df83d8827d21193156e22f7" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" dependencies = [ - "once_cell", + "windows-targets 0.42.2", ] [[package]] -name = "tracing-futures" -version = "0.2.5" +name = "windows-sys" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "pin-project", - "tracing", + "windows-targets 0.48.5", ] [[package]] -name = "try-lock" -version = "0.2.3" +name = "windows-sys" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] [[package]] -name = "tungstenite" -version = "0.17.3" +name = "windows-sys" +version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e27992fd6a8c29ee7eef28fc78349aa244134e10ad447ce3b9f0ac0ed0fa4ce0" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" dependencies = [ - "base64 0.13.1", - "byteorder", - "bytes", - "http", - "httparse", - "log", - "rand", - "sha-1 0.10.0", - "thiserror", - "url", - "utf-8", + "windows-targets 0.52.6", ] [[package]] -name = "typed-builder" -version = "0.10.0" +name = "windows-sys" +version = "0.60.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89851716b67b937e393b3daa8423e67ddfc4bbbf1654bcf05488e95e0828db0c" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" dependencies = [ - "proc-macro2", - "quote", - "syn", + "windows-targets 0.53.3", ] [[package]] -name = "typenum" -version = "1.15.0" +name = "windows-sys" +version = "0.61.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link 0.2.1", +] [[package]] -name = "uint" -version = "0.9.1" +name = "windows-targets" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6470ab50f482bde894a037a57064480a246dbfdd5960bd65a44824693f08da5f" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" dependencies = [ - "byteorder", - "crunchy", - "hex", - "static_assertions", + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", ] [[package]] -name = "unicase" -version = "2.6.0" +name = "windows-targets" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" dependencies = [ - "version_check", + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", ] [[package]] -name = "unicode-bidi" -version = "0.3.5" +name = "windows-targets" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eeb8be209bb1c96b7c177c7420d26e04eccacb0eeae6b980e35fcb74678107e0" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "matches", + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", ] [[package]] -name = "unicode-ident" -version = "1.0.1" +name = "windows-targets" +version = "0.53.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bd2fe26506023ed7b5e1e315add59d6f584c621d037f9368fea9cfb988f368c" +checksum = "d5fe6031c4041849d7c496a8ded650796e7b6ecc19df1a431c1a363342e5dc91" +dependencies = [ + "windows-link 0.1.3", + "windows_aarch64_gnullvm 0.53.0", + "windows_aarch64_msvc 0.53.0", + "windows_i686_gnu 0.53.0", + "windows_i686_gnullvm 0.53.0", + "windows_i686_msvc 0.53.0", + "windows_x86_64_gnu 0.53.0", + "windows_x86_64_gnullvm 0.53.0", + "windows_x86_64_msvc 0.53.0", +] [[package]] -name = "unicode-normalization" -version = "0.1.19" +name = "windows_aarch64_gnullvm" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d54590932941a9e9266f0832deed84ebe1bf2e4c9e4a3554d393d18f5e854bf9" -dependencies = [ - "tinyvec", -] +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" [[package]] -name = "unicode-segmentation" -version = "1.8.0" +name = "windows_aarch64_gnullvm" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8895849a949e7845e06bd6dc1aa51731a103c42707010a5b591c0038fb73385b" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] -name = "unicode-width" -version = "0.1.8" +name = "windows_aarch64_gnullvm" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" [[package]] -name = "unicode-xid" -version = "0.2.2" +name = "windows_aarch64_gnullvm" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" +checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" [[package]] -name = "unreachable" -version = "1.0.0" +name = "windows_aarch64_msvc" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "382810877fe448991dfc7f0dd6e3ae5d58088fd0ea5e35189655f84e6814fa56" -dependencies = [ - "void", -] +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" [[package]] -name = "unsigned-varint" -version = "0.7.1" +name = "windows_aarch64_msvc" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d86a8dc7f45e4c1b0d30e43038c38f274e77af056aa5f74b93c2cf9eb3c1c836" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] -name = "untrusted" -version = "0.7.1" +name = "windows_aarch64_msvc" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" [[package]] -name = "url" -version = "2.3.1" +name = "windows_aarch64_msvc" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643" -dependencies = [ - "form_urlencoded", - "idna 0.3.0", - "percent-encoding", -] +checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" [[package]] -name = "utf-8" -version = "0.7.6" +name = "windows_i686_gnu" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" [[package]] -name = "uuid" -version = "1.3.0" +name = "windows_i686_gnu" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1674845326ee10d37ca60470760d4288a6f80f304007d92e5c53bab78c9cfd79" -dependencies = [ - "getrandom", -] +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] -name = "vcpkg" -version = "0.2.15" +name = "windows_i686_gnu" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" [[package]] -name = "version_check" -version = "0.9.3" +name = "windows_i686_gnu" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe" +checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3" [[package]] -name = "void" -version = "1.0.2" +name = "windows_i686_gnullvm" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" [[package]] -name = "walkdir" -version = "2.3.2" +name = "windows_i686_gnullvm" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56" -dependencies = [ - "same-file", - "winapi", - "winapi-util", -] +checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" + +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" [[package]] -name = "want" -version = "0.3.0" +name = "windows_x86_64_gnullvm" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" -dependencies = [ - "log", - "try-lock", -] +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" [[package]] -name = "wasi" -version = "0.10.0+wasi-snapshot-preview1" +name = "windows_x86_64_gnullvm" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] -name = "wasm-bindgen" -version = "0.2.82" +name = "windows_x86_64_gnullvm" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc7652e3f6c4706c8d9cd54832c4a4ccb9b5336e2c3bd154d5cccfbf1c1f5f7d" -dependencies = [ - "cfg-if 1.0.0", - "serde", - "serde_json", - "wasm-bindgen-macro", -] +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" [[package]] -name = "wasm-bindgen-backend" -version = "0.2.82" +name = "windows_x86_64_gnullvm" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "662cd44805586bd52971b9586b1df85cdbbd9112e4ef4d8f41559c334dc6ac3f" -dependencies = [ - "bumpalo", - "log", - "once_cell", - "proc-macro2", - "quote", - "syn", - "wasm-bindgen-shared", -] +checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" [[package]] -name = "wasm-bindgen-futures" -version = "0.4.25" +name = "windows_x86_64_msvc" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16646b21c3add8e13fdb8f20172f8a28c3dbf62f45406bcff0233188226cfe0c" -dependencies = [ - "cfg-if 1.0.0", - "js-sys", - "wasm-bindgen", - "web-sys", -] +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" [[package]] -name = "wasm-bindgen-macro" -version = "0.2.82" +name = "windows_x86_64_msvc" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b260f13d3012071dfb1512849c033b1925038373aea48ced3012c09df952c602" -dependencies = [ - "quote", - "wasm-bindgen-macro-support", -] +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] -name = "wasm-bindgen-macro-support" -version = "0.2.82" +name = "windows_x86_64_msvc" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5be8e654bdd9b79216c2929ab90721aa82faf65c48cdf08bdc4e7f51357b80da" -dependencies = [ - "proc-macro2", - "quote", - "syn", - "wasm-bindgen-backend", - "wasm-bindgen-shared", -] +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] -name = "wasm-bindgen-shared" -version = "0.2.82" +name = "windows_x86_64_msvc" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6598dd0bd3c7d51095ff6531a5b23e02acdc81804e30d8f07afb77b7215a140a" +checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" [[package]] -name = "wasm-instrument" -version = "0.2.0" +name = "winnow" +version = "0.5.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bca81f5279342b38b17d9acbf007a46ddeb73144e2bd5f0a21bfa9fc5d4ab3e" +checksum = "f593a95398737aeed53e489c785df13f3618e41dbcd6718c6addbf1395aa6876" dependencies = [ - "parity-wasm", + "memchr", ] [[package]] -name = "wasmparser" -version = "0.78.2" +name = "winnow" +version = "0.7.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52144d4c78e5cf8b055ceab8e5fa22814ce4315d6002ad32cfd914f37c12fd65" +checksum = "21a0236b59786fed61e2a80582dd500fe61f18b5dca67a4a067d0bc9039339cf" [[package]] -name = "wasmtime" -version = "0.27.0" +name = "winnow" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b310b9d20fcf59385761d1ade7a3ef06aecc380e3d3172035b919eaf7465d9f7" +checksum = "a90e88e4667264a994d34e6d1ab2d26d398dcdca8b7f52bec8668957517fc7d8" dependencies = [ - "anyhow", - "backtrace", - "bincode", - "cfg-if 1.0.0", - "cpp_demangle", - "indexmap", - "lazy_static", - "libc", - "log", - "paste", - "psm", - "region", - "rustc-demangle", - "serde", - "smallvec", - "target-lexicon", - "wasmparser", - "wasmtime-cache", - "wasmtime-environ", - "wasmtime-fiber", - "wasmtime-jit", - "wasmtime-profiling", - "wasmtime-runtime", - "wat", - "winapi", + "memchr", ] [[package]] -name = "wasmtime-cache" -version = "0.27.0" +name = "wiremock" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d14d500d5c3dc5f5c097158feee123d64b3097f0d836a2a27dff9c761c73c843" +checksum = "08db1edfb05d9b3c1542e521aea074442088292f00b5f28e435c714a98f85031" dependencies = [ - "anyhow", - "base64 0.13.1", - "bincode", - "directories-next", - "errno", - "file-per-thread-logger", - "libc", + "assert-json-diff", + "base64", + "deadpool 0.12.3", + "futures 0.3.31", + "http 1.4.2", + "http-body-util", + "hyper", + "hyper-util", "log", + "once_cell", + "regex", "serde", - "sha2 0.9.5", - "toml 0.5.11", - "winapi", - "zstd", + "serde_json", + "tokio", + "url", ] [[package]] -name = "wasmtime-cranelift" -version = "0.27.0" +name = "wit-bindgen" +version = "0.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c525b39f062eada7db3c1298287b96dcb6e472b9f6b22501300b28d9fa7582f6" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" dependencies = [ - "cranelift-codegen", - "cranelift-entity", - "cranelift-frontend", - "cranelift-wasm", - "target-lexicon", - "wasmparser", - "wasmtime-environ", + "wit-bindgen-rust-macro", ] [[package]] -name = "wasmtime-debug" -version = "0.27.0" +name = "wit-bindgen-core" +version = "0.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5d2a763e7a6fc734218e0e463196762a4f409c483063d81e0e85f96343b2e0a" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" dependencies = [ "anyhow", - "gimli 0.24.0", - "more-asserts", - "object 0.24.0", - "target-lexicon", - "thiserror", - "wasmparser", - "wasmtime-environ", + "heck 0.5.0", + "wit-parser 0.244.0", ] [[package]] -name = "wasmtime-environ" -version = "0.27.0" +name = "wit-bindgen-rt" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f64d0c2d881c31b0d65c1f2695e022d71eb60b9fbdd336aacca28208b58eac90" +checksum = "3268f3d866458b787f390cf61f4bbb563b922d091359f9608842999eaee3943c" dependencies = [ - "cfg-if 1.0.0", - "cranelift-codegen", - "cranelift-entity", - "cranelift-wasm", - "gimli 0.24.0", - "indexmap", - "log", - "more-asserts", - "serde", - "thiserror", - "wasmparser", + "bitflags 2.11.1", ] [[package]] -name = "wasmtime-fiber" -version = "0.27.0" +name = "wit-bindgen-rust" +version = "0.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a089d44cd7e2465d41a53b840a5b4fca1bf6d1ecfebc970eac9592b34ea5f0b3" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" dependencies = [ - "cc", - "libc", - "winapi", + "anyhow", + "heck 0.5.0", + "indexmap 2.14.0", + "prettyplease", + "syn 2.0.118", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", ] [[package]] -name = "wasmtime-jit" -version = "0.27.0" +name = "wit-bindgen-rust-macro" +version = "0.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d4539ea734422b7c868107e2187d7746d8affbcaa71916d72639f53757ad707" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" dependencies = [ - "addr2line 0.15.2", "anyhow", - "cfg-if 1.0.0", - "cranelift-codegen", - "cranelift-entity", - "cranelift-frontend", - "cranelift-native", - "cranelift-wasm", - "gimli 0.24.0", - "log", - "more-asserts", - "object 0.24.0", - "rayon", - "region", - "serde", - "target-lexicon", - "thiserror", - "wasmparser", - "wasmtime-cranelift", - "wasmtime-debug", - "wasmtime-environ", - "wasmtime-obj", - "wasmtime-profiling", - "wasmtime-runtime", - "winapi", + "prettyplease", + "proc-macro2", + "quote", + "syn 2.0.118", + "wit-bindgen-core", + "wit-bindgen-rust", ] [[package]] -name = "wasmtime-obj" -version = "0.27.0" +name = "wit-component" +version = "0.244.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e1a8ff85246d091828e2225af521a6208ed28c997bb5c39eb697366dc2e2f2b" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" dependencies = [ "anyhow", - "more-asserts", - "object 0.24.0", - "target-lexicon", - "wasmtime-debug", - "wasmtime-environ", + "bitflags 2.11.1", + "indexmap 2.14.0", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder 0.244.0", + "wasm-metadata", + "wasmparser 0.244.0", + "wit-parser 0.244.0", ] [[package]] -name = "wasmtime-profiling" -version = "0.27.0" +name = "wit-parser" +version = "0.244.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e24364d522dcd67c897c8fffc42e5bdfc57207bbb6d7eeade0da9d4a7d70105b" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" dependencies = [ "anyhow", - "cfg-if 1.0.0", - "gimli 0.24.0", - "lazy_static", - "libc", - "object 0.24.0", - "scroll", + "id-arena", + "indexmap 2.14.0", + "log", + "semver 1.0.28", "serde", - "target-lexicon", - "wasmtime-environ", - "wasmtime-runtime", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser 0.244.0", ] [[package]] -name = "wasmtime-runtime" -version = "0.27.0" +name = "wit-parser" +version = "0.251.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c51e57976e8a19a18a18e002c6eb12e5769554204238e47ff155fda1809ef0f7" +checksum = "e960732e824fab95099971a09e638979347c94ca48568d3c854c945729196947" dependencies = [ "anyhow", - "backtrace", - "cc", - "cfg-if 1.0.0", - "indexmap", - "lazy_static", - "libc", + "hashbrown 0.17.0", + "id-arena", + "indexmap 2.14.0", "log", - "mach", - "memoffset", - "more-asserts", - "rand", - "region", - "thiserror", - "wasmtime-environ", - "wasmtime-fiber", - "winapi", + "semver 1.0.28", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser 0.251.0", ] [[package]] -name = "wast" -version = "37.0.0" +name = "write16" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bc7b9a76845047ded00e031754ff410afee0d50fbdf62b55bdeecd245063d68" -dependencies = [ - "leb128", -] +checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" [[package]] -name = "wat" -version = "1.0.39" +name = "writeable" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ab2cc8d9a69d1ab28a41d9149bb06bb927aba8fc9d56625f8b597a564c83f50" -dependencies = [ - "wast", -] +checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" [[package]] -name = "web-sys" -version = "0.3.52" +name = "ws_stream_wasm" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01c70a82d842c9979078c772d4a1344685045f1a5628f677c2b2eab4dd7d2696" +checksum = "6c173014acad22e83f16403ee360115b38846fe754e735c5d9d3803fe70c6abc" dependencies = [ + "async_io_stream", + "futures 0.3.31", "js-sys", + "log", + "pharos", + "rustc_version 0.4.0", + "send_wrapper", + "thiserror 2.0.18", "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", ] [[package]] -name = "web3" -version = "0.19.0-graph" -source = "git+https://github.com/graphprotocol/rust-web3?branch=graph-patches-onto-0.18#7f8eb6dfcc13a4186f9b42f91de950646bc4a833" +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" dependencies = [ - "arrayvec 0.7.2", - "base64 0.13.1", - "bytes", - "derive_more", - "ethabi", - "ethereum-types", - "futures 0.3.16", - "futures-timer", - "headers", - "hex", - "idna 0.2.3", - "jsonrpc-core", - "log", - "once_cell", - "parking_lot 0.12.1", - "pin-project", - "reqwest", - "rlp", - "secp256k1", - "serde", - "serde_json", - "soketto", - "tiny-keccak 2.0.2", - "tokio", - "tokio-stream", - "tokio-util 0.6.7", - "url", - "web3-async-native-tls", + "tap", ] [[package]] -name = "web3-async-native-tls" -version = "0.4.0" +name = "xxhash-rust" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f6d8d1636b2627fe63518d5a9b38a569405d9c9bc665c43c9c341de57227ebb" -dependencies = [ - "native-tls", - "thiserror", - "tokio", - "url", -] +checksum = "63658493314859b4dfdf3fb8c1defd61587839def09582db50b8a4e93afca6bb" [[package]] -name = "webpki" -version = "0.22.0" +name = "yansi" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd" -dependencies = [ - "ring", - "untrusted", -] +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" [[package]] -name = "which" -version = "4.2.2" +name = "yoke" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea187a8ef279bc014ec368c27a920da2024d2a711109bfbe3440585d5cf27ad9" +checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40" dependencies = [ - "either", - "lazy_static", - "libc", + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", ] [[package]] -name = "winapi" -version = "0.3.9" +name = "yoke-derive" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", + "proc-macro2", + "quote", + "syn 2.0.118", + "synstructure", ] [[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - -[[package]] -name = "winapi-util" -version = "0.1.5" +name = "zerocopy" +version = "0.8.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +checksum = "668f5168d10b9ee831de31933dc111a459c97ec93225beb307aed970d1372dfd" dependencies = [ - "winapi", + "zerocopy-derive", ] [[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - -[[package]] -name = "windows-sys" -version = "0.32.0" +name = "zerocopy-derive" +version = "0.8.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3df6e476185f92a12c072be4a189a0210dcdcf512a1891d6dff9edb874deadc6" +checksum = "2c7962b26b0a8685668b671ee4b54d007a67d4eaf05fda79ac0ecf41e32270f1" dependencies = [ - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_msvc", + "proc-macro2", + "quote", + "syn 2.0.118", ] [[package]] -name = "windows_aarch64_msvc" -version = "0.32.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8e92753b1c443191654ec532f14c199742964a061be25d77d7a96f09db20bf5" - -[[package]] -name = "windows_i686_gnu" -version = "0.32.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a711c68811799e017b6038e0922cb27a5e2f43a2ddb609fe0b6f3eeda9de615" - -[[package]] -name = "windows_i686_msvc" -version = "0.32.0" +name = "zerofrom" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "146c11bb1a02615db74680b32a68e2d61f553cc24c4eb5b4ca10311740e44172" +checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +dependencies = [ + "zerofrom-derive", +] [[package]] -name = "windows_x86_64_gnu" -version = "0.32.0" +name = "zerofrom-derive" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c912b12f7454c6620635bbff3450962753834be2a594819bd5e945af18ec64bc" +checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", + "synstructure", +] [[package]] -name = "windows_x86_64_msvc" -version = "0.32.0" +name = "zeroize" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "504a2476202769977a040c6364301a3f65d0cc9e3fb08600b2bda150a0488316" +checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" +dependencies = [ + "zeroize_derive", +] [[package]] -name = "winreg" -version = "0.7.0" +name = "zeroize_derive" +version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69" +checksum = "85a5b4158499876c763cb03bc4e49185d3cccbabb15b33c627f7884f43db852e" dependencies = [ - "winapi", + "proc-macro2", + "quote", + "syn 2.0.118", ] [[package]] -name = "wyz" -version = "0.5.0" +name = "zerovec" +version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30b31594f29d27036c383b53b59ed3476874d518f0efb151b27a4c275141390e" +checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" dependencies = [ - "tap", + "yoke", + "zerofrom", + "zerovec-derive", ] [[package]] -name = "xxhash-rust" -version = "0.8.5" +name = "zerovec-derive" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "074914ea4eec286eb8d1fd745768504f420a1f7b7919185682a4a267bed7d2e7" +checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] [[package]] -name = "yaml-rust" -version = "0.4.5" +name = "zlib-rs" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85" -dependencies = [ - "linked-hash-map", -] +checksum = "c745c48e1007337ed136dc99df34128b9faa6ed542d80a1c673cf55a6d7236c8" [[package]] -name = "yansi" -version = "0.5.1" +name = "zmij" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" [[package]] name = "zstd" -version = "0.6.1+zstd.1.4.9" +version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5de55e77f798f205d8561b8fe2ef57abfb6e0ff2abe7fd3c089e119cdb5631a3" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" dependencies = [ "zstd-safe", ] [[package]] name = "zstd-safe" -version = "3.0.1+zstd.1.4.9" +version = "7.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1387cabcd938127b30ce78c4bf00b30387dddf704e3f0881dbc4ff62b5566f8c" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" dependencies = [ - "libc", "zstd-sys", ] [[package]] name = "zstd-sys" -version = "1.4.20+zstd.1.4.9" +version = "2.0.15+zstd.1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebd5b733d7cf2d9447e2c3e76a5589b4f5e5ae065c22a2bc0b023cbc331b6c8e" +checksum = "eb81183ddd97d0c74cedf1d50d85c8d08c1b8b68ee863bdee9e706eedba1a237" dependencies = [ "cc", - "libc", + "pkg-config", ] diff --git a/Cargo.toml b/Cargo.toml index 1808e6f3f5b..a956a514a95 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,20 +1,33 @@ [workspace] +resolver = "2" members = [ "core", - "chain/*", + "core/graphman", + "core/graphman_store", + "chain/common", + "chain/ethereum", + "chain/near", + "gnd", "graphql", - "mock", "node", - "runtime/*", - "server/*", - "store/*", + "runtime/derive", + "runtime/test", + "runtime/wasm", + "server/graphman", + "server/http", + "server/index-node", + "server/json-rpc", + "server/metrics", + "store/postgres", + "store/test-store", "graph", "tests", + "graph/derive", ] [workspace.package] -version = "0.30.0" -edition = "2021" +version = "0.45.0" +edition = "2024" authors = ["The Graph core developers & contributors"] readme = "README.md" homepage = "https://thegraph.com" @@ -22,14 +35,107 @@ repository = "https://github.com/graphprotocol/graph-node" license = "MIT OR Apache-2.0" [workspace.dependencies] -prost = "0.11.6" -prost-types = "0.11.6" -tonic = { version = "0.8.3", features = ["tls-roots", "gzip"] } -tonic-build = { version = "0.8.4", features = ["prost"] } +alloy = { version = "2.0.5", features = ["dyn-abi", "json-abi", "full", "arbitrary", "json-rpc", "serde"] } +alloy-rpc-types = "2.0.0" +# rustls is pulled in transitively by alloy (aws_lc_rs) and object_store via +# reqwest (ring). With both providers linked, rustls 0.23 requires an explicit +# default provider to be installed before any TLS use. We install aws_lc_rs +# explicitly in each binary/test entry point. +rustls = { version = "0.23", default-features = false, features = ["aws_lc_rs"] } +anyhow = "1.0" +async-graphql = { version = "7.2.1", features = ["chrono"] } +async-graphql-axum = "7.2.1" +async-trait = "0.1.74" +axum = "0.8.9" +chrono = "0.4.44" +bs58 = "0.5.1" +clap = { version = "4.5.4", features = ["derive", "env", "wrap_help"] } +clap_complete = "4" +console = "0.16" +derive_more = { version = "2.1.1", default-features = false } +diesel = { version = "2.2.7", features = [ + "postgres", + "serde_json", + "numeric", + "r2d2", + "chrono", + "i-implement-a-third-party-backend-and-opt-into-breaking-changes", +] } +diesel-async = { version = "0.9.0", features = ["deadpool", "async-connection-wrapper", "tokio", "postgres"] } +diesel-derive-enum = { version = "2.1.0", features = ["postgres"] } +diesel-dynamic-schema = { version = "0.2.3", features = ["postgres"] } +diesel_derives = "2.3.9" +diesel_migrations = "2.3.2" +env_logger = "0.11.10" +envconfig = "0.11.1" +git-testament = "0.2" +graph = { path = "./graph" } +graph-core = { path = "./core" } +graph-store-postgres = { path = "./store/postgres" } +graphman-server = { path = "./server/graphman" } +graphman = { path = "./core/graphman" } +graphman-store = { path = "./core/graphman_store" } +graphql-tools = "0.5.1" +indicatif = "0.18" +Inflector = "0.11.3" +itertools = "0.15.0" +lazy_static = "1.5.0" +prost = "0.14" +prost-types = "0.14" +redis = { version = "1.2.2", features = [ + "aio", + "connection-manager", + "tokio-comp", +] } +regex = "1.5.4" +reqwest = "0.12.23" +semver = { version = "1", features = ["serde"] } +serde = { version = "1.0.126", features = ["rc"] } +serde_derive = "1.0.125" +serde_json = { version = "1.0", features = ["arbitrary_precision"] } +serde_regex = "1.1.0" +serde_yaml = "0.9.21" +slog = { version = "2.8.2", features = ["release_max_level_trace", "max_level_trace"] } +slog-async = "2.5.0" +slog-term = "2.7.0" +sqlparser = { version = "0.62.0", features = ["visitor"] } +strum = { version = "0.28", features = ["derive"] } +syn = { version = "2.0.117", features = ["full"] } +test-store = { path = "./store/test-store" } +thiserror = "2.0.18" +deadpool = { version = "0.13", features = ["rt_tokio_1", "managed"] } +tokio = { version = "1.50.0", features = ["full"] } +tokio-stream = { version = "0.1.18", features = ["sync"] } +tokio-retry = "0.3.0" -# Incremental compilation on Rust 1.58 causes an ICE on build. As soon as graph node builds again, these can be removed. -[profile.test] -incremental = false +tonic = { version = "0.14", features = ["tls-native-roots", "gzip"] } +tonic-prost = "0.14" +tonic-prost-build = "0.14" +tower-http = { version = "0.7.0", features = ["cors"] } +tower = { version = "0.5.1", features = ["full"] } +wasmparser = "0.118.1" +wasmtime = { version = "46.0.0", features = ["async"] } +rand = { version = "0.9.2", features = ["os_rng"] } +prometheus = "0.14.0" +url = "2.5.8" -[profile.dev] -incremental = false +# Dependencies related to Amp subgraphs +ahash = "0.8.11" +arrow = { version = "=59.0.0" } +arrow-flight = { version = "=59.0.0", features = ["flight-sql-experimental"] } +parquet = { version = "=59.0.0" } +futures = "0.3.31" +half = "2.7.1" +indoc = "2.0.7" +lazy-regex = "3.6.0" +parking_lot = "0.12.4" +sqlparser-latest = { version = "0.57.0", package = "sqlparser", features = ["visitor"] } +tokio-util = "0.7.15" + +[workspace.lints.clippy] +too_many_arguments = "allow" +type_complexity = "allow" + +[profile.release] +opt-level = 's' +strip = "debuginfo" diff --git a/FUNDING.json b/FUNDING.json new file mode 100644 index 00000000000..273d2cfb684 --- /dev/null +++ b/FUNDING.json @@ -0,0 +1,7 @@ +{ + "drips": { + "ethereum": { + "ownedBy": "0x7630586acda59C53e6b1421B7e097512B74C5236" + } + } +} diff --git a/NEWS.md b/NEWS.md index bec220e7112..8068ecbc9fa 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,6 +1,772 @@ # NEWS -## Unreleased +## v0.45.0 + +``` +$ docker pull graphprotocol/graph-node:v0.45.0 +``` + +### Breaking Changes + +- **`GRAPH_GETH_ETH_CALL_ERRORS` renamed to `GRAPH_RPC_ETH_CALL_ERRORS`.** The env var (and internal config field) controlling which `eth_call` error messages are treated as reverts is renamed for provider-neutral naming. Update it before upgrading, or the setting is silently ignored and defaults apply. ([#6487](https://github.com/graphprotocol/graph-node/pull/6487)) + +### What's New + +- **Postponed attribute index creation re-enabled, now built in the background.** `GRAPH_POSTPONE_ATTRIBUTE_INDEX_CREATION` (introduced in v0.44.0 but inert — hardcoded off) now actually defers non-critical attribute indexes until a new deployment nears chain head (`GRAPH_POSTPONE_INDEXES_CREATION_THRESHOLD`, default 10000 blocks from head). Index creation now runs in a background task instead of blocking block processing, and interrupted `CREATE INDEX CONCURRENTLY` runs are detected and rebuilt on retry instead of being silently skipped. ([#6608](https://github.com/graphprotocol/graph-node/pull/6608)) +- **`ethereum.decodeParams` host function.** Decodes calldata/event data whose top-level type is a dynamic tuple (e.g. Gnosis Safe's `execTransaction`), which `ethereum.decode` cannot handle since it expects a single ABI value with a leading offset word. Gated behind apiVersion 0.0.10; `ethereum.decode` itself is unchanged. ([#6649](https://github.com/graphprotocol/graph-node/pull/6649)) + +### Bug Fixes + +- Fixed `EthereumLogFilter` merging per-handler `receipt` flags with overwrite instead of OR semantics, so a handler declaring `receipt: true` could silently run with `event.receipt == null` if another handler on the same filter key declared `receipt: false`. ([#6558](https://github.com/graphprotocol/graph-node/pull/6558)) +- Fixed grafts silently succeeding when the graft block was below the base deployment's earliest available block, leaving the destination missing pre-graft entity versions and producing `unexpected null in handler` errors on every event; `Graft::validate` now rejects these upfront. ([#6610](https://github.com/graphprotocol/graph-node/pull/6610)) +- Fixed `_logs` queries returning an empty result (indistinguishable from a subgraph producing no logs) when no log store is configured; now returns a `NotSupported` error. ([#6637](https://github.com/graphprotocol/graph-node/pull/6637)) +- Fixed startup panicking when a shard is configured with `pool_size = 0`; such shards' chains are now skipped instead of erroring. ([#6648](https://github.com/graphprotocol/graph-node/pull/6648)) +- Fixed the Firehose extended-blocks check matching on `optimism-mainnet` instead of the `optimism` chain ID used by Firehose endpoints, so Optimism's extended block data went undetected. ([#6662](https://github.com/graphprotocol/graph-node/pull/6662)) +- Fixed fulltext (`tsvector`) search columns being silently dropped when copying or grafting a subgraph, leaving the destination's fulltext index empty. ([#6687](https://github.com/graphprotocol/graph-node/pull/6687)) +- Fixed Reth `StackUnderflow`/`StackOverflow`/`OpcodeNotFound` EVM halts being misclassified as non-deterministic — Reth surfaces them via its `Debug` format (e.g. `EVM error: StackUnderflow`), which didn't match graph-node's space-separated deterministic-error patterns, so subgraphs indexing via Reth-backed RPC providers stalled and retried instead of recording a deterministic revert. ([#6645](https://github.com/graphprotocol/graph-node/pull/6645)) + +### Graphman + +- Fixed `graphman dump` panicking on tables with more than ~2 GiB in a single string/binary column; batches are now split into byte-bounded slices before conversion to Arrow. ([#6646](https://github.com/graphprotocol/graph-node/pull/6646)) + +### gnd (Graph Node Dev) + +- Fixed `gnd add` producing a subgraph that failed `codegen`/`build` because it never wrote new event entities to `schema.graphql`; unified `add` and `init` onto one scaffolding path, which also fixes event-name collisions across data sources, overloaded-event disambiguation, small-integer type mapping, and reserved-word parameter names. ([#6660](https://github.com/graphprotocol/graph-node/pull/6660)) + +### Upgrade Notes + +- If you set `GRAPH_GETH_ETH_CALL_ERRORS`, rename it to `GRAPH_RPC_ETH_CALL_ERRORS` before upgrading. ([#6487](https://github.com/graphprotocol/graph-node/pull/6487)) +- `GRAPH_POSTPONE_ATTRIBUTE_INDEX_CREATION` now actually takes effect (it was a no-op in v0.44.0). If you already set it expecting no behavior change, be aware new deployments will now defer some attribute indexes until near chain head. ([#6608](https://github.com/graphprotocol/graph-node/pull/6608)) +- If a subgraph was grafted from a base block below the base deployment's earliest available block before this release, it may be missing pre-graft entity versions; re-create the graft from a valid earliest block. ([#6610](https://github.com/graphprotocol/graph-node/pull/6610)) + +### Contributors + +Thanks to all contributors for this release: @cargopete, @dimitrovmaksim, @fordN, @incrypto32, @lutter, @YaroShkvorets + +**Full Changelog**: https://github.com/graphprotocol/graph-node/compare/v0.44.0...v0.45.0 + +## v0.44.0 + +``` +$ docker pull graphprotocol/graph-node:v0.44.0 +``` + +### Critical Fix + +- **`EntityCache::load_related` returned wrong derived-collection membership on same-block parent reassignment.** Membership was decided against intermediate cache layers rather than the entity's final state, so entities reassigned to a new parent within the same block could be left in the old collection, omitted from the new one, or wrongly retained after a cross-handler revert. See Upgrade Notes. ([#6548](https://github.com/graphprotocol/graph-node/pull/6548)) + +### Breaking Changes + +- **Elasticsearch CLI flags removed.** `--elasticsearch-url`, `--elasticsearch-user`, `--elasticsearch-password` (and the matching `ELASTICSEARCH_*` env vars) are gone. Configure Elasticsearch under the new `[log_store]` section in `graph-node.toml`. See Upgrade Notes. ([#6278](https://github.com/graphprotocol/graph-node/pull/6278)) + +### What's New + +- **Query subgraph logs over GraphQL.** A new `_logs` field exposes subgraph-emitted logs (mapping `log.*` calls, runtime, system) with filters for level, timestamp range, and text search, plus pagination and `orderDirection`. The old Elasticsearch sink for subgraph logs is replaced by a unified `LogStore` abstraction with three backends configured under a new `[log_store]` section in `graph-node.toml`: File (JSON Lines), Loki, or Elasticsearch. Omit `[log_store]` to disable; logs still go to stdout/stderr. ([#6278](https://github.com/graphprotocol/graph-node/pull/6278)) + + + + +### Improvements + +- Revert entity versions during copy instead of in a separate post-copy pass, speeding up subgraph copy and graft. ([#6472](https://github.com/graphprotocol/graph-node/pull/6472)) + +### Bug Fixes + +- Fixed `trace_filter` deserialization failing on providers that omit `result.output` (notably Sonic), which caused repeated trace ingestion retries. Fixed upstream in alloy 2.0.5 ([alloy#3931](https://github.com/alloy-rs/alloy/pull/3931)) and picked up via the bump in this release. ([#6576](https://github.com/graphprotocol/graph-node/pull/6576)) +- Fixed runner panics on out-of-range `BigInt` values: `to_signed_u256` replaced by fallible `to_i256`, and `to_unsigned_u256` now errors on values `>= 2^256` instead of panicking. ([#6560](https://github.com/graphprotocol/graph-node/pull/6560)) +- Fixed `graphman chain change-shard` failing with a duplicate-key error on revert to the original shard. Backups now use unique names and revert reuses the existing `-old`. ([#6199](https://github.com/graphprotocol/graph-node/pull/6199)) + +### Upgrade Notes + +- v0.44.0 changes `EntityCache::load_related` results for same-block parent-reassignment edge cases ([#6548](https://github.com/graphprotocol/graph-node/pull/6548)). Subgraphs that read derived fields within handlers may diverge in POI from v0.43.0 on blocks exhibiting this pattern; resync from before the affected blocks if affected. The v0.44.0 result is canonical. +- If you set `--elasticsearch-*` flags or `ELASTICSEARCH_*` env vars, migrate to a `[log_store]` section in `graph-node.toml` before upgrading. See `docs/log-store.md`. ([#6278](https://github.com/graphprotocol/graph-node/pull/6278)) + +### Contributors + +Thanks to all contributors for this release: @erayack, @fordN, @incrypto32, @lutter + +**Full Changelog**: https://github.com/graphprotocol/graph-node/compare/v0.43.0...v0.44.0 + +## v0.43.0 + +### What's New + +- **`skipDuplicates` for immutable entities.** A new `skipDuplicates` parameter on the `@entity` directive (`@entity(immutable: true, skipDuplicates: true)`) silently skips duplicate inserts instead of failing the subgraph. This unblocks subgraph composition on Amp-powered subgraphs where SQL queries can produce the same entities across block ranges. ([#6458](https://github.com/graphprotocol/graph-node/pull/6458)) +- **Per-chain RPC settings via TOML config.** Chain-specific tuning parameters (`json_rpc_timeout`, `request_retries`, `max_block_range_size`, `polling_interval`, `block_batch_size`, etc.) can now be set per chain in `config.toml` instead of relying on global environment variables. Fully backwards compatible — env vars remain the fallback. ([#6459](https://github.com/graphprotocol/graph-node/pull/6459)) +- **RPC provider failover for block ingestor.** When the current RPC provider becomes unreachable during block ingestion polling, graph-node now automatically switches to a healthy alternative provider. ([#6430](https://github.com/graphprotocol/graph-node/pull/6430)) +- Warn on startup when running a debug build, with `[DEBUG-BUILD]` prefix on all log lines. ([#6488](https://github.com/graphprotocol/graph-node/pull/6488)) + +### Improvements + +- **Call cache eviction rewrite.** Replaced the old per-contract iteration approach with ctid-based join deletes against the `call_meta` table, significantly faster on large `call_cache` tables. The old `--ttl-max-contracts` flag has been replaced by `--max-contracts`, which computes an effective TTL cutoff instead of limiting per-invocation iteration. Eviction now also returns stats on the number of contracts and entries removed. ([#6476](https://github.com/graphprotocol/graph-node/pull/6476)) ([#6477](https://github.com/graphprotocol/graph-node/pull/6477)) +- Reduced unnecessary `eth_getBlockByHash` and `eth_getBlockByNumber` RPC calls by checking the block cache first in `block_pointer_from_number` and `fetch_full_block_with_rpc`. ([#6491](https://github.com/graphprotocol/graph-node/pull/6491), partially reverted by [#6537](https://github.com/graphprotocol/graph-node/pull/6537)) +- Header-only `ChainStore` query methods (`ancestor_block_ptr`, `block_parent_ptr`) skip deserializing the full block `data` JSONB column when only hash/number/parent are needed. ([#6456](https://github.com/graphprotocol/graph-node/pull/6456)) +- Batch checking for update attempts on immutable entities, reducing per-entity overhead. +- Node name is now used as the PostgreSQL `application_name` when `PGAPPNAME` is not set, making it easier to identify graph-node connections in `pg_stat_activity`. + +### Bug Fixes + +- Fixed dropped block trigger when `once` and `polling` filters match the same block — only one trigger type was firing. ([#6530](https://github.com/graphprotocol/graph-node/pull/6530)) +- Fixed block stream ignoring configured `endBlock` in two cases: the block-skip optimization bypassing the `max_end_block` check, and `max_end_block` being set to `None` when multiple data sources share the same `endBlock`. ([#6474](https://github.com/graphprotocol/graph-node/pull/6474)) +- Fixed GraphQL introspection not returning `isDeprecated: false` for `__InputValue`, which caused some client libraries to fail. ([#6475](https://github.com/graphprotocol/graph-node/pull/6475)) +- Fixed IPC provider connections failing when configured with `ipc://` or `file://` URLs — the URL was passed directly to the transport instead of extracting the file path. ([#6443](https://github.com/graphprotocol/graph-node/pull/6443)) +- Fixed `graphman config pools` not working due to hardcoded pool size override. ([#6444](https://github.com/graphprotocol/graph-node/pull/6444)) +- Fixed unfail retry mechanism stopping after the first attempt when the deployment head was still behind the error block. ([#6529](https://github.com/graphprotocol/graph-node/pull/6529)) + +### Note on `ethereum.decode()` whitespace handling + +The migration from `ethabi` to `alloy` in v0.42.0 ([#6063](https://github.com/graphprotocol/graph-node/pull/6063)) incidentally fixed a long-standing parsing bug in `ethabi` where type strings containing whitespace before a type name (e.g. `" address"` with a leading space) were silently decoded as `Uint(8)` instead of the intended type. `alloy` parses these correctly. + +Subgraphs that relied on the incorrect `Uint(8)` decoding to subsequently call `.toBigInt()` on what is actually an `Address` value will abort on v0.42.0+ with: + +``` +Mapping aborted ... Ethereum value is not an int or uint. +``` + +This is not a graph-node regression. Recompile the subgraph with the correct accessor (`.toAddress()` for addresses) to fix. See [#6461](https://github.com/graphprotocol/graph-node/issues/6461) for details. + +### gnd (Graph Node Dev) + +- `gnd indexer` command that delegates to `graph-indexer`, allowing indexer management (allocations, rules, cost models, status) directly through gnd. ([#6492](https://github.com/graphprotocol/graph-node/pull/6492)) +- `gnd deploy` now prompts for `--version-label` in interactive mode and requires it in non-interactive mode. ([#6532](https://github.com/graphprotocol/graph-node/pull/6532)) +- Test framework improvements: partial receipt.logs support, mock IPFS/Arweave clients for file data source testing. ([#6442](https://github.com/graphprotocol/graph-node/pull/6442)) + +### Contributors + +Thanks to all contributors for this release: @aayushbaluni, @dimitrovmaksim, @incrypto32, @isum, @lutter, @suntzu + +**Full Changelog**: https://github.com/graphprotocol/graph-node/compare/v0.42.1...v0.43.0 + +## v0.42.1 + +### Bug Fixes + +- Fixed multi-shard deployments becoming unresponsive due to `mirror_primary_tables` holding locks indefinitely, exhausting the connection pool. Added `statement_timeout` and `lock_timeout` to prevent stuck queries from cascading into a full lockup. ([#6446](https://github.com/graphprotocol/graph-node/pull/6446)) +- Fixed the `ingestor` setting in `[chains.*]` config sections having no effect — block ingestion ran regardless of its value. ([#6447](https://github.com/graphprotocol/graph-node/pull/6447)) + +## v0.42.0 + +### Breaking Changes + +- **Substreams support removed.** Substreams have been unsupported on the network for some time. All substreams-related code has been removed, simplifying the codebase significantly. Substreams-based subgraphs will no longer work. ([#6261](https://github.com/graphprotocol/graph-node/pull/6261)) +- **Migrated from rust-web3 to alloy.** The Ethereum RPC layer now uses the alloy crate instead of rust-web3. This should be transparent to most users, but chains that do not return EIP-2718 typed transaction fields may need the `no_eip2718` provider feature flag. ([#6063](https://github.com/graphprotocol/graph-node/pull/6063)) ([#6317](https://github.com/graphprotocol/graph-node/pull/6317)) + +### What's New + +- **Amp-powered subgraphs (experimental).** A new kind of subgraph powered by [Amp](https://github.com/edgeandnode/amp), the blockchain native database. Amp-powered subgraphs index data directly from Amp servers instead of processing blocks individually, reducing indexing time from days/weeks to minutes/hours. See `docs/amp-powered-subgraphs.md` for details. ([#6218](https://github.com/graphprotocol/graph-node/pull/6218)) ([#6293](https://github.com/graphprotocol/graph-node/pull/6293)) ([#6369](https://github.com/graphprotocol/graph-node/pull/6369)) +- **SQL query interface (experimental).** A new experimental SQL query interface allows querying subgraph data using SQL syntax via GraphQL. Documentation in `docs/implementation/sql-interface.md`. ([#6172](https://github.com/graphprotocol/graph-node/pull/6172)) +- **Async store.** All store and database interactions are now fully async using `diesel_async`, eliminating blocking database calls from tokio tasks. This improves throughput and reduces the risk of thread starvation under heavy load. ([#6194](https://github.com/graphprotocol/graph-node/pull/6194)) ([#6185](https://github.com/graphprotocol/graph-node/pull/6185)) +- **RPC request compression.** Configurable compression for outgoing JSON-RPC requests to upstream providers, supporting gzip, brotli, and deflate. Configured via provider `features` in the config file. ([#6084](https://github.com/graphprotocol/graph-node/pull/6084)) +- Query the current (partially filled) aggregation bucket with `current: include` in GraphQL requests. ([#6293](https://github.com/graphprotocol/graph-node/pull/6293)) +- Extended `first` and `last` aggregation support to `String`, `Bytes`, and entity reference types. ([#6225](https://github.com/graphprotocol/graph-node/pull/6225)) +- Automated account-like table optimization — an optional background job that detects and marks tables with high entity update ratios as account-like. Enabled via `GRAPH_STORE_ACCOUNT_LIKE_SCAN_INTERVAL_HOURS`, `GRAPH_STORE_ACCOUNT_LIKE_MIN_VERSIONS_COUNT`, and `GRAPH_STORE_ACCOUNT_LIKE_MAX_UNIQUE_RATIO`. ([#6209](https://github.com/graphprotocol/graph-node/pull/6209)) +- Subgraph table sizes and row counts are now exposed in `SubgraphIndexingStatus` via the index-node API. ([#6201](https://github.com/graphprotocol/graph-node/pull/6201)) +- Configurable database setup timeout via `GRAPH_STORE_SETUP_TIMEOUT`. +- Option to disable the store call cache via `GRAPH_STORE_DISABLE_CALL_CACHE`, useful for indexers with locally hosted RPC nodes. ([#6214](https://github.com/graphprotocol/graph-node/pull/6214)) +- Docker: added profiling support to the `graph-node-debug` image with frame pointers and `linux-perf`. ([#6418](https://github.com/graphprotocol/graph-node/pull/6418)) + +### Improvements + +- Optimized WASM trigger instantiation by caching linker and compilation artifacts, reducing per-trigger overhead. ([#6364](https://github.com/graphprotocol/graph-node/pull/6364)) +- Optimized log filter matching and trigger allocation, reducing CPU usage during block processing. ([#6419](https://github.com/graphprotocol/graph-node/pull/6419)) +- Reduced unnecessary entity clones during block processing. ([#6362](https://github.com/graphprotocol/graph-node/pull/6362)) +- Replaced `std::sync::RwLock` with `parking_lot::RwLock` across multiple components for better performance under contention. +- Deadlock prevention under heavy load, especially during index node startup — connection pool and store semaphore improvements. ([#6224](https://github.com/graphprotocol/graph-node/pull/6224)) +- Raw subgraph manifests are now loaded from the store cache during startup, reducing IPFS dependency. ([#6223](https://github.com/graphprotocol/graph-node/pull/6223)) +- Deployment hashes are now logged when removing a subgraph. ([#6405](https://github.com/graphprotocol/graph-node/pull/6405)) + +### Bug Fixes + +- Fixed duplicate VID when multiple offchain triggers (file/IPFS data sources) fire in the same block, causing unique constraint violations. Also fixed in `ipfs.map()` callbacks. ([#6336](https://github.com/graphprotocol/graph-node/pull/6336)) ([#6416](https://github.com/graphprotocol/graph-node/pull/6416)) +- Fixed incorrect `nocase` filter generation for non-String types (e.g., `Bytes`), which caused SQL errors like `operator does not exist: bytea ~~* bytea`. ([#6351](https://github.com/graphprotocol/graph-node/pull/6351)) +- Fixed `can_copy_from` failures not fully rolling back deployment creation, leaving orphaned records. ([#6228](https://github.com/graphprotocol/graph-node/pull/6228)) +- Fixed multi-column index validation bug where column order mismatch caused valid indexes to be silently dropped during subgraph copy/graft. ([#6341](https://github.com/graphprotocol/graph-node/pull/6341)) +- Fixed grafting failure ("Unexpected null for non-null column") when source tables have too few rows for PostgreSQL to generate `histogram_bounds` statistics. ([#6275](https://github.com/graphprotocol/graph-node/pull/6275)) +- Fixed panic when converting a negative number to `U256` in the runtime — now returns an error instead. ([#6219](https://github.com/graphprotocol/graph-node/pull/6219)) +- Fixed `graphman copy` copying pruned `earliest_block_number` from source, which could leave the destination deployment in an invalid state. ([#6384](https://github.com/graphprotocol/graph-node/pull/6384)) +- Fixed calls returning `0x` being incorrectly cached in the call cache, preventing stale empty results from persisting. ([#6187](https://github.com/graphprotocol/graph-node/pull/6187)) + +### Graphman + +- **`graphman dump` and `graphman restore` (experimental).** New commands for exporting and importing subgraph data in Parquet format, enabling backup and migration of deployment data across shards. Supports incremental dumps and progress reporting. See `docs/dump.md` for details. ([#6397](https://github.com/graphprotocol/graph-node/pull/6397)) +- `graphman chain call-cache remove` now supports `--ttl-days` to remove stale call cache entries that haven't been accessed within a specified number of days. ([#6186](https://github.com/graphprotocol/graph-node/pull/6186)) + +### gnd (Graph Node Dev) + +- **[experimental]** Major expansion of `gnd` as a drop-in replacement for `graph-cli`: `init`, `add`, `codegen`, `build`, `publish`, `deploy`, `create`, `remove`, `auth`, `clean`, and `test` commands. ([#6282](https://github.com/graphprotocol/graph-node/pull/6282)) + +### Contributors + +Thanks to all contributors for this release: @DaMandal0rian, @dimitrovmaksim, @fubhy, @hudsonhrh, @incrypto32, @isum, @lutter, @shiyasmohd + +**Full Changelog**: https://github.com/graphprotocol/graph-node/compare/v0.41.2...v0.42.0 + +## v0.41.2 + +### Bug Fixes + +- Fixed entity corruption after rewinding pruned subgraphs — rewinding a subgraph with `history_blocks` (pruning) enabled could leave frequently-updated entities with no open version, causing deterministic `unexpected null` errors and making the subgraph unrecoverable without a full re-index ([#6340](https://github.com/graphprotocol/graph-node/pull/6340)) +- Fixed `graphman rewind` and `graphman truncate` targeting the wrong deployment when multiple instances of the same subgraph exist ([#6299](https://github.com/graphprotocol/graph-node/pull/6299)) +- Fixed batch write memoization bug where entity lookups could incorrectly return `None`, causing missed entity updates during indexing ([#6314](https://github.com/graphprotocol/graph-node/pull/6314)) +- Expanded deterministic RPC error handling for Reth and Erigon — added `vm execution error`, `invalidjump`, `notactivated`, and `invalidfeopcode` to recognized deterministic errors. Indexers using these clients no longer need to set `GRAPH_GETH_ETH_CALL_ERRORS` manually for these cases ([#6355](https://github.com/graphprotocol/graph-node/pull/6355)) + +**Full changelog:** + +## v0.41.1 + +### Bug Fixes + +- Fixed a regression in v0.41.0 where the indexing status endpoint would return an empty list when querying all subgraphs without specific deployment filters ([#6210](https://github.com/graphprotocol/graph-node/pull/6210)) + +**Full changelog:** + +## v0.41.0 + +### Bug Fixes + +- Fix non-deterministic handling of transaction receipts in event handlers; receipts are only processed when explicitly declared ([#6200](https://github.com/graphprotocol/graph-node/pull/6200)) + +### New Features + +- Added support for struct field access in declarative calls — enables accessing nested struct fields by field name using dot notation in manifest call declarations ([#6099](https://github.com/graphprotocol/graph-node/pull/6099)) +- New standalone `gnd` (Graph Node Dev) crate — a graph-node binary optimized for local development with minimal setup required ([#6167](https://github.com/graphprotocol/graph-node/pull/6167)) +- Extended IPFS usage metrics and logging capabilities ([#6058](https://github.com/graphprotocol/graph-node/pull/6058)) +- Extended support for additional IPFS content path formats ([#6058](https://github.com/graphprotocol/graph-node/pull/6058)) + +### Improvements + +- Deferred IPFS manifest fetching when starting subgraphs — fixes issue where slow IPFS responses could block assignment event processing ([#6170](https://github.com/graphprotocol/graph-node/pull/6170)) +- Added Content Identifier (CID) to IPFS retry logs for better debugging ([#6144](https://github.com/graphprotocol/graph-node/pull/6144)) +- Fixed handling of empty arrays in indexing status queries ([#6143](https://github.com/graphprotocol/graph-node/pull/6143)) +- Fixed case-insensitive array inputs in queries ([#6075](https://github.com/graphprotocol/graph-node/pull/6075)) +- Fixed panic when handling empty lists in `list_values` ([#6100](https://github.com/graphprotocol/graph-node/pull/6100)) +- Enhanced logging for subgraph assignment processing ([#6169](https://github.com/graphprotocol/graph-node/pull/6169)) +- Added logging for `MAX_BLOCKING_THREADS` configuration setting ([#6161](https://github.com/graphprotocol/graph-node/pull/6161)) + +**Full changelog:** + +## v0.40.1 + +### Improvements + +- Enhanced IPFS logging to include Content Identifiers (CIDs) in operation names for better debugging + +**Full changelog:** + +## v0.40.0 + +### Critical Bugfix + +- Fixed GraphQL query panic with empty arrays: Resolved a critical bug where GraphQL queries using `_in` filters with empty arrays would cause the graph-node to panic ([#6100](https://github.com/graphprotocol/graph-node/pull/6100)) + +### New Features + +- IPFS files can now be cached to disk using `GRAPH_IPFS_CACHE_LOCATION` — reduces IPFS requests after restarts ([#6031](https://github.com/graphprotocol/graph-node/pull/6031)) + +### Improvements + +- Speed up appending changes to batch operations ([#6025](https://github.com/graphprotocol/graph-node/pull/6025)) +- Improved backoff strategy for offchain data sources, configurable via `GRAPH_FDS_MAX_BACKOFF` ([#6043](https://github.com/graphprotocol/graph-node/pull/6043)) +- Better error messages for OR operator usage with column filters ([#6078](https://github.com/graphprotocol/graph-node/pull/6078)) +- Update Wasmtime version for improved WebAssembly performance ([#6050](https://github.com/graphprotocol/graph-node/pull/6050)) + +### Bug Fixes + +- Fixed subgraph composition sync failures with different ID types ([#6080](https://github.com/graphprotocol/graph-node/pull/6080)) +- Fixed pruning status reporting accuracy ([#6062](https://github.com/graphprotocol/graph-node/pull/6062)) + +**Full changelog:** + +## v0.39.1 + +### Critical Fix + +- Reverted `event.transactionLogIndex` behavior change from v0.38.0 that caused subgraph failures with duplicate ID errors and POI divergence between indexers ([#6042](https://github.com/graphprotocol/graph-node/pull/6042)) + +**Full changelog:** + +## v0.39.0 + +### Breaking Changes + +- **Database schema migration**: `subgraphs.subgraph_deployment` table split into `subgraphs.head` and `subgraphs.deployment`. External tools accessing this table will need updates. See [documentation](https://github.com/graphprotocol/graph-node/blob/master/docs/implementation/metadata.md#subgraphshead) ([#6003](https://github.com/graphprotocol/graph-node/pull/6003)) +- **Arweave blockchain support removed** — Arweave subgraphs are no longer supported (file data sources remain unaffected) ([#5951](https://github.com/graphprotocol/graph-node/pull/5951)) +- **`graphman drop` removed** — Use `graphman remove` followed by `graphman unused record && graphman unused remove` ([#5974](https://github.com/graphprotocol/graph-node/pull/5974)) +- **Failed subgraphs are now paused instead of unassigned** ([#5971](https://github.com/graphprotocol/graph-node/pull/5971)) + +### New Features + +- Pruning status tracking with new graphman commands: `prune status`, `prune run`, `prune set` ([#5949](https://github.com/graphprotocol/graph-node/pull/5949)) +- Pruning timeout protection via `GRAPH_STORE_BATCH_TIMEOUT` ([#6002](https://github.com/graphprotocol/graph-node/pull/6002)) +- `graphman chain ingest` for manually ingesting specific blocks ([#5945](https://github.com/graphprotocol/graph-node/pull/5945)) +- Configurable IPFS retry limits via `GRAPH_IPFS_MAX_ATTEMPTS` and `GRAPH_IPFS_REQUEST_TIMEOUT` ([#5998](https://github.com/graphprotocol/graph-node/pull/5998)) + +### Bug Fixes + +- Fixed VID sequence naming to comply with PostgreSQL 63-character limit +- Fixed pruning of tables with VID sequences using CASCADE drops ([#5968](https://github.com/graphprotocol/graph-node/pull/5968)) +- Fixed numerical precision issues with large VID values ([#5970](https://github.com/graphprotocol/graph-node/pull/5970)) + +**Full changelog:** + +## v0.38.0 + +### What's new + +- A new `deployment_synced` metric is added [(#5816)](https://github.com/graphprotocol/graph-node/pull/5816) + that indicates whether a deployment has reached the chain head since it was deployed. + + **Possible values for the metric:** + - `0` - means that the deployment is not synced; + - `1` - means that the deployment is synced; + + _If a deployment is not running, the metric reports no value for that deployment._ + +## v0.37.0 + +### What's new + +- A new `deployment_status` metric is added [(#5720)](https://github.com/graphprotocol/graph-node/pull/5720) with the + following behavior: + - Once graph-node has figured out that it should index a deployment, `deployment_status` is set to `1` _(starting)_; + - When the block stream is created and blocks are ready to be processed, `deployment_status` is set to `2` _( + running)_; + - When a deployment is unassigned, `deployment_status` is set to `3` _(stopped)_; + - If a temporary or permanent failure occurs, `deployment_status` is set to `4` _(failed)_; + - If indexing manages to recover from a temporary failure, the `deployment_status` is set back to `2` _( + running)_; + +### Breaking changes + +- The `deployment_failed` metric is removed and the failures are reported by the new `deployment_status` + metric. [(#5720)](https://github.com/graphprotocol/graph-node/pull/5720) + +## v0.36.0 + +### Note on Firehose Extended Block Details + +By default, all Firehose providers are required to support extended block details, as this is the +safest option for a graph-node operator. Firehose providers that do not support extended block +details for enabled chains are considered invalid and will not be used. + +To disable checks for one or more chains, simply specify their names +in `GRAPH_NODE_FIREHOSE_DISABLE_EXTENDED_BLOCKS_FOR_CHAINS` as a comma separated list of chain +names. Graph Node defaults to an empty list, which means that this feature is enabled for all +chains. + +### What's new + +- Add support for substreams using 'index modules', 'block filters', 'store:sum_set'. [(#5463)](https://github.com/graphprotocol/graph-node/pull/5463) +- Implement new IPFS client [(#5600)](https://github.com/graphprotocol/graph-node/pull/5600) +- Add `timestamp` support to substreams. [(#5641)](https://github.com/graphprotocol/graph-node/pull/5641) +- Add graph-indexed header to query responses. [(#5710)](https://github.com/graphprotocol/graph-node/pull/5710) +- Use the new Firehose info endpoint. [(#5672)](https://github.com/graphprotocol/graph-node/pull/5672) +- Store `synced_at_block_number` when a deployment syncs. [(#5610)](https://github.com/graphprotocol/graph-node/pull/5610) +- Create nightly docker builds from master branch. [(#5400)](https://github.com/graphprotocol/graph-node/pull/5400) +- Make sure `transact_block_operations` does not go backwards. [(#5419)](https://github.com/graphprotocol/graph-node/pull/5419) +- Improve error message when store write fails. [(#5420)](https://github.com/graphprotocol/graph-node/pull/5420) +- Allow generating map of section nesting in debug builds. [(#5279)](https://github.com/graphprotocol/graph-node/pull/5279) +- Ensure substream module name is valid. [(#5424)](https://github.com/graphprotocol/graph-node/pull/5424) +- Improve error message when resolving references. [(#5385)](https://github.com/graphprotocol/graph-node/pull/5385) +- Check if subgraph head exists before trying to unfail. [(#5409)](https://github.com/graphprotocol/graph-node/pull/5409) +- Check for EIP 1898 support when checking block receipts support. [(#5406)](https://github.com/graphprotocol/graph-node/pull/5406) +- Use latest block hash for `check_block_receipts`. [(#5427)](https://github.com/graphprotocol/graph-node/pull/5427) +- Handle null blocks from Lotus. [(#5294)](https://github.com/graphprotocol/graph-node/pull/5294) +- Increase firehose grpc max decode size. [(#5483)](https://github.com/graphprotocol/graph-node/pull/5483) +- Improve Environment variable docs, rename `GRAPH_ETHEREUM_BLOCK_RECEIPTS_TIMEOUT` to `GRAPH_ETHEREUM_BLOCK_RECEIPTS_CHECK_TIMEOUT`. [(#5468)](https://github.com/graphprotocol/graph-node/pull/5468) +- Remove provider checks at startup. [(#5337)](https://github.com/graphprotocol/graph-node/pull/5337) +- Track more features in subgraph features table. [(#5479)](https://github.com/graphprotocol/graph-node/pull/5479) +- Implement is_duplicate_of for substreams. [(#5482)](https://github.com/graphprotocol/graph-node/pull/5482) +- Add docs for `GRAPH_POSTPONE_ATTRIBUTE_INDEX_CREATION`. [(#5515)](https://github.com/graphprotocol/graph-node/pull/5515) +- Improve error message for missing template during grafting. [(#5464)](https://github.com/graphprotocol/graph-node/pull/5464) +- Enable "hard-coded" values in declarative eth_calls. [(#5498)](https://github.com/graphprotocol/graph-node/pull/5498) +- Respect causality region in derived fields. [(#5488)](https://github.com/graphprotocol/graph-node/pull/5488) +- Improve net_identifiers call with timeout. [(#5549)](https://github.com/graphprotocol/graph-node/pull/5549) +- Add arbitrum-sepolia chain ID to GRAPH_ETH_CALL_NO_GAS default value. [(#5504)](https://github.com/graphprotocol/graph-node/pull/5504) +- Disable genesis validation by default. [(#5565)](https://github.com/graphprotocol/graph-node/pull/5565) +- Timeout when trying to get `net_identifiers` at startup. [(#5568)](https://github.com/graphprotocol/graph-node/pull/5568) +- Only start substreams if no other block investor is available. [(#5569)](https://github.com/graphprotocol/graph-node/pull/5569) +- Allow running a single test case for integration tests. [(#5577)](https://github.com/graphprotocol/graph-node/pull/5577) +- Store timestamp when marking subgraph as synced. [(#5566)](https://github.com/graphprotocol/graph-node/pull/5566) +- Document missing env vars. [(#5580)](https://github.com/graphprotocol/graph-node/pull/5580) +- Return more features in status API. [(#5582)](https://github.com/graphprotocol/graph-node/pull/5582) +- Respect substreams datasource `startBlock`. [(#5617)](https://github.com/graphprotocol/graph-node/pull/5617) +- Update flagged dependencies. [(#5659)](https://github.com/graphprotocol/graph-node/pull/5659) +- Add more debug logs when subgraph is marked unhealthy. [(#5662)](https://github.com/graphprotocol/graph-node/pull/5662) +- Add config option for cache stores. [(#5716)](https://github.com/graphprotocol/graph-node/pull/5716) + +### Bug fixes + +- Add safety check when rewinding. [(#5423)](https://github.com/graphprotocol/graph-node/pull/5423) +- Fix rewind for deployments with multiple names. [(#5502)](https://github.com/graphprotocol/graph-node/pull/5502) +- Improve `graphman copy` performance [(#5425)](https://github.com/graphprotocol/graph-node/pull/5425) +- Fix retrieving chain info with graphman for some edge cases. [(#5516)](https://github.com/graphprotocol/graph-node/pull/5516) +- Improve `graphman restart` to handle multiple subgraph names for a deployment. [(#5674)](https://github.com/graphprotocol/graph-node/pull/5674) +- Improve adapter startup. [(#5503)](https://github.com/graphprotocol/graph-node/pull/5503) +- Detect Nethermind eth_call reverts. [(#5533)](https://github.com/graphprotocol/graph-node/pull/5533) +- Fix genesis block fetching for substreams. [(#5548)](https://github.com/graphprotocol/graph-node/pull/5548) +- Fix subgraph_resume being mislabelled as pause. [(#5588)](https://github.com/graphprotocol/graph-node/pull/5588) +- Make `SubgraphIndexingStatus.paused` nullable. [(#5551)](https://github.com/graphprotocol/graph-node/pull/5551) +- Fix a count aggregation bug. [(#5639)](https://github.com/graphprotocol/graph-node/pull/5639) +- Fix prost generated file. [(#5450)](https://github.com/graphprotocol/graph-node/pull/5450) +- Fix `deployment_head` metrics not progressing for substreams. [(#5522)](https://github.com/graphprotocol/graph-node/pull/5522) +- Enable graft validation checks in debug builds. [(#5584)](https://github.com/graphprotocol/graph-node/pull/5584) +- Use correct store when loading indexes for graft base. [(#5616)](https://github.com/graphprotocol/graph-node/pull/5616) +- Sanitise columns in SQL. [(#5578)](https://github.com/graphprotocol/graph-node/pull/5578) +- Truncate `subgraph_features` table before migrating. [(#5505)](https://github.com/graphprotocol/graph-node/pull/5505) +- Consistently apply max decode size. [(#5520)](https://github.com/graphprotocol/graph-node/pull/5520) +- Various docker packaging improvements [(#5709)](https://github.com/graphprotocol/graph-node/pull/5709) [(#5711)](https://github.com/graphprotocol/graph-node/pull/5711) [(#5712)](https://github.com/graphprotocol/graph-node/pull/5712) [(#5620)](https://github.com/graphprotocol/graph-node/pull/5620) [(#5621)](https://github.com/graphprotocol/graph-node/pull/5621) +- Retry IPFS requests on Cloudflare 521 Web Server Down. [(#5687)](https://github.com/graphprotocol/graph-node/pull/5687) +- Optimize IPFS retries. [(#5698)](https://github.com/graphprotocol/graph-node/pull/5698) +- Exclude full-text search columns from entity queries. [(#5693)](https://github.com/graphprotocol/graph-node/pull/5693) +- Do not allow multiple active runners for a subgraph. [(#5715)](https://github.com/graphprotocol/graph-node/pull/5715) +- Stop subgraphs passing max endBlock. [(#5583)](https://github.com/graphprotocol/graph-node/pull/5583) +- Do not repeat a rollup after restart in some corner cases. [(#5675)](https://github.com/graphprotocol/graph-node/pull/5675) + +### Graphman + +- Add command to update genesis block for a chain and to check genesis information against all providers. [(#5517)](https://github.com/graphprotocol/graph-node/pull/5517) +- Create GraphQL API to execute commands [(#5554)](https://github.com/graphprotocol/graph-node/pull/5554) +- Add graphman create/remove commands to GraphQL API. [(#5685)](https://github.com/graphprotocol/graph-node/pull/5685) + +### Contributors + +Thanks to all contributors for this release: @dwerner, @encalypto, @incrypto32, @isum, @leoyvens, @lutter, @mangas, @sduchesneau, @Shiyasmohd, @shuaibbapputty, @YaroShkvorets, @ziyadonji, @zorancv + +**Full Changelog**: https://github.com/graphprotocol/graph-node/compare/v0.35.1...v0.36.0 + +## v0.35.0 +### What's new + +- **Aggregations** - Declarative aggregations defined in the subgraph schema allow the developer to aggregate values on specific intervals using flexible aggregation functions. [(#5082)](https://github.com/graphprotocol/graph-node/pull/5082) [(#5184)](https://github.com/graphprotocol/graph-node/pull/5184) [(#5209)](https://github.com/graphprotocol/graph-node/pull/5209) [(#5242)](https://github.com/graphprotocol/graph-node/pull/5242) [(#5208)](https://github.com/graphprotocol/graph-node/pull/5208) +- **Add pause and resume to admin JSON-RPC API** - Adds support for explicit pausing and resuming of subgraph deployments with a field tracking the paused state in `indexerStatuses`. [(#5190)](https://github.com/graphprotocol/graph-node/pull/5190) +- **Support eth_getBalance calls in subgraph mappings** - Enables fetching the Eth balance of an address from the mappings using `ethereum.getBalance(address)`. [(#5202)](https://github.com/graphprotocol/graph-node/pull/5202) +- **Add parentHash to _meta query** - Particularly useful when polling for data each block to verify the sequence of blocks. [(#5232)](https://github.com/graphprotocol/graph-node/pull/5232) +- **Parallel execution of all top-level queries in a single query body** [(#5273)](https://github.com/graphprotocol/graph-node/pull/5273) +- The ElasticSearch index to which `graph-node` logs can now be configured with the `GRAPH_ELASTIC_SEARCH_INDEX` environment variable which defaults to `subgraph`. [(#5210)](https://github.com/graphprotocol/graph-node/pull/5210) +- Some small prefetch simplifications. [(#5132)](https://github.com/graphprotocol/graph-node/pull/5132) +- Migration changing the type of health column to text. [(#5077)](https://github.com/graphprotocol/graph-node/pull/5077) +- Disable eth_call_execution_time metric by default. [(#5164)](https://github.com/graphprotocol/graph-node/pull/5164) +- Call revert_state_to whenever blockstream is restarted. [(#5187)](https://github.com/graphprotocol/graph-node/pull/5187) +- Pruning performance improvement: only analyze when rebuilding. [(#5186)](https://github.com/graphprotocol/graph-node/pull/5186) +- Disallow grafts within the reorg threshold. [(#5135)](https://github.com/graphprotocol/graph-node/pull/5135) +- Optimize subgraph synced check-less. [(#5198)](https://github.com/graphprotocol/graph-node/pull/5198) +- Improve error log. [(#5217)](https://github.com/graphprotocol/graph-node/pull/5217) +- Update provider docs. [(#5216)](https://github.com/graphprotocol/graph-node/pull/5216) +- Downgrade 'Entity cache statistics' log to trace. [(#5241)](https://github.com/graphprotocol/graph-node/pull/5241) +- Do not clone MappingEventHandlers in match_and_decode. [(#5244)](https://github.com/graphprotocol/graph-node/pull/5244) +- Make batching conditional on caught-up status. [(#5252)](https://github.com/graphprotocol/graph-node/pull/5252) +- Remove hack in chain_head_listener. [(#5240)](https://github.com/graphprotocol/graph-node/pull/5240) +- Increase sleep time in write queue processing. [(#5266)](https://github.com/graphprotocol/graph-node/pull/5266) +- Memoize Batch.indirect_weight. [(#5276)](https://github.com/graphprotocol/graph-node/pull/5276) +- Optionally track detailed indexing gas metrics in csv. [(#5215)](https://github.com/graphprotocol/graph-node/pull/5215) +- store: Do not use prefix comparisons for primary keys. [(#5289)](https://github.com/graphprotocol/graph-node/pull/5289) + +### Graphman + +- Add ability to list removed unused deployment by id. [(#5152)](https://github.com/graphprotocol/graph-node/pull/5152) +- Add command to change block cache shard. [(#5169)](https://github.com/graphprotocol/graph-node/pull/5169) + +### Firehose and Substreams + +- **Add key-based authentication for Firehose/Substreams providers.** [(#5259)](https://github.com/graphprotocol/graph-node/pull/5259) +- Increase blockstream buffer size for substreams. [(#5182)](https://github.com/graphprotocol/graph-node/pull/5182) +- Improve substreams error handling. [(#5160)](https://github.com/graphprotocol/graph-node/pull/5160) +- Reset substreams/firehose block ingestor backoff. [(#5047)](https://github.com/graphprotocol/graph-node/pull/5047) + +### Bug Fixes + +- Fix graphiql issue when querying subgraph names with multiple path segments. [(#5136)](https://github.com/graphprotocol/graph-node/pull/5136) +- Fix change_health_column migration for sharded setup. [(#5183)](https://github.com/graphprotocol/graph-node/pull/5183) +- Fix conversion of BlockTime for NEAR. [(#5206)](https://github.com/graphprotocol/graph-node/pull/5206) +- Call revert_state_to to last good block instead of current block. [(#5195)](https://github.com/graphprotocol/graph-node/pull/5195) +- Fix Action::block_finished. [(#5218)](https://github.com/graphprotocol/graph-node/pull/5218) +- Fix runtime timeouts. [(#5236)](https://github.com/graphprotocol/graph-node/pull/5236) +- Remove panic from rewind and truncate. [(#5233)](https://github.com/graphprotocol/graph-node/pull/5233) +- Fix version stats for huge number of versions. [(#5261)](https://github.com/graphprotocol/graph-node/pull/5261) +- Fix _meta query failure due to incorrect selection set use. [(#5265)](https://github.com/graphprotocol/graph-node/pull/5265) + +### Major dependency upgrades + +- Update to diesel 2. [(#5002)](https://github.com/graphprotocol/graph-node/pull/5002) +- bump rust version. [(#4985)](https://github.com/graphprotocol/graph-node/pull/4985) + +### Contributors + +Thank you to all the contributors! `@incrypto32`, `@mangas`, `@lutter`, `@leoyvens`, `@zorancv`, `@YaroShkvorets`, `@seem-less` + +**Full Changelog**: https://github.com/graphprotocol/graph-node/compare/v0.34.1...v0.35.0 + + + +## v0.34.1 +## Bug fixes +- Fixed an issue that caused an increase in data size of /metrics endpoint of graph-node. [(#5161)](https://github.com/graphprotocol/graph-node/issues/5161) +- Fixed an issue that caused subgraphs with file data sources to skip non-deterministic errors that occurred in a file data source mapping handler. + +## v0.34.0 +### What's New + +- **Substreams as Source of Triggers for Subgraphs** - This update significantly enhances subgraph functionality by enabling substreams to act as a source of triggers for running subgraph mappings. Developers can now directly run subgraph mappings on the data output from substreams, facilitating a more integrated and efficient workflow.[(#4887)](https://github.com/graphprotocol/graph-node/pull/4887) [(#4916)](https://github.com/graphprotocol/graph-node/pull/4916) +- **`indexerHints` in Manifest for Automated Pruning** - This update introduces the ability for subgraph authors to specify `indexerHints` with a field `prune` in their manifest, indicating the desired extent of historical block data retention. This feature enables graph-node to automatically prune subgraphs when the stored history exceeds the specified limit, significantly improving query performance. This automated process eliminates the need for manual action by indexers for each subgraph. Indexers can also override user-set historyBlocks with the environment variable `GRAPH_HISTORY_BLOCKS_OVERRIDE` [(#5032](https://github.com/graphprotocol/graph-node/pull/5032) [(#5117)](https://github.com/graphprotocol/graph-node/pull/5117) +- **Initial Starknet Support** - Introducing initial Starknet support for graph-node, expanding indexing capabilities to the Starknet ecosystem. The current integration is in its early stages, with notable areas for development including the implementation of trigger filters and data source template support. Future updates will also bring substream support. [(#4895)](https://github.com/graphprotocol/graph-node/pull/4895) +- **`endBlock` Feature in Data Sources** - This update adds the `endBlock` field for dataSources in subgraph manifest. By setting an `endBlock`, subgraph authors can define the exact block at which a data source will cease processing, ensuring no further triggers are processed beyond this point. [(#4787](https://github.com/graphprotocol/graph-node/pull/4787) +- **Autogenerated `Int8` IDs in graph-node** - Introduced support for using `Int8` as the ID type for entities, with the added capability to auto-generate these IDs, enhancing flexibility and functionality in entity management. [(#5029)](https://github.com/graphprotocol/graph-node/pull/5029) +- **GraphiQL V2 Update** - Updated GraphiQL query interface of graph-node to version 2. [(#4677)](https://github.com/graphprotocol/graph-node/pull/4677) +- **Sharding Guide for Graph-Node** - A new guide has been added to graph-node documentation, explaining how to scale graph-node installations using sharding with multiple Postgres instances. [Sharding Guide](https://github.com/graphprotocol/graph-node/blob/master/docs/sharding.md) +- Per-chain polling interval configuration for RPC Block Ingestors [(#5066)](https://github.com/graphprotocol/graph-node/pull/5066) +- Metrics Enhancements[(#5055)](https://github.com/graphprotocol/graph-node/pull/5055) [(#4937)](https://github.com/graphprotocol/graph-node/pull/4937) +- graph-node now avoids creating GIN indexes on array attributes to enhance database write performance, addressing the issue of expensive updates and underutilization in queries. [(#4933)](https://github.com/graphprotocol/graph-node/pull/4933) +- The `subgraphFeatures` endpoint in graph-node has been updated to load features from subgraphs prior to their deployment. [(#4864)](https://github.com/graphprotocol/graph-node/pull/4864) +- Improved log filtering performance in blockstream. [(#5015)](https://github.com/graphprotocol/graph-node/pull/5015) +- Enhanced GraphQL error reporting by including `__schema` and `__type` fields in the results during indexing errors [(#4968)](https://github.com/graphprotocol/graph-node/pull/4968) + +### Bug fixes + +- Addressed a bug in the deduplication logic for Cosmos events, ensuring all distinct events are properly indexed and handled, especially when similar but not identical events occur within the same block. [(#5112)](https://github.com/graphprotocol/graph-node/pull/5112) +- Fixed compatibility issues with ElasticSearch 8.X, ensuring proper log functionality. [(#5013)](https://github.com/graphprotocol/graph-node/pull/5013) + - Resolved an issue when rewinding data sources across multiple blocks. In rare cases, when a subgraph had been rewound by multiple blocks, data sources 'from the future' could have been left behind. This release adds a database migration that fixes that. With very unlucky timing this migration might miss some subgraphs, which will later lead to an error `assertion failed: self.hosts.last().and_then(|h| h.creation_block_number()) <= data_source.creation_block()`. Should that happen, the [migration script](https://github.com/graphprotocol/graph-node/blob/master/store/postgres/migrations/2024-01-05-170000_ds_corruption_fix_up/up.sql) should be rerun against the affected shard. [(#5083)](https://github.com/graphprotocol/graph-node/pull/5083) +- Increased the base backoff time for RPC, enhancing stability and reliability under load. [(#4984)](https://github.com/graphprotocol/graph-node/pull/4984) +- Resolved an issue related to spawning offchain data sources from existing offchain data source mappings. [(#5051)](https://github.com/graphprotocol/graph-node/pull/5051)[(#5092)](https://github.com/graphprotocol/graph-node/pull/5092) +- Resolved an issue where eth-call results for reverted calls were being cached in call cache. [(#4879)](https://github.com/graphprotocol/graph-node/pull/4879) +- Fixed a bug in graphman's index creation to ensure entire String and Bytes columns are indexed rather than just their prefixes, resulting in optimized query performance and accuracy. [(#4995)](https://github.com/graphprotocol/graph-node/pull/4995) +- Adjusted `SubstreamsBlockIngestor` to initiate at the chain's head block instead of starting at block zero when no cursor exists. [(#4951)](https://github.com/graphprotocol/graph-node/pull/4951) +- Fixed a bug that caused incorrect progress reporting when copying subgraphs, ensuring accurate status updates. [(#5075)](https://github.com/graphprotocol/graph-node/pull/5075) + + +### Graphman + +- **Graphman Deploy Command** - A new `graphman deploy` command has been introduced, simplifying the process of deploying subgraphs to graph-node. [(#4930)](https://github.com/graphprotocol/graph-node/pull/4930) + + + +**Full Changelog**: https://github.com/graphprotocol/graph-node/compare/v0.33.0...v0.34.0 + +## v0.33.0 + +### What's New + +- **Arweave file data sources** - Arweave file data sources allow subgraph developers to access offchain data from Arweave from within the subgraph mappings.[(#4789)](https://github.com/graphprotocol/graph-node/pull/4789) +- **Major performance boost for substreams-based subgraphs** - Significant performance improvements have been achieved for substreams-based subgraphs by moving substreams processing to the block stream.[(#4851)](https://github.com/graphprotocol/graph-node/pull/4851) +- **Polling block handler** - A new block handler filter `polling` for `ethereum` data sources which enables subgraph developers to run a block handler at defined block intervals. This is useful for use cases such as taking periodic snapshots of the contract state.[(#4725)](https://github.com/graphprotocol/graph-node/pull/4725) +- **Initialization handler** - A new block handler filter `once` for `ethereum` data sources which enables subgraph developers to create a handler which will be called only once before all other handlers run. This configuration allows the subgraph to use the handler as an initialization handler, performing specific tasks at the start of indexing. [(#4725)](https://github.com/graphprotocol/graph-node/pull/4725) +- **DataSourceContext in manifest** - `DataSourceContext` in Manifest - DataSourceContext can now be defined in the subgraph manifest. It's a free-form map accessible from the mapping. This feature is useful for templating chain-specific data in subgraphs that use the same codebase across multiple chains.[(#4848)](https://github.com/graphprotocol/graph-node/pull/4848) +- `graph-node` version in index node API - The Index Node API now features a new query, Version, which can be used to query the current graph-node version and commit. [(#4852)](https://github.com/graphprotocol/graph-node/pull/4852) +- Added a '`paused`' field to Index Node API, a boolean indicating the subgraph’s pause status. [(#4779)](https://github.com/graphprotocol/graph-node/pull/4779) +- Proof of Indexing logs now include block number [(#4798)](https://github.com/graphprotocol/graph-node/pull/4798) +- `subgraph_features` table now tracks details about handlers used in a subgraph [(#4820)](https://github.com/graphprotocol/graph-node/pull/4820) +- Configurable SSL for Postgres in Dockerfile - ssl-mode for Postgres can now be configured via the connection string when deploying through Docker, offering enhanced flexibility in database security settings.[(#4840)](https://github.com/graphprotocol/graph-node/pull/4840) +- Introspection Schema Update - The introspection schema has been updated to align with the October 2021 GraphQL specification update.[(#4676)](https://github.com/graphprotocol/graph-node/pull/4676) +- `trace_id` Added to Substreams Logger [(#4868)](https://github.com/graphprotocol/graph-node/pull/4868) +- New apiVersion for Mapping Validation - The latest apiVersion 0.0.8 validates that fields set in entities from the mappings are actually defined in the schema. This fixes a source of non-deterministic PoI. Subgraphs using this new API version will fail if they try to set undefined schema fields in the mappings. Its strongly recommended updating to 0.0.8 to avoid these issues. [(#4894)](https://github.com/graphprotocol/graph-node/pull/4894) +- Substreams Block Ingestor Support - Added the ability to run a pure substreams chain by introducing a block ingestor for substreams-only chains. This feature allows users to run a chain with just a single substreams endpoint, enhancing support beyond RPC and firehose. Prior to this, a pure substreams chain couldn’t be synced.[(#4839)](https://github.com/graphprotocol/graph-node/pull/4839) + +### Bug fixes + +- Fix for rewinding dynamic data source - Resolved an issue where a rewind would fail to properly remove dynamic data sources when using `graphman rewind`. This has been fixed to ensure correct behavior.[(#4810)](https://github.com/graphprotocol/graph-node/pull/4810) +- Improved Deployment Reliability with Retry Mechanism - A retry feature has been added to the block_pointer_from_number function to enhance the robustness of subgraph deployments. This resolves occasional failures encountered during deployment processes.[(#4812)](https://github.com/graphprotocol/graph-node/pull/4812) +- Fixed Cross-Shard Grafting Issue - Addressed a bug that prevented cross-shard grafting from starting, causing the copy operation to stall at 0% progress. This issue occurred when a new shard was added after the primary shard had already been configured. The fix ensures that foreign tables and schemas are correctly set up in new shards. For existing installations experiencing this issue, it can be resolved by running `graphman database remap`.[(#4845)](https://github.com/graphprotocol/graph-node/pull/4845) +- Fixed a Full-text search regression - Reverted a previous commit (ad1c6ea) that inadvertently limited the number of populated search indexes per entity.[(#4808)](https://github.com/graphprotocol/graph-node/pull/4808) +- Attestable Error for Nested Child Filters - Nested child filter queries now return an attestable `ChildFilterNestingNotSupportedError`, improving error reporting for users.[(#4828)](https://github.com/graphprotocol/graph-node/pull/4828) + +### Graphman + +- **Index on prefixed fields** - The graphman index create command now correctly indexes prefixed fields of type String and Bytes for more query-efficient combined indexes. Note: For fields that are references to entities, the behavior may differ. The command may create an index using left(..) when it should index the column directly. +- **Partial Indexing for Recent Blocks** - The graphman index create command now includes a `--after $recent_block` flag for creating partial indexes focused on recent blocks. This enhances query performance similar to the effects of pruning. Queries using these partial indexes must include a specific clause for optimal performance.[(#4830)](https://github.com/graphprotocol/graph-node/pull/4830) + + + +**Full Changelog**: https://github.com/graphprotocol/graph-node/compare/v0.33.0...e253ee14cda2d8456a86ae8f4e3f74a1a7979953 + +## v0.32.0 + +### What's New + +- **Derived fields getter**: Derived fields can now be accessed from within the mapping code during indexing. ([#4434](https://github.com/graphprotocol/graph-node/pull/4434)) +- **Sorting interfaces by child entity**: Interfaces can now be sorted by non-derived child entities. ([#4058](https://github.com/graphprotocol/graph-node/pull/4058)) +- **File data sources can now be spawned from handlers of other file data sources**: This enables the use of file data sources for scenarios where a file data source needs to be spawned from another one. One practical application of this feature is in handling NFT metadata. In such cases, the metadata itself is stored as a file on IPFS and contains embedded IPFS CID for the actual file for the NFT. ([#4713](https://github.com/graphprotocol/graph-node/pull/4713)) +- Allow redeployment of grafted subgraphs even when graft_base is not available: This will allow renaming of already synced grafted subgraphs even when the graft base is not available, which previously failed due to `graft-base` validation errors. ([#4695](https://github.com/graphprotocol/graph-node/pull/4695)) +- `history_blocks` is now available in the index-node API. ([#4662](https://github.com/graphprotocol/graph-node/pull/4662)) +- Added a new `subgraph features` table in `primary` to easily track information like `apiVersion`, `specVersion`, `features`, and data source kinds used by subgraphs. ([#4679](https://github.com/graphprotocol/graph-node/pull/4679)) +- `subgraphFeatures` endpoint now includes data from `subgraph_features` table. +- `ens_name_by_hash` is now undeprecated: This reintroduces support for fetching ENS names by their hash, dependent on the availability of the underlying [Rainbow Table](https://github.com/graphprotocol/ens-rainbow) ([#4751](https://github.com/graphprotocol/graph-node/pull/4751)). +- Deterministically failed subgraphs now return valid POIs for subsequent blocks after the block at which it failed. ([#4774](https://github.com/graphprotocol/graph-node/pull/4774)) +- `eth-call` logs now include block hash and block number: This enables easier debugging of eth-call issues. ([#4718](https://github.com/graphprotocol/graph-node/pull/4718)) +- Enabled support for substreams on already supported networks. ([#4767](https://github.com/graphprotocol/graph-node/pull/4767)) +- Add new GraphQL scalar type `Int8`. This new scalar type allows subgraph developers to represent 8-bit signed integers. ([#4511](https://github.com/graphprotocol/graph-node/pull/4511)) +- Add support for overriding module params for substreams-based subgraphs when params are provided in the subgraph manifest. ([#4759](https://github.com/graphprotocol/graph-node/pull/4759)) + +### Breaking changes + +- Duplicate provider labels are not allowed in graph-node config anymore + +### Bug fixes + +- Fixed `PublicProofsOfIndexing` returning the error `Null value resolved for non-null field proofOfIndexing` when fetching POIs for blocks that are not in the cache ([#4768](https://github.com/graphprotocol/graph-node/pull/4768)) +- Fixed an issue where Block stream would fail when switching back to an RPC-based block ingestor from a Firehose ingestor. ([#4790](https://github.com/graphprotocol/graph-node/pull/4790)) +- Fixed an issue where derived loaders were not working with entities with Bytes as IDs ([#4773](https://github.com/graphprotocol/graph-node/pull/4773)) +- Firehose connection test now retries for 30 secs before setting the provider status to `Broken` ([#4754](https://github.com/graphprotocol/graph-node/pull/4754)) +- Fixed the `nonFatalErrors` field not populating in the index node API. ([#4615](https://github.com/graphprotocol/graph-node/pull/4615)) +- Fixed `graph-node` panicking on the first startup when both Firehose and RPC providers are configured together. ([#4680](https://github.com/graphprotocol/graph-node/pull/4680)) +- Fixed block ingestor failing to startup with the error `net version for chain mainnet has changed from 0 to 1` when switching from Firehose to an RPC provider. ([#4692](https://github.com/graphprotocol/graph-node/pull/4692)) +- Fixed Firehose endpoints getting rate-limited due to duplicated providers during connection pool initialization. ([#4778](https://github.com/graphprotocol/graph-node/pull/4778)) +- Fixed a determinism issue where stale entities were being returned when using `get_many` and `get_derived` ([#4801]https://github.com/graphprotocol/graph-node/pull/4801) + +### Graphman + +- Added two new `graphman` commands `pause` and `resume`: Instead of reassigning to a non-existent node these commands can now be used for pausing and resuming subgraphs. ([#4642](https://github.com/graphprotocol/graph-node/pull/4642)) +- Added a new `graphman` command `restart` to restart a subgraph. ([#4742](https://github.com/graphprotocol/graph-node/pull/4742)) + +**Full Changelog**: https://github.com/graphprotocol/graph-node/compare/v0.31.0...c350e4f35c49bcf8a8b521851f790234ba2c0295 + + + +## v0.31.0 + +### What's new + +- **Fulltext searches can now be combined with `where` filtering**, further narrowing down search results. [#4442](https://github.com/graphprotocol/graph-node/pull/4442) +- Tweaked how RPC provider limiting rules are interpreted from configurations. In particular, node IDs that don't match any rules of a provider won't have access to said provider instead of having access to it for an unlimited number of subgraphs. Read the [docs](https://github.com/graphprotocol/graph-node/pull/4353/files) for more information. [#4353](https://github.com/graphprotocol/graph-node/pull/4353) +- Introduced WASM host function `store.get_in_block`, which is a much faster variant of `store.get` limited to entities created or updated in the current block. [#4540](https://github.com/graphprotocol/graph-node/pull/4540) +- The entity cache that `graph-node` keeps around is much more efficient, meaning more cache entries fit in the same amount of memory resulting in a performance increase under a wide range of workloads. [#4485](https://github.com/graphprotocol/graph-node/pull/4485) +- The `subgraph_deploy` JSON-RPC method now accepts a `history_blocks` parameter, which indexers can use to set default amounts of history to keep. [#4564](https://github.com/graphprotocol/graph-node/pull/4564) +- IPFS requests for polling file data sources are not throttled anymore (also known as concurrency or burst limiting), only rate-limited. [#4570](https://github.com/graphprotocol/graph-node/pull/4570) +- Exponential requests backoff when retrying failed subgraphs is now "jittered", smoothing out request spikes. [#4476](https://github.com/graphprotocol/graph-node/pull/4476) +- RPC provider responses that decrease the chain head block number (non-monotonic) are now ignored, increasing resiliency against inconsistent provider data. [#4354](https://github.com/graphprotocol/graph-node/pull/4354) +- It's now possible to to have a Firehose-only chain with no RPC provider at all in the configuration. [#4508](https://github.com/graphprotocol/graph-node/pull/4508), [#4553](https://github.com/graphprotocol/graph-node/pull/4553) +- The materialized views in the `info` schema (`table_sizes`, `subgraph_sizes`, and `chain_sizes`) that provide information about the size of various database objects are now automatically refreshed every 6 hours. [#4461](https://github.com/graphprotocol/graph-node/pull/4461) +- Adapter selection now takes error rates into account, preferring adapters with lower error rates. [#4468](https://github.com/graphprotocol/graph-node/pull/4468) +- The substreams protocol has been updated to `sf.substreams.rpc.v2.Stream/Blocks`. [#4556](https://github.com/graphprotocol/graph-node/pull/4556) +- Removed support for `GRAPH_ETHEREUM_IS_FIREHOSE_PREFERRED`, `REVERSIBLE_ORDER_BY_OFF`, and `GRAPH_STORE_CONNECTION_TRY_ALWAYS` env. variables. [#4375](https://github.om/graphprotocol/graph-node/pull/4375), [#4436](https://github.com/graphprotocol/graph-node/pull/4436) + +### Bug fixes + +- Fixed a bug that would cause subgraphs to fail with a `subgraph writer poisoned by previous error` message following certain database errors. [#4533](https://github.com/graphprotocol/graph-node/pull/4533) +- Fixed a bug that would cause subgraphs to fail with a `store error: no connection to the server` message when database connection e.g. gets killed. [#4435](https://github.com/graphprotocol/graph-node/pull/4435) +- The `subgraph_reassign` JSON-RPC method doesn't fail anymore when multiple deployment copies are found: only the active copy is reassigned, the others are ignored. [#4395](https://github.com/graphprotocol/graph-node/pull/4395) +- Fixed a bug that would cause `on_sync` handlers on copied deployments to fail with the message `Subgraph instance failed to run: deployment not found [...]`. [#4396](https://github.com/graphprotocol/graph-node/pull/4396) +- Fixed a bug that would cause the copying or grafting of a subgraph while pruning it to incorrectly set `earliest_block` in the destination deployment. [#4502](https://github.com/graphprotocol/graph-node/pull/4502) +- Handler timeouts would sometimes be reported as deterministic errors with the error message `Subgraph instance failed to run: Failed to call 'asc_type_id' with [...] wasm backtrace [...]`; this error is now nondeterministic and recoverable. [#4475](https://github.com/graphprotocol/graph-node/pull/4475) +- Fixed faulty exponential request backoff behavior after many minutes of failed requests, caused by an overflow. [#4421](https://github.com/graphprotocol/graph-node/pull/4421) +- `json.fromBytes` and all `BigInt` operations now require more gas, protecting against malicious subgraphs. [#4594](https://github.com/graphprotocol/graph-node/pull/4594), [#4595](https://github.com/graphprotocol/graph-node/pull/4595) +- Fixed faulty `startBlock` selection logic in substreams. [#4463](https://github.com/graphprotocol/graph-node/pull/4463) + +### Graphman + +- The behavior for `graphman prune` has changed: running just `graphman prune` will mark the subgraph for ongoing pruning in addition to performing an initial pruning. To avoid ongoing pruning, use `graphman prune --once` ([docs](./docs/implementation/pruning.md)). [#4429](https://github.com/graphprotocol/graph-node/pull/4429) +- The env. var. `GRAPH_STORE_HISTORY_COPY_THRESHOLD` –which serves as a configuration setting for `graphman prune`– has been renamed to `GRAPH_STORE_HISTORY_REBUILD_THRESHOLD`. [#4505](https://github.com/graphprotocol/graph-node/pull/4505) +- You can now list all existing deployments via `graphman info --all`. [#4347](https://github.com/graphprotocol/graph-node/pull/4347) +- The command `graphman chain call-cache remove` now requires `--remove-entire-cache` as an explicit flag, protecting against accidental destructive command invocations. [#4397](https://github.com/graphprotocol/graph-node/pull/4397) +- `graphman copy create` accepts two new flags, `--activate` and `--replace`, which make moving of subgraphs across shards much easier. [#4374](https://github.com/graphprotocol/graph-node/pull/4374) +- The log level for `graphman` is now set via `GRAPHMAN_LOG` or command line instead of `GRAPH_LOG`. [#4462](https://github.com/graphprotocol/graph-node/pull/4462) +- `graphman reassign` now emits a warning when it suspects a typo in node IDs. [#4377](https://github.com/graphprotocol/graph-node/pull/4377) + +### Metrics and logging + +- Subgraph syncing time metric `deployment_sync_secs` now stops updating once the subgraph has synced. [#4489](https://github.com/graphprotocol/graph-node/pull/4489) +- New `endpoint_request` metric to track error rates of different providers. [#4490](https://github.com/graphprotocol/graph-node/pull/4490), [#4504](https://github.com/graphprotocol/graph-node/pull/4504), [#4430](https://github.com/graphprotocol/graph-node/pull/4430) +- New metrics `chain_head_cache_num_blocks`, `chain_head_cache_oldest_block`, `chain_head_cache_latest_block`, `chain_head_cache_hits`, and `chain_head_cache_misses` to monitor the effectiveness of `graph-node`'s in-memory chain head caches. [#4440](https://github.com/graphprotocol/graph-node/pull/4440) +- The subgraph error message `store error: Failed to remove entities` is now more detailed and contains more useful information. [#4367](https://github.com/graphprotocol/graph-node/pull/4367) +- `eth_call` logs now include the provider string. [#4548](https://github.com/graphprotocol/graph-node/pull/4548) +- Tweaks and small changes to log messages when resolving data sources, mappings, and manifests. [#4399](https://github.com/graphprotocol/graph-node/pull/4399) +- `FirehoseBlockStream` and `FirehoseBlockIngestor` now log adapter names. [#4411](https://github.com/graphprotocol/graph-node/pull/4411) +- The `deployment_count` metric has been split into `deployment_running_count` and `deployment_count`. [#4401](https://github.com/grahprotocol/graph-node/pull/4401), [#4398](https://github.com/graphprotocol/graph-node/pul/4398) + + + +**Full Changelog**: https://github.com/graphprotocol/graph-node/compare/v0.30.0...aa6677a38 ## v0.30.0 @@ -22,7 +788,7 @@ New `graph-node` installations now **mandate** PostgreSQL to use C locale and UT - Lots of visual and filtering improvements to [#4232](https://github.com/graphprotocol/graph-node/pull/4232) - More aggressive in-memory caching of blocks close the chain head, potentially alleviating database load. [#4215](https://github.com/graphprotocol/graph-node/pull/4215) - New counter Prometheus metric `query_validation_error_counter`, labelled by deployment ID and error code. [#4230](https://github.com/graphprotocol/graph-node/pull/4230) -graph_elasticsearch_logs_sent + graph_elasticsearch_logs_sent - Turned "Flushing logs to Elasticsearch" log into a Prometheus metric (`graph_elasticsearch_logs_sent`) to reduce log noise. [#4333](https://github.com/graphprotocol/graph-node/pull/4333) - New materialized view `info.chain_sizes`, which works the same way as the already existing `info.subgraph_sizes` and `info.table_sizes`. [#4318](https://github.com/graphprotocol/graph-node/pull/4318) - New `graphman stats` subcommands `set-target` and `target` to manage statistics targets for specific deployments (i.e. how much data PostgreSQL samples when analyzing a table). [#4092](https://github.com/graphprotocol/graph-node/pull/4092) @@ -87,15 +853,15 @@ Dependency upgrades: - `Qmccst5mbV5a6vT6VvJMLPKMAA1VRgT6NGbxkLL8eDRsE7` - `Qmd9nZKCH8UZU1pBzk7G8ECJr3jX3a2vAf3vowuTwFvrQg` - - Here's an example [manifest](https://ipfs.io/ipfs/Qmd9nZKCH8UZU1pBzk7G8ECJr3jX3a2vAf3vowuTwFvrQg), taking a look at the data sources of name `ERC721` and `CryptoKitties`, both listen to the `Transfer(...)` event. Considering a block where there's only one occurence of this event, `graph-node` would duplicate it and call `handleTransfer` twice. Now this is fixed and it will be called only once per event/call that happened on chain. + + Here's an example [manifest](https://ipfs.io/ipfs/Qmd9nZKCH8UZU1pBzk7G8ECJr3jX3a2vAf3vowuTwFvrQg), taking a look at the data sources of name `ERC721` and `CryptoKitties`, both listen to the `Transfer(...)` event. Considering a block where there's only one occurrence of this event, `graph-node` would duplicate it and call `handleTransfer` twice. Now this is fixed and it will be called only once per event/call that happened on chain. In the case you're indexing one of the impacted subgraphs, you should first upgrade the `graph-node` version, then rewind the affected subgraphs to the smallest `startBlock` of their subgraph manifest. To achieve that the `graphman rewind` CLI command can be used. See [#4055](https://github.com/graphprotocol/graph-node/pull/4055) for more information. * This release fixes another determinism bug that affects a handful of subgraphs. The bug affects all subgraphs which have an `apiVersion` **older than** 0.0.5 using call handlers. While call handlers prior to 0.0.5 should be triggered by both failed and successful transactions, in some cases failed transactions would not trigger the handlers. This resulted in nondeterministic behavior. With this version of `graph-node`, call handlers with an `apiVersion` older than 0.0.5 will always be triggered by both successful and failed transactions. Behavior for `apiVersion` 0.0.5 onward is not affected. - + The affected subgraphs are: - `QmNY7gDNXHECV8SXoEY7hbfg4BX1aDMxTBDiFuG4huaSGA` @@ -111,41 +877,44 @@ Dependency upgrades: ### What's new -* Grafted subgraphs can now add their own data sources. [#3989](https://github.com/graphprotocol/graph-node/pull/3989), [#4027](https://github.com/graphprotocol/graph-node/pull/4027), [#4030](https://github.com/graphprotocol/graph-node/pull/4030) -* Add support for filtering by nested interfaces. [#3677](https://github.com/graphprotocol/graph-node/pull/3677) -* Add support for message handlers in Cosmos [#3975](https://github.com/graphprotocol/graph-node/pull/3975) -* Dynamic data sources for Firehose-backed subgraphs. [#4075](https://github.com/graphprotocol/graph-node/pull/4075) -* Various logging improvements. [#4078](https://github.com/graphprotocol/graph-node/pull/4078), [#4084](https://github.com/graphprotocol/graph-node/pull/4084), [#4031](https://github.com/graphprotocol/graph-node/pull/4031), [#4144](https://github.com/graphprotocol/graph-node/pull/4144), [#3990](https://github.com/graphprotocol/graph-node/pull/3990) -* Some DB queries now have GCP Cloud Insight -compliant tags that show where the query originated from. [#4079](https://github.com/graphprotocol/graph-node/pull/4079) -* New configuration variable `GRAPH_STATIC_FILTERS_THRESHOLD` to conditionally enable static filtering based on the number of dynamic data sources. [#4008](https://github.com/graphprotocol/graph-node/pull/4008) -* New configuration variable `GRAPH_STORE_BATCH_TARGET_DURATION`. [#4133](https://github.com/graphprotocol/graph-node/pull/4133) +- Grafted subgraphs can now add their own data sources. [#3989](https://github.com/graphprotocol/graph-node/pull/3989), [#4027](https://github.com/graphprotocol/graph-node/pull/4027), [#4030](https://github.com/graphprotocol/graph-node/pull/4030) +- Add support for filtering by nested interfaces. [#3677](https://github.com/graphprotocol/graph-node/pull/3677) +- Add support for message handlers in Cosmos [#3975](https://github.com/graphprotocol/graph-node/pull/3975) +- Dynamic data sources for Firehose-backed subgraphs. [#4075](https://github.com/graphprotocol/graph-node/pull/4075) +- Various logging improvements. [#4078](https://github.com/graphprotocol/graph-node/pull/4078), [#4084](https://github.com/graphprotocol/graph-node/pull/4084), [#4031](https://github.com/graphprotocol/graph-node/pull/4031), [#4144](https://github.com/graphprotocol/graph-node/pull/4144), [#3990](https://github.com/graphprotocol/graph-node/pull/3990) +- Some DB queries now have GCP Cloud Insight -compliant tags that show where the query originated from. [#4079](https://github.com/graphprotocol/graph-node/pull/4079) +- New configuration variable `GRAPH_STATIC_FILTERS_THRESHOLD` to conditionally enable static filtering based on the number of dynamic data sources. [#4008](https://github.com/graphprotocol/graph-node/pull/4008) +- New configuration variable `GRAPH_STORE_BATCH_TARGET_DURATION`. [#4133](https://github.com/graphprotocol/graph-node/pull/4133) #### Docker image -* The official Docker image now runs on Debian 11 "Bullseye". [#4081](https://github.com/graphprotocol/graph-node/pull/4081) -* We now ship [`envsubst`](https://github.com/a8m/envsubst) with the official Docker image, allowing you to easily run templating logic on your configuration files. [#3974](https://github.com/graphprotocol/graph-node/pull/3974) + +- The official Docker image now runs on Debian 11 "Bullseye". [#4081](https://github.com/graphprotocol/graph-node/pull/4081) +- We now ship [`envsubst`](https://github.com/a8m/envsubst) with the official Docker image, allowing you to easily run templating logic on your configuration files. [#3974](https://github.com/graphprotocol/graph-node/pull/3974) #### Graphman We have a new documentation page for `graphman`, check it out [here](https://github.com/graphprotocol/graph-node/blob/2da697b1af17b1c947679d1b1a124628146545a6/docs/graphman.md)! -* Subgraph pruning with `graphman`! [#3898](https://github.com/graphprotocol/graph-node/pull/3898), [#4125](https://github.com/graphprotocol/graph-node/pull/4125), [#4153](https://github.com/graphprotocol/graph-node/pull/4153), [#4152](https://github.com/graphprotocol/graph-node/pull/4152), [#4156](https://github.com/graphprotocol/graph-node/pull/4156), [#4041](https://github.com/graphprotocol/graph-node/pull/4041) -* New command `graphman drop` to hastily delete a subgraph deployment. [#4035](https://github.com/graphprotocol/graph-node/pull/4035) -* New command `graphman chain call-cache` for clearing the call cache for a given chain. [#4066](https://github.com/graphprotocol/graph-node/pull/4066) -* Add `--delete-duplicates` flag to `graphman check-blocks` by @tilacog in https://github.com/graphprotocol/graph-node/pull/3988 +- Subgraph pruning with `graphman`! [#3898](https://github.com/graphprotocol/graph-node/pull/3898), [#4125](https://github.com/graphprotocol/graph-node/pull/4125), [#4153](https://github.com/graphprotocol/graph-node/pull/4153), [#4152](https://github.com/graphprotocol/graph-node/pull/4152), [#4156](https://github.com/graphprotocol/graph-node/pull/4156), [#4041](https://github.com/graphprotocol/graph-node/pull/4041) +- New command `graphman drop` to hastily delete a subgraph deployment. [#4035](https://github.com/graphprotocol/graph-node/pull/4035) +- New command `graphman chain call-cache` for clearing the call cache for a given chain. [#4066](https://github.com/graphprotocol/graph-node/pull/4066) +- Add `--delete-duplicates` flag to `graphman check-blocks` by @tilacog in https://github.com/graphprotocol/graph-node/pull/3988 #### Performance -* Restarting a node now takes much less time because `postgres_fdw` user mappings are only rebuilt upon schema changes. If necessary, you can also use the new commands `graphman database migrate` and `graphman database remap` to respectively apply schema migrations or run remappings manually. [#4009](https://github.com/graphprotocol/graph-node/pull/4009), [#4076](https://github.com/graphprotocol/graph-node/pull/4076) -* Database replicas now won't fall behind as much when copying subgraph data. [#3966](https://github.com/graphprotocol/graph-node/pull/3966) [#3986](https://github.com/graphprotocol/graph-node/pull/3986) -* Block handlers optimization with Firehose >= 1.1.0. [#3971](https://github.com/graphprotocol/graph-node/pull/3971) -* Reduced the amount of data that a non-primary shard has to mirror from the primary shard. [#4015](https://github.com/graphprotocol/graph-node/pull/4015) -* We now use advisory locks to lock deployments' tables against concurrent writes. [#4010](https://github.com/graphprotocol/graph-node/pull/4010) + +- Restarting a node now takes much less time because `postgres_fdw` user mappings are only rebuilt upon schema changes. If necessary, you can also use the new commands `graphman database migrate` and `graphman database remap` to respectively apply schema migrations or run remappings manually. [#4009](https://github.com/graphprotocol/graph-node/pull/4009), [#4076](https://github.com/graphprotocol/graph-node/pull/4076) +- Database replicas now won't fall behind as much when copying subgraph data. [#3966](https://github.com/graphprotocol/graph-node/pull/3966) [#3986](https://github.com/graphprotocol/graph-node/pull/3986) +- Block handlers optimization with Firehose >= 1.1.0. [#3971](https://github.com/graphprotocol/graph-node/pull/3971) +- Reduced the amount of data that a non-primary shard has to mirror from the primary shard. [#4015](https://github.com/graphprotocol/graph-node/pull/4015) +- We now use advisory locks to lock deployments' tables against concurrent writes. [#4010](https://github.com/graphprotocol/graph-node/pull/4010) #### Bug fixes -* Fixed a bug that would cause some failed subgraphs to never restart. [#3959](https://github.com/graphprotocol/graph-node/pull/3959) -* Fixed a bug that would cause bad POIs for Firehose-backed subgraphs when processing `CREATE` calls. [#4085](https://github.com/graphprotocol/graph-node/pull/4085) -* Fixed a bug which would cause failure to redeploy a subgraph immediately after deletion. [#4044](https://github.com/graphprotocol/graph-node/pull/4044) -* Firehose connections are now load-balanced. [#4083](https://github.com/graphprotocol/graph-node/pull/4083) -* Determinism fixes. **See above.** [#4055](https://github.com/graphprotocol/graph-node/pull/4055), [#4149](https://github.com/graphprotocol/graph-node/pull/4149) + +- Fixed a bug that would cause some failed subgraphs to never restart. [#3959](https://github.com/graphprotocol/graph-node/pull/3959) +- Fixed a bug that would cause bad POIs for Firehose-backed subgraphs when processing `CREATE` calls. [#4085](https://github.com/graphprotocol/graph-node/pull/4085) +- Fixed a bug which would cause failure to redeploy a subgraph immediately after deletion. [#4044](https://github.com/graphprotocol/graph-node/pull/4044) +- Firehose connections are now load-balanced. [#4083](https://github.com/graphprotocol/graph-node/pull/4083) +- Determinism fixes. **See above.** [#4055](https://github.com/graphprotocol/graph-node/pull/4055), [#4149](https://github.com/graphprotocol/graph-node/pull/4149) #### Dependency updates @@ -431,12 +1200,11 @@ These are some of the features that will probably be helpful for indexers 😊 - A token can be set via `GRAPH_POI_ACCESS_TOKEN` to limit access to the POI route - The new `graphman` commands 🙂 - ### Api Version 0.0.7 and Spec Version 0.0.5 + This release brings API Version 0.0.7 in mappings, which allows Ethereum event handlers to require transaction receipts to be present in the `Event` object. Refer to [PR #3373](https://github.com/graphprotocol/graph-node/pull/3373) for instructions on how to enable that. - ## 0.25.2 This release includes two changes: @@ -458,20 +1226,22 @@ We strongly recommend updating to this version as quickly as possible. ## 0.25.0 ### Api Version 0.0.6 + This release ships support for API version 0.0.6 in mappings: + - Added `nonce` field for `Transaction` objects. - Added `baseFeePerGas` field for `Block` objects ([EIP-1559](https://eips.ethereum.org/EIPS/eip-1559)). #### Block Cache Invalidation and Reset -All cached block data must be refetched to account for the new `Block` and `Trasaction` +All cached block data must be refetched to account for the new `Block` and `Transaction` struct versions, so this release includes a `graph-node` startup check that will: + 1. Truncate all block cache tables. 2. Bump the `db_version` value from `2` to `3`. _(Table truncation is a fast operation and no downtime will occur because of that.)_ - ### Ethereum - 'Out of gas' errors on contract calls are now considered deterministic errors, @@ -483,10 +1253,12 @@ _(Table truncation is a fast operation and no downtime will occur because of tha is now hardcoded to 50 million. ### Multiblockchain + - Initial support for NEAR subgraphs. - Added `FirehoseBlockStream` implementation of `BlockStream` (#2716) ### Misc + - Rust docker image is now based on Debian Buster. - Optimizations to the PostgreSQL notification queue. - Improve PostgreSQL robustness in multi-sharded setups. (#2815) @@ -501,7 +1273,6 @@ _(Table truncation is a fast operation and no downtime will occur because of tha - Handle revert cases from Hardhat and Ganache (#2984) - Fix bug on experimental prefetching optimization feature (#2899) - ## 0.24.2 This release only adds a fix for an issue where certain GraphQL queries @@ -538,7 +1309,9 @@ For instance, the following query... ```graphql { - subgraphFeatures(subgraphId: "QmW9ajg2oTyPfdWKyUkxc7cTJejwdyCbRrSivfryTfFe5D") { + subgraphFeatures( + subgraphId: "QmW9ajg2oTyPfdWKyUkxc7cTJejwdyCbRrSivfryTfFe5D" + ) { features errors } @@ -552,10 +1325,7 @@ For instance, the following query... "data": { "subgraphFeatures": { "errors": [], - "features": [ - "nonFatalErrors", - "ipfsOnEthereumContracts" - ] + "features": ["nonFatalErrors", "ipfsOnEthereumContracts"] } } } @@ -595,14 +1365,17 @@ and the long awaited AssemblyScript version upgrade! resolving issue [#2409](https://github.com/graphprotocol/graph-node/issues/2409). Done in [#2511](https://github.com/graphprotocol/graph-node/pull/2511). ### Logs + - The log `"Skipping handler because the event parameters do not match the event signature."` was downgraded from info to trace level. - Some block ingestor error logs were upgrded from debug to info level [#2666](https://github.com/graphprotocol/graph-node/pull/2666). ### Metrics + - `query_semaphore_wait_ms` is now by shard, and has the `pool` and `shard` labels. - `deployment_failed` metric added, it is `1` if the subgraph has failed and `0` otherwise. ### Other + - Upgrade to tokio 1.0 and futures 0.3 [#2679](https://github.com/graphprotocol/graph-node/pull/2679), the first major contribution by StreamingFast! - Support Celo block reward events [#2670](https://github.com/graphprotocol/graph-node/pull/2670). - Reduce the maximum WASM stack size and make it configurable [#2719](https://github.com/graphprotocol/graph-node/pull/2719). @@ -637,14 +1410,17 @@ In the meantime, here are the changes for this release: - Using `ethereum.call` in mappings in globals is deprecated ### Graphman + Graphman is a CLI tool to manage your subgraphs. It is now included in the Docker container [#2289](https://github.com/graphprotocol/graph-node/pull/2289). And new commands have been added: + - `graphman copy` can copy subgraphs across DB shards [#2313](https://github.com/graphprotocol/graph-node/pull/2313). - `graphman rewind` to rewind a deployment to a given block [#2373](https://github.com/graphprotocol/graph-node/pull/2373). - `graphman query` to log info about a GraphQL query [#2206](https://github.com/graphprotocol/graph-node/pull/2206). - `graphman create` to create a subgraph name [#2419](https://github.com/graphprotocol/graph-node/pull/2419). ### Metrics + - The `deployment_blocks_behind` metric has been removed, and a `deployment_head` metric has been added. To see how far a deployment is behind, use the difference between `ethereum_chain_head_number` and @@ -654,6 +1430,7 @@ Graphman is a CLI tool to manage your subgraphs. It is now included in the Docke ## 0.22.0 ### Feature: Block store sharding + This release makes it possible to [shard the block and call cache](./docs/config.md) for chain data across multiple independent Postgres databases. **This feature is considered experimental. We encourage users to try this out in a test environment, but do not recommend it yet for production @@ -661,17 +1438,20 @@ use.** In particular, the details of how sharding is configured may change in ba ways in the future. ### Feature: Non-fatal errors update + Non-fatal errors (see release 0.20 for details) is documented and can now be enabled on graph-cli. Various related bug fixes have been made #2121 #2136 #2149 #2160. ### Improvements + - Add bitwise operations and string constructor to BigInt #2151. - docker: Allow custom ethereum poll interval #2139. - Deterministic error work in preparation for gas #2112 ### Bug fixes + - Fix not contains filter #2146. -- Resolve __typename in _meta field #2118 +- Resolve \_\_typename in \_meta field #2118 - Add CORS for all HTTP responses #2196 ## 0.21.1 @@ -690,7 +1470,7 @@ storage](./docs/config.md) and spread subgraph deployments, and the load coming from indexing and querying them across multiple independent Postgres databases. -**This feature is considered experimenatal. We encourage users to try this +**This feature is considered experimental. We encourage users to try this out in a test environment, but do not recommend it yet for production use** In particular, the details of how sharding is configured may change in backwards-incompatible ways in the future. diff --git a/README.md b/README.md index 76cb9f6f392..667c606a045 100644 --- a/README.md +++ b/README.md @@ -1,189 +1,157 @@ # Graph Node [![Build Status](https://github.com/graphprotocol/graph-node/actions/workflows/ci.yml/badge.svg)](https://github.com/graphprotocol/graph-node/actions/workflows/ci.yml?query=branch%3Amaster) -[![Getting Started Docs](https://img.shields.io/badge/docs-getting--started-brightgreen.svg)](docs/getting-started.md) +[![Docs](https://img.shields.io/badge/docs-graph--node-green.svg)](docs/) +[![Subgraphs](https://img.shields.io/badge/docs-subgraphs-green.svg)](https://thegraph.com/docs/en/subgraphs/quick-start/) -[The Graph](https://thegraph.com/) is a protocol for building decentralized applications (dApps) quickly on Ethereum and IPFS using GraphQL. +## Overview -Graph Node is an open source Rust implementation that event sources the Ethereum blockchain to deterministically update a data store that can be queried via the GraphQL endpoint. +[The Graph](https://thegraph.com/) is a decentralized protocol that organizes and distributes blockchain data across the leading Web3 networks. A key component of The Graph's tech stack is Graph Node. -For detailed instructions and more context, check out the [Getting Started Guide](docs/getting-started.md). +Before using `graph-node,` it is highly recommended that you read the [official Graph documentation](https://thegraph.com/docs/en/subgraphs/quick-start/) to understand Subgraphs, which are the central mechanism for extracting and organizing blockchain data. -## Quick Start +This guide is for: -### Prerequisites +1. Subgraph developers who want to run `graph-node` locally to test their Subgraphs during development +2. Contributors who want to add features or fix bugs to `graph-node` itself -To build and run this project you need to have the following installed on your system: +## Running `graph-node` from Docker images -- Rust (latest stable) – [How to install Rust](https://www.rust-lang.org/en-US/install.html) - - Note that `rustfmt`, which is part of the default Rust installation, is a build-time requirement. -- PostgreSQL – [PostgreSQL Downloads](https://www.postgresql.org/download/) -- IPFS – [Installing IPFS](https://docs.ipfs.io/install/) +For subgraph developers, it is highly recommended to use prebuilt Docker +images to set up a local `graph-node` environment. Please read [these +instructions](./docker/README.md) to learn how to do that. -For Ethereum network data, you can either run your own Ethereum node or use an Ethereum node provider of your choice. +## Running `graph-node` from source -**Minimum Hardware Requirements:** +This is usually only needed for developers who want to contribute to `graph-node`. -- To build graph-node with `cargo`, 8GB RAM are required. +### Prerequisites -### Running a Local Graph Node +To build and run this project, you need to have the following installed on your system: -This is a quick example to show a working Graph Node. It is a [subgraph for Gravatars](https://github.com/graphprotocol/example-subgraph). +- Rust (latest stable): Follow [How to install + Rust](https://rust-lang.org/tools/install/). Run `rustup install +stable` in _this directory_ to make sure all required components are + installed. The `graph-node` code assumes that the latest available + `stable` compiler is used. +- PostgreSQL: [PostgreSQL Downloads](https://www.postgresql.org/download/) lists + downloads for almost all operating systems. + - For OSX: We highly recommend [Postgres.app](https://postgresapp.com/). + - For Linux: Use the Postgres version that comes with the distribution. +- IPFS: [Installing IPFS](https://docs.ipfs.io/install/) +- Protobuf Compiler: [Installing Protobuf](https://grpc.io/docs/protoc-installation/) -1. Install IPFS and run `ipfs init` followed by `ipfs daemon`. -2. Install PostgreSQL and run `initdb -D .postgres` followed by `pg_ctl -D .postgres -l logfile start` and `createdb graph-node`. -3. If using Ubuntu, you may need to install additional packages: - - `sudo apt-get install -y clang libpq-dev libssl-dev pkg-config` -4. In the terminal, clone https://github.com/graphprotocol/example-subgraph, and install dependencies and generate types for contract ABIs: +For Ethereum network data, you can either run your own Ethereum node or use an Ethereum node provider of your choice. -``` -yarn -yarn codegen -``` +### Create a database -5. In the terminal, clone https://github.com/graphprotocol/graph-node, and run `cargo build`. +Once Postgres is running, you need to issue the following commands to create a database +and configure it for use with `graph-node`. -Once you have all the dependencies set up, you can run the following: - -``` -cargo run -p graph-node --release -- \ - --postgres-url postgresql://USERNAME[:PASSWORD]@localhost:5432/graph-node \ - --ethereum-rpc NETWORK_NAME:[CAPABILITIES]:URL \ - --ipfs 127.0.0.1:5001 -``` - -Try your OS username as `USERNAME` and `PASSWORD`. For details on setting -the connection string, check the [Postgres -documentation](https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING). -`graph-node` uses a few Postgres extensions. If the Postgres user with which -you run `graph-node` is a superuser, `graph-node` will enable these -extensions when it initalizes the database. If the Postgres user is not a -superuser, you will need to create the extensions manually since only -superusers are allowed to do that. To create them you need to connect as a -superuser, which in many installations is the `postgres` user: +The name of the `SUPERUSER` depends on your installation, but is usually `postgres` or your username. ```bash - psql -q -X -U graph-node < <'; +create database "graph-node" with owner=graph template=template0 encoding='UTF8' locale='C'; create extension pg_trgm; -create extension pg_stat_statements; create extension btree_gist; create extension postgres_fdw; -grant usage on foreign data wrapper postgres_fdw to ; +grant usage on foreign data wrapper postgres_fdw to graph; EOF - ``` -This will also spin up a GraphiQL interface at `http://127.0.0.1:8000/`. - -6. With this Gravatar example, to get the subgraph working locally run: +For convenience, set the connection string to the database in an environment +variable, and save it, e.g., in `~/.bashrc`: -``` -yarn create-local +```bash +export POSTGRES_URL=postgresql://graph:@localhost:5432/graph-node ``` -Then you can deploy the subgraph: +Use the `POSTGRES_URL` from above to have `graph-node` connect to the +database. If you ever need to manually inspect the contents of your +database, you can do that by running `psql $POSTGRES_URL`. Running this +command is also a convenient way to check that the database is up and +running and that the connection string is correct. -``` -yarn deploy-local +### Build and Run `graph-node` + +Clone this repository and run this command at the root of the repository: + +```bash +export GRAPH_LOG=debug +cargo run -p graph-node --release -- \ + --postgres-url $POSTGRES_URL \ + --ethereum-rpc NETWORK_NAME:[CAPABILITIES]:URL \ + --ipfs 127.0.0.1:5001 ``` -This will build and deploy the subgraph to the Graph Node. It should start indexing the subgraph immediately. +The argument for `--ethereum-rpc` contains a network name (e.g. `mainnet`) and +a list of provider capabilities (e.g. `archive,traces`). The URL is the address +of the Ethereum node you want to connect to, usually a `https` URL, so that the +entire argument might be `mainnet:archive,traces:https://provider.io/some/path`. -### Command-Line Interface +When `graph-node` starts, it prints the various ports that it is listening on. +The most important of these is the GraphQL HTTP server, which by default +is at `http://localhost:8000`. You can use routes like `/subgraphs/name/` +and `/subgraphs/id/` to query subgraphs once you have deployed them. -``` -USAGE: - graph-node [FLAGS] [OPTIONS] --ethereum-ipc --ethereum-rpc --ethereum-ws --ipfs --postgres-url +### Deploying a Subgraph -FLAGS: - --debug Enable debug logging - -h, --help Prints help information - -V, --version Prints version information +Follow the [Subgraph deployment +guide](https://thegraph.com/docs/en/subgraphs/developing/introduction/). +After setting up `graph-cli` as described, you can deploy a Subgraph to your +local Graph Node instance. -OPTIONS: - --admin-port Port for the JSON-RPC admin server [default: 8020] - --elasticsearch-password - Password to use for Elasticsearch logging [env: ELASTICSEARCH_PASSWORD] +### Advanced Configuration - --elasticsearch-url - Elasticsearch service to write subgraph logs to [env: ELASTICSEARCH_URL=] +The command line arguments generally are all that is needed to run a +`graph-node` instance. For advanced uses, various aspects of `graph-node` +can further be configured through [environment +variables](https://github.com/graphprotocol/graph-node/blob/master/docs/environment-variables.md). - --elasticsearch-user User to use for Elasticsearch logging [env: ELASTICSEARCH_USER=] - --ethereum-ipc - Ethereum network name (e.g. 'mainnet'), optional comma-seperated capabilities (eg full,archive), and an Ethereum IPC pipe, separated by a ':' +Very large `graph-node` instances can also be configured using a +[configuration file](./docs/config.md) That is usually only necessary when +the `graph-node` needs to connect to multiple chains or if the work of +indexing and querying needs to be split across [multiple databases](./docs/config.md). - --ethereum-polling-interval - How often to poll the Ethereum node for new blocks [env: ETHEREUM_POLLING_INTERVAL=] [default: 500] +#### Log Storage - --ethereum-rpc - Ethereum network name (e.g. 'mainnet'), optional comma-seperated capabilities (eg 'full,archive'), and an Ethereum RPC URL, separated by a ':' +`graph-node` supports storing and querying subgraph logs through multiple backends: - --ethereum-ws - Ethereum network name (e.g. 'mainnet'), optional comma-seperated capabilities (eg `full,archive), and an Ethereum WebSocket URL, separated by a ':' +- **File**: Local JSON Lines files (recommended for local development) +- **Elasticsearch**: Enterprise-grade search and analytics (for production) +- **Loki**: Grafana's log aggregation system (for production) +- **Disabled**: No log storage (default) - --node-id - A unique identifier for this node instance. Should have the same value between consecutive node restarts [default: default] +**Quick example (file-based logs for local development):** +```bash +mkdir -p ./graph-logs - --http-port Port for the GraphQL HTTP server [default: 8000] - --ipfs HTTP address of an IPFS node - --postgres-url Location of the Postgres database used for storing entities - --subgraph <[NAME:]IPFS_HASH> Name and IPFS hash of the subgraph manifest - --ws-port Port for the GraphQL WebSocket server [default: 8001] +cargo run -p graph-node --release -- \ + --postgres-url $POSTGRES_URL \ + --ethereum-rpc mainnet:archive:https://... \ + --ipfs 127.0.0.1:5001 \ + --log-store-backend file \ + --log-store-file-dir ./graph-logs ``` -### Advanced Configuration - -The command line arguments generally are all that is needed to run a -`graph-node` instance. For advanced uses, various aspects of `graph-node` -can further be configured through [environment -variables](https://github.com/graphprotocol/graph-node/blob/master/docs/environment-variables.md). Very -large `graph-node` instances can also split the work of querying and -indexing across [multiple databases](./docs/config.md). - -## Project Layout - -- `node` — A local Graph Node. -- `graph` — A library providing traits for system components and types for - common data. -- `core` — A library providing implementations for core components, used by all - nodes. -- `chain/ethereum` — A library with components for obtaining data from - Ethereum. -- `graphql` — A GraphQL implementation with API schema generation, - introspection, and more. -- `mock` — A library providing mock implementations for all system components. -- `runtime/wasm` — A library for running WASM data-extraction scripts. -- `server/http` — A library providing a GraphQL server over HTTP. -- `store/postgres` — A Postgres store with a GraphQL-friendly interface - and audit logs. - -## Roadmap - -🔨 = In Progress - -🛠 = Feature complete. Additional testing required. - -✅ = Feature complete - - -| Feature | Status | -| ------- | :------: | -| **Ethereum** | | -| Indexing smart contract events | ✅ | -| Handle chain reorganizations | ✅ | -| **Mappings** | | -| WASM-based mappings| ✅ | -| TypeScript-to-WASM toolchain | ✅ | -| Autogenerated TypeScript types | ✅ | -| **GraphQL** | | -| Query entities by ID | ✅ | -| Query entity collections | ✅ | -| Pagination | ✅ | -| Filtering | ✅ | -| Block-based Filtering | ✅ | -| Entity relationships | ✅ | -| Subscriptions | ✅ | +Logs are queried via GraphQL at `http://localhost:8000/graphql`: +```graphql +query { + _logs(subgraphId: "QmYourSubgraphHash", level: ERROR, first: 10) { + timestamp + level + text + } +} +``` +**For complete documentation**, see the **[Log Store Guide](./docs/log-store.md)**, which covers: +- How to configure each backend (Elasticsearch, Loki, File) +- Complete GraphQL query examples +- Choosing the right backend for your use case +- Performance considerations and best practices ## Contributing diff --git a/chain/arweave/.gitignore b/chain/arweave/.gitignore deleted file mode 100644 index 97442b5f148..00000000000 --- a/chain/arweave/.gitignore +++ /dev/null @@ -1 +0,0 @@ -google.protobuf.rs \ No newline at end of file diff --git a/chain/arweave/Cargo.toml b/chain/arweave/Cargo.toml deleted file mode 100644 index c969e0a400a..00000000000 --- a/chain/arweave/Cargo.toml +++ /dev/null @@ -1,21 +0,0 @@ -[package] -name = "graph-chain-arweave" -version.workspace = true -edition.workspace = true - -[build-dependencies] -tonic-build = { workspace = true } - -[dependencies] -base64-url = "1.4.13" -graph = { path = "../../graph" } -prost = { workspace = true } -prost-types = { workspace = true } -serde = "1.0" -sha2 = "0.10.6" - -graph-runtime-wasm = { path = "../../runtime/wasm" } -graph-runtime-derive = { path = "../../runtime/derive" } - -[dev-dependencies] -diesel = { version = "1.4.7", features = ["postgres", "serde_json", "numeric", "r2d2"] } diff --git a/chain/arweave/build.rs b/chain/arweave/build.rs deleted file mode 100644 index e2ede2acef2..00000000000 --- a/chain/arweave/build.rs +++ /dev/null @@ -1,7 +0,0 @@ -fn main() { - println!("cargo:rerun-if-changed=proto"); - tonic_build::configure() - .out_dir("src/protobuf") - .compile(&["proto/type.proto"], &["proto"]) - .expect("Failed to compile Firehose Arweave proto(s)"); -} diff --git a/chain/arweave/proto/type.proto b/chain/arweave/proto/type.proto deleted file mode 100644 index b3a41a4a56a..00000000000 --- a/chain/arweave/proto/type.proto +++ /dev/null @@ -1,108 +0,0 @@ -syntax = "proto3"; - -package sf.arweave.type.v1; - -option go_package = "github.com/ChainSafe/firehose-arweave/pb/sf/arweave/type/v1;pbcodec"; - -message BigInt { - bytes bytes = 1; -} - -message Block { - // Firehose block version (unrelated to Arweave block version) - uint32 ver = 1; - // The block identifier - bytes indep_hash = 2; - // The nonce chosen to solve the mining problem - bytes nonce = 3; - // `indep_hash` of the previous block in the weave - bytes previous_block = 4; - // POSIX time of block discovery - uint64 timestamp = 5; - // POSIX time of the last difficulty retarget - uint64 last_retarget = 6; - // Mining difficulty; the number `hash` must be greater than. - BigInt diff = 7; - // How many blocks have passed since the genesis block - uint64 height = 8; - // Mining solution hash of the block; must satisfy the mining difficulty - bytes hash = 9; - // Merkle root of the tree of Merkle roots of block's transactions' data. - bytes tx_root = 10; - // Transactions contained within this block - repeated Transaction txs = 11; - // The root hash of the Merkle Patricia Tree containing - // all wallet (account) balances and the identifiers - // of the last transactions posted by them; if any. - bytes wallet_list = 12; - // (string or) Address of the account to receive the block rewards. Can also be unclaimed which is encoded as a null byte - bytes reward_addr = 13; - // Tags that a block producer can add to a block - repeated Tag tags = 14; - // Size of reward pool - BigInt reward_pool = 15; - // Size of the weave in bytes - BigInt weave_size = 16; - // Size of this block in bytes - BigInt block_size = 17; - // Required after the version 1.8 fork. Zero otherwise. - // The sum of the average number of hashes computed - // by the network to produce the past blocks including this one. - BigInt cumulative_diff = 18; - // Required after the version 1.8 fork. Null byte otherwise. - // The Merkle root of the block index - the list of {`indep_hash`; `weave_size`; `tx_root`} triplets - bytes hash_list_merkle = 20; - // The proof of access; Used after v2.4 only; set as defaults otherwise - ProofOfAccess poa = 21; -} - -// A succinct proof of access to a recall byte found in a TX -message ProofOfAccess { - // The recall byte option chosen; global offset of index byte - string option = 1; - // The path through the Merkle tree of transactions' `data_root`s; - // from the `data_root` being proven to the corresponding `tx_root` - bytes tx_path = 2; - // The path through the Merkle tree of identifiers of chunks of the - // corresponding transaction; from the chunk being proven to the - // corresponding `data_root`. - bytes data_path = 3; - // The data chunk. - bytes chunk = 4; -} - -message Transaction { - // 1 or 2 for v1 or v2 transactions. More allowable in the future - uint32 format = 1; - // The transaction identifier. - bytes id = 2; - // Either the identifier of the previous transaction from the same - // wallet or the identifier of one of the last ?MAX_TX_ANCHOR_DEPTH blocks. - bytes last_tx = 3; - // The public key the transaction is signed with. - bytes owner = 4; - // A list of arbitrary key-value pairs - repeated Tag tags = 5; - // The address of the recipient; if any. The SHA2-256 hash of the public key. - bytes target = 6; - // The amount of Winstons to send to the recipient; if any. - BigInt quantity = 7; - // The data to upload; if any. For v2 transactions; the field is optional - // - a fee is charged based on the `data_size` field; - // data may be uploaded any time later in chunks. - bytes data = 8; - // Size in bytes of the transaction data. - BigInt data_size = 9; - // The Merkle root of the Merkle tree of data chunks. - bytes data_root = 10; - // The signature. - bytes signature = 11; - // The fee in Winstons. - BigInt reward = 12; -} - - -message Tag { - bytes name = 1; - bytes value = 2; -} diff --git a/chain/arweave/src/adapter.rs b/chain/arweave/src/adapter.rs deleted file mode 100644 index fd2d962e31e..00000000000 --- a/chain/arweave/src/adapter.rs +++ /dev/null @@ -1,258 +0,0 @@ -use crate::{data_source::DataSource, Chain}; -use graph::blockchain as bc; -use graph::prelude::*; -use sha2::{Digest, Sha256}; -use std::collections::HashSet; - -const MATCH_ALL_WILDCARD: &str = ""; -// Size of sha256(pubkey) -const SHA256_LEN: usize = 32; - -#[derive(Clone, Debug, Default)] -pub struct TriggerFilter { - pub(crate) block_filter: ArweaveBlockFilter, - pub(crate) transaction_filter: ArweaveTransactionFilter, -} - -impl bc::TriggerFilter for TriggerFilter { - fn extend<'a>(&mut self, data_sources: impl Iterator + Clone) { - let TriggerFilter { - block_filter, - transaction_filter, - } = self; - - block_filter.extend(ArweaveBlockFilter::from_data_sources(data_sources.clone())); - transaction_filter.extend(ArweaveTransactionFilter::from_data_sources(data_sources)); - } - - fn node_capabilities(&self) -> bc::EmptyNodeCapabilities { - bc::EmptyNodeCapabilities::default() - } - - fn extend_with_template( - &mut self, - _data_source: impl Iterator::DataSourceTemplate>, - ) { - } - - fn to_firehose_filter(self) -> Vec { - vec![] - } -} - -/// ArweaveBlockFilter will match every block regardless of source being set. -/// see docs: https://thegraph.com/docs/en/supported-networks/arweave/ -#[derive(Clone, Debug, Default)] -pub(crate) struct ArweaveTransactionFilter { - owners_pubkey: HashSet>, - owners_sha: HashSet>, - match_all: bool, -} - -impl ArweaveTransactionFilter { - pub fn matches(&self, owner: &[u8]) -> bool { - if self.match_all { - return true; - } - - if owner.len() == SHA256_LEN { - return self.owners_sha.contains(owner); - } - - self.owners_pubkey.contains(owner) || self.owners_sha.contains(&sha256(owner)) - } - - pub fn from_data_sources<'a>(iter: impl IntoIterator) -> Self { - let owners: Vec> = iter - .into_iter() - .filter(|data_source| { - data_source.source.owner.is_some() - && !data_source.mapping.transaction_handlers.is_empty() - }) - .map(|ds| match &ds.source.owner { - Some(str) if MATCH_ALL_WILDCARD.eq(str) => MATCH_ALL_WILDCARD.as_bytes().to_owned(), - owner => base64_url::decode(&owner.clone().unwrap_or_default()).unwrap_or_default(), - }) - .collect(); - - let (owners_sha, long) = owners - .into_iter() - .partition::>, _>(|owner| owner.len() == SHA256_LEN); - - let (owners_pubkey, wildcard) = long - .into_iter() - .partition::>, _>(|long| long.len() != MATCH_ALL_WILDCARD.len()); - - let match_all = !wildcard.is_empty(); - - let owners_sha: Vec> = owners_sha - .into_iter() - .chain::>>(owners_pubkey.iter().map(|long| sha256(long)).collect()) - .collect(); - - Self { - match_all, - owners_pubkey: HashSet::from_iter(owners_pubkey), - owners_sha: HashSet::from_iter(owners_sha), - } - } - - pub fn extend(&mut self, other: ArweaveTransactionFilter) { - let ArweaveTransactionFilter { - owners_pubkey, - owners_sha, - match_all, - } = self; - - owners_pubkey.extend(other.owners_pubkey); - owners_sha.extend(other.owners_sha); - *match_all = *match_all || other.match_all; - } -} - -/// ArweaveBlockFilter will match every block regardless of source being set. -/// see docs: https://thegraph.com/docs/en/supported-networks/arweave/ -#[derive(Clone, Debug, Default)] -pub(crate) struct ArweaveBlockFilter { - pub trigger_every_block: bool, -} - -impl ArweaveBlockFilter { - pub fn from_data_sources<'a>(iter: impl IntoIterator) -> Self { - Self { - trigger_every_block: iter - .into_iter() - .any(|data_source| !data_source.mapping.block_handlers.is_empty()), - } - } - - pub fn extend(&mut self, other: ArweaveBlockFilter) { - self.trigger_every_block = self.trigger_every_block || other.trigger_every_block; - } -} - -fn sha256(bs: &[u8]) -> Vec { - let mut hasher = Sha256::new(); - hasher.update(bs); - let res = hasher.finalize(); - res.to_vec() -} - -#[cfg(test)] -mod test { - use std::sync::Arc; - - use graph::{prelude::Link, semver::Version}; - - use crate::data_source::{DataSource, Mapping, Source, TransactionHandler}; - - use super::{ArweaveTransactionFilter, MATCH_ALL_WILDCARD}; - - const ARWEAVE_PUBKEY_EXAMPLE: &str = "x-62w7g2yKACOgP_d04bhG8IX-AWgPrxHl2JgZBDdNLfAsidiiAaoIZPeM8K5gGvl7-8QVk79YV4OC878Ey0gXi7Atj5BouRyXnFMjJcPVXVyBoYCBuG7rJDDmh4_Ilon6vVOuHVIZ47Vb0tcgsxgxdvVFC2mn9N_SBl23pbeICNJZYOH57kf36gicuV_IwYSdqlQ0HQ_psjmg8EFqO7xzvAMP5HKW3rqTrYZxbCew2FkM734ysWckT39TpDBPx3HrFOl6obUdQWkHNOeKyzcsKFDywNgVWZOb89CYU7JFYlwX20io39ZZv0UJUOEFNjtVHkT_s0_A2O9PltsrZLLlQXZUuYASdbAPD2g_qXfhmPBZ0SXPWCDY-UVwVN1ncwYmk1F_i35IA8kAKsajaltD2wWDQn9g5mgJAWWn2xhLqkbwGbdwQMRD0-0eeuy1uzCooJQCC_bPJksoqkYwB9SGOjkayf4r4oZ2QDY4FicCsswz4Od_gud30ZWyHjWgqGzSFYFzawDBS1Gr_nu_q5otFrv20ZGTxYqGsLHWq4VHs6KjsQvzgBjfyb0etqHQEPJJmbQmY3LSogR4bxdReUHhj2EK9xIB-RKzDvDdL7fT5K0V9MjbnC2uktA0VjLlvwJ64_RhbQhxdp_zR39r-zyCXT-brPEYW1-V7Ey9K3XUE"; - const ARWEAVE_SHA_EXAMPLE: &str = "ahLxjCMCHr1ZE72VDDoaK4IKiLUUpeuo8t-M6y23DXw"; - - #[test] - fn transaction_filter_wildcard_matches_all() { - let dss = vec![ - new_datasource(None, 10), - new_datasource(Some(base64_url::encode(MATCH_ALL_WILDCARD)), 10), - new_datasource(Some(base64_url::encode("owner")), 10), - new_datasource(Some(ARWEAVE_PUBKEY_EXAMPLE.into()), 10), - ]; - - let dss: Vec<&DataSource> = dss.iter().collect(); - - let filter = ArweaveTransactionFilter::from_data_sources(dss); - assert_eq!(true, filter.matches("asdas".as_bytes())) - } - - #[test] - fn transaction_filter_match() { - let dss = vec![ - new_datasource(None, 10), - new_datasource(Some(ARWEAVE_PUBKEY_EXAMPLE.into()), 10), - ]; - - let dss: Vec<&DataSource> = dss.iter().collect(); - - let filter = ArweaveTransactionFilter::from_data_sources(dss); - assert_eq!(false, filter.matches("asdas".as_bytes())); - assert_eq!( - true, - filter.matches( - &base64_url::decode(ARWEAVE_SHA_EXAMPLE).expect("failed to parse sha example") - ) - ); - assert_eq!( - true, - filter.matches( - &base64_url::decode(ARWEAVE_PUBKEY_EXAMPLE).expect("failed to parse PK example") - ) - ) - } - - #[test] - fn transaction_filter_extend_match() { - let dss = vec![ - new_datasource(None, 10), - new_datasource(Some(ARWEAVE_SHA_EXAMPLE.into()), 10), - ]; - - let dss: Vec<&DataSource> = dss.iter().collect(); - - let filter = ArweaveTransactionFilter::from_data_sources(dss); - assert_eq!(false, filter.matches("asdas".as_bytes())); - assert_eq!( - true, - filter.matches( - &base64_url::decode(ARWEAVE_SHA_EXAMPLE).expect("failed to parse sha example") - ) - ); - assert_eq!( - true, - filter.matches( - &base64_url::decode(ARWEAVE_PUBKEY_EXAMPLE).expect("failed to parse PK example") - ) - ) - } - - #[test] - fn transaction_filter_extend_wildcard_matches_all() { - let dss = vec![ - new_datasource(None, 10), - new_datasource(Some(MATCH_ALL_WILDCARD.into()), 10), - new_datasource(Some("owner".into()), 10), - ]; - - let dss: Vec<&DataSource> = dss.iter().collect(); - - let mut filter = ArweaveTransactionFilter::default(); - - filter.extend(ArweaveTransactionFilter::from_data_sources(dss)); - assert_eq!(true, filter.matches("asdas".as_bytes())); - assert_eq!(true, filter.matches(ARWEAVE_PUBKEY_EXAMPLE.as_bytes())); - assert_eq!(true, filter.matches(ARWEAVE_SHA_EXAMPLE.as_bytes())) - } - - fn new_datasource(owner: Option, start_block: i32) -> DataSource { - DataSource { - kind: "".into(), - network: None, - name: "".into(), - source: Source { owner, start_block }, - mapping: Mapping { - api_version: Version::new(1, 2, 3), - language: "".into(), - entities: vec![], - block_handlers: vec![], - transaction_handlers: vec![TransactionHandler { - handler: "my_handler".into(), - }], - runtime: Arc::new(vec![]), - link: Link { link: "".into() }, - }, - context: Arc::new(None), - creation_block: None, - } - } -} diff --git a/chain/arweave/src/chain.rs b/chain/arweave/src/chain.rs deleted file mode 100644 index 86038c1110f..00000000000 --- a/chain/arweave/src/chain.rs +++ /dev/null @@ -1,340 +0,0 @@ -use graph::blockchain::client::ChainClient; -use graph::blockchain::{Block, BlockchainKind, EmptyNodeCapabilities}; -use graph::cheap_clone::CheapClone; -use graph::data::subgraph::UnifiedMappingApiVersion; -use graph::firehose::{FirehoseEndpoint, FirehoseEndpoints}; -use graph::prelude::MetricsRegistry; -use graph::{ - blockchain::{ - block_stream::{ - BlockStreamEvent, BlockWithTriggers, FirehoseError, - FirehoseMapper as FirehoseMapperTrait, TriggersAdapter as TriggersAdapterTrait, - }, - firehose_block_stream::FirehoseBlockStream, - BlockHash, BlockPtr, Blockchain, IngestorError, RuntimeAdapter as RuntimeAdapterTrait, - }, - components::store::DeploymentLocator, - firehose::{self as firehose, ForkStep}, - prelude::{async_trait, o, BlockNumber, ChainStore, Error, Logger, LoggerFactory}, -}; -use prost::Message; -use std::sync::Arc; - -use crate::adapter::TriggerFilter; -use crate::data_source::{DataSourceTemplate, UnresolvedDataSourceTemplate}; -use crate::runtime::RuntimeAdapter; -use crate::trigger::{self, ArweaveTrigger}; -use crate::{ - codec, - data_source::{DataSource, UnresolvedDataSource}, -}; -use graph::blockchain::block_stream::{BlockStream, FirehoseCursor}; - -pub struct Chain { - logger_factory: LoggerFactory, - name: String, - client: Arc>, - chain_store: Arc, - metrics_registry: Arc, -} - -impl std::fmt::Debug for Chain { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "chain: arweave") - } -} - -impl Chain { - pub fn new( - logger_factory: LoggerFactory, - name: String, - chain_store: Arc, - firehose_endpoints: FirehoseEndpoints, - metrics_registry: Arc, - ) -> Self { - Chain { - logger_factory, - name, - client: Arc::new(ChainClient::::new_firehose(firehose_endpoints)), - chain_store, - metrics_registry, - } - } -} - -#[async_trait] -impl Blockchain for Chain { - const KIND: BlockchainKind = BlockchainKind::Arweave; - - type Client = (); - type Block = codec::Block; - - type DataSource = DataSource; - - type UnresolvedDataSource = UnresolvedDataSource; - - type DataSourceTemplate = DataSourceTemplate; - - type UnresolvedDataSourceTemplate = UnresolvedDataSourceTemplate; - - type TriggerData = crate::trigger::ArweaveTrigger; - - type MappingTrigger = crate::trigger::ArweaveTrigger; - - type TriggerFilter = crate::adapter::TriggerFilter; - - type NodeCapabilities = EmptyNodeCapabilities; - - fn triggers_adapter( - &self, - _loc: &DeploymentLocator, - _capabilities: &Self::NodeCapabilities, - _unified_api_version: UnifiedMappingApiVersion, - ) -> Result>, Error> { - let adapter = TriggersAdapter {}; - Ok(Arc::new(adapter)) - } - - fn is_refetch_block_required(&self) -> bool { - false - } - - async fn refetch_firehose_block( - &self, - _logger: &Logger, - _cursor: FirehoseCursor, - ) -> Result { - unimplemented!("This chain does not support Dynamic Data Sources. is_refetch_block_required always returns false, this shouldn't be called.") - } - - async fn new_firehose_block_stream( - &self, - deployment: DeploymentLocator, - block_cursor: FirehoseCursor, - start_blocks: Vec, - subgraph_current_block: Option, - filter: Arc, - unified_api_version: UnifiedMappingApiVersion, - ) -> Result>, Error> { - let adapter = self - .triggers_adapter( - &deployment, - &EmptyNodeCapabilities::default(), - unified_api_version, - ) - .unwrap_or_else(|_| panic!("no adapter for network {}", self.name)); - - let firehose_endpoint = self.client.firehose_endpoint()?; - let logger = self - .logger_factory - .subgraph_logger(&deployment) - .new(o!("component" => "FirehoseBlockStream")); - - let firehose_mapper = Arc::new(FirehoseMapper {}); - - Ok(Box::new(FirehoseBlockStream::new( - deployment.hash, - firehose_endpoint, - subgraph_current_block, - block_cursor, - firehose_mapper, - adapter, - filter, - start_blocks, - logger, - self.metrics_registry.clone(), - ))) - } - - async fn new_polling_block_stream( - &self, - _deployment: DeploymentLocator, - _start_blocks: Vec, - _subgraph_current_block: Option, - _filter: Arc, - _unified_api_version: UnifiedMappingApiVersion, - ) -> Result>, Error> { - panic!("Arweave does not support polling block stream") - } - - fn chain_store(&self) -> Arc { - self.chain_store.clone() - } - - async fn block_pointer_from_number( - &self, - logger: &Logger, - number: BlockNumber, - ) -> Result { - self.client - .firehose_endpoint()? - .block_ptr_for_number::(logger, number) - .await - .map_err(Into::into) - } - - fn runtime_adapter(&self) -> Arc> { - Arc::new(RuntimeAdapter {}) - } - - fn chain_client(&self) -> Arc> { - self.client.clone() - } -} - -pub struct TriggersAdapter {} - -#[async_trait] -impl TriggersAdapterTrait for TriggersAdapter { - async fn scan_triggers( - &self, - _from: BlockNumber, - _to: BlockNumber, - _filter: &TriggerFilter, - ) -> Result>, Error> { - panic!("Should never be called since not used by FirehoseBlockStream") - } - - async fn triggers_in_block( - &self, - logger: &Logger, - block: codec::Block, - filter: &TriggerFilter, - ) -> Result, Error> { - // TODO: Find the best place to introduce an `Arc` and avoid this clone. - let shared_block = Arc::new(block.clone()); - - let TriggerFilter { - block_filter, - transaction_filter, - } = filter; - - let txs = block - .clone() - .txs - .into_iter() - .filter(|tx| transaction_filter.matches(&tx.owner)) - .map(|tx| trigger::TransactionWithBlockPtr { - tx: Arc::new(tx), - block: shared_block.clone(), - }) - .collect::>(); - - let mut trigger_data: Vec<_> = txs - .into_iter() - .map(|tx| ArweaveTrigger::Transaction(Arc::new(tx))) - .collect(); - - if block_filter.trigger_every_block { - trigger_data.push(ArweaveTrigger::Block(shared_block.cheap_clone())); - } - - Ok(BlockWithTriggers::new(block, trigger_data, logger)) - } - - async fn is_on_main_chain(&self, _ptr: BlockPtr) -> Result { - panic!("Should never be called since not used by FirehoseBlockStream") - } - - async fn ancestor_block( - &self, - _ptr: BlockPtr, - _offset: BlockNumber, - ) -> Result, Error> { - panic!("Should never be called since FirehoseBlockStream cannot resolve it") - } - - /// Panics if `block` is genesis. - /// But that's ok since this is only called when reverting `block`. - async fn parent_ptr(&self, block: &BlockPtr) -> Result, Error> { - // FIXME (Arweave): Might not be necessary for Arweave support for now - Ok(Some(BlockPtr { - hash: BlockHash::from(vec![0xff; 48]), - number: block.number.saturating_sub(1), - })) - } -} - -pub struct FirehoseMapper {} - -#[async_trait] -impl FirehoseMapperTrait for FirehoseMapper { - async fn to_block_stream_event( - &self, - logger: &Logger, - response: &firehose::Response, - adapter: &Arc>, - filter: &TriggerFilter, - ) -> Result, FirehoseError> { - let step = ForkStep::from_i32(response.step).unwrap_or_else(|| { - panic!( - "unknown step i32 value {}, maybe you forgot update & re-regenerate the protobuf definitions?", - response.step - ) - }); - - let any_block = response - .block - .as_ref() - .expect("block payload information should always be present"); - - // Right now, this is done in all cases but in reality, with how the BlockStreamEvent::Revert - // is defined right now, only block hash and block number is necessary. However, this information - // is not part of the actual bstream::BlockResponseV2 payload. As such, we need to decode the full - // block which is useless. - // - // Check about adding basic information about the block in the bstream::BlockResponseV2 or maybe - // define a slimmed down stuct that would decode only a few fields and ignore all the rest. - let block = codec::Block::decode(any_block.value.as_ref())?; - - use ForkStep::*; - match step { - StepNew => Ok(BlockStreamEvent::ProcessBlock( - adapter.triggers_in_block(logger, block, filter).await?, - FirehoseCursor::from(response.cursor.clone()), - )), - - StepUndo => { - let parent_ptr = block - .parent_ptr() - .expect("Genesis block should never be reverted"); - - Ok(BlockStreamEvent::Revert( - parent_ptr, - FirehoseCursor::from(response.cursor.clone()), - )) - } - - StepFinal => { - panic!("irreversible step is not handled and should not be requested in the Firehose request") - } - - StepUnset => { - panic!("unknown step should not happen in the Firehose response") - } - } - } - - async fn block_ptr_for_number( - &self, - logger: &Logger, - endpoint: &Arc, - number: BlockNumber, - ) -> Result { - endpoint - .block_ptr_for_number::(logger, number) - .await - } - - // # FIXME - // - // the final block of arweave is itself in the current implementation - async fn final_block_ptr_for( - &self, - _logger: &Logger, - _endpoint: &Arc, - block: &codec::Block, - ) -> Result { - Ok(block.ptr()) - } -} diff --git a/chain/arweave/src/codec.rs b/chain/arweave/src/codec.rs deleted file mode 100644 index 09da7fee1b0..00000000000 --- a/chain/arweave/src/codec.rs +++ /dev/null @@ -1,37 +0,0 @@ -#[rustfmt::skip] -#[path = "protobuf/sf.arweave.r#type.v1.rs"] -mod pbcodec; - -use graph::{blockchain::Block as BlockchainBlock, blockchain::BlockPtr, prelude::BlockNumber}; - -pub use pbcodec::*; - -impl BlockchainBlock for Block { - fn number(&self) -> i32 { - BlockNumber::try_from(self.height).unwrap() - } - - fn ptr(&self) -> BlockPtr { - BlockPtr { - hash: self.indep_hash.clone().into(), - number: self.number(), - } - } - - fn parent_ptr(&self) -> Option { - if self.height == 0 { - return None; - } - - Some(BlockPtr { - hash: self.previous_block.clone().into(), - number: self.number().saturating_sub(1), - }) - } -} - -impl AsRef<[u8]> for BigInt { - fn as_ref(&self) -> &[u8] { - self.bytes.as_ref() - } -} diff --git a/chain/arweave/src/data_source.rs b/chain/arweave/src/data_source.rs deleted file mode 100644 index 1bf6ac72caa..00000000000 --- a/chain/arweave/src/data_source.rs +++ /dev/null @@ -1,369 +0,0 @@ -use graph::blockchain::{Block, TriggerWithHandler}; -use graph::components::store::StoredDynamicDataSource; -use graph::data::subgraph::DataSourceContext; -use graph::prelude::SubgraphManifestValidationError; -use graph::{ - anyhow::{anyhow, Error}, - blockchain::{self, Blockchain}, - prelude::{ - async_trait, info, BlockNumber, CheapClone, DataSourceTemplateInfo, Deserialize, Link, - LinkResolver, Logger, - }, - semver, -}; -use std::sync::Arc; - -use crate::chain::Chain; -use crate::trigger::ArweaveTrigger; - -pub const ARWEAVE_KIND: &str = "arweave"; - -/// Runtime representation of a data source. -#[derive(Clone, Debug)] -pub struct DataSource { - pub kind: String, - pub network: Option, - pub name: String, - pub(crate) source: Source, - pub mapping: Mapping, - pub context: Arc>, - pub creation_block: Option, -} - -impl blockchain::DataSource for DataSource { - fn from_template_info(_info: DataSourceTemplateInfo) -> Result { - Err(anyhow!("Arweave subgraphs do not support templates")) - } - - // FIXME - // - // need to decode the base64url encoding? - fn address(&self) -> Option<&[u8]> { - self.source.owner.as_ref().map(String::as_bytes) - } - - fn start_block(&self) -> BlockNumber { - self.source.start_block - } - - fn match_and_decode( - &self, - trigger: &::TriggerData, - block: &Arc<::Block>, - _logger: &Logger, - ) -> Result>, Error> { - if self.source.start_block > block.number() { - return Ok(None); - } - - let handler = match trigger { - // A block trigger matches if a block handler is present. - ArweaveTrigger::Block(_) => match self.handler_for_block() { - Some(handler) => &handler.handler, - None => return Ok(None), - }, - // A transaction trigger matches if a transaction handler is present. - ArweaveTrigger::Transaction(_) => match self.handler_for_transaction() { - Some(handler) => &handler.handler, - None => return Ok(None), - }, - }; - - Ok(Some(TriggerWithHandler::::new( - trigger.cheap_clone(), - handler.clone(), - block.ptr(), - ))) - } - - fn name(&self) -> &str { - &self.name - } - - fn kind(&self) -> &str { - &self.kind - } - - fn network(&self) -> Option<&str> { - self.network.as_deref() - } - - fn context(&self) -> Arc> { - self.context.cheap_clone() - } - - fn creation_block(&self) -> Option { - self.creation_block - } - - fn is_duplicate_of(&self, other: &Self) -> bool { - let DataSource { - kind, - network, - name, - source, - mapping, - context, - - // The creation block is ignored for detection duplicate data sources. - // Contract ABI equality is implicit in `source` and `mapping.abis` equality. - creation_block: _, - } = self; - - // mapping_request_sender, host_metrics, and (most of) host_exports are operational structs - // used at runtime but not needed to define uniqueness; each runtime host should be for a - // unique data source. - kind == &other.kind - && network == &other.network - && name == &other.name - && source == &other.source - && mapping.block_handlers == other.mapping.block_handlers - && context == &other.context - } - - fn as_stored_dynamic_data_source(&self) -> StoredDynamicDataSource { - // FIXME (Arweave): Implement me! - todo!() - } - - fn from_stored_dynamic_data_source( - _template: &DataSourceTemplate, - _stored: StoredDynamicDataSource, - ) -> Result { - // FIXME (Arweave): Implement me correctly - todo!() - } - - fn validate(&self) -> Vec { - let mut errors = Vec::new(); - - if self.kind != ARWEAVE_KIND { - errors.push(anyhow!( - "data source has invalid `kind`, expected {} but found {}", - ARWEAVE_KIND, - self.kind - )) - } - - // Validate that there is a `source` address if there are transaction handlers - let no_source_address = self.address().is_none(); - let has_transaction_handlers = !self.mapping.transaction_handlers.is_empty(); - if no_source_address && has_transaction_handlers { - errors.push(SubgraphManifestValidationError::SourceAddressRequired.into()); - }; - - // Validate that there are no more than one of both block handlers and transaction handlers - if self.mapping.block_handlers.len() > 1 { - errors.push(anyhow!("data source has duplicated block handlers")); - } - if self.mapping.transaction_handlers.len() > 1 { - errors.push(anyhow!("data source has duplicated transaction handlers")); - } - - errors - } - - fn api_version(&self) -> semver::Version { - self.mapping.api_version.clone() - } - - fn runtime(&self) -> Option>> { - Some(self.mapping.runtime.cheap_clone()) - } -} - -impl DataSource { - fn from_manifest( - kind: String, - network: Option, - name: String, - source: Source, - mapping: Mapping, - context: Option, - ) -> Result { - // Data sources in the manifest are created "before genesis" so they have no creation block. - let creation_block = None; - - Ok(DataSource { - kind, - network, - name, - source, - mapping, - context: Arc::new(context), - creation_block, - }) - } - - fn handler_for_block(&self) -> Option<&MappingBlockHandler> { - self.mapping.block_handlers.first() - } - - fn handler_for_transaction(&self) -> Option<&TransactionHandler> { - self.mapping.transaction_handlers.first() - } -} - -#[derive(Clone, Debug, Eq, PartialEq, Deserialize)] -pub struct UnresolvedDataSource { - pub kind: String, - pub network: Option, - pub name: String, - pub(crate) source: Source, - pub mapping: UnresolvedMapping, - pub context: Option, -} - -#[async_trait] -impl blockchain::UnresolvedDataSource for UnresolvedDataSource { - async fn resolve( - self, - resolver: &Arc, - logger: &Logger, - _manifest_idx: u32, - ) -> Result { - let UnresolvedDataSource { - kind, - network, - name, - source, - mapping, - context, - } = self; - - info!(logger, "Resolve data source"; "name" => &name, "source_address" => format_args!("{:?}", base64_url::encode(&source.owner.clone().unwrap_or_default())), "source_start_block" => source.start_block); - - let mapping = mapping.resolve(resolver, logger).await?; - - DataSource::from_manifest(kind, network, name, source, mapping, context) - } -} - -#[derive(Clone, Debug, Default, Hash, Eq, PartialEq, Deserialize)] -pub struct BaseDataSourceTemplate { - pub kind: String, - pub network: Option, - pub name: String, - pub mapping: M, -} - -pub type UnresolvedDataSourceTemplate = BaseDataSourceTemplate; -pub type DataSourceTemplate = BaseDataSourceTemplate; - -#[async_trait] -impl blockchain::UnresolvedDataSourceTemplate for UnresolvedDataSourceTemplate { - async fn resolve( - self, - resolver: &Arc, - logger: &Logger, - _manifest_idx: u32, - ) -> Result { - let UnresolvedDataSourceTemplate { - kind, - network, - name, - mapping, - } = self; - - info!(logger, "Resolve data source template"; "name" => &name); - - Ok(DataSourceTemplate { - kind, - network, - name, - mapping: mapping.resolve(resolver, logger).await?, - }) - } -} - -impl blockchain::DataSourceTemplate for DataSourceTemplate { - fn name(&self) -> &str { - &self.name - } - - fn api_version(&self) -> semver::Version { - self.mapping.api_version.clone() - } - - fn runtime(&self) -> Option>> { - Some(self.mapping.runtime.cheap_clone()) - } - - fn manifest_idx(&self) -> u32 { - unreachable!("arweave does not support dynamic data sources") - } -} - -#[derive(Clone, Debug, Default, Hash, Eq, PartialEq, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct UnresolvedMapping { - pub api_version: String, - pub language: String, - pub entities: Vec, - #[serde(default)] - pub block_handlers: Vec, - #[serde(default)] - pub transaction_handlers: Vec, - pub file: Link, -} - -impl UnresolvedMapping { - pub async fn resolve( - self, - resolver: &Arc, - logger: &Logger, - ) -> Result { - let UnresolvedMapping { - api_version, - language, - entities, - block_handlers, - transaction_handlers, - file: link, - } = self; - - let api_version = semver::Version::parse(&api_version)?; - - info!(logger, "Resolve mapping"; "link" => &link.link); - let module_bytes = resolver.cat(logger, &link).await?; - - Ok(Mapping { - api_version, - language, - entities, - block_handlers, - transaction_handlers, - runtime: Arc::new(module_bytes), - link, - }) - } -} - -#[derive(Clone, Debug)] -pub struct Mapping { - pub api_version: semver::Version, - pub language: String, - pub entities: Vec, - pub block_handlers: Vec, - pub transaction_handlers: Vec, - pub runtime: Arc>, - pub link: Link, -} - -#[derive(Clone, Debug, Hash, Eq, PartialEq, Deserialize)] -pub struct MappingBlockHandler { - pub handler: String, -} - -#[derive(Clone, Debug, Hash, Eq, PartialEq, Deserialize)] -pub struct TransactionHandler { - pub handler: String, -} - -#[derive(Clone, Debug, Hash, Eq, PartialEq, Deserialize)] -pub(crate) struct Source { - // A data source that does not have an owner can only have block handlers. - pub(crate) owner: Option, - #[serde(rename = "startBlock", default)] - pub(crate) start_block: BlockNumber, -} diff --git a/chain/arweave/src/lib.rs b/chain/arweave/src/lib.rs deleted file mode 100644 index 77e63bc51ab..00000000000 --- a/chain/arweave/src/lib.rs +++ /dev/null @@ -1,9 +0,0 @@ -mod adapter; -mod chain; -mod codec; -mod data_source; -mod runtime; -mod trigger; - -pub use crate::chain::Chain; -pub use codec::Block; diff --git a/chain/arweave/src/protobuf/sf.arweave.r#type.v1.rs b/chain/arweave/src/protobuf/sf.arweave.r#type.v1.rs deleted file mode 100644 index fba41614f1b..00000000000 --- a/chain/arweave/src/protobuf/sf.arweave.r#type.v1.rs +++ /dev/null @@ -1,146 +0,0 @@ -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct BigInt { - #[prost(bytes = "vec", tag = "1")] - pub bytes: ::prost::alloc::vec::Vec, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Block { - /// Firehose block version (unrelated to Arweave block version) - #[prost(uint32, tag = "1")] - pub ver: u32, - /// The block identifier - #[prost(bytes = "vec", tag = "2")] - pub indep_hash: ::prost::alloc::vec::Vec, - /// The nonce chosen to solve the mining problem - #[prost(bytes = "vec", tag = "3")] - pub nonce: ::prost::alloc::vec::Vec, - /// `indep_hash` of the previous block in the weave - #[prost(bytes = "vec", tag = "4")] - pub previous_block: ::prost::alloc::vec::Vec, - /// POSIX time of block discovery - #[prost(uint64, tag = "5")] - pub timestamp: u64, - /// POSIX time of the last difficulty retarget - #[prost(uint64, tag = "6")] - pub last_retarget: u64, - /// Mining difficulty; the number `hash` must be greater than. - #[prost(message, optional, tag = "7")] - pub diff: ::core::option::Option, - /// How many blocks have passed since the genesis block - #[prost(uint64, tag = "8")] - pub height: u64, - /// Mining solution hash of the block; must satisfy the mining difficulty - #[prost(bytes = "vec", tag = "9")] - pub hash: ::prost::alloc::vec::Vec, - /// Merkle root of the tree of Merkle roots of block's transactions' data. - #[prost(bytes = "vec", tag = "10")] - pub tx_root: ::prost::alloc::vec::Vec, - /// Transactions contained within this block - #[prost(message, repeated, tag = "11")] - pub txs: ::prost::alloc::vec::Vec, - /// The root hash of the Merkle Patricia Tree containing - /// all wallet (account) balances and the identifiers - /// of the last transactions posted by them; if any. - #[prost(bytes = "vec", tag = "12")] - pub wallet_list: ::prost::alloc::vec::Vec, - /// (string or) Address of the account to receive the block rewards. Can also be unclaimed which is encoded as a null byte - #[prost(bytes = "vec", tag = "13")] - pub reward_addr: ::prost::alloc::vec::Vec, - /// Tags that a block producer can add to a block - #[prost(message, repeated, tag = "14")] - pub tags: ::prost::alloc::vec::Vec, - /// Size of reward pool - #[prost(message, optional, tag = "15")] - pub reward_pool: ::core::option::Option, - /// Size of the weave in bytes - #[prost(message, optional, tag = "16")] - pub weave_size: ::core::option::Option, - /// Size of this block in bytes - #[prost(message, optional, tag = "17")] - pub block_size: ::core::option::Option, - /// Required after the version 1.8 fork. Zero otherwise. - /// The sum of the average number of hashes computed - /// by the network to produce the past blocks including this one. - #[prost(message, optional, tag = "18")] - pub cumulative_diff: ::core::option::Option, - /// Required after the version 1.8 fork. Null byte otherwise. - /// The Merkle root of the block index - the list of {`indep_hash`; `weave_size`; `tx_root`} triplets - #[prost(bytes = "vec", tag = "20")] - pub hash_list_merkle: ::prost::alloc::vec::Vec, - /// The proof of access; Used after v2.4 only; set as defaults otherwise - #[prost(message, optional, tag = "21")] - pub poa: ::core::option::Option, -} -/// A succinct proof of access to a recall byte found in a TX -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ProofOfAccess { - /// The recall byte option chosen; global offset of index byte - #[prost(string, tag = "1")] - pub option: ::prost::alloc::string::String, - /// The path through the Merkle tree of transactions' `data_root`s; - /// from the `data_root` being proven to the corresponding `tx_root` - #[prost(bytes = "vec", tag = "2")] - pub tx_path: ::prost::alloc::vec::Vec, - /// The path through the Merkle tree of identifiers of chunks of the - /// corresponding transaction; from the chunk being proven to the - /// corresponding `data_root`. - #[prost(bytes = "vec", tag = "3")] - pub data_path: ::prost::alloc::vec::Vec, - /// The data chunk. - #[prost(bytes = "vec", tag = "4")] - pub chunk: ::prost::alloc::vec::Vec, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Transaction { - /// 1 or 2 for v1 or v2 transactions. More allowable in the future - #[prost(uint32, tag = "1")] - pub format: u32, - /// The transaction identifier. - #[prost(bytes = "vec", tag = "2")] - pub id: ::prost::alloc::vec::Vec, - /// Either the identifier of the previous transaction from the same - /// wallet or the identifier of one of the last ?MAX_TX_ANCHOR_DEPTH blocks. - #[prost(bytes = "vec", tag = "3")] - pub last_tx: ::prost::alloc::vec::Vec, - /// The public key the transaction is signed with. - #[prost(bytes = "vec", tag = "4")] - pub owner: ::prost::alloc::vec::Vec, - /// A list of arbitrary key-value pairs - #[prost(message, repeated, tag = "5")] - pub tags: ::prost::alloc::vec::Vec, - /// The address of the recipient; if any. The SHA2-256 hash of the public key. - #[prost(bytes = "vec", tag = "6")] - pub target: ::prost::alloc::vec::Vec, - /// The amount of Winstons to send to the recipient; if any. - #[prost(message, optional, tag = "7")] - pub quantity: ::core::option::Option, - /// The data to upload; if any. For v2 transactions; the field is optional - /// - a fee is charged based on the `data_size` field; - /// data may be uploaded any time later in chunks. - #[prost(bytes = "vec", tag = "8")] - pub data: ::prost::alloc::vec::Vec, - /// Size in bytes of the transaction data. - #[prost(message, optional, tag = "9")] - pub data_size: ::core::option::Option, - /// The Merkle root of the Merkle tree of data chunks. - #[prost(bytes = "vec", tag = "10")] - pub data_root: ::prost::alloc::vec::Vec, - /// The signature. - #[prost(bytes = "vec", tag = "11")] - pub signature: ::prost::alloc::vec::Vec, - /// The fee in Winstons. - #[prost(message, optional, tag = "12")] - pub reward: ::core::option::Option, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Tag { - #[prost(bytes = "vec", tag = "1")] - pub name: ::prost::alloc::vec::Vec, - #[prost(bytes = "vec", tag = "2")] - pub value: ::prost::alloc::vec::Vec, -} diff --git a/chain/arweave/src/runtime/abi.rs b/chain/arweave/src/runtime/abi.rs deleted file mode 100644 index ea495c6e5ff..00000000000 --- a/chain/arweave/src/runtime/abi.rs +++ /dev/null @@ -1,191 +0,0 @@ -use crate::codec; -use crate::trigger::TransactionWithBlockPtr; -use graph::runtime::gas::GasCounter; -use graph::runtime::{asc_new, AscHeap, AscPtr, DeterministicHostError, ToAscObj}; -use graph_runtime_wasm::asc_abi::class::{Array, Uint8Array}; - -pub(crate) use super::generated::*; - -impl ToAscObj for codec::Tag { - fn to_asc_obj( - &self, - heap: &mut H, - gas: &GasCounter, - ) -> Result { - Ok(AscTag { - name: asc_new(heap, self.name.as_slice(), gas)?, - value: asc_new(heap, self.value.as_slice(), gas)?, - }) - } -} - -impl ToAscObj for Vec> { - fn to_asc_obj( - &self, - heap: &mut H, - gas: &GasCounter, - ) -> Result { - let content = self - .iter() - .map(|x| asc_new(heap, x.as_slice(), gas)) - .collect::>, _>>()?; - Ok(AscTransactionArray(Array::new(&content, heap, gas)?)) - } -} - -impl ToAscObj for Vec { - fn to_asc_obj( - &self, - heap: &mut H, - gas: &GasCounter, - ) -> Result { - let content = self - .iter() - .map(|x| asc_new(heap, x, gas)) - .collect::, _>>()?; - Ok(AscTagArray(Array::new(&content, heap, gas)?)) - } -} - -impl ToAscObj for codec::ProofOfAccess { - fn to_asc_obj( - &self, - heap: &mut H, - gas: &GasCounter, - ) -> Result { - Ok(AscProofOfAccess { - option: asc_new(heap, &self.option, gas)?, - tx_path: asc_new(heap, self.tx_path.as_slice(), gas)?, - data_path: asc_new(heap, self.data_path.as_slice(), gas)?, - chunk: asc_new(heap, self.chunk.as_slice(), gas)?, - }) - } -} - -impl ToAscObj for codec::Transaction { - fn to_asc_obj( - &self, - heap: &mut H, - gas: &GasCounter, - ) -> Result { - Ok(AscTransaction { - format: self.format, - id: asc_new(heap, self.id.as_slice(), gas)?, - last_tx: asc_new(heap, self.last_tx.as_slice(), gas)?, - owner: asc_new(heap, self.owner.as_slice(), gas)?, - tags: asc_new(heap, &self.tags, gas)?, - target: asc_new(heap, self.target.as_slice(), gas)?, - quantity: asc_new( - heap, - self.quantity - .as_ref() - .map(|b| b.as_ref()) - .unwrap_or_default(), - gas, - )?, - data: asc_new(heap, self.data.as_slice(), gas)?, - data_size: asc_new( - heap, - self.data_size - .as_ref() - .map(|b| b.as_ref()) - .unwrap_or_default(), - gas, - )?, - data_root: asc_new(heap, self.data_root.as_slice(), gas)?, - signature: asc_new(heap, self.signature.as_slice(), gas)?, - reward: asc_new( - heap, - self.reward.as_ref().map(|b| b.as_ref()).unwrap_or_default(), - gas, - )?, - }) - } -} - -impl ToAscObj for codec::Block { - fn to_asc_obj( - &self, - heap: &mut H, - gas: &GasCounter, - ) -> Result { - Ok(AscBlock { - indep_hash: asc_new(heap, self.indep_hash.as_slice(), gas)?, - nonce: asc_new(heap, self.nonce.as_slice(), gas)?, - previous_block: asc_new(heap, self.previous_block.as_slice(), gas)?, - timestamp: self.timestamp, - last_retarget: self.last_retarget, - diff: asc_new( - heap, - self.diff.as_ref().map(|b| b.as_ref()).unwrap_or_default(), - gas, - )?, - height: self.height, - hash: asc_new(heap, self.hash.as_slice(), gas)?, - tx_root: asc_new(heap, self.tx_root.as_slice(), gas)?, - txs: asc_new( - heap, - &self - .txs - .iter() - .map(|tx| tx.id.clone()) - .collect::>>(), - gas, - )?, - wallet_list: asc_new(heap, self.wallet_list.as_slice(), gas)?, - reward_addr: asc_new(heap, self.reward_addr.as_slice(), gas)?, - tags: asc_new(heap, &self.tags, gas)?, - reward_pool: asc_new( - heap, - self.reward_pool - .as_ref() - .map(|b| b.as_ref()) - .unwrap_or_default(), - gas, - )?, - weave_size: asc_new( - heap, - self.weave_size - .as_ref() - .map(|b| b.as_ref()) - .unwrap_or_default(), - gas, - )?, - block_size: asc_new( - heap, - self.block_size - .as_ref() - .map(|b| b.as_ref()) - .unwrap_or_default(), - gas, - )?, - cumulative_diff: asc_new( - heap, - self.cumulative_diff - .as_ref() - .map(|b| b.as_ref()) - .unwrap_or_default(), - gas, - )?, - hash_list_merkle: asc_new(heap, self.hash_list_merkle.as_slice(), gas)?, - poa: self - .poa - .as_ref() - .map(|poa| asc_new(heap, poa, gas)) - .unwrap_or(Ok(AscPtr::null()))?, - }) - } -} - -impl ToAscObj for TransactionWithBlockPtr { - fn to_asc_obj( - &self, - heap: &mut H, - gas: &GasCounter, - ) -> Result { - Ok(AscTransactionWithBlockPtr { - tx: asc_new(heap, &self.tx.as_ref(), gas)?, - block: asc_new(heap, self.block.as_ref(), gas)?, - }) - } -} diff --git a/chain/arweave/src/runtime/generated.rs b/chain/arweave/src/runtime/generated.rs deleted file mode 100644 index e8a10fdb158..00000000000 --- a/chain/arweave/src/runtime/generated.rs +++ /dev/null @@ -1,128 +0,0 @@ -use graph::runtime::{AscIndexId, AscPtr, AscType, DeterministicHostError, IndexForAscTypeId}; -use graph::semver::Version; -use graph_runtime_derive::AscType; -use graph_runtime_wasm::asc_abi::class::{Array, AscString, Uint8Array}; - -#[repr(C)] -#[derive(AscType, Default)] -pub struct AscBlock { - pub timestamp: u64, - pub last_retarget: u64, - pub height: u64, - pub indep_hash: AscPtr, - pub nonce: AscPtr, - pub previous_block: AscPtr, - pub diff: AscPtr, - pub hash: AscPtr, - pub tx_root: AscPtr, - pub txs: AscPtr, - pub wallet_list: AscPtr, - pub reward_addr: AscPtr, - pub tags: AscPtr, - pub reward_pool: AscPtr, - pub weave_size: AscPtr, - pub block_size: AscPtr, - pub cumulative_diff: AscPtr, - pub hash_list_merkle: AscPtr, - pub poa: AscPtr, -} - -impl AscIndexId for AscBlock { - const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::ArweaveBlock; -} - -#[repr(C)] -#[derive(AscType)] -pub struct AscProofOfAccess { - pub option: AscPtr, - pub tx_path: AscPtr, - pub data_path: AscPtr, - pub chunk: AscPtr, -} - -impl AscIndexId for AscProofOfAccess { - const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::ArweaveProofOfAccess; -} - -#[repr(C)] -#[derive(AscType)] -pub struct AscTransaction { - pub format: u32, - pub id: AscPtr, - pub last_tx: AscPtr, - pub owner: AscPtr, - pub tags: AscPtr, - pub target: AscPtr, - pub quantity: AscPtr, - pub data: AscPtr, - pub data_size: AscPtr, - pub data_root: AscPtr, - pub signature: AscPtr, - pub reward: AscPtr, -} - -impl AscIndexId for AscTransaction { - const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::ArweaveTransaction; -} - -#[repr(C)] -#[derive(AscType)] -pub struct AscTag { - pub name: AscPtr, - pub value: AscPtr, -} - -impl AscIndexId for AscTag { - const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::ArweaveTag; -} - -#[repr(C)] -pub struct AscTransactionArray(pub(crate) Array>); - -impl AscType for AscTransactionArray { - fn to_asc_bytes(&self) -> Result, DeterministicHostError> { - self.0.to_asc_bytes() - } - - fn from_asc_bytes( - asc_obj: &[u8], - api_version: &Version, - ) -> Result { - Ok(Self(Array::from_asc_bytes(asc_obj, api_version)?)) - } -} - -impl AscIndexId for AscTransactionArray { - const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::ArweaveTransactionArray; -} - -#[repr(C)] -pub struct AscTagArray(pub(crate) Array>); - -impl AscType for AscTagArray { - fn to_asc_bytes(&self) -> Result, DeterministicHostError> { - self.0.to_asc_bytes() - } - - fn from_asc_bytes( - asc_obj: &[u8], - api_version: &Version, - ) -> Result { - Ok(Self(Array::from_asc_bytes(asc_obj, api_version)?)) - } -} - -impl AscIndexId for AscTagArray { - const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::ArweaveTagArray; -} - -#[repr(C)] -#[derive(AscType)] -pub struct AscTransactionWithBlockPtr { - pub tx: AscPtr, - pub block: AscPtr, -} - -impl AscIndexId for AscTransactionWithBlockPtr { - const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::ArweaveTransactionWithBlockPtr; -} diff --git a/chain/arweave/src/runtime/mod.rs b/chain/arweave/src/runtime/mod.rs deleted file mode 100644 index f44391caffd..00000000000 --- a/chain/arweave/src/runtime/mod.rs +++ /dev/null @@ -1,6 +0,0 @@ -pub use runtime_adapter::RuntimeAdapter; - -pub mod abi; -pub mod runtime_adapter; - -mod generated; diff --git a/chain/arweave/src/runtime/runtime_adapter.rs b/chain/arweave/src/runtime/runtime_adapter.rs deleted file mode 100644 index c5fa9e15059..00000000000 --- a/chain/arweave/src/runtime/runtime_adapter.rs +++ /dev/null @@ -1,11 +0,0 @@ -use crate::{data_source::DataSource, Chain}; -use blockchain::HostFn; -use graph::{anyhow::Error, blockchain}; - -pub struct RuntimeAdapter {} - -impl blockchain::RuntimeAdapter for RuntimeAdapter { - fn host_fns(&self, _ds: &DataSource) -> Result, Error> { - Ok(vec![]) - } -} diff --git a/chain/arweave/src/trigger.rs b/chain/arweave/src/trigger.rs deleted file mode 100644 index 9d2f7ad3a4d..00000000000 --- a/chain/arweave/src/trigger.rs +++ /dev/null @@ -1,137 +0,0 @@ -use graph::blockchain::Block; -use graph::blockchain::TriggerData; -use graph::cheap_clone::CheapClone; -use graph::prelude::web3::types::H256; -use graph::prelude::BlockNumber; -use graph::runtime::asc_new; -use graph::runtime::gas::GasCounter; -use graph::runtime::AscHeap; -use graph::runtime::AscPtr; -use graph::runtime::DeterministicHostError; -use graph_runtime_wasm::module::ToAscPtr; -use std::{cmp::Ordering, sync::Arc}; - -use crate::codec; - -// Logging the block is too verbose, so this strips the block from the trigger for Debug. -impl std::fmt::Debug for ArweaveTrigger { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - #[derive(Debug)] - pub enum MappingTriggerWithoutBlock { - Block, - Transaction(Arc), - } - - let trigger_without_block = match self { - ArweaveTrigger::Block(_) => MappingTriggerWithoutBlock::Block, - ArweaveTrigger::Transaction(tx) => { - MappingTriggerWithoutBlock::Transaction(tx.tx.clone()) - } - }; - - write!(f, "{:?}", trigger_without_block) - } -} - -impl ToAscPtr for ArweaveTrigger { - fn to_asc_ptr( - self, - heap: &mut H, - gas: &GasCounter, - ) -> Result, DeterministicHostError> { - Ok(match self { - ArweaveTrigger::Block(block) => asc_new(heap, block.as_ref(), gas)?.erase(), - ArweaveTrigger::Transaction(tx) => asc_new(heap, tx.as_ref(), gas)?.erase(), - }) - } -} - -#[derive(Clone)] -pub enum ArweaveTrigger { - Block(Arc), - Transaction(Arc), -} - -impl CheapClone for ArweaveTrigger { - fn cheap_clone(&self) -> ArweaveTrigger { - match self { - ArweaveTrigger::Block(block) => ArweaveTrigger::Block(block.cheap_clone()), - ArweaveTrigger::Transaction(tx) => ArweaveTrigger::Transaction(tx.cheap_clone()), - } - } -} - -impl PartialEq for ArweaveTrigger { - fn eq(&self, other: &Self) -> bool { - match (self, other) { - (Self::Block(a_ptr), Self::Block(b_ptr)) => a_ptr == b_ptr, - (Self::Transaction(a_tx), Self::Transaction(b_tx)) => a_tx.tx.id == b_tx.tx.id, - _ => false, - } - } -} - -impl Eq for ArweaveTrigger {} - -impl ArweaveTrigger { - pub fn block_number(&self) -> BlockNumber { - match self { - ArweaveTrigger::Block(block) => block.number(), - ArweaveTrigger::Transaction(tx) => tx.block.number(), - } - } - - pub fn block_hash(&self) -> H256 { - match self { - ArweaveTrigger::Block(block) => block.ptr().hash_as_h256(), - ArweaveTrigger::Transaction(tx) => tx.block.ptr().hash_as_h256(), - } - } -} - -impl Ord for ArweaveTrigger { - fn cmp(&self, other: &Self) -> Ordering { - match (self, other) { - // Keep the order when comparing two block triggers - (Self::Block(..), Self::Block(..)) => Ordering::Equal, - - // Block triggers always come last - (Self::Block(..), _) => Ordering::Greater, - (_, Self::Block(..)) => Ordering::Less, - - // Execution outcomes have no intrinsic ordering information so we keep the order in - // which they are included in the `txs` field of `Block`. - (Self::Transaction(..), Self::Transaction(..)) => Ordering::Equal, - } - } -} - -impl PartialOrd for ArweaveTrigger { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl TriggerData for ArweaveTrigger { - fn error_context(&self) -> std::string::String { - match self { - ArweaveTrigger::Block(..) => { - format!("Block #{} ({})", self.block_number(), self.block_hash()) - } - ArweaveTrigger::Transaction(tx) => { - format!( - "Tx #{}, block #{}({})", - base64_url::encode(&tx.tx.id), - self.block_number(), - self.block_hash() - ) - } - } - } -} - -pub struct TransactionWithBlockPtr { - // REVIEW: Do we want to actually also have those two below behind an `Arc` wrapper? - pub tx: Arc, - pub block: Arc, -} diff --git a/chain/common/Cargo.toml b/chain/common/Cargo.toml index 7ebb131d62e..1f945ea984e 100644 --- a/chain/common/Cargo.toml +++ b/chain/common/Cargo.toml @@ -7,6 +7,9 @@ edition.workspace = true [dependencies] protobuf = "3.0.2" -protobuf-parse = "3.2.0" +protobuf-parse = "3.7.2" anyhow = "1" -heck = "0.4" +heck = "0.5" + +[lints] +workspace = true diff --git a/chain/common/proto/near-filter-substreams.proto b/chain/common/proto/near-filter-substreams.proto new file mode 100644 index 00000000000..d7e4a822573 --- /dev/null +++ b/chain/common/proto/near-filter-substreams.proto @@ -0,0 +1,15 @@ +syntax = "proto3"; + +import "near.proto"; + +package receipts.v1; + +message BlockAndReceipts { + sf.near.codec.v1.Block block = 1; + repeated sf.near.codec.v1.ExecutionOutcomeWithId outcome = 2; + repeated sf.near.codec.v1.Receipt receipt = 3; +} + + + + diff --git a/chain/common/src/lib.rs b/chain/common/src/lib.rs index b8f2ae47eb4..395dd440c84 100644 --- a/chain/common/src/lib.rs +++ b/chain/common/src/lib.rs @@ -2,13 +2,13 @@ use std::collections::HashMap; use std::fmt::Debug; use anyhow::Error; -use protobuf::descriptor::field_descriptor_proto::Label; -use protobuf::descriptor::field_descriptor_proto::Type; +use protobuf::Message; +use protobuf::UnknownValueRef; use protobuf::descriptor::DescriptorProto; use protobuf::descriptor::FieldDescriptorProto; use protobuf::descriptor::OneofDescriptorProto; -use protobuf::Message; -use protobuf::UnknownValueRef; +use protobuf::descriptor::field_descriptor_proto::Label; +use protobuf::descriptor::field_descriptor_proto::Type; use std::convert::From; use std::path::Path; diff --git a/chain/common/tests/test-acme.rs b/chain/common/tests/test-acme.rs index 554e4ecbd5c..2945fea337e 100644 --- a/chain/common/tests/test-acme.rs +++ b/chain/common/tests/test-acme.rs @@ -7,8 +7,8 @@ fn check_repeated_type_ok() { let types = parse_proto_file(PROTO_FILE).expect("Unable to read proto file!"); let array_types = types - .iter() - .flat_map(|(_, t)| t.fields.iter()) + .values() + .flat_map(|t| t.fields.iter()) .filter(|t| t.is_array) .map(|t| t.type_name.clone()) .collect::>(); @@ -63,7 +63,7 @@ fn required_ok() { !f.required, "Transaction.events field should NOT be required!" ), - _ => assert!(false, "Unexpected message field [{}]!", f.name), + _ => panic!("Unexpected message field [{}]!", f.name), }; }); } diff --git a/chain/cosmos/Cargo.toml b/chain/cosmos/Cargo.toml deleted file mode 100644 index c932b5185ee..00000000000 --- a/chain/cosmos/Cargo.toml +++ /dev/null @@ -1,19 +0,0 @@ -[package] -name = "graph-chain-cosmos" -version.workspace = true -edition = "2018" - -[build-dependencies] -tonic-build = { workspace = true } -graph-chain-common = { path = "../common" } - -[dependencies] -graph = { path = "../../graph" } -prost = { workspace = true } -prost-types = { workspace = true } -serde = "1.0" -anyhow = "1.0" -semver = "1.0.16" - -graph-runtime-wasm = { path = "../../runtime/wasm" } -graph-runtime-derive = { path = "../../runtime/derive" } diff --git a/chain/cosmos/build.rs b/chain/cosmos/build.rs deleted file mode 100644 index fc07b4907e0..00000000000 --- a/chain/cosmos/build.rs +++ /dev/null @@ -1,54 +0,0 @@ -const PROTO_FILE: &str = "proto/type.proto"; - -fn main() { - println!("cargo:rerun-if-changed=proto"); - - let types = - graph_chain_common::parse_proto_file(PROTO_FILE).expect("Unable to parse proto file!"); - - let array_types = types - .iter() - .flat_map(|(_, t)| t.fields.iter()) - .filter(|t| t.is_array) - .map(|t| t.type_name.clone()) - .collect::>(); - - let mut builder = tonic_build::configure().out_dir("src/protobuf"); - - for (name, ptype) in types { - //generate Asc - builder = builder.type_attribute( - name.clone(), - format!( - "#[graph_runtime_derive::generate_asc_type({})]", - ptype.fields().unwrap_or_default() - ), - ); - - //generate data index id - builder = builder.type_attribute( - name.clone(), - "#[graph_runtime_derive::generate_network_type_id(Cosmos)]", - ); - - //generate conversion from rust type to asc - builder = builder.type_attribute( - name.clone(), - format!( - "#[graph_runtime_derive::generate_from_rust_type({})]", - ptype.fields().unwrap_or_default() - ), - ); - - if array_types.contains(&ptype.name) { - builder = builder.type_attribute( - name.clone(), - "#[graph_runtime_derive::generate_array_type(Cosmos)]", - ); - } - } - - builder - .compile(&["proto/type.proto"], &["proto"]) - .expect("Failed to compile Firehose Cosmos proto(s)"); -} diff --git a/chain/cosmos/proto/cosmos_proto/cosmos.proto b/chain/cosmos/proto/cosmos_proto/cosmos.proto deleted file mode 100644 index 5c63b86f063..00000000000 --- a/chain/cosmos/proto/cosmos_proto/cosmos.proto +++ /dev/null @@ -1,97 +0,0 @@ -syntax = "proto3"; -package cosmos_proto; - -import "google/protobuf/descriptor.proto"; - -option go_package = "github.com/cosmos/cosmos-proto;cosmos_proto"; - -extend google.protobuf.MessageOptions { - - // implements_interface is used to indicate the type name of the interface - // that a message implements so that it can be used in google.protobuf.Any - // fields that accept that interface. A message can implement multiple - // interfaces. Interfaces should be declared using a declare_interface - // file option. - repeated string implements_interface = 93001; -} - -extend google.protobuf.FieldOptions { - - // accepts_interface is used to annotate that a google.protobuf.Any - // field accepts messages that implement the specified interface. - // Interfaces should be declared using a declare_interface file option. - string accepts_interface = 93001; - - // scalar is used to indicate that this field follows the formatting defined - // by the named scalar which should be declared with declare_scalar. Code - // generators may choose to use this information to map this field to a - // language-specific type representing the scalar. - string scalar = 93002; -} - -extend google.protobuf.FileOptions { - - // declare_interface declares an interface type to be used with - // accepts_interface and implements_interface. Interface names are - // expected to follow the following convention such that their declaration - // can be discovered by tools: for a given interface type a.b.C, it is - // expected that the declaration will be found in a protobuf file named - // a/b/interfaces.proto in the file descriptor set. - repeated InterfaceDescriptor declare_interface = 793021; - - // declare_scalar declares a scalar type to be used with - // the scalar field option. Scalar names are - // expected to follow the following convention such that their declaration - // can be discovered by tools: for a given scalar type a.b.C, it is - // expected that the declaration will be found in a protobuf file named - // a/b/scalars.proto in the file descriptor set. - repeated ScalarDescriptor declare_scalar = 793022; -} - -// InterfaceDescriptor describes an interface type to be used with -// accepts_interface and implements_interface and declared by declare_interface. -message InterfaceDescriptor { - - // name is the name of the interface. It should be a short-name (without - // a period) such that the fully qualified name of the interface will be - // package.name, ex. for the package a.b and interface named C, the - // fully-qualified name will be a.b.C. - string name = 1; - - // description is a human-readable description of the interface and its - // purpose. - string description = 2; -} - -// ScalarDescriptor describes an scalar type to be used with -// the scalar field option and declared by declare_scalar. -// Scalars extend simple protobuf built-in types with additional -// syntax and semantics, for instance to represent big integers. -// Scalars should ideally define an encoding such that there is only one -// valid syntactical representation for a given semantic meaning, -// i.e. the encoding should be deterministic. -message ScalarDescriptor { - - // name is the name of the scalar. It should be a short-name (without - // a period) such that the fully qualified name of the scalar will be - // package.name, ex. for the package a.b and scalar named C, the - // fully-qualified name will be a.b.C. - string name = 1; - - // description is a human-readable description of the scalar and its - // encoding format. For instance a big integer or decimal scalar should - // specify precisely the expected encoding format. - string description = 2; - - // field_type is the type of field with which this scalar can be used. - // Scalars can be used with one and only one type of field so that - // encoding standards and simple and clear. Currently only string and - // bytes fields are supported for scalars. - repeated ScalarType field_type = 3; -} - -enum ScalarType { - SCALAR_TYPE_UNSPECIFIED = 0; - SCALAR_TYPE_STRING = 1; - SCALAR_TYPE_BYTES = 2; -} diff --git a/chain/cosmos/proto/firehose/annotations.proto b/chain/cosmos/proto/firehose/annotations.proto deleted file mode 100644 index 1476c1ab08d..00000000000 --- a/chain/cosmos/proto/firehose/annotations.proto +++ /dev/null @@ -1,11 +0,0 @@ -syntax = "proto3"; - -package firehose; - -option go_package = "github.com/streamingfast/pbgo/sf/firehose/v1;pbfirehose"; - -import "google/protobuf/descriptor.proto"; - -extend google.protobuf.FieldOptions { - optional bool required = 77001; -} diff --git a/chain/cosmos/proto/gogoproto/gogo.proto b/chain/cosmos/proto/gogoproto/gogo.proto deleted file mode 100644 index 49e78f99fe5..00000000000 --- a/chain/cosmos/proto/gogoproto/gogo.proto +++ /dev/null @@ -1,145 +0,0 @@ -// Protocol Buffers for Go with Gadgets -// -// Copyright (c) 2013, The GoGo Authors. All rights reserved. -// http://github.com/gogo/protobuf -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -syntax = "proto2"; -package gogoproto; - -import "google/protobuf/descriptor.proto"; - -option java_package = "com.google.protobuf"; -option java_outer_classname = "GoGoProtos"; -option go_package = "github.com/gogo/protobuf/gogoproto"; - -extend google.protobuf.EnumOptions { - optional bool goproto_enum_prefix = 62001; - optional bool goproto_enum_stringer = 62021; - optional bool enum_stringer = 62022; - optional string enum_customname = 62023; - optional bool enumdecl = 62024; -} - -extend google.protobuf.EnumValueOptions { - optional string enumvalue_customname = 66001; -} - -extend google.protobuf.FileOptions { - optional bool goproto_getters_all = 63001; - optional bool goproto_enum_prefix_all = 63002; - optional bool goproto_stringer_all = 63003; - optional bool verbose_equal_all = 63004; - optional bool face_all = 63005; - optional bool gostring_all = 63006; - optional bool populate_all = 63007; - optional bool stringer_all = 63008; - optional bool onlyone_all = 63009; - - optional bool equal_all = 63013; - optional bool description_all = 63014; - optional bool testgen_all = 63015; - optional bool benchgen_all = 63016; - optional bool marshaler_all = 63017; - optional bool unmarshaler_all = 63018; - optional bool stable_marshaler_all = 63019; - - optional bool sizer_all = 63020; - - optional bool goproto_enum_stringer_all = 63021; - optional bool enum_stringer_all = 63022; - - optional bool unsafe_marshaler_all = 63023; - optional bool unsafe_unmarshaler_all = 63024; - - optional bool goproto_extensions_map_all = 63025; - optional bool goproto_unrecognized_all = 63026; - optional bool gogoproto_import = 63027; - optional bool protosizer_all = 63028; - optional bool compare_all = 63029; - optional bool typedecl_all = 63030; - optional bool enumdecl_all = 63031; - - optional bool goproto_registration = 63032; - optional bool messagename_all = 63033; - - optional bool goproto_sizecache_all = 63034; - optional bool goproto_unkeyed_all = 63035; -} - -extend google.protobuf.MessageOptions { - optional bool goproto_getters = 64001; - optional bool goproto_stringer = 64003; - optional bool verbose_equal = 64004; - optional bool face = 64005; - optional bool gostring = 64006; - optional bool populate = 64007; - optional bool stringer = 67008; - optional bool onlyone = 64009; - - optional bool equal = 64013; - optional bool description = 64014; - optional bool testgen = 64015; - optional bool benchgen = 64016; - optional bool marshaler = 64017; - optional bool unmarshaler = 64018; - optional bool stable_marshaler = 64019; - - optional bool sizer = 64020; - - optional bool unsafe_marshaler = 64023; - optional bool unsafe_unmarshaler = 64024; - - optional bool goproto_extensions_map = 64025; - optional bool goproto_unrecognized = 64026; - - optional bool protosizer = 64028; - optional bool compare = 64029; - - optional bool typedecl = 64030; - - optional bool messagename = 64033; - - optional bool goproto_sizecache = 64034; - optional bool goproto_unkeyed = 64035; -} - -extend google.protobuf.FieldOptions { - optional bool nullable = 65001; - optional bool embed = 65002; - optional string customtype = 65003; - optional string customname = 65004; - optional string jsontag = 65005; - optional string moretags = 65006; - optional string casttype = 65007; - optional string castkey = 65008; - optional string castvalue = 65009; - - optional bool stdtime = 65010; - optional bool stdduration = 65011; - optional bool wktpointer = 65012; - - optional string castrepeated = 65013; -} diff --git a/chain/cosmos/proto/type.proto b/chain/cosmos/proto/type.proto deleted file mode 100644 index c32502da1e9..00000000000 --- a/chain/cosmos/proto/type.proto +++ /dev/null @@ -1,368 +0,0 @@ -syntax = "proto3"; - -package sf.cosmos.type.v1; - -option go_package = "github.com/figment-networks/proto-cosmos/pb/sf/cosmos/type/v1;pbcosmos"; - -import "google/protobuf/descriptor.proto"; -import "google/protobuf/any.proto"; -import "gogoproto/gogo.proto"; -import "cosmos_proto/cosmos.proto"; -import "firehose/annotations.proto"; - -message Block { - Header header = 1 [(firehose.required) = true, (gogoproto.nullable) = false]; - EvidenceList evidence = 2 [(gogoproto.nullable) = false]; - Commit last_commit = 3; - ResponseBeginBlock result_begin_block = 4 [(firehose.required) = true]; - ResponseEndBlock result_end_block = 5 [(firehose.required) = true]; - repeated TxResult transactions = 7; - repeated Validator validator_updates = 8; -} - -// HeaderOnlyBlock is a standard [Block] structure where all other fields are -// removed so that hydrating that object from a [Block] bytes payload will -// drastically reduce allocated memory required to hold the full block. -// -// This can be used to unpack a [Block] when only the [Header] information -// is required and greatly reduce required memory. -message HeaderOnlyBlock { - Header header = 1 [(firehose.required) = true, (gogoproto.nullable) = false]; -} - -message EventData { - Event event = 1 [(firehose.required) = true]; - HeaderOnlyBlock block = 2 [(firehose.required) = true]; - TransactionContext tx = 3; -} - -message TransactionData { - TxResult tx = 1 [(firehose.required) = true]; - HeaderOnlyBlock block = 2 [(firehose.required) = true]; -} - -message MessageData { - google.protobuf.Any message = 1 [(firehose.required) = true]; - HeaderOnlyBlock block = 2 [(firehose.required) = true]; - TransactionContext tx = 3 [(firehose.required) = true]; -} - -message TransactionContext { - bytes hash = 1; - uint32 index = 2; - uint32 code = 3; - int64 gas_wanted = 4; - int64 gas_used = 5; -} - -message Header { - Consensus version = 1 [(gogoproto.nullable) = false]; - string chain_id = 2 [(gogoproto.customname) = "ChainID"]; - uint64 height = 3; - Timestamp time = 4 [(gogoproto.nullable) = false, (gogoproto.stdtime) = true]; - BlockID last_block_id = 5 [(firehose.required) = true, (gogoproto.nullable) = false]; - bytes last_commit_hash = 6; - bytes data_hash = 7; - bytes validators_hash = 8; - bytes next_validators_hash = 9; - bytes consensus_hash = 10; - bytes app_hash = 11; - bytes last_results_hash = 12; - bytes evidence_hash = 13; - bytes proposer_address = 14; - bytes hash = 15; -} - -message Consensus { - option (gogoproto.equal) = true; - - uint64 block = 1; - uint64 app = 2; -} - -message Timestamp { - int64 seconds = 1; - int32 nanos = 2; -} - -message BlockID { - bytes hash = 1; - PartSetHeader part_set_header = 2 [(gogoproto.nullable) = false]; -} - -message PartSetHeader { - uint32 total = 1; - bytes hash = 2; -} - -message EvidenceList { - repeated Evidence evidence = 1 [(gogoproto.nullable) = false]; -} - -message Evidence { - oneof sum { - DuplicateVoteEvidence duplicate_vote_evidence = 1; - LightClientAttackEvidence light_client_attack_evidence = 2; - } -} - -message DuplicateVoteEvidence { - EventVote vote_a = 1; - EventVote vote_b = 2; - int64 total_voting_power = 3; - int64 validator_power = 4; - Timestamp timestamp = 5 [(gogoproto.nullable) = false, (gogoproto.stdtime) = true]; -} - -message EventVote { - SignedMsgType event_vote_type = 1 [json_name = "type"]; - uint64 height = 2; - int32 round = 3; - BlockID block_id = 4 [(gogoproto.nullable) = false, (gogoproto.customname) = "BlockID"]; - Timestamp timestamp = 5 [(gogoproto.nullable) = false, (gogoproto.stdtime) = true]; - bytes validator_address = 6; - int32 validator_index = 7; - bytes signature = 8; -} - -enum SignedMsgType { - option (gogoproto.goproto_enum_stringer) = true; - option (gogoproto.goproto_enum_prefix) = false; - - SIGNED_MSG_TYPE_UNKNOWN = 0 [(gogoproto.enumvalue_customname) = "UnknownType"]; - SIGNED_MSG_TYPE_PREVOTE = 1 [(gogoproto.enumvalue_customname) = "PrevoteType"]; - SIGNED_MSG_TYPE_PRECOMMIT = 2 [(gogoproto.enumvalue_customname) = "PrecommitType"]; - SIGNED_MSG_TYPE_PROPOSAL = 32 [(gogoproto.enumvalue_customname) = "ProposalType"]; -} - -message LightClientAttackEvidence { - LightBlock conflicting_block = 1; - int64 common_height = 2; - repeated Validator byzantine_validators = 3; - int64 total_voting_power = 4; - Timestamp timestamp = 5 [(gogoproto.nullable) = false, (gogoproto.stdtime) = true]; -} - -message LightBlock { - SignedHeader signed_header = 1; - ValidatorSet validator_set = 2; -} - -message SignedHeader { - Header header = 1; - Commit commit = 2; -} - -message Commit { - int64 height = 1; - int32 round = 2; - BlockID block_id = 3 [(gogoproto.nullable) = false, (gogoproto.customname) = "BlockID"]; - repeated CommitSig signatures = 4 [(gogoproto.nullable) = false]; -} - -message CommitSig { - BlockIDFlag block_id_flag = 1; - bytes validator_address = 2; - Timestamp timestamp = 3 [(gogoproto.nullable) = false, (gogoproto.stdtime) = true]; - bytes signature = 4; -} - -enum BlockIDFlag { - option (gogoproto.goproto_enum_stringer) = true; - option (gogoproto.goproto_enum_prefix) = false; - - BLOCK_ID_FLAG_UNKNOWN = 0 [(gogoproto.enumvalue_customname) = "BlockIDFlagUnknown"]; - BLOCK_ID_FLAG_ABSENT = 1 [(gogoproto.enumvalue_customname) = "BlockIDFlagAbsent"]; - BLOCK_ID_FLAG_COMMIT = 2 [(gogoproto.enumvalue_customname) = "BlockIDFlagCommit"]; - BLOCK_ID_FLAG_NIL = 3 [(gogoproto.enumvalue_customname) = "BlockIDFlagNil"]; -} - -message ValidatorSet { - repeated Validator validators = 1; - Validator proposer = 2; - int64 total_voting_power = 3; -} - -message Validator { - bytes address = 1; - PublicKey pub_key = 2 [(gogoproto.nullable) = false]; - int64 voting_power = 3; - int64 proposer_priority = 4; -} - -message PublicKey { - option (gogoproto.compare) = true; - option (gogoproto.equal) = true; - - oneof sum { - bytes ed25519 = 1; - bytes secp256k1 = 2; - } -} - -message ResponseBeginBlock { - repeated Event events = 1 [(gogoproto.nullable) = false, (gogoproto.jsontag) = "events,omitempty"]; -} - -message Event { - string event_type = 1 [json_name = "type"]; - repeated EventAttribute attributes = 2 [(gogoproto.nullable) = false, (gogoproto.jsontag) = "attributes,omitempty"]; -} - -message EventAttribute { - string key = 1; - string value = 2; - bool index = 3; -} - -message ResponseEndBlock { - repeated ValidatorUpdate validator_updates = 1; - ConsensusParams consensus_param_updates = 2; - repeated Event events = 3; -} - -message ValidatorUpdate { - bytes address = 1; - PublicKey pub_key = 2 [(gogoproto.nullable) = false]; - int64 power = 3; -} - -message ConsensusParams { - BlockParams block = 1 [(gogoproto.nullable) = false]; - EvidenceParams evidence = 2 [(gogoproto.nullable) = false]; - ValidatorParams validator = 3 [(gogoproto.nullable) = false]; - VersionParams version = 4 [(gogoproto.nullable) = false]; -} - -message BlockParams { - int64 max_bytes = 1; - int64 max_gas = 2; -} - -message EvidenceParams { - int64 max_age_num_blocks = 1; - Duration max_age_duration = 2 [(gogoproto.nullable) = false, (gogoproto.stdduration) = true]; - int64 max_bytes = 3; -} - -message Duration { - int64 seconds = 1; - int32 nanos = 2; -} - -message ValidatorParams { - option (gogoproto.populate) = true; - option (gogoproto.equal) = true; - - repeated string pub_key_types = 1; -} - -message VersionParams { - option (gogoproto.populate) = true; - option (gogoproto.equal) = true; - - uint64 app_version = 1; -} - -message TxResult { - uint64 height = 1; - uint32 index = 2; - Tx tx = 3 [(firehose.required) = true]; - ResponseDeliverTx result = 4 [(firehose.required) = true]; - bytes hash = 5; -} - -message Tx { - TxBody body = 1 [(firehose.required) = true]; - AuthInfo auth_info = 2; - repeated bytes signatures = 3; -} - -message TxBody { - repeated google.protobuf.Any messages = 1; - string memo = 2; - uint64 timeout_height = 3; - repeated google.protobuf.Any extension_options = 1023; - repeated google.protobuf.Any non_critical_extension_options = 2047; -} - -message Any { - string type_url = 1; - bytes value = 2; -} - -message AuthInfo { - repeated SignerInfo signer_infos = 1; - Fee fee = 2; - Tip tip = 3; -} - -message SignerInfo { - google.protobuf.Any public_key = 1; - ModeInfo mode_info = 2; - uint64 sequence = 3; -} - -message ModeInfo { - oneof sum { - ModeInfoSingle single = 1; - ModeInfoMulti multi = 2; - } -} - -message ModeInfoSingle { - SignMode mode = 1; -} - -enum SignMode { - SIGN_MODE_UNSPECIFIED = 0; - SIGN_MODE_DIRECT = 1; - SIGN_MODE_TEXTUAL = 2; - SIGN_MODE_LEGACY_AMINO_JSON = 127; -} - -message ModeInfoMulti { - CompactBitArray bitarray = 1; - repeated ModeInfo mode_infos = 2; -} - -message CompactBitArray { - option (gogoproto.goproto_stringer) = false; - - uint32 extra_bits_stored = 1; - bytes elems = 2; -} - -message Fee { - repeated Coin amount = 1 [(gogoproto.nullable) = false, (gogoproto.castrepeated) = "github.com/cosmos/cosmos-sdk/types.Coins"]; - uint64 gas_limit = 2; - string payer = 3 [(cosmos_proto.scalar) = "cosmos.AddressString"]; - string granter = 4 [(cosmos_proto.scalar) = "cosmos.AddressString"]; -} - -message Coin { - option (gogoproto.equal) = true; - - string denom = 1; - string amount = 2 [(gogoproto.customtype) = "Int", (gogoproto.nullable) = false]; -} - -message Tip { - repeated Coin amount = 1 [(gogoproto.nullable) = false, (gogoproto.castrepeated) = "github.com/cosmos/cosmos-sdk/types.Coins"]; - string tipper = 2 [(cosmos_proto.scalar) = "cosmos.AddressString"]; -} - -message ResponseDeliverTx { - uint32 code = 1; - bytes data = 2; - string log = 3; - string info = 4; - int64 gas_wanted = 5; - int64 gas_used = 6; - repeated Event events = 7 [(gogoproto.nullable) = false, (gogoproto.jsontag) = "events,omitempty"]; - string codespace = 8; -} - -message ValidatorSetUpdates { - repeated Validator validator_updates = 1; -} diff --git a/chain/cosmos/src/adapter.rs b/chain/cosmos/src/adapter.rs deleted file mode 100644 index 746c91e2e07..00000000000 --- a/chain/cosmos/src/adapter.rs +++ /dev/null @@ -1,159 +0,0 @@ -use std::collections::HashSet; - -use prost::Message; -use prost_types::Any; - -use crate::{data_source::DataSource, Chain}; -use graph::blockchain as bc; -use graph::firehose::EventTypeFilter; -use graph::prelude::*; - -const EVENT_TYPE_FILTER_TYPE_URL: &str = - "type.googleapis.com/sf.cosmos.transform.v1.EventTypeFilter"; - -#[derive(Clone, Debug, Default)] -pub struct TriggerFilter { - pub(crate) event_type_filter: CosmosEventTypeFilter, - pub(crate) block_filter: CosmosBlockFilter, -} - -impl bc::TriggerFilter for TriggerFilter { - fn extend<'a>(&mut self, data_sources: impl Iterator + Clone) { - self.event_type_filter - .extend_from_data_sources(data_sources.clone()); - self.block_filter.extend_from_data_sources(data_sources); - } - - fn node_capabilities(&self) -> bc::EmptyNodeCapabilities { - bc::EmptyNodeCapabilities::default() - } - - fn extend_with_template( - &mut self, - _data_source: impl Iterator::DataSourceTemplate>, - ) { - } - - fn to_firehose_filter(self) -> Vec { - if self.block_filter.trigger_every_block { - return vec![]; - } - - if self.event_type_filter.event_types.is_empty() { - return vec![]; - } - - let filter = EventTypeFilter { - event_types: Vec::from_iter(self.event_type_filter.event_types), - }; - - vec![Any { - type_url: EVENT_TYPE_FILTER_TYPE_URL.to_string(), - value: filter.encode_to_vec(), - }] - } -} - -pub type EventType = String; - -#[derive(Clone, Debug, Default)] -pub(crate) struct CosmosEventTypeFilter { - pub event_types: HashSet, -} - -impl CosmosEventTypeFilter { - pub(crate) fn matches(&self, event_type: &EventType) -> bool { - self.event_types.contains(event_type) - } - - fn extend_from_data_sources<'a>(&mut self, data_sources: impl Iterator) { - self.event_types.extend( - data_sources.flat_map(|data_source| data_source.events().map(ToString::to_string)), - ); - } -} - -#[derive(Clone, Debug, Default)] -pub(crate) struct CosmosBlockFilter { - pub trigger_every_block: bool, -} - -impl CosmosBlockFilter { - fn extend_from_data_sources<'a>( - &mut self, - mut data_sources: impl Iterator, - ) { - if !self.trigger_every_block { - self.trigger_every_block = data_sources.any(DataSource::has_block_handler); - } - } -} - -#[cfg(test)] -mod test { - use graph::blockchain::TriggerFilter as _; - - use super::*; - - #[test] - fn test_trigger_filters() { - let cases = [ - (TriggerFilter::test_new(false, &[]), None), - (TriggerFilter::test_new(true, &[]), None), - (TriggerFilter::test_new(true, &["event_1", "event_2"]), None), - ( - TriggerFilter::test_new(false, &["event_1", "event_2", "event_3"]), - Some(event_type_filter_with(&["event_1", "event_2", "event_3"])), - ), - ]; - - for (trigger_filter, expected_filter) in cases { - let firehose_filter = trigger_filter.to_firehose_filter(); - let decoded_filter = decode_filter(firehose_filter); - - assert_eq!(decoded_filter.is_some(), expected_filter.is_some()); - - if let (Some(mut expected_filter), Some(mut decoded_filter)) = - (expected_filter, decoded_filter) - { - // event types may be in different order - expected_filter.event_types.sort(); - decoded_filter.event_types.sort(); - - assert_eq!(decoded_filter, expected_filter); - } - } - } - - impl TriggerFilter { - pub(crate) fn test_new(trigger_every_block: bool, event_types: &[&str]) -> TriggerFilter { - TriggerFilter { - event_type_filter: CosmosEventTypeFilter { - event_types: event_types.iter().map(ToString::to_string).collect(), - }, - block_filter: CosmosBlockFilter { - trigger_every_block, - }, - } - } - } - - fn event_type_filter_with(event_types: &[&str]) -> EventTypeFilter { - EventTypeFilter { - event_types: event_types.iter().map(ToString::to_string).collect(), - } - } - - fn decode_filter(proto_filters: Vec) -> Option { - assert!(proto_filters.len() <= 1); - - let proto_filter = proto_filters.get(0)?; - - assert_eq!(proto_filter.type_url, EVENT_TYPE_FILTER_TYPE_URL); - - let firehose_filter = EventTypeFilter::decode(&*proto_filter.value) - .expect("Could not decode EventTypeFilter from protobuf Any"); - - Some(firehose_filter) - } -} diff --git a/chain/cosmos/src/chain.rs b/chain/cosmos/src/chain.rs deleted file mode 100644 index f4a8b2953b1..00000000000 --- a/chain/cosmos/src/chain.rs +++ /dev/null @@ -1,645 +0,0 @@ -use std::sync::Arc; - -use graph::blockchain::block_stream::FirehoseCursor; -use graph::blockchain::client::ChainClient; -use graph::cheap_clone::CheapClone; -use graph::data::subgraph::UnifiedMappingApiVersion; -use graph::prelude::MetricsRegistry; -use graph::{ - blockchain::{ - block_stream::{ - BlockStream, BlockStreamEvent, BlockWithTriggers, FirehoseError, - FirehoseMapper as FirehoseMapperTrait, TriggersAdapter as TriggersAdapterTrait, - }, - firehose_block_stream::FirehoseBlockStream, - Block as _, BlockHash, BlockPtr, Blockchain, BlockchainKind, EmptyNodeCapabilities, - IngestorError, RuntimeAdapter as RuntimeAdapterTrait, - }, - components::store::DeploymentLocator, - firehose::{self, FirehoseEndpoint, FirehoseEndpoints, ForkStep}, - prelude::{async_trait, o, BlockNumber, ChainStore, Error, Logger, LoggerFactory}, -}; -use prost::Message; - -use crate::data_source::{ - DataSource, DataSourceTemplate, EventOrigin, UnresolvedDataSource, UnresolvedDataSourceTemplate, -}; -use crate::trigger::CosmosTrigger; -use crate::RuntimeAdapter; -use crate::{codec, TriggerFilter}; - -pub struct Chain { - logger_factory: LoggerFactory, - name: String, - client: Arc>, - chain_store: Arc, - metrics_registry: Arc, -} - -impl std::fmt::Debug for Chain { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "chain: cosmos") - } -} - -impl Chain { - pub fn new( - logger_factory: LoggerFactory, - name: String, - chain_store: Arc, - firehose_endpoints: FirehoseEndpoints, - metrics_registry: Arc, - ) -> Self { - Chain { - logger_factory, - name, - client: Arc::new(ChainClient::new_firehose(firehose_endpoints)), - chain_store, - metrics_registry, - } - } -} - -#[async_trait] -impl Blockchain for Chain { - const KIND: BlockchainKind = BlockchainKind::Cosmos; - - type Client = (); - type Block = codec::Block; - - type DataSource = DataSource; - - type UnresolvedDataSource = UnresolvedDataSource; - - type DataSourceTemplate = DataSourceTemplate; - - type UnresolvedDataSourceTemplate = UnresolvedDataSourceTemplate; - - type TriggerData = CosmosTrigger; - - type MappingTrigger = CosmosTrigger; - - type TriggerFilter = TriggerFilter; - - type NodeCapabilities = EmptyNodeCapabilities; - - fn is_refetch_block_required(&self) -> bool { - false - } - async fn refetch_firehose_block( - &self, - _logger: &Logger, - _cursor: FirehoseCursor, - ) -> Result { - unimplemented!("This chain does not support Dynamic Data Sources. is_refetch_block_required always returns false, this shouldn't be called.") - } - - fn triggers_adapter( - &self, - _loc: &DeploymentLocator, - _capabilities: &Self::NodeCapabilities, - _unified_api_version: UnifiedMappingApiVersion, - ) -> Result>, Error> { - let adapter = TriggersAdapter {}; - Ok(Arc::new(adapter)) - } - - async fn new_firehose_block_stream( - &self, - deployment: DeploymentLocator, - block_cursor: FirehoseCursor, - start_blocks: Vec, - subgraph_current_block: Option, - filter: Arc, - unified_api_version: UnifiedMappingApiVersion, - ) -> Result>, Error> { - let adapter = self - .triggers_adapter( - &deployment, - &EmptyNodeCapabilities::default(), - unified_api_version, - ) - .unwrap_or_else(|_| panic!("no adapter for network {}", self.name)); - - let firehose_endpoint = self.client.firehose_endpoint()?; - - let logger = self - .logger_factory - .subgraph_logger(&deployment) - .new(o!("component" => "FirehoseBlockStream")); - - let firehose_mapper = Arc::new(FirehoseMapper {}); - - Ok(Box::new(FirehoseBlockStream::new( - deployment.hash, - firehose_endpoint, - subgraph_current_block, - block_cursor, - firehose_mapper, - adapter, - filter, - start_blocks, - logger, - self.metrics_registry.clone(), - ))) - } - - async fn new_polling_block_stream( - &self, - _deployment: DeploymentLocator, - _start_blocks: Vec, - _subgraph_start_block: Option, - _filter: Arc, - _unified_api_version: UnifiedMappingApiVersion, - ) -> Result>, Error> { - panic!("Cosmos does not support polling block stream") - } - - fn chain_store(&self) -> Arc { - self.chain_store.cheap_clone() - } - - async fn block_pointer_from_number( - &self, - logger: &Logger, - number: BlockNumber, - ) -> Result { - let firehose_endpoint = self.client.firehose_endpoint()?; - - firehose_endpoint - .block_ptr_for_number::(logger, number) - .await - .map_err(Into::into) - } - - fn runtime_adapter(&self) -> Arc> { - Arc::new(RuntimeAdapter {}) - } - - fn chain_client(&self) -> Arc> { - self.client.clone() - } -} - -pub struct TriggersAdapter {} - -#[async_trait] -impl TriggersAdapterTrait for TriggersAdapter { - async fn ancestor_block( - &self, - _ptr: BlockPtr, - _offset: BlockNumber, - ) -> Result, Error> { - panic!("Should never be called since not used by FirehoseBlockStream") - } - - async fn scan_triggers( - &self, - _from: BlockNumber, - _to: BlockNumber, - _filter: &TriggerFilter, - ) -> Result>, Error> { - panic!("Should never be called since not used by FirehoseBlockStream") - } - - async fn triggers_in_block( - &self, - logger: &Logger, - block: codec::Block, - filter: &TriggerFilter, - ) -> Result, Error> { - let shared_block = Arc::new(block.clone()); - - let header_only_block = codec::HeaderOnlyBlock::from(&block); - - let mut triggers: Vec<_> = shared_block - .begin_block_events()? - .cloned() - // FIXME (Cosmos): Optimize. Should use an Arc instead of cloning the - // block. This is not currently possible because EventData is automatically - // generated. - .filter_map(|event| { - filter_event_trigger( - filter, - event, - &header_only_block, - None, - EventOrigin::BeginBlock, - ) - }) - .chain(shared_block.transactions().flat_map(|tx| { - tx.result - .as_ref() - .unwrap() - .events - .iter() - .filter_map(|e| { - filter_event_trigger( - filter, - e.clone(), - &header_only_block, - Some(build_tx_context(tx)), - EventOrigin::DeliverTx, - ) - }) - .collect::>() - })) - .chain( - shared_block - .end_block_events()? - .cloned() - .filter_map(|event| { - filter_event_trigger( - filter, - event, - &header_only_block, - None, - EventOrigin::EndBlock, - ) - }), - ) - .collect(); - - triggers.extend(shared_block.transactions().cloned().flat_map(|tx_result| { - let mut triggers: Vec<_> = Vec::new(); - if let Some(tx) = tx_result.tx.clone() { - if let Some(tx_body) = tx.body { - triggers.extend(tx_body.messages.into_iter().map(|message| { - CosmosTrigger::with_message( - message, - header_only_block.clone(), - build_tx_context(&tx_result), - ) - })); - } - } - triggers.push(CosmosTrigger::with_transaction( - tx_result, - header_only_block.clone(), - )); - triggers - })); - - if filter.block_filter.trigger_every_block { - triggers.push(CosmosTrigger::Block(shared_block.cheap_clone())); - } - - Ok(BlockWithTriggers::new(block, triggers, logger)) - } - - async fn is_on_main_chain(&self, _ptr: BlockPtr) -> Result { - panic!("Should never be called since not used by FirehoseBlockStream") - } - - /// Panics if `block` is genesis. - /// But that's ok since this is only called when reverting `block`. - async fn parent_ptr(&self, block: &BlockPtr) -> Result, Error> { - Ok(Some(BlockPtr { - hash: BlockHash::from(vec![0xff; 32]), - number: block.number.saturating_sub(1), - })) - } -} - -/// Returns a new event trigger only if the given event matches the event filter. -fn filter_event_trigger( - filter: &TriggerFilter, - event: codec::Event, - block: &codec::HeaderOnlyBlock, - tx_context: Option, - origin: EventOrigin, -) -> Option { - if filter.event_type_filter.matches(&event.event_type) { - Some(CosmosTrigger::with_event( - event, - block.clone(), - tx_context, - origin, - )) - } else { - None - } -} - -fn build_tx_context(tx: &codec::TxResult) -> codec::TransactionContext { - codec::TransactionContext { - hash: tx.hash.clone(), - index: tx.index, - code: tx.result.as_ref().unwrap().code, - gas_wanted: tx.result.as_ref().unwrap().gas_wanted, - gas_used: tx.result.as_ref().unwrap().gas_used, - } -} - -pub struct FirehoseMapper {} - -#[async_trait] -impl FirehoseMapperTrait for FirehoseMapper { - async fn to_block_stream_event( - &self, - logger: &Logger, - response: &firehose::Response, - adapter: &Arc>, - filter: &TriggerFilter, - ) -> Result, FirehoseError> { - let step = ForkStep::from_i32(response.step).unwrap_or_else(|| { - panic!( - "unknown step i32 value {}, maybe you forgot update & re-regenerate the protobuf definitions?", - response.step - ) - }); - - let any_block = response - .block - .as_ref() - .expect("block payload information should always be present"); - - // Right now, this is done in all cases but in reality, with how the BlockStreamEvent::Revert - // is defined right now, only block hash and block number is necessary. However, this information - // is not part of the actual bstream::BlockResponseV2 payload. As such, we need to decode the full - // block which is useless. - // - // Check about adding basic information about the block in the bstream::BlockResponseV2 or maybe - // define a slimmed down struct that would decode only a few fields and ignore all the rest. - let sp = codec::Block::decode(any_block.value.as_ref())?; - - match step { - ForkStep::StepNew => Ok(BlockStreamEvent::ProcessBlock( - adapter.triggers_in_block(logger, sp, filter).await?, - FirehoseCursor::from(response.cursor.clone()), - )), - - ForkStep::StepUndo => { - let parent_ptr = sp - .parent_ptr() - .map_err(FirehoseError::from)? - .expect("Genesis block should never be reverted"); - - Ok(BlockStreamEvent::Revert( - parent_ptr, - FirehoseCursor::from(response.cursor.clone()), - )) - } - - ForkStep::StepFinal => { - panic!( - "final step is not handled and should not be requested in the Firehose request" - ) - } - - ForkStep::StepUnset => { - panic!("unknown step should not happen in the Firehose response") - } - } - } - - async fn block_ptr_for_number( - &self, - logger: &Logger, - endpoint: &Arc, - number: BlockNumber, - ) -> Result { - endpoint - .block_ptr_for_number::(logger, number) - .await - } - - async fn final_block_ptr_for( - &self, - logger: &Logger, - endpoint: &Arc, - block: &codec::Block, - ) -> Result { - // Cosmos provides instant block finality. - self.block_ptr_for_number(logger, endpoint, block.number()) - .await - } -} - -#[cfg(test)] -mod test { - use graph::prelude::{ - slog::{o, Discard, Logger}, - tokio, - }; - - use super::*; - - use codec::{ - Block, Event, Header, HeaderOnlyBlock, ResponseBeginBlock, ResponseDeliverTx, - ResponseEndBlock, TxResult, - }; - - #[tokio::test] - async fn test_trigger_filters() { - let adapter = TriggersAdapter {}; - let logger = Logger::root(Discard, o!()); - - let block_with_events = Block::test_with_event_types( - vec!["begin_event_1", "begin_event_2", "begin_event_3"], - vec!["tx_event_1", "tx_event_2", "tx_event_3"], - vec!["end_event_1", "end_event_2", "end_event_3"], - ); - - let header_only_block = HeaderOnlyBlock::from(&block_with_events); - - let cases = [ - ( - Block::test_new(), - TriggerFilter::test_new(false, &[]), - vec![], - ), - ( - Block::test_new(), - TriggerFilter::test_new(true, &[]), - vec![CosmosTrigger::Block(Arc::new(Block::test_new()))], - ), - ( - Block::test_new(), - TriggerFilter::test_new(false, &["event_1", "event_2", "event_3"]), - vec![], - ), - ( - block_with_events.clone(), - TriggerFilter::test_new(false, &["begin_event_3", "tx_event_3", "end_event_3"]), - vec![ - CosmosTrigger::with_event( - Event::test_with_type("begin_event_3"), - header_only_block.clone(), - None, - EventOrigin::BeginBlock, - ), - CosmosTrigger::with_event( - Event::test_with_type("tx_event_3"), - header_only_block.clone(), - Some(build_tx_context(&block_with_events.transactions[2])), - EventOrigin::DeliverTx, - ), - CosmosTrigger::with_event( - Event::test_with_type("end_event_3"), - header_only_block.clone(), - None, - EventOrigin::EndBlock, - ), - CosmosTrigger::with_transaction( - TxResult::test_with_event_type("tx_event_1"), - header_only_block.clone(), - ), - CosmosTrigger::with_transaction( - TxResult::test_with_event_type("tx_event_2"), - header_only_block.clone(), - ), - CosmosTrigger::with_transaction( - TxResult::test_with_event_type("tx_event_3"), - header_only_block.clone(), - ), - ], - ), - ( - block_with_events.clone(), - TriggerFilter::test_new(true, &["begin_event_3", "tx_event_2", "end_event_1"]), - vec![ - CosmosTrigger::Block(Arc::new(block_with_events.clone())), - CosmosTrigger::with_event( - Event::test_with_type("begin_event_3"), - header_only_block.clone(), - None, - EventOrigin::BeginBlock, - ), - CosmosTrigger::with_event( - Event::test_with_type("tx_event_2"), - header_only_block.clone(), - Some(build_tx_context(&block_with_events.transactions[1])), - EventOrigin::DeliverTx, - ), - CosmosTrigger::with_event( - Event::test_with_type("end_event_1"), - header_only_block.clone(), - None, - EventOrigin::EndBlock, - ), - CosmosTrigger::with_transaction( - TxResult::test_with_event_type("tx_event_1"), - header_only_block.clone(), - ), - CosmosTrigger::with_transaction( - TxResult::test_with_event_type("tx_event_2"), - header_only_block.clone(), - ), - CosmosTrigger::with_transaction( - TxResult::test_with_event_type("tx_event_3"), - header_only_block.clone(), - ), - ], - ), - ]; - - for (block, trigger_filter, expected_triggers) in cases { - let triggers = adapter - .triggers_in_block(&logger, block, &trigger_filter) - .await - .expect("failed to get triggers in block"); - - assert_eq!( - triggers.trigger_data.len(), - expected_triggers.len(), - "Expected trigger list to contain exactly {:?}, but it didn't: {:?}", - expected_triggers, - triggers.trigger_data - ); - - // they may not be in the same order - for trigger in expected_triggers { - assert!( - triggers.trigger_data.contains(&trigger), - "Expected trigger list to contain {:?}, but it only contains: {:?}", - trigger, - triggers.trigger_data - ); - } - } - } - - impl Block { - fn test_new() -> Block { - Block::test_with_event_types(vec![], vec![], vec![]) - } - - fn test_with_event_types( - begin_event_types: Vec<&str>, - tx_event_types: Vec<&str>, - end_event_types: Vec<&str>, - ) -> Block { - Block { - header: Some(Header { - version: None, - chain_id: "test".to_string(), - height: 1, - time: None, - last_block_id: None, - last_commit_hash: vec![], - data_hash: vec![], - validators_hash: vec![], - next_validators_hash: vec![], - consensus_hash: vec![], - app_hash: vec![], - last_results_hash: vec![], - evidence_hash: vec![], - proposer_address: vec![], - hash: vec![], - }), - evidence: None, - last_commit: None, - result_begin_block: Some(ResponseBeginBlock { - events: begin_event_types - .into_iter() - .map(Event::test_with_type) - .collect(), - }), - result_end_block: Some(ResponseEndBlock { - validator_updates: vec![], - consensus_param_updates: None, - events: end_event_types - .into_iter() - .map(Event::test_with_type) - .collect(), - }), - transactions: tx_event_types - .into_iter() - .map(TxResult::test_with_event_type) - .collect(), - validator_updates: vec![], - } - } - } - - impl Event { - fn test_with_type(event_type: &str) -> Event { - Event { - event_type: event_type.to_string(), - attributes: vec![], - } - } - } - - impl TxResult { - fn test_with_event_type(event_type: &str) -> TxResult { - TxResult { - height: 1, - index: 1, - tx: None, - result: Some(ResponseDeliverTx { - code: 1, - data: vec![], - log: "".to_string(), - info: "".to_string(), - gas_wanted: 1, - gas_used: 1, - codespace: "".to_string(), - events: vec![Event::test_with_type(event_type)], - }), - hash: vec![], - } - } - } -} diff --git a/chain/cosmos/src/codec.rs b/chain/cosmos/src/codec.rs deleted file mode 100644 index fae145a449e..00000000000 --- a/chain/cosmos/src/codec.rs +++ /dev/null @@ -1,188 +0,0 @@ -pub(crate) use crate::protobuf::pbcodec::*; - -use graph::blockchain::Block as BlockchainBlock; -use graph::{ - blockchain::BlockPtr, - prelude::{anyhow::anyhow, BlockNumber, Error}, -}; - -use std::convert::TryFrom; - -impl Block { - pub fn header(&self) -> Result<&Header, Error> { - self.header - .as_ref() - .ok_or_else(|| anyhow!("block data missing header field")) - } - - pub fn begin_block_events(&self) -> Result, Error> { - let events = self - .result_begin_block - .as_ref() - .ok_or_else(|| anyhow!("block data missing result_begin_block field"))? - .events - .iter(); - - Ok(events) - } - - pub fn end_block_events(&self) -> Result, Error> { - let events = self - .result_end_block - .as_ref() - .ok_or_else(|| anyhow!("block data missing result_end_block field"))? - .events - .iter(); - - Ok(events) - } - - pub fn transactions(&self) -> impl Iterator { - self.transactions.iter() - } - - pub fn parent_ptr(&self) -> Result, Error> { - let header = self.header()?; - - Ok(header - .last_block_id - .as_ref() - .map(|last_block_id| BlockPtr::from((last_block_id.hash.clone(), header.height - 1)))) - } -} - -impl TryFrom for BlockPtr { - type Error = Error; - - fn try_from(b: Block) -> Result { - BlockPtr::try_from(&b) - } -} - -impl<'a> TryFrom<&'a Block> for BlockPtr { - type Error = Error; - - fn try_from(b: &'a Block) -> Result { - let header = b.header()?; - Ok(BlockPtr::from((header.hash.clone(), header.height))) - } -} - -impl BlockchainBlock for Block { - fn number(&self) -> i32 { - BlockNumber::try_from(self.header().unwrap().height).unwrap() - } - - fn ptr(&self) -> BlockPtr { - BlockPtr::try_from(self).unwrap() - } - - fn parent_ptr(&self) -> Option { - self.parent_ptr().unwrap() - } -} - -impl HeaderOnlyBlock { - pub fn header(&self) -> Result<&Header, Error> { - self.header - .as_ref() - .ok_or_else(|| anyhow!("block data missing header field")) - } - - pub fn parent_ptr(&self) -> Result, Error> { - let header = self.header()?; - - Ok(header - .last_block_id - .as_ref() - .map(|last_block_id| BlockPtr::from((last_block_id.hash.clone(), header.height - 1)))) - } -} - -impl From<&Block> for HeaderOnlyBlock { - fn from(b: &Block) -> HeaderOnlyBlock { - HeaderOnlyBlock { - header: b.header.clone(), - } - } -} - -impl TryFrom for BlockPtr { - type Error = Error; - - fn try_from(b: HeaderOnlyBlock) -> Result { - BlockPtr::try_from(&b) - } -} - -impl<'a> TryFrom<&'a HeaderOnlyBlock> for BlockPtr { - type Error = Error; - - fn try_from(b: &'a HeaderOnlyBlock) -> Result { - let header = b.header()?; - - Ok(BlockPtr::from((header.hash.clone(), header.height))) - } -} - -impl BlockchainBlock for HeaderOnlyBlock { - fn number(&self) -> i32 { - BlockNumber::try_from(self.header().unwrap().height).unwrap() - } - - fn ptr(&self) -> BlockPtr { - BlockPtr::try_from(self).unwrap() - } - - fn parent_ptr(&self) -> Option { - self.parent_ptr().unwrap() - } -} - -impl EventData { - pub fn event(&self) -> Result<&Event, Error> { - self.event - .as_ref() - .ok_or_else(|| anyhow!("event data missing event field")) - } - pub fn block(&self) -> Result<&HeaderOnlyBlock, Error> { - self.block - .as_ref() - .ok_or_else(|| anyhow!("event data missing block field")) - } -} - -impl TransactionData { - pub fn tx_result(&self) -> Result<&TxResult, Error> { - self.tx - .as_ref() - .ok_or_else(|| anyhow!("transaction data missing tx field")) - } - - pub fn response_deliver_tx(&self) -> Result<&ResponseDeliverTx, Error> { - self.tx_result()? - .result - .as_ref() - .ok_or_else(|| anyhow!("transaction data missing result field")) - } - - pub fn block(&self) -> Result<&HeaderOnlyBlock, Error> { - self.block - .as_ref() - .ok_or_else(|| anyhow!("transaction data missing block field")) - } -} - -impl MessageData { - pub fn message(&self) -> Result<&prost_types::Any, Error> { - self.message - .as_ref() - .ok_or_else(|| anyhow!("message data missing message field")) - } - - pub fn block(&self) -> Result<&HeaderOnlyBlock, Error> { - self.block - .as_ref() - .ok_or_else(|| anyhow!("message data missing block field")) - } -} diff --git a/chain/cosmos/src/data_source.rs b/chain/cosmos/src/data_source.rs deleted file mode 100644 index 3d6043b41bc..00000000000 --- a/chain/cosmos/src/data_source.rs +++ /dev/null @@ -1,666 +0,0 @@ -use std::collections::{HashMap, HashSet}; -use std::sync::Arc; - -use anyhow::{Error, Result}; - -use graph::{ - blockchain::{self, Block, Blockchain, TriggerWithHandler}, - components::store::StoredDynamicDataSource, - data::subgraph::DataSourceContext, - prelude::{ - anyhow, async_trait, info, BlockNumber, CheapClone, DataSourceTemplateInfo, Deserialize, - Link, LinkResolver, Logger, - }, -}; - -use crate::chain::Chain; -use crate::codec; -use crate::trigger::CosmosTrigger; - -pub const COSMOS_KIND: &str = "cosmos"; - -const DYNAMIC_DATA_SOURCE_ERROR: &str = "Cosmos subgraphs do not support dynamic data sources"; -const TEMPLATE_ERROR: &str = "Cosmos subgraphs do not support templates"; - -/// Runtime representation of a data source. -// Note: Not great for memory usage that this needs to be `Clone`, considering how there may be tens -// of thousands of data sources in memory at once. -#[derive(Clone, Debug)] -pub struct DataSource { - pub kind: String, - pub network: Option, - pub name: String, - pub source: Source, - pub mapping: Mapping, - pub context: Arc>, - pub creation_block: Option, -} - -impl blockchain::DataSource for DataSource { - fn from_template_info(_template_info: DataSourceTemplateInfo) -> Result { - Err(anyhow!(TEMPLATE_ERROR)) - } - - fn address(&self) -> Option<&[u8]> { - None - } - - fn start_block(&self) -> BlockNumber { - self.source.start_block - } - - fn match_and_decode( - &self, - trigger: &::TriggerData, - block: &Arc<::Block>, - _logger: &Logger, - ) -> Result>> { - if self.source.start_block > block.number() { - return Ok(None); - } - - let handler = match trigger { - CosmosTrigger::Block(_) => match self.handler_for_block() { - Some(handler) => handler.handler, - None => return Ok(None), - }, - - CosmosTrigger::Event { event_data, origin } => { - match self.handler_for_event(event_data.event()?, *origin) { - Some(handler) => handler.handler, - None => return Ok(None), - } - } - - CosmosTrigger::Transaction(_) => match self.handler_for_transaction() { - Some(handler) => handler.handler, - None => return Ok(None), - }, - - CosmosTrigger::Message(message_data) => { - match self.handler_for_message(message_data.message()?) { - Some(handler) => handler.handler, - None => return Ok(None), - } - } - }; - - Ok(Some(TriggerWithHandler::::new( - trigger.cheap_clone(), - handler, - block.ptr(), - ))) - } - - fn name(&self) -> &str { - &self.name - } - - fn kind(&self) -> &str { - &self.kind - } - - fn network(&self) -> Option<&str> { - self.network.as_deref() - } - - fn context(&self) -> Arc> { - self.context.cheap_clone() - } - - fn creation_block(&self) -> Option { - self.creation_block - } - - fn is_duplicate_of(&self, other: &Self) -> bool { - let DataSource { - kind, - network, - name, - source, - mapping, - context, - - // The creation block is ignored for detection duplicate data sources. - // Contract ABI equality is implicit in `source` and `mapping.abis` equality. - creation_block: _, - } = self; - - // mapping_request_sender, host_metrics, and (most of) host_exports are operational structs - // used at runtime but not needed to define uniqueness; each runtime host should be for a - // unique data source. - kind == &other.kind - && network == &other.network - && name == &other.name - && source == &other.source - && mapping.block_handlers == other.mapping.block_handlers - && mapping.event_handlers == other.mapping.event_handlers - && mapping.transaction_handlers == other.mapping.transaction_handlers - && mapping.message_handlers == other.mapping.message_handlers - && context == &other.context - } - - fn as_stored_dynamic_data_source(&self) -> StoredDynamicDataSource { - unimplemented!("{}", DYNAMIC_DATA_SOURCE_ERROR); - } - - fn from_stored_dynamic_data_source( - _template: &DataSourceTemplate, - _stored: StoredDynamicDataSource, - ) -> Result { - Err(anyhow!(DYNAMIC_DATA_SOURCE_ERROR)) - } - - fn validate(&self) -> Vec { - let mut errors = Vec::new(); - - if self.kind != COSMOS_KIND { - errors.push(anyhow!( - "data source has invalid `kind`, expected {} but found {}", - COSMOS_KIND, - self.kind - )) - } - - // Ensure there is only one block handler - if self.mapping.block_handlers.len() > 1 { - errors.push(anyhow!("data source has duplicated block handlers")); - } - - // Ensure there is only one transaction handler - if self.mapping.transaction_handlers.len() > 1 { - errors.push(anyhow!("data source has duplicated transaction handlers")); - } - - // Ensure that each event type + origin filter combination has only one handler - - // group handler origin filters by event type - let mut event_types = HashMap::with_capacity(self.mapping.event_handlers.len()); - for event_handler in self.mapping.event_handlers.iter() { - let origins = event_types - .entry(&event_handler.event) - // 3 is the maximum number of valid handlers for an event type (1 for each origin) - .or_insert(HashSet::with_capacity(3)); - - // insert returns false if value was already in the set - if !origins.insert(event_handler.origin) { - errors.push(multiple_origin_err( - &event_handler.event, - event_handler.origin, - )) - } - } - - // Ensure each event type either has: - // 1 handler with no origin filter - // OR - // 1 or more handlers with origin filter - for (event_type, origins) in event_types.iter() { - if origins.len() > 1 && !origins.iter().all(Option::is_some) { - errors.push(combined_origins_err(event_type)) - } - } - - // Ensure each message handlers is unique - let mut message_type_urls = HashSet::with_capacity(self.mapping.message_handlers.len()); - for message_handler in self.mapping.message_handlers.iter() { - if !message_type_urls.insert(message_handler.message.clone()) { - errors.push(duplicate_url_type(&message_handler.message)) - } - } - - errors - } - - fn api_version(&self) -> semver::Version { - self.mapping.api_version.clone() - } - - fn runtime(&self) -> Option>> { - Some(self.mapping.runtime.cheap_clone()) - } -} - -impl DataSource { - fn from_manifest( - kind: String, - network: Option, - name: String, - source: Source, - mapping: Mapping, - context: Option, - ) -> Result { - // Data sources in the manifest are created "before genesis" so they have no creation block. - let creation_block = None; - - Ok(DataSource { - kind, - network, - name, - source, - mapping, - context: Arc::new(context), - creation_block, - }) - } - - fn handler_for_block(&self) -> Option { - self.mapping.block_handlers.first().cloned() - } - - fn handler_for_transaction(&self) -> Option { - self.mapping.transaction_handlers.first().cloned() - } - - fn handler_for_message(&self, message: &::prost_types::Any) -> Option { - self.mapping - .message_handlers - .iter() - .find(|handler| handler.message == message.type_url) - .cloned() - } - - fn handler_for_event( - &self, - event: &codec::Event, - event_origin: EventOrigin, - ) -> Option { - self.mapping - .event_handlers - .iter() - .find(|handler| { - let event_type_matches = event.event_type == handler.event; - - if let Some(handler_origin) = handler.origin { - event_type_matches && event_origin == handler_origin - } else { - event_type_matches - } - }) - .cloned() - } - - pub(crate) fn has_block_handler(&self) -> bool { - !self.mapping.block_handlers.is_empty() - } - - /// Return an iterator over all event types from event handlers. - pub(crate) fn events(&self) -> impl Iterator { - self.mapping - .event_handlers - .iter() - .map(|handler| handler.event.as_str()) - } -} - -#[derive(Clone, Debug, Eq, PartialEq, Deserialize)] -pub struct UnresolvedDataSource { - pub kind: String, - pub network: Option, - pub name: String, - pub source: Source, - pub mapping: UnresolvedMapping, - pub context: Option, -} - -#[async_trait] -impl blockchain::UnresolvedDataSource for UnresolvedDataSource { - async fn resolve( - self, - resolver: &Arc, - logger: &Logger, - _manifest_idx: u32, - ) -> Result { - let UnresolvedDataSource { - kind, - network, - name, - source, - mapping, - context, - } = self; - - info!(logger, "Resolve data source"; "name" => &name, "source" => &source.start_block); - - let mapping = mapping.resolve(resolver, logger).await?; - - DataSource::from_manifest(kind, network, name, source, mapping, context) - } -} - -#[derive(Clone, Debug, Default, Hash, Eq, PartialEq, Deserialize)] -pub struct BaseDataSourceTemplate { - pub kind: String, - pub network: Option, - pub name: String, - pub mapping: M, -} - -pub type UnresolvedDataSourceTemplate = BaseDataSourceTemplate; -pub type DataSourceTemplate = BaseDataSourceTemplate; - -#[async_trait] -impl blockchain::UnresolvedDataSourceTemplate for UnresolvedDataSourceTemplate { - async fn resolve( - self, - _resolver: &Arc, - _logger: &Logger, - _manifest_idx: u32, - ) -> Result { - Err(anyhow!(TEMPLATE_ERROR)) - } -} - -impl blockchain::DataSourceTemplate for DataSourceTemplate { - fn name(&self) -> &str { - unimplemented!("{}", TEMPLATE_ERROR); - } - - fn api_version(&self) -> semver::Version { - unimplemented!("{}", TEMPLATE_ERROR); - } - - fn runtime(&self) -> Option>> { - unimplemented!("{}", TEMPLATE_ERROR); - } - - fn manifest_idx(&self) -> u32 { - unimplemented!("{}", TEMPLATE_ERROR); - } -} - -#[derive(Clone, Debug, Default, Hash, Eq, PartialEq, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct UnresolvedMapping { - pub api_version: String, - pub language: String, - pub entities: Vec, - #[serde(default)] - pub block_handlers: Vec, - #[serde(default)] - pub event_handlers: Vec, - #[serde(default)] - pub transaction_handlers: Vec, - #[serde(default)] - pub message_handlers: Vec, - pub file: Link, -} - -impl UnresolvedMapping { - pub async fn resolve( - self, - resolver: &Arc, - logger: &Logger, - ) -> Result { - let UnresolvedMapping { - api_version, - language, - entities, - block_handlers, - event_handlers, - transaction_handlers, - message_handlers, - file: link, - } = self; - - let api_version = semver::Version::parse(&api_version)?; - - info!(logger, "Resolve mapping"; "link" => &link.link); - let module_bytes = resolver.cat(logger, &link).await?; - - Ok(Mapping { - api_version, - language, - entities, - block_handlers: block_handlers.clone(), - event_handlers: event_handlers.clone(), - transaction_handlers: transaction_handlers.clone(), - message_handlers: message_handlers.clone(), - runtime: Arc::new(module_bytes), - link, - }) - } -} - -#[derive(Clone, Debug)] -pub struct Mapping { - pub api_version: semver::Version, - pub language: String, - pub entities: Vec, - pub block_handlers: Vec, - pub event_handlers: Vec, - pub transaction_handlers: Vec, - pub message_handlers: Vec, - pub runtime: Arc>, - pub link: Link, -} - -#[derive(Clone, Debug, Hash, Eq, PartialEq, Deserialize)] -pub struct MappingBlockHandler { - pub handler: String, -} - -#[derive(Clone, Debug, Hash, Eq, PartialEq, Deserialize)] -pub struct MappingEventHandler { - pub event: String, - pub origin: Option, - pub handler: String, -} - -#[derive(Clone, Debug, Hash, Eq, PartialEq, Deserialize)] -pub struct MappingTransactionHandler { - pub handler: String, -} - -#[derive(Clone, Debug, Hash, Eq, PartialEq, Deserialize)] -pub struct MappingMessageHandler { - pub message: String, - pub handler: String, -} - -#[derive(Clone, Debug, Hash, Eq, PartialEq, Deserialize)] -pub struct Source { - #[serde(rename = "startBlock", default)] - pub start_block: BlockNumber, -} - -#[derive(Clone, Copy, Debug, Hash, Eq, PartialEq, Deserialize)] -pub enum EventOrigin { - BeginBlock, - DeliverTx, - EndBlock, -} - -fn multiple_origin_err(event_type: &str, origin: Option) -> Error { - let origin_err_name = match origin { - Some(origin) => format!("{:?}", origin), - None => "no".to_string(), - }; - - anyhow!( - "data source has multiple {} event handlers with {} origin", - event_type, - origin_err_name, - ) -} - -fn combined_origins_err(event_type: &str) -> Error { - anyhow!( - "data source has combined origin and no-origin {} event handlers", - event_type - ) -} - -fn duplicate_url_type(message: &str) -> Error { - anyhow!( - "data source has more than one message handler for message {} ", - message - ) -} - -#[cfg(test)] -mod tests { - use super::*; - - use graph::blockchain::DataSource as _; - - #[test] - fn test_event_handlers_origin_validation() { - let cases = [ - ( - DataSource::with_event_handlers(vec![ - MappingEventHandler::with_origin("event_1", None), - MappingEventHandler::with_origin("event_2", None), - MappingEventHandler::with_origin("event_3", None), - ]), - vec![], - ), - ( - DataSource::with_event_handlers(vec![ - MappingEventHandler::with_origin("event_1", Some(EventOrigin::BeginBlock)), - MappingEventHandler::with_origin("event_2", Some(EventOrigin::BeginBlock)), - MappingEventHandler::with_origin("event_1", Some(EventOrigin::DeliverTx)), - MappingEventHandler::with_origin("event_1", Some(EventOrigin::EndBlock)), - MappingEventHandler::with_origin("event_2", Some(EventOrigin::DeliverTx)), - MappingEventHandler::with_origin("event_2", Some(EventOrigin::EndBlock)), - ]), - vec![], - ), - ( - DataSource::with_event_handlers(vec![ - MappingEventHandler::with_origin("event_1", None), - MappingEventHandler::with_origin("event_1", None), - MappingEventHandler::with_origin("event_2", None), - MappingEventHandler::with_origin("event_2", Some(EventOrigin::BeginBlock)), - MappingEventHandler::with_origin("event_3", Some(EventOrigin::EndBlock)), - MappingEventHandler::with_origin("event_3", Some(EventOrigin::EndBlock)), - ]), - vec![ - multiple_origin_err("event_1", None), - combined_origins_err("event_2"), - multiple_origin_err("event_3", Some(EventOrigin::EndBlock)), - ], - ), - ]; - - for (data_source, errors) in &cases { - let validation_errors = data_source.validate(); - - assert_eq!(errors.len(), validation_errors.len()); - - for error in errors.iter() { - assert!( - validation_errors - .iter() - .any(|validation_error| validation_error.to_string() == error.to_string()), - r#"expected "{}" to be in validation errors, but it wasn't"#, - error - ); - } - } - } - - #[test] - fn test_message_handlers_duplicate() { - let cases = [ - ( - DataSource::with_message_handlers(vec![ - MappingMessageHandler { - handler: "handler".to_string(), - message: "message_0".to_string(), - }, - MappingMessageHandler { - handler: "handler".to_string(), - message: "message_1".to_string(), - }, - ]), - vec![], - ), - ( - DataSource::with_message_handlers(vec![ - MappingMessageHandler { - handler: "handler".to_string(), - message: "message_0".to_string(), - }, - MappingMessageHandler { - handler: "handler".to_string(), - message: "message_0".to_string(), - }, - ]), - vec![duplicate_url_type("message_0")], - ), - ]; - - for (data_source, errors) in &cases { - let validation_errors = data_source.validate(); - - assert_eq!(errors.len(), validation_errors.len()); - - for error in errors.iter() { - assert!( - validation_errors - .iter() - .any(|validation_error| validation_error.to_string() == error.to_string()), - r#"expected "{}" to be in validation errors, but it wasn't"#, - error - ); - } - } - } - - impl DataSource { - fn with_event_handlers(event_handlers: Vec) -> DataSource { - DataSource { - kind: "cosmos".to_string(), - network: None, - name: "Test".to_string(), - source: Source { start_block: 1 }, - mapping: Mapping { - api_version: semver::Version::new(0, 0, 0), - language: "".to_string(), - entities: vec![], - block_handlers: vec![], - event_handlers, - transaction_handlers: vec![], - message_handlers: vec![], - runtime: Arc::new(vec![]), - link: "test".to_string().into(), - }, - context: Arc::new(None), - creation_block: None, - } - } - - fn with_message_handlers(message_handlers: Vec) -> DataSource { - DataSource { - kind: "cosmos".to_string(), - network: None, - name: "Test".to_string(), - source: Source { start_block: 1 }, - mapping: Mapping { - api_version: semver::Version::new(0, 0, 0), - language: "".to_string(), - entities: vec![], - block_handlers: vec![], - event_handlers: vec![], - transaction_handlers: vec![], - message_handlers, - runtime: Arc::new(vec![]), - link: "test".to_string().into(), - }, - context: Arc::new(None), - creation_block: None, - } - } - } - - impl MappingEventHandler { - fn with_origin(event_type: &str, origin: Option) -> MappingEventHandler { - MappingEventHandler { - event: event_type.to_string(), - origin, - handler: "handler".to_string(), - } - } - } -} diff --git a/chain/cosmos/src/lib.rs b/chain/cosmos/src/lib.rs deleted file mode 100644 index 6d84b61947e..00000000000 --- a/chain/cosmos/src/lib.rs +++ /dev/null @@ -1,18 +0,0 @@ -mod adapter; -pub mod chain; -pub mod codec; -mod data_source; -mod protobuf; -pub mod runtime; -mod trigger; - -pub use self::runtime::RuntimeAdapter; - -// ETHDEP: These concrete types should probably not be exposed. -pub use data_source::{DataSource, DataSourceTemplate}; - -pub use crate::adapter::TriggerFilter; -pub use crate::chain::Chain; - -pub use protobuf::pbcodec; -pub use protobuf::pbcodec::Block; diff --git a/chain/cosmos/src/protobuf/.gitignore b/chain/cosmos/src/protobuf/.gitignore deleted file mode 100644 index 96786948080..00000000000 --- a/chain/cosmos/src/protobuf/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -/google.protobuf.rs -/gogoproto.rs -/cosmos_proto.rs -/firehose.rs diff --git a/chain/cosmos/src/protobuf/mod.rs b/chain/cosmos/src/protobuf/mod.rs deleted file mode 100644 index c3292e66c4b..00000000000 --- a/chain/cosmos/src/protobuf/mod.rs +++ /dev/null @@ -1,8 +0,0 @@ -#[rustfmt::skip] -#[path = "sf.cosmos.r#type.v1.rs"] -pub mod pbcodec; - -pub use graph_runtime_wasm::asc_abi::class::{Array, AscEnum, AscString, Uint8Array}; - -pub use crate::runtime::abi::*; -pub use pbcodec::*; diff --git a/chain/cosmos/src/protobuf/sf.cosmos.r#type.v1.rs b/chain/cosmos/src/protobuf/sf.cosmos.r#type.v1.rs deleted file mode 100644 index d60de8086b1..00000000000 --- a/chain/cosmos/src/protobuf/sf.cosmos.r#type.v1.rs +++ /dev/null @@ -1,838 +0,0 @@ -#[graph_runtime_derive::generate_asc_type( - __required__{header:Header, - result_begin_block:ResponseBeginBlock, - result_end_block:ResponseEndBlock} -)] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type( - __required__{header:Header, - result_begin_block:ResponseBeginBlock, - result_end_block:ResponseEndBlock} -)] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Block { - #[prost(message, optional, tag = "1")] - pub header: ::core::option::Option
, - #[prost(message, optional, tag = "2")] - pub evidence: ::core::option::Option, - #[prost(message, optional, tag = "3")] - pub last_commit: ::core::option::Option, - #[prost(message, optional, tag = "4")] - pub result_begin_block: ::core::option::Option, - #[prost(message, optional, tag = "5")] - pub result_end_block: ::core::option::Option, - #[prost(message, repeated, tag = "7")] - pub transactions: ::prost::alloc::vec::Vec, - #[prost(message, repeated, tag = "8")] - pub validator_updates: ::prost::alloc::vec::Vec, -} -/// HeaderOnlyBlock is a standard \[Block\] structure where all other fields are -/// removed so that hydrating that object from a \[Block\] bytes payload will -/// drastically reduce allocated memory required to hold the full block. -/// -/// This can be used to unpack a \[Block\] when only the \[Header\] information -/// is required and greatly reduce required memory. -#[graph_runtime_derive::generate_asc_type(__required__{header:Header})] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type(__required__{header:Header})] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct HeaderOnlyBlock { - #[prost(message, optional, tag = "1")] - pub header: ::core::option::Option
, -} -#[graph_runtime_derive::generate_asc_type( - __required__{event:Event, - block:HeaderOnlyBlock} -)] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type( - __required__{event:Event, - block:HeaderOnlyBlock} -)] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct EventData { - #[prost(message, optional, tag = "1")] - pub event: ::core::option::Option, - #[prost(message, optional, tag = "2")] - pub block: ::core::option::Option, - #[prost(message, optional, tag = "3")] - pub tx: ::core::option::Option, -} -#[graph_runtime_derive::generate_asc_type( - __required__{tx:TxResult, - block:HeaderOnlyBlock} -)] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type( - __required__{tx:TxResult, - block:HeaderOnlyBlock} -)] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct TransactionData { - #[prost(message, optional, tag = "1")] - pub tx: ::core::option::Option, - #[prost(message, optional, tag = "2")] - pub block: ::core::option::Option, -} -#[graph_runtime_derive::generate_asc_type( - __required__{message:Any, - block:HeaderOnlyBlock, - tx:TransactionContext} -)] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type( - __required__{message:Any, - block:HeaderOnlyBlock, - tx:TransactionContext} -)] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct MessageData { - #[prost(message, optional, tag = "1")] - pub message: ::core::option::Option<::prost_types::Any>, - #[prost(message, optional, tag = "2")] - pub block: ::core::option::Option, - #[prost(message, optional, tag = "3")] - pub tx: ::core::option::Option, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct TransactionContext { - #[prost(bytes = "vec", tag = "1")] - pub hash: ::prost::alloc::vec::Vec, - #[prost(uint32, tag = "2")] - pub index: u32, - #[prost(uint32, tag = "3")] - pub code: u32, - #[prost(int64, tag = "4")] - pub gas_wanted: i64, - #[prost(int64, tag = "5")] - pub gas_used: i64, -} -#[graph_runtime_derive::generate_asc_type(__required__{last_block_id:BlockID})] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type(__required__{last_block_id:BlockID})] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Header { - #[prost(message, optional, tag = "1")] - pub version: ::core::option::Option, - #[prost(string, tag = "2")] - pub chain_id: ::prost::alloc::string::String, - #[prost(uint64, tag = "3")] - pub height: u64, - #[prost(message, optional, tag = "4")] - pub time: ::core::option::Option, - #[prost(message, optional, tag = "5")] - pub last_block_id: ::core::option::Option, - #[prost(bytes = "vec", tag = "6")] - pub last_commit_hash: ::prost::alloc::vec::Vec, - #[prost(bytes = "vec", tag = "7")] - pub data_hash: ::prost::alloc::vec::Vec, - #[prost(bytes = "vec", tag = "8")] - pub validators_hash: ::prost::alloc::vec::Vec, - #[prost(bytes = "vec", tag = "9")] - pub next_validators_hash: ::prost::alloc::vec::Vec, - #[prost(bytes = "vec", tag = "10")] - pub consensus_hash: ::prost::alloc::vec::Vec, - #[prost(bytes = "vec", tag = "11")] - pub app_hash: ::prost::alloc::vec::Vec, - #[prost(bytes = "vec", tag = "12")] - pub last_results_hash: ::prost::alloc::vec::Vec, - #[prost(bytes = "vec", tag = "13")] - pub evidence_hash: ::prost::alloc::vec::Vec, - #[prost(bytes = "vec", tag = "14")] - pub proposer_address: ::prost::alloc::vec::Vec, - #[prost(bytes = "vec", tag = "15")] - pub hash: ::prost::alloc::vec::Vec, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Consensus { - #[prost(uint64, tag = "1")] - pub block: u64, - #[prost(uint64, tag = "2")] - pub app: u64, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Timestamp { - #[prost(int64, tag = "1")] - pub seconds: i64, - #[prost(int32, tag = "2")] - pub nanos: i32, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct BlockId { - #[prost(bytes = "vec", tag = "1")] - pub hash: ::prost::alloc::vec::Vec, - #[prost(message, optional, tag = "2")] - pub part_set_header: ::core::option::Option, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct PartSetHeader { - #[prost(uint32, tag = "1")] - pub total: u32, - #[prost(bytes = "vec", tag = "2")] - pub hash: ::prost::alloc::vec::Vec, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct EvidenceList { - #[prost(message, repeated, tag = "1")] - pub evidence: ::prost::alloc::vec::Vec, -} -#[graph_runtime_derive::generate_asc_type( - sum{duplicate_vote_evidence:DuplicateVoteEvidence, - light_client_attack_evidence:LightClientAttackEvidence} -)] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type( - sum{duplicate_vote_evidence:DuplicateVoteEvidence, - light_client_attack_evidence:LightClientAttackEvidence} -)] -#[graph_runtime_derive::generate_array_type(Cosmos)] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Evidence { - #[prost(oneof = "evidence::Sum", tags = "1, 2")] - pub sum: ::core::option::Option, -} -/// Nested message and enum types in `Evidence`. -pub mod evidence { - #[allow(clippy::derive_partial_eq_without_eq)] - #[derive(Clone, PartialEq, ::prost::Oneof)] - pub enum Sum { - #[prost(message, tag = "1")] - DuplicateVoteEvidence(super::DuplicateVoteEvidence), - #[prost(message, tag = "2")] - LightClientAttackEvidence(super::LightClientAttackEvidence), - } -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct DuplicateVoteEvidence { - #[prost(message, optional, tag = "1")] - pub vote_a: ::core::option::Option, - #[prost(message, optional, tag = "2")] - pub vote_b: ::core::option::Option, - #[prost(int64, tag = "3")] - pub total_voting_power: i64, - #[prost(int64, tag = "4")] - pub validator_power: i64, - #[prost(message, optional, tag = "5")] - pub timestamp: ::core::option::Option, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct EventVote { - #[prost(enumeration = "SignedMsgType", tag = "1")] - pub event_vote_type: i32, - #[prost(uint64, tag = "2")] - pub height: u64, - #[prost(int32, tag = "3")] - pub round: i32, - #[prost(message, optional, tag = "4")] - pub block_id: ::core::option::Option, - #[prost(message, optional, tag = "5")] - pub timestamp: ::core::option::Option, - #[prost(bytes = "vec", tag = "6")] - pub validator_address: ::prost::alloc::vec::Vec, - #[prost(int32, tag = "7")] - pub validator_index: i32, - #[prost(bytes = "vec", tag = "8")] - pub signature: ::prost::alloc::vec::Vec, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct LightClientAttackEvidence { - #[prost(message, optional, tag = "1")] - pub conflicting_block: ::core::option::Option, - #[prost(int64, tag = "2")] - pub common_height: i64, - #[prost(message, repeated, tag = "3")] - pub byzantine_validators: ::prost::alloc::vec::Vec, - #[prost(int64, tag = "4")] - pub total_voting_power: i64, - #[prost(message, optional, tag = "5")] - pub timestamp: ::core::option::Option, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct LightBlock { - #[prost(message, optional, tag = "1")] - pub signed_header: ::core::option::Option, - #[prost(message, optional, tag = "2")] - pub validator_set: ::core::option::Option, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct SignedHeader { - #[prost(message, optional, tag = "1")] - pub header: ::core::option::Option
, - #[prost(message, optional, tag = "2")] - pub commit: ::core::option::Option, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Commit { - #[prost(int64, tag = "1")] - pub height: i64, - #[prost(int32, tag = "2")] - pub round: i32, - #[prost(message, optional, tag = "3")] - pub block_id: ::core::option::Option, - #[prost(message, repeated, tag = "4")] - pub signatures: ::prost::alloc::vec::Vec, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[graph_runtime_derive::generate_array_type(Cosmos)] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct CommitSig { - #[prost(enumeration = "BlockIdFlag", tag = "1")] - pub block_id_flag: i32, - #[prost(bytes = "vec", tag = "2")] - pub validator_address: ::prost::alloc::vec::Vec, - #[prost(message, optional, tag = "3")] - pub timestamp: ::core::option::Option, - #[prost(bytes = "vec", tag = "4")] - pub signature: ::prost::alloc::vec::Vec, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ValidatorSet { - #[prost(message, repeated, tag = "1")] - pub validators: ::prost::alloc::vec::Vec, - #[prost(message, optional, tag = "2")] - pub proposer: ::core::option::Option, - #[prost(int64, tag = "3")] - pub total_voting_power: i64, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[graph_runtime_derive::generate_array_type(Cosmos)] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Validator { - #[prost(bytes = "vec", tag = "1")] - pub address: ::prost::alloc::vec::Vec, - #[prost(message, optional, tag = "2")] - pub pub_key: ::core::option::Option, - #[prost(int64, tag = "3")] - pub voting_power: i64, - #[prost(int64, tag = "4")] - pub proposer_priority: i64, -} -#[graph_runtime_derive::generate_asc_type(sum{ed25519:Vec, secp256k1:Vec})] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type(sum{ed25519:Vec, secp256k1:Vec})] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct PublicKey { - #[prost(oneof = "public_key::Sum", tags = "1, 2")] - pub sum: ::core::option::Option, -} -/// Nested message and enum types in `PublicKey`. -pub mod public_key { - #[allow(clippy::derive_partial_eq_without_eq)] - #[derive(Clone, PartialEq, ::prost::Oneof)] - pub enum Sum { - #[prost(bytes, tag = "1")] - Ed25519(::prost::alloc::vec::Vec), - #[prost(bytes, tag = "2")] - Secp256k1(::prost::alloc::vec::Vec), - } -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ResponseBeginBlock { - #[prost(message, repeated, tag = "1")] - pub events: ::prost::alloc::vec::Vec, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[graph_runtime_derive::generate_array_type(Cosmos)] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Event { - #[prost(string, tag = "1")] - pub event_type: ::prost::alloc::string::String, - #[prost(message, repeated, tag = "2")] - pub attributes: ::prost::alloc::vec::Vec, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[graph_runtime_derive::generate_array_type(Cosmos)] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct EventAttribute { - #[prost(string, tag = "1")] - pub key: ::prost::alloc::string::String, - #[prost(string, tag = "2")] - pub value: ::prost::alloc::string::String, - #[prost(bool, tag = "3")] - pub index: bool, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ResponseEndBlock { - #[prost(message, repeated, tag = "1")] - pub validator_updates: ::prost::alloc::vec::Vec, - #[prost(message, optional, tag = "2")] - pub consensus_param_updates: ::core::option::Option, - #[prost(message, repeated, tag = "3")] - pub events: ::prost::alloc::vec::Vec, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[graph_runtime_derive::generate_array_type(Cosmos)] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ValidatorUpdate { - #[prost(bytes = "vec", tag = "1")] - pub address: ::prost::alloc::vec::Vec, - #[prost(message, optional, tag = "2")] - pub pub_key: ::core::option::Option, - #[prost(int64, tag = "3")] - pub power: i64, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ConsensusParams { - #[prost(message, optional, tag = "1")] - pub block: ::core::option::Option, - #[prost(message, optional, tag = "2")] - pub evidence: ::core::option::Option, - #[prost(message, optional, tag = "3")] - pub validator: ::core::option::Option, - #[prost(message, optional, tag = "4")] - pub version: ::core::option::Option, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct BlockParams { - #[prost(int64, tag = "1")] - pub max_bytes: i64, - #[prost(int64, tag = "2")] - pub max_gas: i64, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct EvidenceParams { - #[prost(int64, tag = "1")] - pub max_age_num_blocks: i64, - #[prost(message, optional, tag = "2")] - pub max_age_duration: ::core::option::Option, - #[prost(int64, tag = "3")] - pub max_bytes: i64, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Duration { - #[prost(int64, tag = "1")] - pub seconds: i64, - #[prost(int32, tag = "2")] - pub nanos: i32, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ValidatorParams { - #[prost(string, repeated, tag = "1")] - pub pub_key_types: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct VersionParams { - #[prost(uint64, tag = "1")] - pub app_version: u64, -} -#[graph_runtime_derive::generate_asc_type(__required__{tx:Tx, result:ResponseDeliverTx})] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type( - __required__{tx:Tx, - result:ResponseDeliverTx} -)] -#[graph_runtime_derive::generate_array_type(Cosmos)] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct TxResult { - #[prost(uint64, tag = "1")] - pub height: u64, - #[prost(uint32, tag = "2")] - pub index: u32, - #[prost(message, optional, tag = "3")] - pub tx: ::core::option::Option, - #[prost(message, optional, tag = "4")] - pub result: ::core::option::Option, - #[prost(bytes = "vec", tag = "5")] - pub hash: ::prost::alloc::vec::Vec, -} -#[graph_runtime_derive::generate_asc_type(__required__{body:TxBody})] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type(__required__{body:TxBody})] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Tx { - #[prost(message, optional, tag = "1")] - pub body: ::core::option::Option, - #[prost(message, optional, tag = "2")] - pub auth_info: ::core::option::Option, - #[prost(bytes = "vec", repeated, tag = "3")] - pub signatures: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec>, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct TxBody { - #[prost(message, repeated, tag = "1")] - pub messages: ::prost::alloc::vec::Vec<::prost_types::Any>, - #[prost(string, tag = "2")] - pub memo: ::prost::alloc::string::String, - #[prost(uint64, tag = "3")] - pub timeout_height: u64, - #[prost(message, repeated, tag = "1023")] - pub extension_options: ::prost::alloc::vec::Vec<::prost_types::Any>, - #[prost(message, repeated, tag = "2047")] - pub non_critical_extension_options: ::prost::alloc::vec::Vec<::prost_types::Any>, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[graph_runtime_derive::generate_array_type(Cosmos)] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Any { - #[prost(string, tag = "1")] - pub type_url: ::prost::alloc::string::String, - #[prost(bytes = "vec", tag = "2")] - pub value: ::prost::alloc::vec::Vec, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct AuthInfo { - #[prost(message, repeated, tag = "1")] - pub signer_infos: ::prost::alloc::vec::Vec, - #[prost(message, optional, tag = "2")] - pub fee: ::core::option::Option, - #[prost(message, optional, tag = "3")] - pub tip: ::core::option::Option, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[graph_runtime_derive::generate_array_type(Cosmos)] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct SignerInfo { - #[prost(message, optional, tag = "1")] - pub public_key: ::core::option::Option<::prost_types::Any>, - #[prost(message, optional, tag = "2")] - pub mode_info: ::core::option::Option, - #[prost(uint64, tag = "3")] - pub sequence: u64, -} -#[graph_runtime_derive::generate_asc_type( - sum{single:ModeInfoSingle, - multi:ModeInfoMulti} -)] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type( - sum{single:ModeInfoSingle, - multi:ModeInfoMulti} -)] -#[graph_runtime_derive::generate_array_type(Cosmos)] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ModeInfo { - #[prost(oneof = "mode_info::Sum", tags = "1, 2")] - pub sum: ::core::option::Option, -} -/// Nested message and enum types in `ModeInfo`. -pub mod mode_info { - #[allow(clippy::derive_partial_eq_without_eq)] - #[derive(Clone, PartialEq, ::prost::Oneof)] - pub enum Sum { - #[prost(message, tag = "1")] - Single(super::ModeInfoSingle), - #[prost(message, tag = "2")] - Multi(super::ModeInfoMulti), - } -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ModeInfoSingle { - #[prost(enumeration = "SignMode", tag = "1")] - pub mode: i32, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ModeInfoMulti { - #[prost(message, optional, tag = "1")] - pub bitarray: ::core::option::Option, - #[prost(message, repeated, tag = "2")] - pub mode_infos: ::prost::alloc::vec::Vec, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct CompactBitArray { - #[prost(uint32, tag = "1")] - pub extra_bits_stored: u32, - #[prost(bytes = "vec", tag = "2")] - pub elems: ::prost::alloc::vec::Vec, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Fee { - #[prost(message, repeated, tag = "1")] - pub amount: ::prost::alloc::vec::Vec, - #[prost(uint64, tag = "2")] - pub gas_limit: u64, - #[prost(string, tag = "3")] - pub payer: ::prost::alloc::string::String, - #[prost(string, tag = "4")] - pub granter: ::prost::alloc::string::String, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[graph_runtime_derive::generate_array_type(Cosmos)] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Coin { - #[prost(string, tag = "1")] - pub denom: ::prost::alloc::string::String, - #[prost(string, tag = "2")] - pub amount: ::prost::alloc::string::String, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Tip { - #[prost(message, repeated, tag = "1")] - pub amount: ::prost::alloc::vec::Vec, - #[prost(string, tag = "2")] - pub tipper: ::prost::alloc::string::String, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ResponseDeliverTx { - #[prost(uint32, tag = "1")] - pub code: u32, - #[prost(bytes = "vec", tag = "2")] - pub data: ::prost::alloc::vec::Vec, - #[prost(string, tag = "3")] - pub log: ::prost::alloc::string::String, - #[prost(string, tag = "4")] - pub info: ::prost::alloc::string::String, - #[prost(int64, tag = "5")] - pub gas_wanted: i64, - #[prost(int64, tag = "6")] - pub gas_used: i64, - #[prost(message, repeated, tag = "7")] - pub events: ::prost::alloc::vec::Vec, - #[prost(string, tag = "8")] - pub codespace: ::prost::alloc::string::String, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ValidatorSetUpdates { - #[prost(message, repeated, tag = "1")] - pub validator_updates: ::prost::alloc::vec::Vec, -} -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] -#[repr(i32)] -pub enum SignedMsgType { - Unknown = 0, - Prevote = 1, - Precommit = 2, - Proposal = 32, -} -impl SignedMsgType { - /// String value of the enum field names used in the ProtoBuf definition. - /// - /// The values are not transformed in any way and thus are considered stable - /// (if the ProtoBuf definition does not change) and safe for programmatic use. - pub fn as_str_name(&self) -> &'static str { - match self { - SignedMsgType::Unknown => "SIGNED_MSG_TYPE_UNKNOWN", - SignedMsgType::Prevote => "SIGNED_MSG_TYPE_PREVOTE", - SignedMsgType::Precommit => "SIGNED_MSG_TYPE_PRECOMMIT", - SignedMsgType::Proposal => "SIGNED_MSG_TYPE_PROPOSAL", - } - } - /// Creates an enum from field names used in the ProtoBuf definition. - pub fn from_str_name(value: &str) -> ::core::option::Option { - match value { - "SIGNED_MSG_TYPE_UNKNOWN" => Some(Self::Unknown), - "SIGNED_MSG_TYPE_PREVOTE" => Some(Self::Prevote), - "SIGNED_MSG_TYPE_PRECOMMIT" => Some(Self::Precommit), - "SIGNED_MSG_TYPE_PROPOSAL" => Some(Self::Proposal), - _ => None, - } - } -} -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] -#[repr(i32)] -pub enum BlockIdFlag { - Unknown = 0, - Absent = 1, - Commit = 2, - Nil = 3, -} -impl BlockIdFlag { - /// String value of the enum field names used in the ProtoBuf definition. - /// - /// The values are not transformed in any way and thus are considered stable - /// (if the ProtoBuf definition does not change) and safe for programmatic use. - pub fn as_str_name(&self) -> &'static str { - match self { - BlockIdFlag::Unknown => "BLOCK_ID_FLAG_UNKNOWN", - BlockIdFlag::Absent => "BLOCK_ID_FLAG_ABSENT", - BlockIdFlag::Commit => "BLOCK_ID_FLAG_COMMIT", - BlockIdFlag::Nil => "BLOCK_ID_FLAG_NIL", - } - } - /// Creates an enum from field names used in the ProtoBuf definition. - pub fn from_str_name(value: &str) -> ::core::option::Option { - match value { - "BLOCK_ID_FLAG_UNKNOWN" => Some(Self::Unknown), - "BLOCK_ID_FLAG_ABSENT" => Some(Self::Absent), - "BLOCK_ID_FLAG_COMMIT" => Some(Self::Commit), - "BLOCK_ID_FLAG_NIL" => Some(Self::Nil), - _ => None, - } - } -} -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] -#[repr(i32)] -pub enum SignMode { - Unspecified = 0, - Direct = 1, - Textual = 2, - LegacyAminoJson = 127, -} -impl SignMode { - /// String value of the enum field names used in the ProtoBuf definition. - /// - /// The values are not transformed in any way and thus are considered stable - /// (if the ProtoBuf definition does not change) and safe for programmatic use. - pub fn as_str_name(&self) -> &'static str { - match self { - SignMode::Unspecified => "SIGN_MODE_UNSPECIFIED", - SignMode::Direct => "SIGN_MODE_DIRECT", - SignMode::Textual => "SIGN_MODE_TEXTUAL", - SignMode::LegacyAminoJson => "SIGN_MODE_LEGACY_AMINO_JSON", - } - } - /// Creates an enum from field names used in the ProtoBuf definition. - pub fn from_str_name(value: &str) -> ::core::option::Option { - match value { - "SIGN_MODE_UNSPECIFIED" => Some(Self::Unspecified), - "SIGN_MODE_DIRECT" => Some(Self::Direct), - "SIGN_MODE_TEXTUAL" => Some(Self::Textual), - "SIGN_MODE_LEGACY_AMINO_JSON" => Some(Self::LegacyAminoJson), - _ => None, - } - } -} diff --git a/chain/cosmos/src/runtime/abi.rs b/chain/cosmos/src/runtime/abi.rs deleted file mode 100644 index 3c5f0dd5353..00000000000 --- a/chain/cosmos/src/runtime/abi.rs +++ /dev/null @@ -1,79 +0,0 @@ -use crate::protobuf::*; -pub use graph::semver::Version; - -pub use graph::runtime::{ - asc_new, gas::GasCounter, AscHeap, AscIndexId, AscPtr, AscType, AscValue, - DeterministicHostError, IndexForAscTypeId, ToAscObj, -}; -/* -TODO: AscBytesArray seem to be generic to all chains, but AscIndexId pins it to Cosmos -****************** this can be moved to runtime graph/runtime/src/asc_heap.rs, but IndexForAscTypeId::CosmosBytesArray ****** -*/ -pub struct AscBytesArray(pub Array>); - -impl ToAscObj for Vec> { - fn to_asc_obj( - &self, - heap: &mut H, - gas: &GasCounter, - ) -> Result { - let content: Result, _> = self - .iter() - .map(|x| asc_new(heap, &graph_runtime_wasm::asc_abi::class::Bytes(x), gas)) - .collect(); - - Ok(AscBytesArray(Array::new(&content?, heap, gas)?)) - } -} - -//this can be moved to runtime -impl AscType for AscBytesArray { - fn to_asc_bytes(&self) -> Result, DeterministicHostError> { - self.0.to_asc_bytes() - } - - fn from_asc_bytes( - asc_obj: &[u8], - api_version: &Version, - ) -> Result { - Ok(Self(Array::from_asc_bytes(asc_obj, api_version)?)) - } -} - -//we will have to keep this chain specific (Inner/Outer) -impl AscIndexId for AscBytesArray { - const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::CosmosBytesArray; -} - -/************************************************************************** */ -// this can be moved to runtime - prost_types::Any -impl ToAscObj for prost_types::Any { - fn to_asc_obj( - &self, - heap: &mut H, - gas: &GasCounter, - ) -> Result { - Ok(AscAny { - type_url: asc_new(heap, &self.type_url, gas)?, - value: asc_new( - heap, - &graph_runtime_wasm::asc_abi::class::Bytes(&self.value), - gas, - )?, - ..Default::default() - }) - } -} - -//this can be moved to runtime - prost_types::Any -impl ToAscObj for Vec { - fn to_asc_obj( - &self, - heap: &mut H, - gas: &GasCounter, - ) -> Result { - let content: Result, _> = self.iter().map(|x| asc_new(heap, x, gas)).collect(); - - Ok(AscAnyArray(Array::new(&content?, heap, gas)?)) - } -} diff --git a/chain/cosmos/src/runtime/mod.rs b/chain/cosmos/src/runtime/mod.rs deleted file mode 100644 index 77702f3ba90..00000000000 --- a/chain/cosmos/src/runtime/mod.rs +++ /dev/null @@ -1,351 +0,0 @@ -pub use runtime_adapter::RuntimeAdapter; - -pub mod abi; -pub mod runtime_adapter; - -#[cfg(test)] -mod test { - use crate::protobuf::*; - - use graph::semver::Version; - - /// A macro that takes an ASC struct value definition and calls AscBytes methods to check that - /// memory layout is padded properly. - macro_rules! assert_asc_bytes { - ($struct_name:ident { - $($field:ident : $field_value:expr),+ - $(,)? // trailing - }) => { - let value = $struct_name { - $($field: $field_value),+ - }; - - // just call the function. it will panic on misalignments - let asc_bytes = value.to_asc_bytes().unwrap(); - - let value_004 = $struct_name::from_asc_bytes(&asc_bytes, &Version::new(0, 0, 4)).unwrap(); - let value_005 = $struct_name::from_asc_bytes(&asc_bytes, &Version::new(0, 0, 5)).unwrap(); - - // turn the values into bytes again to verify that they are the same as the original - // because these types usually don't implement PartialEq - assert_eq!( - asc_bytes, - value_004.to_asc_bytes().unwrap(), - "Expected {} v0.0.4 asc bytes to be the same", - stringify!($struct_name) - ); - assert_eq!( - asc_bytes, - value_005.to_asc_bytes().unwrap(), - "Expected {} v0.0.5 asc bytes to be the same", - stringify!($struct_name) - ); - }; - } - - #[test] - fn test_asc_type_alignment() { - // TODO: automatically generate these tests for each struct in derive(AscType) macro - - assert_asc_bytes!(AscBlock { - header: new_asc_ptr(), - evidence: new_asc_ptr(), - last_commit: new_asc_ptr(), - result_begin_block: new_asc_ptr(), - result_end_block: new_asc_ptr(), - transactions: new_asc_ptr(), - validator_updates: new_asc_ptr(), - }); - - assert_asc_bytes!(AscHeaderOnlyBlock { - header: new_asc_ptr(), - }); - - assert_asc_bytes!(AscEventData { - event: new_asc_ptr(), - block: new_asc_ptr(), - tx: new_asc_ptr(), - }); - - assert_asc_bytes!(AscTransactionData { - tx: new_asc_ptr(), - block: new_asc_ptr(), - }); - - assert_asc_bytes!(AscMessageData { - message: new_asc_ptr(), - block: new_asc_ptr(), - tx: new_asc_ptr(), - }); - - assert_asc_bytes!(AscTransactionContext { - hash: new_asc_ptr(), - index: 20, - code: 20, - gas_wanted: 20, - gas_used: 20, - }); - - assert_asc_bytes!(AscHeader { - version: new_asc_ptr(), - chain_id: new_asc_ptr(), - height: 20, - time: new_asc_ptr(), - last_block_id: new_asc_ptr(), - last_commit_hash: new_asc_ptr(), - data_hash: new_asc_ptr(), - validators_hash: new_asc_ptr(), - next_validators_hash: new_asc_ptr(), - consensus_hash: new_asc_ptr(), - app_hash: new_asc_ptr(), - last_results_hash: new_asc_ptr(), - evidence_hash: new_asc_ptr(), - proposer_address: new_asc_ptr(), - hash: new_asc_ptr(), - }); - - assert_asc_bytes!(AscConsensus { block: 0, app: 0 }); - - assert_asc_bytes!(AscTimestamp { - seconds: 20, - nanos: 20, - }); - - assert_asc_bytes!(AscBlockId { - hash: new_asc_ptr(), - part_set_header: new_asc_ptr(), - }); - - assert_asc_bytes!(AscPartSetHeader { - total: 20, - hash: new_asc_ptr(), - }); - - assert_asc_bytes!(AscEvidenceList { - evidence: new_asc_ptr(), - }); - - assert_asc_bytes!(AscEvidence { - duplicate_vote_evidence: new_asc_ptr(), - light_client_attack_evidence: new_asc_ptr(), - }); - - assert_asc_bytes!(AscDuplicateVoteEvidence { - vote_a: new_asc_ptr(), - vote_b: new_asc_ptr(), - total_voting_power: 20, - validator_power: 20, - timestamp: new_asc_ptr(), - }); - - assert_asc_bytes!(AscEventVote { - event_vote_type: 20, - height: 20, - round: 20, - block_id: new_asc_ptr(), - timestamp: new_asc_ptr(), - validator_address: new_asc_ptr(), - validator_index: 20, - signature: new_asc_ptr(), - }); - - assert_asc_bytes!(AscLightClientAttackEvidence { - conflicting_block: new_asc_ptr(), - common_height: 20, - total_voting_power: 20, - byzantine_validators: new_asc_ptr(), - timestamp: new_asc_ptr(), - }); - - assert_asc_bytes!(AscLightBlock { - signed_header: new_asc_ptr(), - validator_set: new_asc_ptr(), - }); - - assert_asc_bytes!(AscSignedHeader { - header: new_asc_ptr(), - commit: new_asc_ptr(), - }); - - assert_asc_bytes!(AscCommit { - height: 20, - round: 20, - block_id: new_asc_ptr(), - signatures: new_asc_ptr(), - }); - - assert_asc_bytes!(AscCommitSig { - block_id_flag: 20, - validator_address: new_asc_ptr(), - timestamp: new_asc_ptr(), - signature: new_asc_ptr(), - }); - - assert_asc_bytes!(AscValidatorSet { - validators: new_asc_ptr(), - proposer: new_asc_ptr(), - total_voting_power: 20, - }); - - assert_asc_bytes!(AscValidator { - address: new_asc_ptr(), - pub_key: new_asc_ptr(), - voting_power: 20, - proposer_priority: 20, - }); - - assert_asc_bytes!(AscPublicKey { - ed25519: new_asc_ptr(), - secp256k1: new_asc_ptr(), - }); - - assert_asc_bytes!(AscResponseBeginBlock { - events: new_asc_ptr(), - }); - - assert_asc_bytes!(AscEvent { - event_type: new_asc_ptr(), - attributes: new_asc_ptr(), - }); - - assert_asc_bytes!(AscEventAttribute { - key: new_asc_ptr(), - value: new_asc_ptr(), - index: true, - }); - - assert_asc_bytes!(AscResponseEndBlock { - validator_updates: new_asc_ptr(), - consensus_param_updates: new_asc_ptr(), - events: new_asc_ptr(), - }); - - assert_asc_bytes!(AscValidatorUpdate { - address: new_asc_ptr(), - pub_key: new_asc_ptr(), - power: 20, - }); - - assert_asc_bytes!(AscConsensusParams { - block: new_asc_ptr(), - evidence: new_asc_ptr(), - validator: new_asc_ptr(), - version: new_asc_ptr(), - }); - - assert_asc_bytes!(AscBlockParams { - max_bytes: 20, - max_gas: 20, - }); - - assert_asc_bytes!(AscEvidenceParams { - max_age_num_blocks: 20, - max_age_duration: new_asc_ptr(), - max_bytes: 20, - }); - - assert_asc_bytes!(AscDuration { - seconds: 20, - nanos: 20, - }); - - assert_asc_bytes!(AscValidatorParams { - pub_key_types: new_asc_ptr(), - }); - - assert_asc_bytes!(AscVersionParams { app_version: 20 }); - - assert_asc_bytes!(AscTxResult { - height: 20, - index: 20, - tx: new_asc_ptr(), - result: new_asc_ptr(), - hash: new_asc_ptr(), - }); - - assert_asc_bytes!(AscTx { - body: new_asc_ptr(), - auth_info: new_asc_ptr(), - signatures: new_asc_ptr(), - }); - - assert_asc_bytes!(AscTxBody { - messages: new_asc_ptr(), - memo: new_asc_ptr(), - timeout_height: 20, - extension_options: new_asc_ptr(), - non_critical_extension_options: new_asc_ptr(), - }); - - assert_asc_bytes!(AscAny { - type_url: new_asc_ptr(), - value: new_asc_ptr(), - }); - - assert_asc_bytes!(AscAuthInfo { - signer_infos: new_asc_ptr(), - fee: new_asc_ptr(), - tip: new_asc_ptr(), - }); - - assert_asc_bytes!(AscSignerInfo { - public_key: new_asc_ptr(), - mode_info: new_asc_ptr(), - sequence: 20, - }); - - assert_asc_bytes!(AscModeInfo { - single: new_asc_ptr(), - multi: new_asc_ptr(), - }); - - assert_asc_bytes!(AscModeInfoSingle { mode: 20 }); - - assert_asc_bytes!(AscModeInfoMulti { - bitarray: new_asc_ptr(), - mode_infos: new_asc_ptr(), - }); - - assert_asc_bytes!(AscCompactBitArray { - extra_bits_stored: 20, - elems: new_asc_ptr(), - }); - - assert_asc_bytes!(AscFee { - amount: new_asc_ptr(), - gas_limit: 20, - payer: new_asc_ptr(), - granter: new_asc_ptr(), - }); - - assert_asc_bytes!(AscCoin { - denom: new_asc_ptr(), - amount: new_asc_ptr(), - }); - - assert_asc_bytes!(AscTip { - amount: new_asc_ptr(), - tipper: new_asc_ptr(), - }); - - assert_asc_bytes!(AscResponseDeliverTx { - code: 20, - data: new_asc_ptr(), - log: new_asc_ptr(), - info: new_asc_ptr(), - gas_wanted: 20, - gas_used: 20, - events: new_asc_ptr(), - codespace: new_asc_ptr(), - }); - - assert_asc_bytes!(AscValidatorSetUpdates { - validator_updates: new_asc_ptr(), - }); - } - - // non-null AscPtr - fn new_asc_ptr() -> AscPtr { - AscPtr::new(12) - } -} diff --git a/chain/cosmos/src/runtime/runtime_adapter.rs b/chain/cosmos/src/runtime/runtime_adapter.rs deleted file mode 100644 index 4bced409e98..00000000000 --- a/chain/cosmos/src/runtime/runtime_adapter.rs +++ /dev/null @@ -1,12 +0,0 @@ -use crate::{Chain, DataSource}; -use anyhow::Result; -use blockchain::HostFn; -use graph::blockchain; - -pub struct RuntimeAdapter {} - -impl blockchain::RuntimeAdapter for RuntimeAdapter { - fn host_fns(&self, _ds: &DataSource) -> Result> { - Ok(vec![]) - } -} diff --git a/chain/cosmos/src/trigger.rs b/chain/cosmos/src/trigger.rs deleted file mode 100644 index 52a64e4b0f2..00000000000 --- a/chain/cosmos/src/trigger.rs +++ /dev/null @@ -1,359 +0,0 @@ -use std::{cmp::Ordering, sync::Arc}; - -use graph::blockchain::{Block, BlockHash, TriggerData}; -use graph::cheap_clone::CheapClone; -use graph::prelude::{BlockNumber, Error}; -use graph::runtime::{asc_new, gas::GasCounter, AscHeap, AscPtr, DeterministicHostError}; -use graph_runtime_wasm::module::ToAscPtr; - -use crate::codec; -use crate::data_source::EventOrigin; - -// Logging the block is too verbose, so this strips the block from the trigger for Debug. -impl std::fmt::Debug for CosmosTrigger { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - #[derive(Debug)] - pub enum MappingTriggerWithoutBlock<'e> { - Block, - Event { - event_type: &'e str, - origin: EventOrigin, - }, - Transaction, - Message, - } - - let trigger_without_block = match self { - CosmosTrigger::Block(_) => MappingTriggerWithoutBlock::Block, - CosmosTrigger::Event { event_data, origin } => MappingTriggerWithoutBlock::Event { - event_type: &event_data.event().map_err(|_| std::fmt::Error)?.event_type, - origin: *origin, - }, - CosmosTrigger::Transaction(_) => MappingTriggerWithoutBlock::Transaction, - CosmosTrigger::Message(_) => MappingTriggerWithoutBlock::Message, - }; - - write!(f, "{:?}", trigger_without_block) - } -} - -impl ToAscPtr for CosmosTrigger { - fn to_asc_ptr( - self, - heap: &mut H, - gas: &GasCounter, - ) -> Result, DeterministicHostError> { - Ok(match self { - CosmosTrigger::Block(block) => asc_new(heap, block.as_ref(), gas)?.erase(), - CosmosTrigger::Event { event_data, .. } => { - asc_new(heap, event_data.as_ref(), gas)?.erase() - } - CosmosTrigger::Transaction(transaction_data) => { - asc_new(heap, transaction_data.as_ref(), gas)?.erase() - } - CosmosTrigger::Message(message_data) => { - asc_new(heap, message_data.as_ref(), gas)?.erase() - } - }) - } -} - -#[derive(Clone)] -pub enum CosmosTrigger { - Block(Arc), - Event { - event_data: Arc, - origin: EventOrigin, - }, - Transaction(Arc), - Message(Arc), -} - -impl CheapClone for CosmosTrigger { - fn cheap_clone(&self) -> CosmosTrigger { - match self { - CosmosTrigger::Block(block) => CosmosTrigger::Block(block.cheap_clone()), - CosmosTrigger::Event { event_data, origin } => CosmosTrigger::Event { - event_data: event_data.cheap_clone(), - origin: *origin, - }, - CosmosTrigger::Transaction(transaction_data) => { - CosmosTrigger::Transaction(transaction_data.cheap_clone()) - } - CosmosTrigger::Message(message_data) => { - CosmosTrigger::Message(message_data.cheap_clone()) - } - } - } -} - -impl PartialEq for CosmosTrigger { - fn eq(&self, other: &Self) -> bool { - match (self, other) { - (Self::Block(a_ptr), Self::Block(b_ptr)) => a_ptr == b_ptr, - ( - Self::Event { - event_data: a_event_data, - origin: a_origin, - }, - Self::Event { - event_data: b_event_data, - origin: b_origin, - }, - ) => { - if let (Ok(a_event), Ok(b_event)) = (a_event_data.event(), b_event_data.event()) { - a_event.event_type == b_event.event_type && a_origin == b_origin - } else { - false - } - } - (Self::Transaction(a_ptr), Self::Transaction(b_ptr)) => a_ptr == b_ptr, - (Self::Message(a_ptr), Self::Message(b_ptr)) => a_ptr == b_ptr, - _ => false, - } - } -} - -impl Eq for CosmosTrigger {} - -impl CosmosTrigger { - pub(crate) fn with_event( - event: codec::Event, - block: codec::HeaderOnlyBlock, - tx_context: Option, - origin: EventOrigin, - ) -> CosmosTrigger { - CosmosTrigger::Event { - event_data: Arc::new(codec::EventData { - event: Some(event), - block: Some(block), - tx: tx_context, - }), - origin, - } - } - - pub(crate) fn with_transaction( - tx_result: codec::TxResult, - block: codec::HeaderOnlyBlock, - ) -> CosmosTrigger { - CosmosTrigger::Transaction(Arc::new(codec::TransactionData { - tx: Some(tx_result), - block: Some(block), - })) - } - - pub(crate) fn with_message( - message: ::prost_types::Any, - block: codec::HeaderOnlyBlock, - tx_context: codec::TransactionContext, - ) -> CosmosTrigger { - CosmosTrigger::Message(Arc::new(codec::MessageData { - message: Some(message), - block: Some(block), - tx: Some(tx_context), - })) - } - - pub fn block_number(&self) -> Result { - match self { - CosmosTrigger::Block(block) => Ok(block.number()), - CosmosTrigger::Event { event_data, .. } => event_data.block().map(|b| b.number()), - CosmosTrigger::Transaction(transaction_data) => { - transaction_data.block().map(|b| b.number()) - } - CosmosTrigger::Message(message_data) => message_data.block().map(|b| b.number()), - } - } - - pub fn block_hash(&self) -> Result { - match self { - CosmosTrigger::Block(block) => Ok(block.hash()), - CosmosTrigger::Event { event_data, .. } => event_data.block().map(|b| b.hash()), - CosmosTrigger::Transaction(transaction_data) => { - transaction_data.block().map(|b| b.hash()) - } - CosmosTrigger::Message(message_data) => message_data.block().map(|b| b.hash()), - } - } -} - -impl Ord for CosmosTrigger { - fn cmp(&self, other: &Self) -> Ordering { - match (self, other) { - // Events have no intrinsic ordering information, so we keep the order in - // which they are included in the `events` field - (Self::Event { .. }, Self::Event { .. }) => Ordering::Equal, - - // Keep the order when comparing two message triggers - (Self::Message(..), Self::Message(..)) => Ordering::Equal, - - // Transactions are ordered by their index inside the block - (Self::Transaction(a), Self::Transaction(b)) => { - if let (Ok(a_tx_result), Ok(b_tx_result)) = (a.tx_result(), b.tx_result()) { - a_tx_result.index.cmp(&b_tx_result.index) - } else { - Ordering::Equal - } - } - - // Keep the order when comparing two block triggers - (Self::Block(..), Self::Block(..)) => Ordering::Equal, - - // Event triggers always come first - (Self::Event { .. }, _) => Ordering::Greater, - (_, Self::Event { .. }) => Ordering::Less, - - // Block triggers always come last - (Self::Block(..), _) => Ordering::Less, - (_, Self::Block(..)) => Ordering::Greater, - - // Message triggers before Transaction triggers - (Self::Message(..), Self::Transaction(..)) => Ordering::Greater, - (Self::Transaction(..), Self::Message(..)) => Ordering::Less, - } - } -} - -impl PartialOrd for CosmosTrigger { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl TriggerData for CosmosTrigger { - fn error_context(&self) -> std::string::String { - match self { - CosmosTrigger::Block(..) => { - if let (Ok(block_number), Ok(block_hash)) = (self.block_number(), self.block_hash()) - { - format!("block #{block_number}, hash {block_hash}") - } else { - "block".to_string() - } - } - CosmosTrigger::Event { event_data, origin } => { - if let (Ok(event), Ok(block_number), Ok(block_hash)) = - (event_data.event(), self.block_number(), self.block_hash()) - { - format!( - "event type {}, origin: {:?}, block #{block_number}, hash {block_hash}", - event.event_type, origin, - ) - } else { - "event".to_string() - } - } - CosmosTrigger::Transaction(transaction_data) => { - if let (Ok(block_number), Ok(block_hash), Ok(response_deliver_tx)) = ( - self.block_number(), - self.block_hash(), - transaction_data.response_deliver_tx(), - ) { - format!( - "block #{block_number}, hash {block_hash}, transaction log: {}", - response_deliver_tx.log - ) - } else { - "transaction".to_string() - } - } - CosmosTrigger::Message(message_data) => { - if let (Ok(message), Ok(block_number), Ok(block_hash)) = ( - message_data.message(), - self.block_number(), - self.block_hash(), - ) { - format!( - "message type {}, block #{block_number}, hash {block_hash}", - message.type_url, - ) - } else { - "message".to_string() - } - } - } - } -} - -#[cfg(test)] -mod tests { - use crate::codec::TxResult; - - use super::*; - - #[test] - fn test_cosmos_trigger_ordering() { - let event_trigger = CosmosTrigger::Event { - event_data: Arc::::new(codec::EventData { - ..Default::default() - }), - origin: EventOrigin::BeginBlock, - }; - let other_event_trigger = CosmosTrigger::Event { - event_data: Arc::::new(codec::EventData { - ..Default::default() - }), - origin: EventOrigin::BeginBlock, - }; - let message_trigger = - CosmosTrigger::Message(Arc::::new(codec::MessageData { - ..Default::default() - })); - let other_message_trigger = - CosmosTrigger::Message(Arc::::new(codec::MessageData { - ..Default::default() - })); - let transaction_trigger = CosmosTrigger::Transaction(Arc::::new( - codec::TransactionData { - block: None, - tx: Some(TxResult { - index: 1, - ..Default::default() - }), - }, - )); - let other_transaction_trigger = CosmosTrigger::Transaction( - Arc::::new(codec::TransactionData { - block: None, - tx: Some(TxResult { - index: 2, - ..Default::default() - }), - }), - ); - let block_trigger = CosmosTrigger::Block(Arc::::new(codec::Block { - ..Default::default() - })); - let other_block_trigger = CosmosTrigger::Block(Arc::::new(codec::Block { - ..Default::default() - })); - - assert_eq!(event_trigger.cmp(&block_trigger), Ordering::Greater); - assert_eq!(event_trigger.cmp(&transaction_trigger), Ordering::Greater); - assert_eq!(event_trigger.cmp(&message_trigger), Ordering::Greater); - assert_eq!(event_trigger.cmp(&other_event_trigger), Ordering::Equal); - - assert_eq!(message_trigger.cmp(&block_trigger), Ordering::Greater); - assert_eq!(message_trigger.cmp(&transaction_trigger), Ordering::Greater); - assert_eq!(message_trigger.cmp(&other_message_trigger), Ordering::Equal); - assert_eq!(message_trigger.cmp(&event_trigger), Ordering::Less); - - assert_eq!(transaction_trigger.cmp(&block_trigger), Ordering::Greater); - assert_eq!( - transaction_trigger.cmp(&other_transaction_trigger), - Ordering::Less - ); - assert_eq!( - other_transaction_trigger.cmp(&transaction_trigger), - Ordering::Greater - ); - assert_eq!(transaction_trigger.cmp(&message_trigger), Ordering::Less); - assert_eq!(transaction_trigger.cmp(&event_trigger), Ordering::Less); - - assert_eq!(block_trigger.cmp(&other_block_trigger), Ordering::Equal); - assert_eq!(block_trigger.cmp(&transaction_trigger), Ordering::Less); - assert_eq!(block_trigger.cmp(&message_trigger), Ordering::Less); - assert_eq!(block_trigger.cmp(&event_trigger), Ordering::Less); - } -} diff --git a/chain/ethereum/Cargo.toml b/chain/ethereum/Cargo.toml index 4a5a1180dcd..a9c9d42a124 100644 --- a/chain/ethereum/Cargo.toml +++ b/chain/ethereum/Cargo.toml @@ -4,30 +4,32 @@ version.workspace = true edition.workspace = true [dependencies] -envconfig = "0.10.0" -futures = "0.1.21" -http = "0.2.4" +async-trait = { workspace = true } +envconfig = { workspace = true } jsonrpc-core = "18.0.0" graph = { path = "../../graph" } -lazy_static = "1.2.0" -serde = "1.0" +serde = { workspace = true } prost = { workspace = true } prost-types = { workspace = true } -dirs-next = "2.0" anyhow = "1.0" -tiny-keccak = "1.5.0" hex = "0.4.3" -semver = "1.0.16" +semver = { workspace = true } +thiserror = { workspace = true } +futures = { workspace = true } +tokio = { workspace = true } +tokio-stream = { workspace = true } +tower = { workspace = true } -itertools = "0.10.5" +itertools = "0.15.0" graph-runtime-wasm = { path = "../../runtime/wasm" } graph-runtime-derive = { path = "../../runtime/derive" } [dev-dependencies] -test-store = { path = "../../store/test-store" } -base64 = "0.20.0" -graph-mock = { path = "../../mock" } +base64 = "0" [build-dependencies] -tonic-build = { workspace = true } +tonic-prost-build = { workspace = true } + +[lints] +workspace = true diff --git a/chain/ethereum/build.rs b/chain/ethereum/build.rs index 0efb360140d..f9d8eb0101b 100644 --- a/chain/ethereum/build.rs +++ b/chain/ethereum/build.rs @@ -1,8 +1,9 @@ fn main() { println!("cargo:rerun-if-changed=proto"); - tonic_build::configure() + tonic_prost_build::configure() .out_dir("src/protobuf") - .compile(&["proto/codec.proto"], &["proto"]) + .protoc_arg("--experimental_allow_proto3_optional") + .compile_protos(&["proto/ethereum.proto"], &["proto"]) .expect("Failed to compile Firehose Ethereum proto(s)"); } diff --git a/chain/ethereum/examples/firehose.rs b/chain/ethereum/examples/firehose.rs index 37acba642e3..3dd21859653 100644 --- a/chain/ethereum/examples/firehose.rs +++ b/chain/ethereum/examples/firehose.rs @@ -1,13 +1,15 @@ use anyhow::Error; use graph::{ + endpoint::EndpointMetrics, env::env_var, - firehose::SubgraphLimit, - prelude::{prost, tokio, tonic}, - {firehose, firehose::FirehoseEndpoint}, + firehose::{self, FirehoseEndpoint, SubgraphLimit}, + log::logger, + prelude::{MetricsRegistry, prost, tokio, tonic}, }; use graph_chain_ethereum::codec; use hex::ToHex; use prost::Message; +use std::slice; use std::sync::Arc; use tonic::Streaming; @@ -20,29 +22,42 @@ async fn main() -> Result<(), Error> { token = Some(token_env); } + let logger = logger(false); + let host = "https://api.streamingfast.io:443".to_string(); + let metrics = Arc::new(EndpointMetrics::new( + logger, + slice::from_ref(&host), + Arc::new(MetricsRegistry::mock()), + )); + let firehose = Arc::new(FirehoseEndpoint::new( "firehose", - "https://api.streamingfast.io:443", + &host, token, + None, false, false, SubgraphLimit::Unlimited, + metrics, )); loop { println!("Connecting to the stream!"); let mut stream: Streaming = match firehose .clone() - .stream_blocks(firehose::Request { - start_block_num: 12369739, - stop_block_num: 12369739, - cursor: match &cursor { - Some(c) => c.clone(), - None => String::from(""), + .stream_blocks( + firehose::Request { + start_block_num: 12369739, + stop_block_num: 12369739, + cursor: match &cursor { + Some(c) => c.clone(), + None => String::from(""), + }, + final_blocks_only: false, + ..Default::default() }, - final_blocks_only: false, - ..Default::default() - }) + &firehose::ConnectionHeaders::new(), + ) .await { Ok(s) => s, diff --git a/chain/ethereum/proto/codec.proto b/chain/ethereum/proto/codec.proto deleted file mode 100644 index 3c9f7378c7d..00000000000 --- a/chain/ethereum/proto/codec.proto +++ /dev/null @@ -1,508 +0,0 @@ -syntax = "proto3"; - -package sf.ethereum.type.v2; - -option go_package = "github.com/streamingfast/sf-ethereum/types/pb/sf/ethereum/type/v2;pbeth"; - -import "google/protobuf/timestamp.proto"; - -message Block { - int32 ver = 1; - bytes hash = 2; - uint64 number = 3; - uint64 size = 4; - BlockHeader header = 5; - - // Uncles represents block produced with a valid solution but were not actually choosen - // as the canonical block for the given height so they are mostly "forked" blocks. - // - // If the Block has been produced using the Proof of Stake consensus algorithm, this - // field will actually be always empty. - repeated BlockHeader uncles = 6; - - repeated TransactionTrace transaction_traces = 10; - repeated BalanceChange balance_changes = 11; - repeated CodeChange code_changes = 20; - - reserved 40; // bool filtering_applied = 40 [deprecated = true]; - reserved 41; // string filtering_include_filter_expr = 41 [deprecated = true]; - reserved 42; // string filtering_exclude_filter_expr = 42 [deprecated = true]; -} - -// HeaderOnlyBlock is used to optimally unpack the [Block] structure (note the -// corresponding message number for the `header` field) while consuming less -// memory, when only the `header` is desired. -// -// WARN: this is a client-side optimization pattern and should be moved in the -// consuming code. -message HeaderOnlyBlock { - BlockHeader header = 5; -} - -// BlockWithRefs is a lightweight block, with traces and transactions -// purged from the `block` within, and only. It is used in transports -// to pass block data around. -message BlockWithRefs { - string id = 1; - Block block = 2; - TransactionRefs transaction_trace_refs = 3; - bool irreversible = 4; -} - -message TransactionRefs { - repeated bytes hashes = 1; -} - -message UnclesHeaders { - repeated BlockHeader uncles = 1; -} - -message BlockRef { - bytes hash = 1; - uint64 number = 2; -} - -message BlockHeader { - bytes parent_hash = 1; - - // Uncle hash of the block, some reference it as `sha3Uncles`, but `sha3`` is badly worded, so we prefer `uncle_hash`, also - // referred as `ommers` in EIP specification. - // - // If the Block containing this `BlockHeader` has been produced using the Proof of Stake - // consensus algorithm, this field will actually be constant and set to `0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347`. - bytes uncle_hash = 2; - - bytes coinbase = 3; - bytes state_root = 4; - bytes transactions_root = 5; - bytes receipt_root = 6; - bytes logs_bloom = 7; - - // Difficulty is the difficulty of the Proof of Work algorithm that was required to compute a solution. - // - // If the Block containing this `BlockHeader` has been produced using the Proof of Stake - // consensus algorithm, this field will actually be constant and set to `0x00`. - BigInt difficulty = 8; - - // TotalDifficulty is the sum of all previous blocks difficulty including this block difficulty. - // - // If the Block containing this `BlockHeader` has been produced using the Proof of Stake - // consensus algorithm, this field will actually be constant and set to the terminal total difficulty - // that was required to transition to Proof of Stake algorithm, which varies per network. It is set to - // 58 750 000 000 000 000 000 000 on Ethereum Mainnet and to 10 790 000 on Ethereum Testnet Goerli. - BigInt total_difficulty = 17; - - uint64 number = 9; - uint64 gas_limit = 10; - uint64 gas_used = 11; - google.protobuf.Timestamp timestamp = 12; - - // ExtraData is free-form bytes included in the block by the "miner". While on Yellow paper of - // Ethereum this value is maxed to 32 bytes, other consensus algorithm like Clique and some other - // forks are using bigger values to carry special consensus data. - // - // If the Block containing this `BlockHeader` has been produced using the Proof of Stake - // consensus algorithm, this field is strictly enforced to be <= 32 bytes. - bytes extra_data = 13; - - // MixHash is used to prove, when combined with the `nonce` that sufficient amount of computation has been - // achieved and that the solution found is valid. - bytes mix_hash = 14; - - // Nonce is used to prove, when combined with the `mix_hash` that sufficient amount of computation has been - // achieved and that the solution found is valid. - // - // If the Block containing this `BlockHeader` has been produced using the Proof of Stake - // consensus algorithm, this field will actually be constant and set to `0`. - uint64 nonce = 15; - - // Hash is the hash of the block which is actually the computation: - // - // Keccak256(rlp([ - // parent_hash, - // uncle_hash, - // coinbase, - // state_root, - // transactions_root, - // receipt_root, - // logs_bloom, - // difficulty, - // number, - // gas_limit, - // gas_used, - // timestamp, - // extra_data, - // mix_hash, - // nonce, - // base_fee_per_gas - // ])) - // - bytes hash = 16; - - // Base fee per gas according to EIP-1559 (e.g. London Fork) rules, only set if London is present/active on the chain. - BigInt base_fee_per_gas = 18; -} - -message BigInt { - bytes bytes = 1; -} - -message TransactionTrace { - // consensus - bytes to = 1; - uint64 nonce = 2; - // GasPrice represents the effective price that has been paid for each gas unit of this transaction. Over time, the - // Ethereum rules changes regarding GasPrice field here. Before London fork, the GasPrice was always set to the - // fixed gas price. After London fork, this value has different meaning depending on the transaction type (see `Type` field). - // - // In cases where `TransactionTrace.Type == TRX_TYPE_LEGACY || TRX_TYPE_ACCESS_LIST`, then GasPrice has the same meaning - // as before the London fork. - // - // In cases where `TransactionTrace.Type == TRX_TYPE_DYNAMIC_FEE`, then GasPrice is the effective gas price paid - // for the transaction which is equals to `BlockHeader.BaseFeePerGas + TransactionTrace.` - BigInt gas_price = 3; - - // GasLimit is the maximum of gas unit the sender of the transaction is willing to consume when perform the EVM - // execution of the whole transaction - uint64 gas_limit = 4; - - // Value is the amount of Ether transferred as part of this transaction. - BigInt value = 5; - - // Input data the transaction will receive for execution of EVM. - bytes input = 6; - - // V is the recovery ID value for the signature Y point. - bytes v = 7; - - // R is the signature's X point on the elliptic curve (32 bytes). - bytes r = 8; - - // S is the signature's Y point on the elliptic curve (32 bytes). - bytes s = 9; - - // GasUsed is the total amount of gas unit used for the whole execution of the transaction. - uint64 gas_used = 10; - - // Type represents the Ethereum transaction type, available only since EIP-2718 & EIP-2930 activation which happened on Berlin fork. - // The value is always set even for transaction before Berlin fork because those before the fork are still legacy transactions. - Type type = 12; - - enum Type { - // All transactions that ever existed prior Berlin fork before EIP-2718 was implemented. - TRX_TYPE_LEGACY = 0; - - // Field that specifies an access list of contract/storage_keys that is going to be used - // in this transaction. - // - // Added in Berlin fork (EIP-2930). - TRX_TYPE_ACCESS_LIST = 1; - - // Transaction that specifies an access list just like TRX_TYPE_ACCESS_LIST but in addition defines the - // max base gas gee and max priority gas fee to pay for this transaction. Transaction's of those type are - // executed against EIP-1559 rules which dictates a dynamic gas cost based on the congestion of the network. - TRX_TYPE_DYNAMIC_FEE = 2; - } - - // AcccessList represents the storage access this transaction has agreed to do in which case those storage - // access cost less gas unit per access. - // - // This will is populated only if `TransactionTrace.Type == TRX_TYPE_ACCESS_LIST || TRX_TYPE_DYNAMIC_FEE` which - // is possible only if Berlin (TRX_TYPE_ACCESS_LIST) nor London (TRX_TYPE_DYNAMIC_FEE) fork are active on the chain. - repeated AccessTuple access_list = 14; - - // MaxFeePerGas is the maximum fee per gas the user is willing to pay for the transaction gas used. - // - // This will is populated only if `TransactionTrace.Type == TRX_TYPE_DYNAMIC_FEE` which is possible only - // if London fork is active on the chain. - BigInt max_fee_per_gas = 11; - - // MaxPriorityFeePerGas is priority fee per gas the user to pay in extra to the miner on top of the block's - // base fee. - // - // This will is populated only if `TransactionTrace.Type == TRX_TYPE_DYNAMIC_FEE` which is possible only - // if London fork is active on the chain. - BigInt max_priority_fee_per_gas = 13; - - // meta - uint32 index = 20; - bytes hash = 21; - bytes from = 22; - bytes return_data = 23; - bytes public_key = 24; - uint64 begin_ordinal = 25; - uint64 end_ordinal = 26; - - TransactionTraceStatus status = 30; - TransactionReceipt receipt = 31; - repeated Call calls = 32; -} - - -// AccessTuple represents a list of storage keys for a given contract's address and is used -// for AccessList construction. -message AccessTuple { - bytes address = 1; - repeated bytes storage_keys = 2; -} - -// TransactionTraceWithBlockRef -message TransactionTraceWithBlockRef { - TransactionTrace trace = 1; - BlockRef block_ref = 2; -} - -enum TransactionTraceStatus { - UNKNOWN = 0; - SUCCEEDED = 1; - FAILED = 2; - REVERTED = 3; -} - -message TransactionReceipt { - // State root is an intermediate state_root hash, computed in-between transactions to make - // **sure** you could build a proof and point to state in the middle of a block. Geth client - // uses `PostState + root + PostStateOrStatus`` while Parity used `status_code, root...`` this piles - // hardforks, see (read the EIPs first): - // - https://github.com/eoscanada/go-ethereum-private/blob/deep-mind/core/types/receipt.go#L147 - // - https://github.com/eoscanada/go-ethereum-private/blob/deep-mind/core/types/receipt.go#L50-L86 - // - https://github.com/ethereum/EIPs/blob/master/EIPS/eip-658.md - // - // Moreover, the notion of `Outcome`` in parity, which segregates the two concepts, which are - // stored in the same field `status_code`` can be computed based on such a hack of the `state_root` - // field, following `EIP-658`. - // - // Before Byzantinium hard fork, this field is always empty. - bytes state_root = 1; - uint64 cumulative_gas_used = 2; - bytes logs_bloom = 3; - repeated Log logs = 4; -} - -message Log { - bytes address = 1; - repeated bytes topics = 2; - bytes data = 3; - - // Index is the index of the log relative to the transaction. This index - // is always populated regardless of the state revertion of the the call - // that emitted this log. - uint32 index = 4; - - // BlockIndex represents the index of the log relative to the Block. - // - // An **important** notice is that this field will be 0 when the call - // that emitted the log has been reverted by the chain. - // - // Currently, there is two locations where a Log can be obtained: - // - block.transaction_traces[].receipt.logs[] - // - block.transaction_traces[].calls[].logs[] - // - // In the `receipt` case, the logs will be populated only when the call - // that emitted them has not been reverted by the chain and when in this - // position, the `blockIndex` is always populated correctly. - // - // In the case of `calls` case, for `call` where `stateReverted == true`, - // the `blockIndex` value will always be 0. - uint32 blockIndex = 6; - - uint64 ordinal = 7; -} - -message Call { - uint32 index = 1; - uint32 parent_index = 2; - uint32 depth = 3; - CallType call_type = 4; - bytes caller = 5; - bytes address = 6; - BigInt value = 7; - uint64 gas_limit = 8; - uint64 gas_consumed = 9; - bytes return_data = 13; - bytes input = 14; - bool executed_code = 15; - bool suicide = 16; - - /* hex representation of the hash -> preimage */ - map keccak_preimages = 20; - repeated StorageChange storage_changes = 21; - repeated BalanceChange balance_changes = 22; - repeated NonceChange nonce_changes = 24; - repeated Log logs = 25; - repeated CodeChange code_changes = 26; - - // Deprecated: repeated bytes created_accounts - reserved 27; - - repeated GasChange gas_changes = 28; - - // Deprecated: repeated GasEvent gas_events - reserved 29; - - // In Ethereum, a call can be either: - // - Successfull, execution passes without any problem encountered - // - Failed, execution failed, and remaining gas should be consumed - // - Reverted, execution failed, but only gas consumed so far is billed, remaining gas is refunded - // - // When a call is either `failed` or `reverted`, the `status_failed` field - // below is set to `true`. If the status is `reverted`, then both `status_failed` - // and `status_reverted` are going to be set to `true`. - bool status_failed = 10; - bool status_reverted = 12; - - // Populated when a call either failed or reverted, so when `status_failed == true`, - // see above for details about those flags. - string failure_reason = 11; - - // This field represents wheter or not the state changes performed - // by this call were correctly recorded by the blockchain. - // - // On Ethereum, a transaction can record state changes even if some - // of its inner nested calls failed. This is problematic however since - // a call will invalidate all its state changes as well as all state - // changes performed by its child call. This means that even if a call - // has a status of `SUCCESS`, the chain might have reverted all the state - // changes it performed. - // - // ```text - // Trx 1 - // Call #1 - // Call #2 - // Call #3 - // |--- Failure here - // Call #4 - // ``` - // - // In the transaction above, while Call #2 and Call #3 would have the - // status `EXECUTED` - bool state_reverted = 30; - - uint64 begin_ordinal = 31; - uint64 end_ordinal = 32; - - repeated AccountCreation account_creations = 33; - - reserved 50; // repeated ERC20BalanceChange erc20_balance_changes = 50 [deprecated = true]; - reserved 51; // repeated ERC20TransferEvent erc20_transfer_events = 51 [deprecated = true]; - reserved 60; // bool filtering_matched = 60 [deprecated = true]; -} - -enum CallType { - UNSPECIFIED = 0; - CALL = 1; // direct? what's the name for `Call` alone? - CALLCODE = 2; - DELEGATE = 3; - STATIC = 4; - CREATE = 5; // create2 ? any other form of calls? -} - -message StorageChange { - bytes address = 1; - bytes key = 2; - bytes old_value = 3; - bytes new_value = 4; - - uint64 ordinal = 5; -} - -message BalanceChange { - bytes address = 1; - BigInt old_value = 2; - BigInt new_value = 3; - Reason reason = 4; - - // Obtain all balanche change reasons under deep mind repository: - // - // ```shell - // ack -ho 'BalanceChangeReason\(".*"\)' | grep -Eo '".*"' | sort | uniq - // ``` - enum Reason { - REASON_UNKNOWN = 0; - REASON_REWARD_MINE_UNCLE = 1; - REASON_REWARD_MINE_BLOCK = 2; - REASON_DAO_REFUND_CONTRACT = 3; - REASON_DAO_ADJUST_BALANCE = 4; - REASON_TRANSFER = 5; - REASON_GENESIS_BALANCE = 6; - REASON_GAS_BUY = 7; - REASON_REWARD_TRANSACTION_FEE = 8; - REASON_REWARD_FEE_RESET = 14; - REASON_GAS_REFUND = 9; - REASON_TOUCH_ACCOUNT = 10; - REASON_SUICIDE_REFUND = 11; - REASON_SUICIDE_WITHDRAW = 13; - REASON_CALL_BALANCE_OVERRIDE = 12; - // Used on chain(s) where some Ether burning happens - REASON_BURN = 15; - } - - uint64 ordinal = 5; -} - -message NonceChange { - bytes address = 1; - uint64 old_value = 2; - uint64 new_value = 3; - uint64 ordinal = 4; -} - -message AccountCreation { - bytes account = 1; - uint64 ordinal = 2; -} - -message CodeChange { - bytes address = 1; - bytes old_hash = 2; - bytes old_code = 3; - bytes new_hash = 4; - bytes new_code = 5; - - uint64 ordinal = 6; -} - -// The gas change model represents the reason why some gas cost has occurred. -// The gas is computed per actual op codes. Doing them completely might prove -// overwhelming in most cases. -// -// Hence, we only index some of them, those that are costy like all the calls -// one, log events, return data, etc. -message GasChange { - uint64 old_value = 1; - uint64 new_value = 2; - Reason reason = 3; - - // Obtain all gas change reasons under deep mind repository: - // - // ```shell - // ack -ho 'GasChangeReason\(".*"\)' | grep -Eo '".*"' | sort | uniq - // ``` - enum Reason { - REASON_UNKNOWN = 0; - REASON_CALL = 1; - REASON_CALL_CODE = 2; - REASON_CALL_DATA_COPY = 3; - REASON_CODE_COPY = 4; - REASON_CODE_STORAGE = 5; - REASON_CONTRACT_CREATION = 6; - REASON_CONTRACT_CREATION2 = 7; - REASON_DELEGATE_CALL = 8; - REASON_EVENT_LOG = 9; - REASON_EXT_CODE_COPY = 10; - REASON_FAILED_EXECUTION = 11; - REASON_INTRINSIC_GAS = 12; - REASON_PRECOMPILED_CONTRACT = 13; - REASON_REFUND_AFTER_EXECUTION = 14; - REASON_RETURN = 15; - REASON_RETURN_DATA_COPY = 16; - REASON_REVERT = 17; - REASON_SELF_DESTRUCT = 18; - REASON_STATIC_CALL = 19; - - // Added in Berlin fork (Geth 1.10+) - REASON_STATE_COLD_ACCESS = 20; - } - - uint64 ordinal = 4; -} \ No newline at end of file diff --git a/chain/ethereum/proto/ethereum.proto b/chain/ethereum/proto/ethereum.proto new file mode 100644 index 00000000000..50c10f921f0 --- /dev/null +++ b/chain/ethereum/proto/ethereum.proto @@ -0,0 +1,1073 @@ +syntax = "proto3"; + +package sf.ethereum.type.v2; + +option go_package = "github.com/streamingfast/firehose-ethereum/types/pb/sf/ethereum/type/v2;pbeth"; + +import "google/protobuf/timestamp.proto"; + +// Block is the representation of the tracing of a block in the Ethereum +// blockchain. A block is a collection of [TransactionTrace] that are grouped +// together and processed as an atomic unit. Each [TransactionTrace] is composed +// of a series of [Call] (a.k.a internal transactions) and there is also at +// least one call per transaction a.k.a the root call which essentially has the +// same parameters as the transaction itself (e.g. `from`, `to`, `gas`, `value`, +// etc.). +// +// The exact tracing method used to build the block must be checked against +// [DetailLevel] field. There is two levels of details available, `BASE` and +// `EXTENDED`. The `BASE` level has been extracted using archive node RPC calls +// and will contain only the block header, transaction receipts and event logs. +// Refers to the Firehose service provider to know which blocks are offered on +// each network. +// +// The `EXTENDED` level has been extracted using the Firehose tracer and all +// fields are available in this Protobuf. +// +// The Ethereum block model is used across many chains which means that it +// happen that certain fields are not available in one chain but are available +// in another. Each field should be documented when necesssary if it's available +// on a subset of chains. +// +// One major concept to get about the Block is the concept of 'ordinal'. The +// ordinal is a number that is used to globally order every element of execution +// that happened throughout the processing of the block like +// [TransactionTracer], [Call], [Log], [BalanceChange], [StateChange], etc. +// Element that have a start and end interval, [Transaction] and [Call], will +// have two ordinals: `begin_ordinal` and `end_ordinal`. Element that are +// executed as "point in time" [Log], [BalanceChange], [StateChange], etc. will +// have only one ordinal named `ordinal`. If you take all of the message in the +// Block that have an 'ordinal' field in an array and you sort each element +// against the `ordinal` field, you will get the exact order of execution of +// each element in the block. +// +// All the 'ordinal' fields in a block are globally unique for the given block, +// it is **not** a chain-wide global ordering. Furthermore, caution must be take +// with reverted elements due to execution failure. For anything attached to a +// [Call] that has a `state_reverted` field set to `true`, the `ordinal` field +// is not reliable and should not be used to order the element against other +// elements in the block as those element might have 0 as the ordinal. Only +// successful calls have a reliable `ordinal` field. +message Block { + // Hash is the block's hash. + bytes hash = 2; + // Number is the block's height at which this block was mined. + uint64 number = 3; + // Size is the size in bytes of the RLP encoding of the block according to Ethereum + // rules. + uint64 size = 4; + // Header contain's the block's header information like its parent hash, the merkel root hash + // and all other information the form a block. + BlockHeader header = 5; + + // Uncles represents block produced with a valid solution but were not actually chosen + // as the canonical block for the given height so they are mostly "forked" blocks. + // + // If the Block has been produced using the Proof of Stake consensus algorithm, this + // field will actually be always empty. + repeated BlockHeader uncles = 6; + + // TransactionTraces hold the execute trace of all the transactions that were executed + // in this block. In in there that you will find most of the Ethereum data model. + // + // They are ordered by the order of execution of the transaction in the block. + repeated TransactionTrace transaction_traces = 10; + + // BalanceChanges here is the array of ETH transfer that happened at the block level + // outside of the normal transaction flow of a block. The best example of this is mining + // reward for the block mined, the transfer of ETH to the miner happens outside the normal + // transaction flow of the chain and is recorded as a `BalanceChange` here since we cannot + // attached it to any transaction. + // + // Only available in DetailLevel: EXTENDED + repeated BalanceChange balance_changes = 11; + + enum DetailLevel{ + DETAILLEVEL_EXTENDED = 0; + // DETAILLEVEL_TRACE = 1; // TBD + DETAILLEVEL_BASE = 2; + } + + // DetailLevel affects the data available in this block. + // + // ## DetailLevel_EXTENDED + // + // Describes the most complete block, with traces, balance changes, storage + // changes. It is extracted during the execution of the block. + // + // ## DetailLevel_BASE + // + // Describes a block that contains only the block header, transaction receipts + // and event logs: everything that can be extracted using the base JSON-RPC + // interface + // (https://ethereum.org/en/developers/docs/apis/json-rpc/#json-rpc-methods) + // Furthermore, the eth_getTransactionReceipt call has been avoided because it + // brings only minimal improvements at the cost of requiring an archive node + // or a full node with complete transaction index. + DetailLevel detail_level = 12; + + // CodeChanges here is the array of smart code change that happened that happened at the block level + // outside of the normal transaction flow of a block. Some Ethereum's fork like BSC and Polygon + // has some capabilities to upgrade internal smart contracts used usually to track the validator + // list. + // + // On hard fork, some procedure runs to upgrade the smart contract code to a new version. In those + // network, a `CodeChange` for each modified smart contract on upgrade would be present here. Note + // that this happen rarely, so the vast majority of block will have an empty list here. + // + // Only available in DetailLevel: EXTENDED + repeated CodeChange code_changes = 20; + + // System calls are introduced in Cancun, along with blobs. They are executed outside of transactions but affect the state. + // + // Only available in DetailLevel: EXTENDED + repeated Call system_calls = 21; + + // Withdrawals represents the list of validator balance withdrawals processed in this block. + // Introduced in the Shanghai hard fork (EIP-4895). + // + // This field has been added because Geth blocks include withdrawals after Shanghai fork, + // but our previous Firehose model didn't capture this data. Currently experimental - + // NOT ready for production use yet as we validate the tracing implementation. + // + // Only available when Shanghai fork is active on the chain. + repeated Withdrawal withdrawals = 22; + + reserved 40; // bool filtering_applied = 40 [deprecated = true]; + reserved 41; // string filtering_include_filter_expr = 41 [deprecated = true]; + reserved 42; // string filtering_exclude_filter_expr = 42 [deprecated = true]; + + // Ver represents that data model version of the block, it is used internally by Firehose on Ethereum + // as a validation that we are reading the correct version. + int32 ver = 1; +} + +message BlockHeader { + bytes parent_hash = 1; + + // Uncle hash of the block, some reference it as `sha3Uncles`, but `sha3`` is badly worded, so we prefer `uncle_hash`, also + // referred as `ommers` in EIP specification. + // + // If the Block containing this `BlockHeader` has been produced using the Proof of Stake + // consensus algorithm, this field will actually be constant and set to `0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347`. + bytes uncle_hash = 2; + + bytes coinbase = 3; + bytes state_root = 4; + bytes transactions_root = 5; + bytes receipt_root = 6; + bytes logs_bloom = 7; + + // Difficulty is the difficulty of the Proof of Work algorithm that was required to compute a solution. + // + // If the Block containing this `BlockHeader` has been produced using the Proof of Stake + // consensus algorithm, this field will actually be constant and set to `0x00`. + BigInt difficulty = 8; + + // TotalDifficulty used to be the sum of all previous blocks difficulty including this block difficulty. + // + // It has been deprecated in geth v1.15.0 but was already removed from the JSON-RPC interface for a while + BigInt total_difficulty = 17 [deprecated = true]; + + uint64 number = 9; + uint64 gas_limit = 10; + uint64 gas_used = 11; + google.protobuf.Timestamp timestamp = 12; + + // ExtraData is free-form bytes included in the block by the "miner". While on Yellow paper of + // Ethereum this value is maxed to 32 bytes, other consensus algorithm like Clique and some other + // forks are using bigger values to carry special consensus data. + // + // If the Block containing this `BlockHeader` has been produced using the Proof of Stake + // consensus algorithm, this field is strictly enforced to be <= 32 bytes. + bytes extra_data = 13; + + // MixHash is used to prove, when combined with the `nonce` that sufficient amount of computation has been + // achieved and that the solution found is valid. + bytes mix_hash = 14; + + // Nonce is used to prove, when combined with the `mix_hash` that sufficient amount of computation has been + // achieved and that the solution found is valid. + // + // If the Block containing this `BlockHeader` has been produced using the Proof of Stake + // consensus algorithm, this field will actually be constant and set to `0`. + uint64 nonce = 15; + + // Hash is the hash of the block which is actually the computation: + // + // Keccak256(rlp([ + // parent_hash, + // uncle_hash, + // coinbase, + // state_root, + // transactions_root, + // receipt_root, + // logs_bloom, + // difficulty, + // number, + // gas_limit, + // gas_used, + // timestamp, + // extra_data, + // mix_hash, + // nonce, + // base_fee_per_gas (to be included only if London fork is active) + // withdrawals_root (to be included only if Shangai fork is active) + // blob_gas_used (to be included only if Cancun fork is active) + // excess_blob_gas (to be included only if Cancun fork is active) + // parent_beacon_root (to be included only if Cancun fork is active) + // requests_hash (to be included only if Prague fork is active) + // ])) + // + bytes hash = 16; + + // Base fee per gas according to EIP-1559 (e.g. London Fork) rules, only set if London is present/active on the chain. + BigInt base_fee_per_gas = 18; + + // Withdrawals root hash according to EIP-4895 (e.g. Shangai Fork) rules, only set if Shangai is present/active on the chain. + // + // Only available in DetailLevel: EXTENDED + bytes withdrawals_root = 19; + + // TxDependency is list of transaction indexes that are dependent on each other in the block + // header. This is metadata only that was used by the internal Polygon parallel execution engine. + // + // This field was available in a few versions on Polygon Mainnet and Polygon Mumbai chains. It was actually + // removed and is not populated anymore. It's now embedded in the `extraData` field, refer to Polygon source + // code to determine how to extract it if you need it. + // + // Only available in DetailLevel: EXTENDED + Uint64NestedArray tx_dependency = 20; + + // BlobGasUsed was added by EIP-4844 and is ignored in legacy headers. + optional uint64 blob_gas_used = 22; + + // ExcessBlobGas was added by EIP-4844 and is ignored in legacy headers. + optional uint64 excess_blob_gas = 23; + + // ParentBeaconRoot was added by EIP-4788 and is ignored in legacy headers. + bytes parent_beacon_root = 24; + + // RequestsHash was added by EIP-7685 and is ignored in legacy headers. + bytes requests_hash = 25; +} + +message Uint64NestedArray { + repeated Uint64Array val = 1; +} + +message Uint64Array { + repeated uint64 val = 1; +} + +message BigInt { + bytes bytes = 1; +} + +// TransactionTrace is full trace of execution of the transaction when the +// it actually executed on chain. +// +// It contains all the transaction details like `from`, `to`, `gas`, etc. +// as well as all the internal calls that were made during the transaction. +// +// The `calls` vector contains Call objects which have balance changes, events +// storage changes, etc. +// +// If ordering is important between elements, almost each message like `Log`, +// `Call`, `StorageChange`, etc. have an ordinal field that is represents "execution" +// order of the said element against all other elements in this block. +// +// Due to how the call tree works doing "naively", looping through all calls then +// through a Call's element like `logs` while not yielding the elements in the order +// they were executed on chain. A log in call could have been done before or after +// another in another call depending on the actual call tree. +// +// The `calls` are ordered by creation order and the call tree can be re-computing +// using fields found in `Call` object (parent/child relationship). +// +// Another important thing to note is that even if a transaction succeed, some calls +// within it could have been reverted internally, if this is important to you, you must +// check the field `state_reverted` on the `Call` to determine if it was fully committed +// to the chain or not. +message TransactionTrace { + // consensus + bytes to = 1; + uint64 nonce = 2; + // GasPrice represents the effective price that has been paid for each gas unit of this transaction. Over time, the + // Ethereum rules changes regarding GasPrice field here. Before London fork, the GasPrice was always set to the + // fixed gas price. After London fork, this value has different meaning depending on the transaction type (see `Type` field). + // + // In cases where `TransactionTrace.Type == TRX_TYPE_LEGACY || TRX_TYPE_ACCESS_LIST`, then GasPrice has the same meaning + // as before the London fork. + // + // In cases where `TransactionTrace.Type == TRX_TYPE_DYNAMIC_FEE`, then GasPrice is the effective gas price paid + // for the transaction which is equals to `BlockHeader.BaseFeePerGas + TransactionTrace.` + BigInt gas_price = 3; + + // GasLimit is the maximum of gas unit the sender of the transaction is willing to consume when perform the EVM + // execution of the whole transaction + uint64 gas_limit = 4; + + // Value is the amount of Ether transferred as part of this transaction. + BigInt value = 5; + + // Input data the transaction will receive for execution of EVM. + bytes input = 6; + + // V is the recovery ID value for the signature Y point. + bytes v = 7; + + // R is the signature's X point on the elliptic curve (32 bytes). + bytes r = 8; + + // S is the signature's Y point on the elliptic curve (32 bytes). + bytes s = 9; + + // GasUsed is the total amount of gas unit used for the whole execution of the transaction. + uint64 gas_used = 10; + + // Type represents the Ethereum transaction type, available only since EIP-2718 & EIP-2930 activation which happened on Berlin fork. + // The value is always set even for transaction before Berlin fork because those before the fork are still legacy transactions. + Type type = 12; + + enum Type { + // All transactions that ever existed prior Berlin fork before EIP-2718 was implemented. + TRX_TYPE_LEGACY = 0; + + // Transaction that specicy an access list of contract/storage_keys that is going to be used + // in this transaction. + // + // Added in Berlin fork (EIP-2930). + TRX_TYPE_ACCESS_LIST = 1; + + // Transaction that specifis an access list just like TRX_TYPE_ACCESS_LIST but in addition defines the + // max base gas gee and max priority gas fee to pay for this transaction. Transaction's of those type are + // executed against EIP-1559 rules which dictates a dynamic gas cost based on the congestion of the network. + TRX_TYPE_DYNAMIC_FEE = 2; + + // Transaction which contain a large amount of data that cannot be accessed by EVM execution, but whose commitment + // can be accessed. The format is intended to be fully compatible with the format that will be used in full sharding. + // + // Transaction that defines an access list just like TRX_TYPE_ACCESS_LIST and enables dynamic fee just like + // TRX_TYPE_DYNAMIC_FEE but in addition defines the fields 'max_fee_per_data_gas' of type 'uint256' and the fields + // 'blob_versioned_hashes' which represents a list of hash outputs from 'kzg_to_versioned_hash'. + // + // Activated in Cancun fork (EIP-4844) + TRX_TYPE_BLOB = 3; + + // Transaction that sets code to an EOA (Externally Owned Accounts) + // + // Activated in Prague (EIP-7702) + TRX_TYPE_SET_CODE = 4; + + // Arbitrum-specific transactions + TRX_TYPE_ARBITRUM_DEPOSIT = 100; + TRX_TYPE_ARBITRUM_UNSIGNED = 101; + TRX_TYPE_ARBITRUM_CONTRACT = 102; + TRX_TYPE_ARBITRUM_RETRY = 104; + TRX_TYPE_ARBITRUM_SUBMIT_RETRYABLE = 105; + TRX_TYPE_ARBITRUM_INTERNAL = 106; + TRX_TYPE_ARBITRUM_LEGACY = 120; + + // OPTIMISM-specific transactions + TRX_TYPE_OPTIMISM_DEPOSIT = 126; + + } + + // AccessList represents the storage access this transaction has agreed to do in which case those storage + // access cost less gas unit per access. + // + // This will is populated only if `TransactionTrace.Type == TRX_TYPE_ACCESS_LIST || TRX_TYPE_DYNAMIC_FEE` which + // is possible only if Berlin (TRX_TYPE_ACCESS_LIST) nor London (TRX_TYPE_DYNAMIC_FEE) fork are active on the chain. + repeated AccessTuple access_list = 14; + + // MaxFeePerGas is the maximum fee per gas the user is willing to pay for the transaction gas used. + // + // This will is populated only if `TransactionTrace.Type == TRX_TYPE_DYNAMIC_FEE` which is possible only + // if London fork is active on the chain. + // + // Only available in DetailLevel: EXTENDED + BigInt max_fee_per_gas = 11; + + // MaxPriorityFeePerGas is priority fee per gas the user to pay in extra to the miner on top of the block's + // base fee. + // + // This will is populated only if `TransactionTrace.Type == TRX_TYPE_DYNAMIC_FEE` which is possible only + // if London fork is active on the chain. + // + // Only available in DetailLevel: EXTENDED + BigInt max_priority_fee_per_gas = 13; + + // meta + uint32 index = 20; + bytes hash = 21; + bytes from = 22; + + // Only available in DetailLevel: EXTENDED + // Known Issues + // - Version 3: + // Field not populated. It will be empty. + // + // Fixed in `Version 4`, see https://docs.substreams.dev/reference-material/chains-and-endpoints/ethereum-data-model for information about block versions. + bytes return_data = 23; + + // Only available in DetailLevel: EXTENDED + bytes public_key = 24; + + // The block's global ordinal when the transaction started executing, refer to + // [Block] documentation for further information about ordinals and total ordering. + uint64 begin_ordinal = 25; + + // The block's global ordinal when the transaction finished executing, refer to + // [Block] documentation for further information about ordinals and total ordering. + uint64 end_ordinal = 26; + + // TransactionTraceStatus is the status of the transaction execution and will let you know if the transaction + // was successful or not. + // + // ## Explanation relevant only for blocks with `DetailLevel: EXTENDED` + // + // A successful transaction has been recorded to the blockchain's state for calls in it that were successful. + // This means it's possible only a subset of the calls were properly recorded, refer to [calls[].state_reverted] field + // to determine which calls were reverted. + // + // A quirks of the Ethereum protocol is that a transaction `FAILED` or `REVERTED` still affects the blockchain's + // state for **some** of the state changes. Indeed, in those cases, the transactions fees are still paid to the miner + // which means there is a balance change for the transaction's emitter (e.g. `from`) to pay the gas fees, an optional + // balance change for gas refunded to the transaction's emitter (e.g. `from`) and a balance change for the miner who + // received the transaction fees. There is also a nonce change for the transaction's emitter (e.g. `from`). + // + // This means that to properly record the state changes for a transaction, you need to conditionally procees the + // transaction's status. + // + // For a `SUCCEEDED` transaction, you iterate over the `calls` array and record the state changes for each call for + // which `state_reverted == false` (if a transaction succeeded, the call at #0 will always `state_reverted == false` + // because it aligns with the transaction). + // + // For a `FAILED` or `REVERTED` transaction, you iterate over the root call (e.g. at #0, will always exist) for + // balance changes you process those where `reason` is either `REASON_GAS_BUY`, `REASON_GAS_REFUND` or + // `REASON_REWARD_TRANSACTION_FEE` and for nonce change, still on the root call, you pick the nonce change which the + // smallest ordinal (if more than one). + TransactionTraceStatus status = 30; + + TransactionReceipt receipt = 31; + + // Only available in DetailLevel: EXTENDED + repeated Call calls = 32; + + // BlobGas is the amount of gas the transaction is going to pay for the blobs, this is a computed value + // equivalent to `self.blob_gas_fee_cap * len(self.blob_hashes)` and provided in the model for convenience. + // + // This is specified by https://eips.ethereum.org/EIPS/eip-4844 + // + // This will is populated only if `TransactionTrace.Type == TRX_TYPE_BLOB` which is possible only + // if Cancun fork is active on the chain. + optional uint64 blob_gas = 33; + + // BlobGasFeeCap is the maximum fee per data gas the user is willing to pay for the data gas used. + // + // This is specified by https://eips.ethereum.org/EIPS/eip-4844 + // + // This will is populated only if `TransactionTrace.Type == TRX_TYPE_BLOB` which is possible only + // if Cancun fork is active on the chain. + optional BigInt blob_gas_fee_cap = 34; + + // BlobHashes field represents a list of hash outputs from 'kzg_to_versioned_hash' which + // essentially is a version byte + the sha256 hash of the blob commitment (e.g. + // `BLOB_COMMITMENT_VERSION_KZG + sha256(commitment)[1:]`. + // + // This is specified by https://eips.ethereum.org/EIPS/eip-4844 + // + // This will is populated only if `TransactionTrace.Type == TRX_TYPE_BLOB` which is possible only + // if Cancun fork is active on the chain. + repeated bytes blob_hashes = 35; + + // SetCodeAuthorizations represents the authorizations of a transaction to set code to an EOA (Externally Owned Accounts) + // as defined in EIP-7702. The list will contain all the authorizations as they were specified in the + // transaction itself regardless of their validity. If you need to determined if a given authorization was + // correctly applied on chain's state, refer to [SetCodeAuthorization.discarded] field that records + // if the authorization was discarded or not by the chain due to invalidity. + // + // This is specified by https://eips.ethereum.org/EIPS/eip-7702 + // + // This will is populated only if `TransactionTrace.Type == TRX_TYPE_SET_CODE` which is possible only + // if Prague fork is active on the chain. + repeated SetCodeAuthorization set_code_authorizations = 36; +} + +// AccessTuple represents a list of storage keys for a given contract's address and is used +// for AccessList construction. +message AccessTuple { + bytes address = 1; + repeated bytes storage_keys = 2; +} + +// SetCodeAuthorization represents the authorization of a transaction to set code of an EOA (Externally Owned Account) +// as defined in EIP-7702. +// +// The 'authority' field is the address that is authorizing the delegation mechanism. The 'authority' value is computed +// from the signature contained in the message using the computation +// `authority = ecrecover(keccak(MAGIC || rlp([chain_id, address, nonce])), y_parity, r, s)` +// where `MAGIC` is `0x5`, `||` is the bytes concatenation operator, `ecrecover` is the Ethereum signature recovery +// and `y_parity` is the recovery ID value denoted `v` in the message below. Checking the go-ethereum implementation +// at https://github.com/ethereum/go-ethereum/blob/v1.15.0/core/types/tx_setcode.go#L117 might prove easier to "read". +// +// We do extract the 'authority' value from the signature in the message and store it in the 'authority' field for +// convenience so you don't need to perform the computation yourself. +message SetCodeAuthorization { + // Discarded determines if this authorization was skipped due to being invalid. As EIP-7702 states, + // if the authorization is invalid (invalid signature, nonce mismatch, etc.) it must be simply + // discarded and the transaction is processed as if the authorization was not present in the + // authorization list. + // + // This boolean records if the authorization was discarded or not by the chain due to invalidity. + bool discarded = 1; + + // ChainID is the chain ID of the chain where the transaction was executed, used + // to recover the authority from the signature. + bytes chain_id = 2; + + // Address contains the address this account is delegating to. This address usually + // contain code that this account essentially "delegates" to. + // + // Note: This was missing when EIP-7702 was first activated on Holesky, Sepolia, BSC Chapel, + // BSC Mainnet and Arbitrum Sepolia but was ready for Ethereum Mainnet hard fork. We will backfill + // those missing values in the near future at which point we will remove this note. + bytes address = 8; + + // Nonce is the nonce of the account that is authorizing delegation mechanism, EIP-7702 rules + // states that nonce should be verified using this rule: + // + // - Verify the nonce of authority is equal to nonce. In case authority does not exist in the trie, + // verify that nonce is equal to 0. + // + // Read SetCodeAuthorization to know how to recover the `authority` value. + uint64 nonce = 3; + + // V is the recovery ID value for the signature Y point. While it's defined as a + // `uint32`, it's actually bounded by a `uint8` data type withing the Ethereum protocol. + uint32 v = 4; + + // R is the signature's X point on the elliptic curve (32 bytes). + bytes r = 5; + + // S is the signature's Y point on the elliptic curve (32 bytes). + bytes s = 6; + + // Authority is the address of the account that is authorizing delegation mechanism, it + // is computed from the signature contained in the message and stored for convenience. + // + // If the authority cannot be recovered from the signature, this field will be empty and + // the `discarded` field will be set to `true`. + optional bytes authority = 7; +} + +enum TransactionTraceStatus { + UNKNOWN = 0; + SUCCEEDED = 1; + FAILED = 2; + REVERTED = 3; +} + +message TransactionReceipt { + // State root is an intermediate state_root hash, computed in-between transactions to make + // **sure** you could build a proof and point to state in the middle of a block. Geth client + // uses `PostState + root + PostStateOrStatus`` while Parity used `status_code, root...`` this piles + // hard forks, see (read the EIPs first): + // - https://github.com/ethereum/EIPs/blob/master/EIPS/eip-658.md + // + // Moreover, the notion of `Outcome`` in parity, which segregates the two concepts, which are + // stored in the same field `status_code`` can be computed based on such a hack of the `state_root` + // field, following `EIP-658`. + // + // Before Byzantinium hard fork, this field is always empty. + bytes state_root = 1; + uint64 cumulative_gas_used = 2; + bytes logs_bloom = 3; + repeated Log logs = 4; + + // BlobGasUsed is the amount of blob gas that has been used within this transaction. At time + // of writing, this is equal to `self.blob_gas_fee_cap * len(self.blob_hashes)`. + // + // This is specified by https://eips.ethereum.org/EIPS/eip-4844 + // + // This will is populated only if `TransactionTrace.Type == TRX_TYPE_BLOB` which is possible only + // if Cancun fork is active on the chain. + optional uint64 blob_gas_used = 5; + + // BlobGasPrice is the amount to pay per blob item in the transaction. + // + // This is specified by https://eips.ethereum.org/EIPS/eip-4844 + // + // This will is populated only if `TransactionTrace.Type == TRX_TYPE_BLOB` which is possible only + // if Cancun fork is active on the chain. + optional BigInt blob_gas_price = 6; +} + +message Log { + bytes address = 1; + repeated bytes topics = 2; + bytes data = 3; + + // Index is the index of the log relative to the transaction. This index + // is always populated regardless of the state reversion of the the call + // that emitted this log. + // + // Only available in DetailLevel: EXTENDED + uint32 index = 4; + + // BlockIndex represents the index of the log relative to the Block. + // + // An **important** notice is that this field will be 0 when the call + // that emitted the log has been reverted by the chain. + // + // Currently, there is two locations where a Log can be obtained: + // - block.transaction_traces[].receipt.logs[] + // - block.transaction_traces[].calls[].logs[] + // + // In the `receipt` case, the logs will be populated only when the call + // that emitted them has not been reverted by the chain and when in this + // position, the `blockIndex` is always populated correctly. + // + // In the case of `calls` case, for `call` where `stateReverted == true`, + // the `blockIndex` value will always be 0. + uint32 blockIndex = 6; + + // The block's global ordinal when the log was recorded, refer to [Block] + // documentation for further information about ordinals and total ordering. + uint64 ordinal = 7; +} + +message Call { + uint32 index = 1; + uint32 parent_index = 2; + uint32 depth = 3; + CallType call_type = 4; + bytes caller = 5; + bytes address = 6; + + // AddressDelegatesTo contains the address from which the actual code to execute will be loaded + // as defined per EIP-7702 rules. If the Call's address value resolves to a code + // that delegates to another address, this field will be populated with the address + // that the call is delegated to. It will be empty in all other situations. + // + // Assumes that a 'SetCode' transaction set address `0xA` to delegates to address `0xB`, + // then when a call is made to `0xA`, the Call object would have: + // + // - caller = + // - address = 0xA + // - address_delegates_to = 0xB + // + // Again, it's important to emphasize that this field relates to EIP-7702, if the call is + // a DELEGATE or CALLCODE type, this field will not be populated and will remain empty. + // + // It will be populated only if EIP-7702 is active on the chain (Prague fork) and if the + // 'address' of the call was pointing to another address at time of execution. + optional bytes address_delegates_to = 34; + + BigInt value = 7; + uint64 gas_limit = 8; + uint64 gas_consumed = 9; + bytes return_data = 13; + + // Known Issues + // - Version 3: + // When call is `CREATE` or `CREATE2`, this field is not populated. A couple of suggestions: + // 1. You can get the contract's code in the `code_changes` field. + // 2. In the root `CREATE` call, you can directly use the `TransactionTrace`'s input field. + // + // Fixed in `Version 4`, see https://docs.substreams.dev/reference-material/chains-and-endpoints/ethereum-data-model for information about block versions. + bytes input = 14; + + // Indicates whether the call executed code. + // + // Known Issues + // - Version 3: + // This may be incorrectly set to `false` for accounts with code handling native value transfers, + // as well as for certain precompiles with no input. + // The value is initially set based on `call.type != CREATE && len(call.input) > 0` + // and later adjusted if the tracer detects an account without code. + // + // Fixed in `Version 4`, see https://docs.substreams.dev/reference-material/chains-and-endpoints/ethereum-data-model for information about block versions. + bool executed_code = 15; + bool suicide = 16; + + /* hex representation of the hash -> preimage */ + map keccak_preimages = 20; + + // Known Issues + // - Version 3: + // The data might be not be in order. + // + // Fixed in `Version 4`, see https://docs.substreams.dev/reference-material/chains-and-endpoints/ethereum-data-model for information about block versions. + repeated StorageChange storage_changes = 21; + repeated BalanceChange balance_changes = 22; + repeated NonceChange nonce_changes = 24; + repeated Log logs = 25; + repeated CodeChange code_changes = 26; + // Deprecated: repeated bytes created_accounts + reserved 27; + + // Known Issues + // - Version 3: + // Some gas changes are not correctly tracked: + // 1. Gas refunded due to data returned to the chain (occurs at the end of a transaction, before buyback). + // 2. Initial gas allocation (0 -> GasLimit) at the start of a call. + // 3. Final gas deduction (LeftOver -> 0) at the end of a call (if applicable). + // Fixed in `Version 4`, see https://docs.substreams.dev/reference-material/chains-and-endpoints/ethereum-data-model for information about block versions. + repeated GasChange gas_changes = 28; + + // Deprecated: repeated GasEvent gas_events + reserved 29; + + // In Ethereum, a call can be either: + // - Successful, execution passes without any problem encountered + // - Failed, execution failed, and remaining gas should be consumed + // - Reverted, execution failed, but only gas consumed so far is billed, remaining gas is refunded + // + // When a call is either `failed` or `reverted`, the `status_failed` field + // below is set to `true`. If the status is `reverted`, then both `status_failed` + // and `status_reverted` are going to be set to `true`. + bool status_failed = 10; + bool status_reverted = 12; + + // Populated when a call either failed or reverted, so when `status_failed == true`, + // see above for details about those flags. + string failure_reason = 11; + + // This field represents whether or not the state changes performed + // by this call were correctly recorded by the blockchain. + // + // On Ethereum, a transaction can record state changes even if some + // of its inner nested calls failed. This is problematic however since + // a call will invalidate all its state changes as well as all state + // changes performed by its child call. This means that even if a call + // has a status of `SUCCESS`, the chain might have reverted all the state + // changes it performed. + // + // ```text + // Trx 1 + // Call #1 + // Call #2 + // Call #3 + // |--- Failure here + // Call #4 + // ``` + // + // In the transaction above, while Call #2 and Call #3 would have the + // status `EXECUTED`. + // + // If you check all calls and check only `state_reverted` flag, you might be missing + // some balance changes and nonce changes. This is because when a full transaction fails + // in ethereum (e.g. `calls.all(x.state_reverted == true)`), there is still the transaction + // fee that are recorded to the chain. + // + // Refer to [TransactionTrace#status] field for more details about the handling you must + // perform. + bool state_reverted = 30; + + // Known Issues + // - Version 3: + // 1. The block's global ordinal when the call started executing, refer to + // [Block] documentation for further information about ordinals and total ordering. + // 2. The transaction root call `begin_ordial` is always `0` (also in the GENESIS block), which can cause issues + // when sorting by this field. To ensure proper execution order, set it as follows: + // `trx.Calls[0].BeginOrdinal = trx.BeginOrdinal`. + // + // Fixed in `Version 4`, see https://docs.substreams.dev/reference-material/chains-and-endpoints/ethereum-data-model for information about block versions. + uint64 begin_ordinal = 31; + + // Known Issues + // - Version 3: + // 1. The block's global ordinal when the call finished executing, refer to + // [Block] documentation for further information about ordinals and total ordering. + // 2. The root call of the GENESIS block is always `0`. To fix it, you can set it as follows: + // `rx.Calls[0].EndOrdinal = max.Uint64`. + // + // Fixed in `Version 4`, see https://docs.substreams.dev/reference-material/chains-and-endpoints/ethereum-data-model for information about block versions. + uint64 end_ordinal = 32; + + // Known Issues + // - Version 4: + // AccountCreations are NOT SUPPORTED anymore. DO NOT rely on them. + repeated AccountCreation account_creations = 33 [deprecated = true]; + + // The identifier 34 is taken by 'address_delegates_to' field above. + + reserved 50; // repeated ERC20BalanceChange erc20_balance_changes = 50 [deprecated = true]; + reserved 51; // repeated ERC20TransferEvent erc20_transfer_events = 51 [deprecated = true]; + reserved 60; // bool filtering_matched = 60 [deprecated = true]; +} + +enum CallType { + UNSPECIFIED = 0; + CALL = 1; // direct? what's the name for `Call` alone? + CALLCODE = 2; + DELEGATE = 3; + STATIC = 4; + CREATE = 5; // create2 ? any other form of calls? +} + +message StorageChange { + bytes address = 1; + bytes key = 2; + bytes old_value = 3; + bytes new_value = 4; + + // The block's global ordinal when the storage change was recorded, refer to [Block] + // documentation for further information about ordinals and total ordering. + uint64 ordinal = 5; +} + +message BalanceChange { + // Address is the address of the account that has changed balance. + bytes address = 1; + + // OldValue is the balance of the address before the change. This value + // can be **nil/null/None** if there was no previous balance for the address. + // It is safe in those case(s) to consider the balance as being 0. + // + // If you consume this from a Substreams, you can safely use: + // + // ```ignore + // let old_value = old_value.unwrap_or_default(); + // ``` + BigInt old_value = 2; + + // NewValue is the balance of the address after the change. This value + // can be **nil/null/None** if there was no previous balance for the address + // after the change. It is safe in those case(s) to consider the balance as being + // 0. + // + // If you consume this from a Substreams, you can safely use: + // + // ```ignore + // let new_value = new_value.unwrap_or_default(); + // ``` + BigInt new_value = 3; + + // Reason is the reason why the balance has changed. This is useful to determine + // why the balance has changed and what is the context of the change. + Reason reason = 4; + + enum Reason { + REASON_UNKNOWN = 0; + REASON_REWARD_MINE_UNCLE = 1; + REASON_REWARD_MINE_BLOCK = 2; + REASON_DAO_REFUND_CONTRACT = 3; + REASON_DAO_ADJUST_BALANCE = 4; + REASON_TRANSFER = 5; + REASON_GENESIS_BALANCE = 6; + REASON_GAS_BUY = 7; + REASON_REWARD_TRANSACTION_FEE = 8; + REASON_REWARD_FEE_RESET = 14; + REASON_GAS_REFUND = 9; + REASON_TOUCH_ACCOUNT = 10; + REASON_SUICIDE_REFUND = 11; + REASON_SUICIDE_WITHDRAW = 13; + REASON_CALL_BALANCE_OVERRIDE = 12; + // Used on chain(s) where some Ether burning happens + REASON_BURN = 15; + REASON_WITHDRAWAL = 16; + + // Rewards for Blob processing on BNB chain added in Tycho hard-fork, refers + // to BNB documentation to check the timestamp at which it was activated. + REASON_REWARD_BLOB_FEE = 17; + + // This reason is used only on Optimism chain. + REASON_INCREASE_MINT = 18; + // This reason is used only on Optimism chain. + REASON_REVERT = 19; + } + + // The block's global ordinal when the balance change was recorded, refer to [Block] + // documentation for further information about ordinals and total ordering. + uint64 ordinal = 5; +} + +message NonceChange { + bytes address = 1; + uint64 old_value = 2; + uint64 new_value = 3; + + // The block's global ordinal when the nonce change was recorded, refer to [Block] + // documentation for further information about ordinals and total ordering. + uint64 ordinal = 4; +} + +message AccountCreation { + bytes account = 1; + + // The block's global ordinal when the account creation was recorded, refer to [Block] + // documentation for further information about ordinals and total ordering. + uint64 ordinal = 2; +} + +message CodeChange { + bytes address = 1; + bytes old_hash = 2; + bytes old_code = 3; + bytes new_hash = 4; + bytes new_code = 5; + + // The block's global ordinal when the code change was recorded, refer to [Block] + // documentation for further information about ordinals and total ordering. + uint64 ordinal = 6; +} + +// The gas change model represents the reason why some gas cost has occurred. +// The gas is computed per actual op codes. Doing them completely might prove +// overwhelming in most cases. +// +// Hence, we only index some of them, those that are costy like all the calls +// one, log events, return data, etc. +message GasChange { + uint64 old_value = 1; + uint64 new_value = 2; + Reason reason = 3; + + enum Reason { + REASON_UNKNOWN = 0; + // REASON_CALL is the amount of gas that will be charged for a 'CALL' opcode executed by the EVM + REASON_CALL = 1; + // REASON_CALL_CODE is the amount of gas that will be charged for a 'CALLCODE' opcode executed by the EVM + REASON_CALL_CODE = 2; + // REASON_CALL_DATA_COPY is the amount of gas that will be charged for a 'CALLDATACOPY' opcode executed by the EVM + REASON_CALL_DATA_COPY = 3; + // REASON_CODE_COPY is the amount of gas that will be charged for a 'CALLDATACOPY' opcode executed by the EVM + REASON_CODE_COPY = 4; + // REASON_CODE_STORAGE is the amount of gas that will be charged for code storage + REASON_CODE_STORAGE = 5; + // REASON_CONTRACT_CREATION is the amount of gas that will be charged for a 'CREATE' opcode executed by the EVM and for the gas + // burned for a CREATE, today controlled by EIP150 rules + REASON_CONTRACT_CREATION = 6; + // REASON_CONTRACT_CREATION2 is the amount of gas that will be charged for a 'CREATE2' opcode executed by the EVM and for the gas + // burned for a CREATE2, today controlled by EIP150 rules + REASON_CONTRACT_CREATION2 = 7; + // REASON_DELEGATE_CALL is the amount of gas that will be charged for a 'DELEGATECALL' opcode executed by the EVM + REASON_DELEGATE_CALL = 8; + // REASON_EVENT_LOG is the amount of gas that will be charged for a 'LOG' opcode executed by the EVM + REASON_EVENT_LOG = 9; + // REASON_EXT_CODE_COPY is the amount of gas that will be charged for a 'LOG' opcode executed by the EVM + REASON_EXT_CODE_COPY = 10; + // REASON_FAILED_EXECUTION is the burning of the remaining gas when the execution failed without a revert + REASON_FAILED_EXECUTION = 11; + // REASON_INTRINSIC_GAS is the amount of gas that will be charged for the intrinsic cost of the transaction, there is + // always exactly one of those per transaction + REASON_INTRINSIC_GAS = 12; + // GasChangePrecompiledContract is the amount of gas that will be charged for a precompiled contract execution + REASON_PRECOMPILED_CONTRACT = 13; + // REASON_REFUND_AFTER_EXECUTION is the amount of gas that will be refunded to the caller after the execution of the call, + // if there is left over at the end of execution + REASON_REFUND_AFTER_EXECUTION = 14; + // REASON_RETURN is the amount of gas that will be charged for a 'RETURN' opcode executed by the EVM + REASON_RETURN = 15; + // REASON_RETURN_DATA_COPY is the amount of gas that will be charged for a 'RETURNDATACOPY' opcode executed by the EVM + REASON_RETURN_DATA_COPY = 16; + // REASON_REVERT is the amount of gas that will be charged for a 'REVERT' opcode executed by the EVM + REASON_REVERT = 17; + // REASON_SELF_DESTRUCT is the amount of gas that will be charged for a 'SELFDESTRUCT' opcode executed by the EVM + REASON_SELF_DESTRUCT = 18; + // REASON_STATIC_CALL is the amount of gas that will be charged for a 'STATICALL' opcode executed by the EVM + REASON_STATIC_CALL = 19; + + // REASON_STATE_COLD_ACCESS is the amount of gas that will be charged for a cold storage access as controlled by EIP2929 rules + // + // Added in Berlin fork (Geth 1.10+) + REASON_STATE_COLD_ACCESS = 20; + + // REASON_TX_INITIAL_BALANCE is the initial balance for the call which will be equal to the gasLimit of the call + // + // Added as new tracing reason in Geth, available only on some chains + REASON_TX_INITIAL_BALANCE = 21; + // REASON_TX_REFUNDS is the sum of all refunds which happened during the tx execution (e.g. storage slot being cleared) + // this generates an increase in gas. There is only one such gas change per transaction. + // + // Added as new tracing reason in Geth, available only on some chains + REASON_TX_REFUNDS = 22; + // REASON_TX_LEFT_OVER_RETURNED is the amount of gas left over at the end of transaction's execution that will be returned + // to the chain. This change will always be a negative change as we "drain" left over gas towards 0. If there was no gas + // left at the end of execution, no such even will be emitted. The returned gas's value in Wei is returned to caller. + // There is at most one of such gas change per transaction. + // + // Added as new tracing reason in Geth, available only on some chains + REASON_TX_LEFT_OVER_RETURNED = 23; + + // REASON_CALL_INITIAL_BALANCE is the initial balance for the call which will be equal to the gasLimit of the call. There is only + // one such gas change per call. + // + // Added as new tracing reason in Geth, available only on some chains + REASON_CALL_INITIAL_BALANCE = 24; + // REASON_CALL_LEFT_OVER_RETURNED is the amount of gas left over that will be returned to the caller, this change will always + // be a negative change as we "drain" left over gas towards 0. If there was no gas left at the end of execution, no such even + // will be emitted. + REASON_CALL_LEFT_OVER_RETURNED = 25; + + // REASON_WITNESS_CONTRACT_INIT flags the event of adding to the witness during the contract creation initialization step. + REASON_WITNESS_CONTRACT_INIT = 26; + + // REASON_WITNESS_CONTRACT_CREATION flags the event of adding to the witness during the contract creation finalization step. + REASON_WITNESS_CONTRACT_CREATION = 27; + // REASON_WITNESS_CODE_CHUNK flags the event of adding one or more contract code chunks to the witness. + REASON_WITNESS_CODE_CHUNK = 28; + // REASON_WITNESS_CONTRACT_COLLISION_CHECK flags the event of adding to the witness when checking for contract address collision. + REASON_WITNESS_CONTRACT_COLLISION_CHECK = 29; + // REASON_TX_DATA_FLOOR is the amount of extra gas the transaction has to pay to reach the minimum gas requirement for the + // transaction data. This change will always be a negative change. + REASON_TX_DATA_FLOOR = 30; + } + + // The block's global ordinal when the gas change was recorded, refer to [Block] + // documentation for further information about ordinals and total ordering. + uint64 ordinal = 4; +} + +// HeaderOnlyBlock is used to optimally unpack the [Block] structure (note the +// corresponding message number for the `header` field) while consuming less +// memory, when only the `header` is desired. +// +// WARN: this is a client-side optimization pattern and should be moved in the +// consuming code. +message HeaderOnlyBlock { + BlockHeader header = 5; +} + +// BlockWithRefs is a lightweight block, with traces and transactions +// purged from the `block` within, and only. It is used in transports +// to pass block data around. +message BlockWithRefs { + string id = 1; + Block block = 2; + TransactionRefs transaction_trace_refs = 3; + bool irreversible = 4; +} + +message TransactionTraceWithBlockRef { + TransactionTrace trace = 1; + BlockRef block_ref = 2; +} + +message TransactionRefs { + repeated bytes hashes = 1; +} + +message BlockRef { + bytes hash = 1; + uint64 number = 2; +} + +// Withdrawal represents a validator withdrawal from the beacon chain to the EVM. +// Introduced in EIP-4895 (Shanghai hard fork). +message Withdrawal { + // Index is the monotonically increasing identifier of the withdrawal + uint64 index = 1; + + // ValidatorIndex is the index of the validator that is withdrawing + uint64 validator_index = 2; + + // Address is the Ethereum address receiving the withdrawn funds + bytes address = 3; + + // Amount is the value of the withdrawal in gwei (1 gwei = 1e9 wei) + uint64 amount = 4; +} \ No newline at end of file diff --git a/chain/ethereum/src/adapter.rs b/chain/ethereum/src/adapter.rs index bf5cde3d3ce..d37d5ba81f7 100644 --- a/chain/ethereum/src/adapter.rs +++ b/chain/ethereum/src/adapter.rs @@ -1,20 +1,26 @@ use anyhow::Error; -use ethabi::{Error as ABIError, Function, ParamType, Token}; -use futures::Future; +use async_trait::async_trait; +use graph::abi; use graph::blockchain::ChainIdentifier; +use graph::components::ethereum::AnyBlock; +use graph::components::subgraph::MappingError; +use graph::data::store::ethereum::call; +use graph::data_source::common::ContractCall; use graph::firehose::CallToFilter; use graph::firehose::CombinedFilter; use graph::firehose::LogFilter; +use graph::prelude::alloy::primitives::keccak256; +use graph::prelude::alloy::primitives::{Address, B256}; +use graph::prelude::alloy::rpc::types::Log; +use graph::prelude::alloy::transports::{RpcError, TransportErrorKind}; use itertools::Itertools; use prost::Message; use prost_types::Any; use std::cmp; use std::collections::{HashMap, HashSet}; use std::fmt; -use std::marker::Unpin; +use std::hash::Hash; use thiserror::Error; -use tiny_keccak::keccak256; -use web3::types::{Address, Log, H256}; use graph::prelude::*; use graph::{ @@ -23,41 +29,117 @@ use graph::{ petgraph::{self, graphmap::GraphMap}, }; +use graph::blockchain::BlockPtr; + const COMBINED_FILTER_TYPE_URL: &str = "type.googleapis.com/sf.ethereum.transform.v1.CombinedFilter"; use crate::capabilities::NodeCapabilities; use crate::data_source::{BlockHandlerFilter, DataSource}; -use crate::{Chain, Mapping, ENV_VARS}; +use crate::{Chain, ENV_VARS, Mapping}; -pub type EventSignature = H256; +pub type EventSignature = B256; pub type FunctionSelector = [u8; 4]; -#[derive(Clone, Debug)] -pub struct EthereumContractCall { - pub address: Address, - pub block_ptr: BlockPtr, - pub function: Function, - pub args: Vec, +/// `EventSignatureWithTopics` is used to match events with +/// indexed arguments when they are defined in the subgraph +/// manifest. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct EventSignatureWithTopics { + pub address: Option
, + pub signature: B256, + pub topic1: Option>, + pub topic2: Option>, + pub topic3: Option>, +} + +impl EventSignatureWithTopics { + pub fn new( + address: Option
, + signature: B256, + topic1: Option>, + topic2: Option>, + topic3: Option>, + ) -> Self { + EventSignatureWithTopics { + address, + signature, + topic1, + topic2, + topic3, + } + } + + /// Checks if an event matches the `EventSignatureWithTopics` + /// If self.address is None, it's considered a wildcard match. + /// Otherwise, it must match the provided address. + /// It must also match the topics if they are Some + pub fn matches(&self, address: Option<&Address>, sig: B256, topics: &[B256]) -> bool { + // If self.address is None, it's considered a wildcard match. Otherwise, it must match the provided address. + let address_matches = match self.address { + Some(ref self_addr) => address == Some(self_addr), + None => true, // self.address is None, so it matches any address. + }; + + address_matches + && self.signature == sig + && self + .topic1 + .as_ref() + .is_none_or(|t1| topics.get(1).is_some_and(|topic| t1.contains(topic))) + && self + .topic2 + .as_ref() + .is_none_or(|t2| topics.get(2).is_some_and(|topic| t2.contains(topic))) + && self + .topic3 + .as_ref() + .is_none_or(|t3| topics.get(3).is_some_and(|topic| t3.contains(topic))) + } } #[derive(Error, Debug)] -pub enum EthereumContractCallError { - #[error("ABI error: {0}")] - ABIError(#[from] ABIError), - /// `Token` is not of expected `ParamType` - #[error("type mismatch, token {0:?} is not of kind {1:?}")] - TypeError(Token, ParamType), - #[error("error encoding input call data: {0}")] - EncodingError(ethabi::Error), +pub enum EthereumRpcError { #[error("call error: {0}")] - Web3Error(web3::Error), - #[error("call reverted: {0}")] - Revert(String), + AlloyError(RpcError), #[error("ethereum node took too long to perform call")] Timeout, } +#[derive(Error, Debug)] +pub enum ContractCallError { + #[error("ABI error: {0:#}")] + ABIError(anyhow::Error), + #[error("type mismatch, decoded value {0:?} is not of kind {1:?}")] + TypeError(abi::DynSolValue, abi::DynSolType), + #[error("error encoding input call data: {0:#}")] + EncodingError(anyhow::Error), + #[error("call error: {0}")] + AlloyError(RpcError), + #[error("ethereum node took too long to perform call")] + Timeout, + #[error("internal error: {0}")] + Internal(String), +} + +impl From for MappingError { + fn from(e: ContractCallError) -> Self { + match e { + // Any error reported by the Ethereum node could be due to the block no longer being on + // the main chain. This is very unespecific but we don't want to risk failing a + // subgraph due to a transient error such as a reorg. + ContractCallError::AlloyError(e) => MappingError::PossibleReorg(anyhow::anyhow!( + "Ethereum node returned an error for an eth_call: {e}" + )), + // Also retry on timeouts. + ContractCallError::Timeout => MappingError::PossibleReorg(anyhow::anyhow!( + "Ethereum node did not respond in time to eth_call" + )), + e => MappingError::Unknown(anyhow::anyhow!("Error when making an eth_call: {e}")), + } + } +} + #[derive(Copy, Clone, Debug, PartialEq, Eq, Ord, PartialOrd, Hash)] enum LogFilterNode { Contract(Address), @@ -68,44 +150,104 @@ enum LogFilterNode { #[derive(Clone, Debug)] pub struct EthGetLogsFilter { pub contracts: Vec
, - pub event_signatures: Vec, + pub event_signatures: Vec, + pub topic1: Option>, + pub topic2: Option>, + pub topic3: Option>, } impl EthGetLogsFilter { + /// Convert to alloy Filter for the given block range + pub fn to_alloy_filter(&self, from: BlockNumber, to: BlockNumber) -> alloy::rpc::types::Filter { + let mut filter_builder = alloy::rpc::types::Filter::new() + .from_block(alloy::rpc::types::BlockNumberOrTag::Number(from as u64)) + .to_block(alloy::rpc::types::BlockNumberOrTag::Number(to as u64)) + .address(self.contracts.clone()) + .event_signature(self.event_signatures.clone()); + + if let Some(ref topic1) = self.topic1 { + filter_builder = filter_builder.topic1(topic1.clone()); + } + if let Some(ref topic2) = self.topic2 { + filter_builder = filter_builder.topic2(topic2.clone()); + } + if let Some(ref topic3) = self.topic3 { + filter_builder = filter_builder.topic3(topic3.clone()); + } + + filter_builder + } + fn from_contract(address: Address) -> Self { EthGetLogsFilter { contracts: vec![address], event_signatures: vec![], + topic1: None, + topic2: None, + topic3: None, } } - fn from_event(event: EventSignature) -> Self { + fn from_event(event: B256) -> Self { EthGetLogsFilter { contracts: vec![], event_signatures: vec![event], + topic1: None, + topic2: None, + topic3: None, + } + } + + fn from_event_with_topics(event: EventSignatureWithTopics) -> Self { + EthGetLogsFilter { + contracts: event.address.map_or(vec![], |a| vec![a]), + event_signatures: vec![event.signature], + topic1: event.topic1, + topic2: event.topic2, + topic3: event.topic3, } } } impl fmt::Display for EthGetLogsFilter { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - if self.contracts.len() == 1 { - write!( - f, + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let base_msg = if self.contracts.len() == 1 { + format!( "contract {:?}, {} events", self.contracts[0], self.event_signatures.len() ) } else if self.event_signatures.len() == 1 { - write!( - f, + format!( "event {:?}, {} contracts", self.event_signatures[0], self.contracts.len() ) } else { - write!(f, "unreachable") - } + "unspecified filter".to_string() + }; + + // Helper to format topics as strings + let format_topics = |topics: &Option>| -> String { + topics.as_ref().map_or_else( + || "None".to_string(), + |ts| { + let signatures: Vec = ts.iter().map(|t| format!("{:?}", t)).collect(); + signatures.join(", ") + }, + ) + }; + + // Constructing topic strings + let topics_msg = format!( + ", topic1: [{}], topic2: [{}], topic3: [{}]", + format_topics(&self.topic1), + format_topics(&self.topic2), + format_topics(&self.topic3), + ); + + // Combine the base message with topic information + write!(f, "{}{}", base_msg, topics_msg) } } @@ -120,6 +262,21 @@ impl TriggerFilter { pub(crate) fn requires_traces(&self) -> bool { !self.call.is_empty() || self.block.requires_traces() } + + #[cfg(debug_assertions)] + pub fn log(&self) -> &EthereumLogFilter { + &self.log + } + + #[cfg(debug_assertions)] + pub fn call(&self) -> &EthereumCallFilter { + &self.call + } + + #[cfg(debug_assertions)] + pub fn block(&self) -> &EthereumBlockFilter { + &self.block + } } impl bc::TriggerFilter for TriggerFilter { @@ -157,10 +314,16 @@ impl bc::TriggerFilter for TriggerFilter { fn to_firehose_filter(self) -> Vec { let EthereumBlockFilter { + polling_intervals, contract_addresses: _contract_addresses, trigger_every_block, } = self.block.clone(); + // If polling_intervals is empty this will return true, else it will be true only if all intervals are 0 + // ie: All triggers are initialization handlers. We do not need firehose to send all block headers for + // initialization handlers + let has_initilization_triggers_only = polling_intervals.iter().all(|(_, i)| *i == 0); + let log_filters: Vec = self.log.into(); let mut call_filters: Vec = self.call.into(); call_filters.extend(Into::>::into(self.block)); @@ -172,7 +335,9 @@ impl bc::TriggerFilter for TriggerFilter { let combined_filter = CombinedFilter { log_filters, call_filters, - send_all_block_headers: trigger_every_block, + // We need firehose to send all block headers when `trigger_every_block` is true and when + // We have polling triggers which are not from initiallization handlers + send_all_block_headers: trigger_every_block || !has_initilization_triggers_only, }; vec![Any { @@ -183,33 +348,135 @@ impl bc::TriggerFilter for TriggerFilter { } #[derive(Clone, Debug, Default)] -pub(crate) struct EthereumLogFilter { +pub struct EthereumLogFilter { /// Log filters can be represented as a bipartite graph between contracts and events. An edge /// exists between a contract and an event if a data source for the contract has a trigger for /// the event. - /// Edges are of `bool` type and indicates when a trigger requires a transaction receipt. - contracts_and_events_graph: GraphMap, + /// Edge weights are booleans indicating whether the trigger requires a transaction receipt. + contracts_and_events_graph: MergeGraph, /// Event sigs with no associated address, matching on all addresses. - /// Maps to a boolean representing if a trigger requires a transaction receipt. - wildcard_events: HashMap, + /// Values are booleans indicating whether the trigger requires a transaction receipt. + wildcard_events: MergeMap, + /// Events with any of the topic filters set. + /// Values are booleans indicating whether the trigger requires a transaction receipt. + events_with_topic_filters: MergeMap, +} + +/// `HashMap` wrapper whose values are OR-merged on every write. +/// +/// The only mutator that writes values is [`MergeMap::or_insert`] — the inner +/// `HashMap` is private so callers cannot bypass the merge via +/// `HashMap::insert`. Used by `EthereumLogFilter` to track per-key receipt +/// requirements where any handler asking for a receipt at a given key forces +/// receipt fetching. +#[derive(Clone, Debug)] +struct MergeMap(HashMap); + +impl Default for MergeMap { + fn default() -> Self { + Self(HashMap::new()) + } +} + +impl MergeMap { + fn or_insert(&mut self, k: K, v: bool) { + self.0.entry(k).and_modify(|e| *e |= v).or_insert(v); + } + + fn get(&self, k: &K) -> Option<&bool> { + self.0.get(k) + } + + fn contains_key(&self, k: &K) -> bool { + self.0.contains_key(k) + } + + fn iter(&self) -> impl Iterator + '_ { + self.0.iter() + } + + fn is_empty(&self) -> bool { + self.0.is_empty() + } +} + +impl IntoIterator for MergeMap { + type Item = (K, bool); + type IntoIter = std::collections::hash_map::IntoIter; + fn into_iter(self) -> Self::IntoIter { + self.0.into_iter() + } +} + +/// `GraphMap` wrapper that OR-merges edge weights on +/// every write. +/// +/// The only mutator that writes edge weights is [`MergeGraph::or_add_edge`] — +/// the inner `GraphMap` is private so callers cannot bypass the merge via +/// `GraphMap::add_edge`. +#[derive(Clone, Debug)] +struct MergeGraph(GraphMap); + +impl Default for MergeGraph { + fn default() -> Self { + Self(GraphMap::new()) + } +} + +impl MergeGraph { + fn or_add_edge(&mut self, a: LogFilterNode, b: LogFilterNode, v: bool) { + // Short-circuit on `v == true`: a `true` weight always wins over any + // prior weight, so we can skip the edge_weight lookup entirely. + let merged = v || self.0.edge_weight(a, b).copied().unwrap_or(false); + self.0.add_edge(a, b, merged); + } + + fn edge_weight(&self, a: LogFilterNode, b: LogFilterNode) -> Option<&bool> { + self.0.edge_weight(a, b) + } + + fn contains_edge(&self, a: LogFilterNode, b: LogFilterNode) -> bool { + self.0.contains_edge(a, b) + } + + fn edge_count(&self) -> usize { + self.0.edge_count() + } + + fn nodes(&self) -> impl Iterator + '_ { + self.0.nodes() + } + + fn neighbors(&self, n: LogFilterNode) -> impl Iterator + '_ { + self.0.neighbors(n) + } + + fn remove_node(&mut self, n: LogFilterNode) -> bool { + self.0.remove_node(n) + } + + fn all_edges(&self) -> impl Iterator { + self.0.all_edges() + } } impl From for Vec { fn from(val: EthereumLogFilter) -> Self { - val.eth_get_logs_filters() + val.eth_get_logs_filters(ENV_VARS.get_logs_max_contracts) .map( |EthGetLogsFilter { contracts, event_signatures, + .. // TODO: Handle events with topic filters for firehose }| LogFilter { addresses: contracts .iter() - .map(|addr| addr.to_fixed_bytes().to_vec()) + .map(|addr| addr.to_vec()) .collect_vec(), event_signatures: event_signatures .iter() - .map(|sig| sig.to_fixed_bytes().to_vec()) + .map(|sig| sig.to_vec()) .collect_vec(), }, ) @@ -221,19 +488,22 @@ impl EthereumLogFilter { /// Check if this filter matches the specified `Log`. pub fn matches(&self, log: &Log) -> bool { // First topic should be event sig - match log.topics.first() { + match log.topics().first() { None => false, Some(sig) => { // The `Log` matches the filter either if the filter contains // a (contract address, event signature) pair that matches the // `Log`, or if the filter contains wildcard event that matches. - let contract = LogFilterNode::Contract(log.address); + let contract = LogFilterNode::Contract(log.address()); let event = LogFilterNode::Event(*sig); self.contracts_and_events_graph - .all_edges() - .any(|(s, t, _)| (s == contract && t == event) || (t == contract && s == event)) + .contains_edge(contract, event) || self.wildcard_events.contains_key(sig) + || self + .events_with_topic_filters + .iter() + .any(|(e, _)| e.matches(Some(&log.address()), *sig, log.topics())) } } } @@ -241,22 +511,43 @@ impl EthereumLogFilter { /// Similar to [`matches`], checks if a transaction receipt is required for this log filter. pub fn requires_transaction_receipt( &self, - event_signature: &H256, + event_signature: &B256, contract_address: Option<&Address>, + topics: &[B256], ) -> bool { - if let Some(true) = self.wildcard_events.get(event_signature) { - true - } else if let Some(address) = contract_address { - let contract = LogFilterNode::Contract(*address); - let event = LogFilterNode::Event(*event_signature); - self.contracts_and_events_graph - .all_edges() - .any(|(s, t, r)| { - *r && (s == contract && t == event) || (t == contract && s == event) - }) - } else { - false + // Check for wildcard events first. + if self.wildcard_events.get(event_signature) == Some(&true) { + return true; } + + // Next, check events with topic filters. + if self + .events_with_topic_filters + .iter() + .any(|(event_with_topics, &requires_receipt)| { + requires_receipt + && event_with_topics.matches(contract_address, *event_signature, topics) + }) + { + return true; + } + + // Finally, check the contracts_and_events_graph if a contract address is specified. + if let Some(address) = contract_address { + let contract_node = LogFilterNode::Contract(*address); + let event_node = LogFilterNode::Event(*event_signature); + + if self + .contracts_and_events_graph + .edge_weight(contract_node, event_node) + == Some(&true) + { + return true; + } + } + + // If none of the conditions above match, return false. + false } pub fn from_data_sources<'a>(iter: impl IntoIterator) -> Self { @@ -265,16 +556,42 @@ impl EthereumLogFilter { for event_handler in ds.mapping.event_handlers.iter() { let event_sig = event_handler.topic0(); match ds.address { - Some(contract) => { - this.contracts_and_events_graph.add_edge( + Some(contract) if !event_handler.has_additional_topics() => { + this.contracts_and_events_graph.or_add_edge( LogFilterNode::Contract(contract), LogFilterNode::Event(event_sig), event_handler.receipt, ); } - None => { + Some(contract) => { + this.events_with_topic_filters.or_insert( + EventSignatureWithTopics::new( + Some(contract), + event_sig, + event_handler.topic1.clone(), + event_handler.topic2.clone(), + event_handler.topic3.clone(), + ), + event_handler.receipt, + ); + } + + None if (!event_handler.has_additional_topics()) => { this.wildcard_events - .insert(event_sig, event_handler.receipt); + .or_insert(event_sig, event_handler.receipt); + } + + None => { + this.events_with_topic_filters.or_insert( + EventSignatureWithTopics::new( + ds.address, + event_sig, + event_handler.topic1.clone(), + event_handler.topic2.clone(), + event_handler.topic3.clone(), + ), + event_handler.receipt, + ); } } } @@ -287,12 +604,18 @@ impl EthereumLogFilter { for event_handler in &mapping.event_handlers { let signature = event_handler.topic0(); this.wildcard_events - .insert(signature, event_handler.receipt); + .or_insert(signature, event_handler.receipt); } this } /// Extends this log filter with another one. + /// + /// The `receipt` flag stored at each filter key is OR-combined across the + /// two filters: a key whose receipt requirement is `true` in either filter + /// stays `true` in the merged result. Overwriting with the incoming value + /// would drop a prior `true` and silently downgrade receipt fetching for + /// any handler that declared `receipt: true` at that key. pub fn extend(&mut self, other: EthereumLogFilter) { if other.is_empty() { return; @@ -302,11 +625,17 @@ impl EthereumLogFilter { let EthereumLogFilter { contracts_and_events_graph, wildcard_events, + events_with_topic_filters, } = other; for (s, t, e) in contracts_and_events_graph.all_edges() { - self.contracts_and_events_graph.add_edge(s, t, *e); + self.contracts_and_events_graph.or_add_edge(s, t, *e); + } + for (k, v) in wildcard_events { + self.wildcard_events.or_insert(k, v); + } + for (k, v) in events_with_topic_filters { + self.events_with_topic_filters.or_insert(k, v); } - self.wildcard_events.extend(wildcard_events); } /// An empty filter is one that never matches. @@ -315,20 +644,35 @@ impl EthereumLogFilter { let EthereumLogFilter { contracts_and_events_graph, wildcard_events, + events_with_topic_filters, } = self; - contracts_and_events_graph.edge_count() == 0 && wildcard_events.is_empty() + contracts_and_events_graph.edge_count() == 0 + && wildcard_events.is_empty() + && events_with_topic_filters.is_empty() } /// Filters for `eth_getLogs` calls. The filters will not return false positives. This attempts /// to balance between having granular filters but too many calls and having few calls but too /// broad filters causing the Ethereum endpoint to timeout. - pub fn eth_get_logs_filters(self) -> impl Iterator { + pub fn eth_get_logs_filters( + self, + get_logs_max_contracts: usize, + ) -> impl Iterator { + let mut filters = Vec::new(); + // Start with the wildcard event filters. - let mut filters = self - .wildcard_events - .into_keys() - .map(EthGetLogsFilter::from_event) - .collect_vec(); + filters.extend( + self.wildcard_events + .into_iter() + .map(|(k, _)| EthGetLogsFilter::from_event(k)), + ); + + // Handle events with topic filters. + filters.extend( + self.events_with_topic_filters + .into_iter() + .map(|(k, _)| EthGetLogsFilter::from_event_with_topics(k)), + ); // The current algorithm is to repeatedly find the maximum cardinality vertex and turn all // of its edges into a filter. This is nice because it is neutral between filtering by @@ -363,7 +707,7 @@ impl EthereumLogFilter { for neighbor in g.neighbors(max_vertex) { match neighbor { LogFilterNode::Contract(address) => { - if filter.contracts.len() == ENV_VARS.get_logs_max_contracts { + if filter.contracts.len() == get_logs_max_contracts { // The batch size was reached, register the filter and start a new one. let event = filter.event_signatures[0]; push_filter(filter); @@ -380,10 +724,20 @@ impl EthereumLogFilter { } filters.into_iter() } + + #[cfg(debug_assertions)] + pub fn contract_addresses(&self) -> impl Iterator + '_ { + self.contracts_and_events_graph + .nodes() + .filter_map(|node| match node { + LogFilterNode::Contract(address) => Some(address), + LogFilterNode::Event(_) => None, + }) + } } #[derive(Clone, Debug, Default)] -pub(crate) struct EthereumCallFilter { +pub struct EthereumCallFilter { // Each call filter has a map of filters keyed by address, each containing a tuple with // start_block and the set of function signatures pub contract_addresses_function_signatures: @@ -392,21 +746,21 @@ pub(crate) struct EthereumCallFilter { pub wildcard_signatures: HashSet, } -impl Into> for EthereumCallFilter { - fn into(self) -> Vec { - if self.is_empty() { +impl From for Vec { + fn from(val: EthereumCallFilter) -> Self { + if val.is_empty() { return Vec::new(); } let EthereumCallFilter { contract_addresses_function_signatures, wildcard_signatures, - } = self; + } = val; let mut filters: Vec = contract_addresses_function_signatures .into_iter() .map(|(addr, (_, sigs))| CallToFilter { - addresses: vec![addr.to_fixed_bytes().to_vec()], + addresses: vec![addr.to_vec()], signatures: sigs.into_iter().map(|x| x.to_vec()).collect_vec(), }) .collect(); @@ -547,16 +901,13 @@ impl FromIterator<(BlockNumber, Address, FunctionSelector)> for EthereumCallFilt let mut lookup: HashMap)> = HashMap::new(); iter.into_iter() .for_each(|(start_block, address, function_signature)| { - lookup + let entry = lookup .entry(address) .or_insert((start_block, HashSet::default())); - lookup.get_mut(&address).map(|set| { - if set.0 > start_block { - set.0 = start_block - } - set.1.insert(function_signature); - set - }); + if entry.0 > start_block { + entry.0 = start_block; + } + entry.1.insert(function_signature); }); EthereumCallFilter { contract_addresses_function_signatures: lookup, @@ -581,20 +932,22 @@ impl From<&EthereumBlockFilter> for EthereumCallFilter { } #[derive(Clone, Debug, Default)] -pub(crate) struct EthereumBlockFilter { +pub struct EthereumBlockFilter { + /// Used for polling block handlers, a hashset of (start_block, polling_interval) + pub polling_intervals: HashSet<(BlockNumber, i32)>, pub contract_addresses: HashSet<(BlockNumber, Address)>, pub trigger_every_block: bool, } -impl Into> for EthereumBlockFilter { - fn into(self) -> Vec { - self.contract_addresses +impl From for Vec { + fn from(val: EthereumBlockFilter) -> Self { + val.contract_addresses .into_iter() .map(|(_, addr)| addr) .sorted() .dedup_by(|x, y| x == y) .map(|addr| CallToFilter { - addresses: vec![addr.to_fixed_bytes().to_vec()], + addresses: vec![addr.to_vec()], signatures: vec![], }) .collect_vec() @@ -608,6 +961,7 @@ impl EthereumBlockFilter { /// which keeps track of deployed contracts and relevant addresses. pub fn from_mapping(mapping: &Mapping) -> Self { Self { + polling_intervals: HashSet::new(), contract_addresses: HashSet::new(), trigger_every_block: !mapping.block_handlers.is_empty(), } @@ -622,9 +976,8 @@ impl EthereumBlockFilter { .block_handlers .clone() .into_iter() - .any(|block_handler| match block_handler.filter { - Some(ref filter) if *filter == BlockHandlerFilter::Call => true, - _ => false, + .any(|block_handler| { + matches!(block_handler.filter, Some(BlockHandlerFilter::Call)) }); let has_block_handler_without_filter = data_source @@ -636,6 +989,19 @@ impl EthereumBlockFilter { filter_opt.extend(Self { trigger_every_block: has_block_handler_without_filter, + polling_intervals: data_source + .mapping + .block_handlers + .clone() + .into_iter() + .filter_map(|block_handler| match block_handler.filter { + Some(BlockHandlerFilter::Polling { every }) => { + Some((data_source.start_block, every.get() as i32)) + } + Some(BlockHandlerFilter::Once) => Some((data_source.start_block, 0)), + _ => None, + }) + .collect(), contract_addresses: if has_block_handler_with_call_filter { vec![(data_source.start_block, data_source.address.unwrap())] .into_iter() @@ -654,6 +1020,7 @@ impl EthereumBlockFilter { }; let EthereumBlockFilter { + polling_intervals, contract_addresses, trigger_every_block, } = other; @@ -678,6 +1045,11 @@ impl EthereumBlockFilter { } } } + + for (other_start_block, other_polling_interval) in &polling_intervals { + self.polling_intervals + .insert((*other_start_block, *other_polling_interval)); + } } fn requires_traces(&self) -> bool { @@ -686,12 +1058,13 @@ impl EthereumBlockFilter { /// An empty filter is one that never matches. pub fn is_empty(&self) -> bool { + let Self { + contract_addresses, + polling_intervals, + trigger_every_block, + } = self; // If we are triggering every block, we are of course not empty - if self.trigger_every_block { - return false; - } - - self.contract_addresses.is_empty() + !*trigger_every_block && contract_addresses.is_empty() && polling_intervals.is_empty() } fn find_contract_address(&self, candidate: &Address) -> Option<(i32, Address)> { @@ -731,7 +1104,7 @@ pub struct ProviderEthRpcMetrics { } impl ProviderEthRpcMetrics { - pub fn new(registry: Arc) -> Self { + pub fn new(registry: Arc) -> Self { let request_duration = registry .new_histogram_vec( "eth_rpc_request_duration", @@ -787,7 +1160,7 @@ pub struct SubgraphEthRpcMetrics { } impl SubgraphEthRpcMetrics { - pub fn new(registry: Arc, subgraph_hash: &str) -> Self { + pub fn new(registry: Arc, subgraph_hash: &str) -> Self { let request_duration = registry .global_gauge_vec( "deployment_eth_rpc_request_duration", @@ -811,13 +1184,13 @@ impl SubgraphEthRpcMetrics { pub fn observe_request(&self, duration: f64, method: &str, provider: &str) { self.request_duration - .with_label_values(&[&self.deployment, method, provider]) + .with_label_values(&[self.deployment.as_str(), method, provider]) .set(duration); } pub fn add_error(&self, method: &str, provider: &str) { self.errors - .with_label_values(&[&self.deployment, method, provider]) + .with_label_values(&[self.deployment.as_str(), method, provider]) .inc(); } } @@ -828,8 +1201,6 @@ impl SubgraphEthRpcMetrics { /// or a remote node over RPC. #[async_trait] pub trait EthereumAdapter: Send + Sync + 'static { - fn url_hostname(&self) -> &str; - /// The `provider.label` from the adapter's configuration fn provider(&self) -> &str; @@ -837,98 +1208,102 @@ pub trait EthereumAdapter: Send + Sync + 'static { /// connected to. async fn net_identifiers(&self) -> Result; - /// Get the latest block, including full transactions. - fn latest_block( - &self, - logger: &Logger, - ) -> Box + Send + Unpin>; - /// Get the latest block, with only the header and transaction hashes. - fn latest_block_header( - &self, - logger: &Logger, - ) -> Box, Error = bc::IngestorError> + Send>; - - fn load_block( - &self, - logger: &Logger, - block_hash: H256, - ) -> Box + Send>; + async fn latest_block_ptr(&self, logger: &Logger) -> Result; /// Load Ethereum blocks in bulk, returning results as they come back as a Stream. /// May use the `chain_store` as a cache. - fn load_blocks( + async fn load_blocks( &self, logger: Logger, chain_store: Arc, - block_hashes: HashSet, - ) -> Box, Error = Error> + Send>; + block_hashes: HashSet, + ) -> Result>, Error>; /// Find a block by its hash. - fn block_by_hash( + async fn block_by_hash( &self, logger: &Logger, - block_hash: H256, - ) -> Box, Error = Error> + Send>; + block_hash: B256, + ) -> Result, Error>; - fn block_by_number( + async fn block_by_number( &self, logger: &Logger, block_number: BlockNumber, - ) -> Box, Error = Error> + Send>; + ) -> Result, Error>; /// Load full information for the specified `block` (in particular, transaction receipts). - fn load_full_block( + async fn load_full_block( &self, logger: &Logger, - block: LightEthereumBlock, - ) -> Pin> + Send>>; + block: AnyBlock, + ) -> Result; - /// Load block pointer for the specified `block number`. - fn block_pointer_from_number( + /// Finds the hash and number of the lowest non-null block with height greater than or equal to + /// the given number. + /// + /// Note that the same caveats on reorgs apply as for `block_hash_by_block_number`, and must + /// also be considered for the resolved block, in case it is higher than the requested number. + async fn next_existing_ptr_to_number( &self, logger: &Logger, block_number: BlockNumber, - ) -> Box + Send>; + ) -> Result; - /// Find a block by its number, according to the Ethereum node. - /// - /// Careful: don't use this function without considering race conditions. - /// Chain reorgs could happen at any time, and could affect the answer received. - /// Generally, it is only safe to use this function with blocks that have received enough - /// confirmations to guarantee no further reorgs, **and** where the Ethereum node is aware of - /// those confirmations. - /// If the Ethereum node is far behind in processing blocks, even old blocks can be subject to - /// reorgs. - fn block_hash_by_block_number( + /// Call the function of a smart contract. A return of `None` indicates + /// that the call reverted. The returned `CallSource` indicates where + /// the result came from for accounting purposes + async fn contract_call( &self, logger: &Logger, - block_number: BlockNumber, - ) -> Box, Error = Error> + Send>; + call: &ContractCall, + cache: Arc, + ) -> Result<(Option>, call::Source), ContractCallError>; - /// Call the function of a smart contract. - fn contract_call( + /// Make multiple contract calls in a single batch. The returned `Vec` + /// has results in the same order as the calls in `calls` on input. The + /// calls must all be for the same block + async fn contract_calls( &self, logger: &Logger, - call: EthereumContractCall, + calls: &[&ContractCall], cache: Arc, - ) -> Box, Error = EthereumContractCallError> + Send>; + ) -> Result>, call::Source)>, ContractCallError>; + + async fn get_balance( + &self, + logger: &Logger, + address: Address, + block_ptr: BlockPtr, + ) -> Result; + + // Returns the compiled bytecode of a smart contract + async fn get_code( + &self, + logger: &Logger, + address: Address, + block_ptr: BlockPtr, + ) -> Result; + + /// Returns a boolean indicating whether the adapter can reach + /// the RPC provider it is configured to use. + /// This is used to determine if a provider should be considered healthy. + async fn is_reachable(&self) -> bool; } #[cfg(test)] mod tests { - use crate::adapter::{FunctionSelector, COMBINED_FILTER_TYPE_URL}; + use crate::adapter::{COMBINED_FILTER_TYPE_URL, FunctionSelector}; use super::{EthereumBlockFilter, LogFilterNode}; use super::{EthereumCallFilter, EthereumLogFilter, TriggerFilter}; + use base64::prelude::*; use graph::blockchain::TriggerFilter as _; use graph::firehose::{CallToFilter, CombinedFilter, LogFilter, MultiLogFilter}; - use graph::petgraph::graphmap::GraphMap; - use graph::prelude::ethabi::ethereum_types::H256; - use graph::prelude::web3::types::Address; - use graph::prelude::web3::types::Bytes; use graph::prelude::EthereumCall; + use graph::prelude::alloy::primitives::{Address, B256, Bytes, U256}; use hex::ToHex; use itertools::Itertools; use prost::Message; @@ -954,7 +1329,7 @@ mod tests { .map(|addr| { format!( "0x{}", - H256::from_str(addr) + B256::from_str(addr) .expect("unable to parse addr") .encode_hex::() ) @@ -965,16 +1340,11 @@ mod tests { let sigs = event_sigs .iter() - .map(|addr| { - H256::from_str(addr) - .expect("unable to parse addr") - .to_fixed_bytes() - .to_vec() - }) + .map(|addr| B256::from_str(addr).expect("unable to parse addr").to_vec()) .collect_vec(); let filter = LogFilter { - addresses: vec![address.to_fixed_bytes().to_vec()], + addresses: vec![address.to_vec()], event_signatures: sigs, }; // This base64 was provided by Streamingfast as a binding example of the expected encoded for the @@ -985,7 +1355,7 @@ mod tests { log_filters: vec![filter], }; - let output = base64::encode(filter.encode_to_vec()); + let output = BASE64_STANDARD.encode(filter.encode_to_vec()); assert_eq!(expected_base64, output); } @@ -1001,10 +1371,11 @@ mod tests { assert_eq!(sig, actual_sig); let filter = LogFilter { - addresses: vec![Address::from_str(hex_addr) - .expect("failed to parse address") - .to_fixed_bytes() - .to_vec()], + addresses: vec![ + Address::from_str(hex_addr) + .expect("failed to parse address") + .to_vec(), + ], event_signatures: vec![fs.to_vec()], }; @@ -1012,19 +1383,15 @@ mod tests { // addresses and signatures above. let expected_base64 = "ChTu0rd1bilakwDlPdBJrrB1GJm64xIEqQWcuw=="; - let output = base64::encode(filter.encode_to_vec()); + let output = BASE64_STANDARD.encode(filter.encode_to_vec()); assert_eq!(expected_base64, output); } #[test] fn ethereum_trigger_filter_to_firehose() { - let address = Address::from_low_u64_be; - let sig = H256::from_low_u64_le; + let sig = |value: u64| B256::from(U256::from(value)); let mut filter = TriggerFilter { - log: EthereumLogFilter { - contracts_and_events_graph: GraphMap::new(), - wildcard_events: HashMap::new(), - }, + log: EthereumLogFilter::default(), call: EthereumCallFilter { contract_addresses_function_signatures: HashMap::from_iter(vec![ (address(0), (0, HashSet::from_iter(vec![[0u8; 4]]))), @@ -1034,6 +1401,7 @@ mod tests { wildcard_signatures: HashSet::new(), }, block: EthereumBlockFilter { + polling_intervals: HashSet::from_iter(vec![(1, 10), (3, 24)]), contract_addresses: HashSet::from_iter([ (100, address(1000)), (200, address(2000)), @@ -1047,42 +1415,42 @@ mod tests { let expected_call_filters = vec![ CallToFilter { - addresses: vec![address(0).to_fixed_bytes().to_vec()], + addresses: vec![address(0).to_vec()], signatures: vec![[0u8; 4].to_vec()], }, CallToFilter { - addresses: vec![address(1).to_fixed_bytes().to_vec()], + addresses: vec![address(1).to_vec()], signatures: vec![[1u8; 4].to_vec()], }, CallToFilter { - addresses: vec![address(2).to_fixed_bytes().to_vec()], + addresses: vec![address(2).to_vec()], signatures: vec![], }, CallToFilter { - addresses: vec![address(1000).to_fixed_bytes().to_vec()], + addresses: vec![address(1000).to_vec()], signatures: vec![], }, CallToFilter { - addresses: vec![address(2000).to_fixed_bytes().to_vec()], + addresses: vec![address(2000).to_vec()], signatures: vec![], }, CallToFilter { - addresses: vec![address(3000).to_fixed_bytes().to_vec()], + addresses: vec![address(3000).to_vec()], signatures: vec![], }, ]; - filter.log.contracts_and_events_graph.add_edge( + filter.log.contracts_and_events_graph.or_add_edge( LogFilterNode::Contract(address(10)), LogFilterNode::Event(sig(100)), false, ); - filter.log.contracts_and_events_graph.add_edge( + filter.log.contracts_and_events_graph.or_add_edge( LogFilterNode::Contract(address(10)), LogFilterNode::Event(sig(101)), false, ); - filter.log.contracts_and_events_graph.add_edge( + filter.log.contracts_and_events_graph.or_add_edge( LogFilterNode::Contract(address(20)), LogFilterNode::Event(sig(100)), false, @@ -1090,15 +1458,12 @@ mod tests { let expected_log_filters = vec![ LogFilter { - addresses: vec![address(10).to_fixed_bytes().to_vec()], - event_signatures: vec![sig(101).to_fixed_bytes().to_vec()], + addresses: vec![address(10).to_vec()], + event_signatures: vec![sig(101).to_vec()], }, LogFilter { - addresses: vec![ - address(10).to_fixed_bytes().to_vec(), - address(20).to_fixed_bytes().to_vec(), - ], - event_signatures: vec![sig(100).to_fixed_bytes().to_vec()], + addresses: vec![address(10).to_vec(), address(20).to_vec()], + event_signatures: vec![sig(100).to_vec()], }, ]; @@ -1137,37 +1502,35 @@ mod tests { filter.event_signatures.sort(); } assert_eq!(expected_log_filters, actual_log_filters); - assert_eq!(false, actual_send_all_block_headers); + assert!(actual_send_all_block_headers); } #[test] fn ethereum_trigger_filter_to_firehose_every_block_plus_logfilter() { - let address = Address::from_low_u64_be; - let sig = H256::from_low_u64_le; + let address = |value: u64| Address::left_padding_from(&value.to_le_bytes()); + let sig = |value: u64| B256::left_padding_from(&value.to_le_bytes()); let mut filter = TriggerFilter { - log: EthereumLogFilter { - contracts_and_events_graph: GraphMap::new(), - wildcard_events: HashMap::new(), - }, + log: EthereumLogFilter::default(), call: EthereumCallFilter { contract_addresses_function_signatures: HashMap::new(), wildcard_signatures: HashSet::new(), }, block: EthereumBlockFilter { + polling_intervals: HashSet::default(), contract_addresses: HashSet::new(), trigger_every_block: true, }, }; - filter.log.contracts_and_events_graph.add_edge( + filter.log.contracts_and_events_graph.or_add_edge( LogFilterNode::Contract(address(10)), LogFilterNode::Event(sig(101)), false, ); let expected_log_filters = vec![LogFilter { - addresses: vec![address(10).to_fixed_bytes().to_vec()], - event_signatures: vec![sig(101).to_fixed_bytes().to_vec()], + addresses: vec![address(10).to_vec()], + event_signatures: vec![sig(101).to_vec()], }]; let firehose_filter = filter.clone().to_firehose_filter(); @@ -1202,7 +1565,7 @@ mod tests { } assert_eq!(expected_log_filters, actual_log_filters); - assert_eq!(true, actual_send_all_block_headers); + assert!(actual_send_all_block_headers); } #[test] @@ -1229,76 +1592,63 @@ mod tests { wildcard_signatures: HashSet::from_iter(vec![[11u8; 4]]), }; - assert_eq!( - false, - filter.matches(&call(address(2), vec![])), + assert!( + !filter.matches(&call(address(2), vec![])), "call with empty bytes are always ignore, whatever the condition" ); - assert_eq!( - false, - filter.matches(&call(address(4), vec![1; 36])), + assert!( + !filter.matches(&call(address(4), vec![1; 36])), "call with incorrect address should be ignored" ); - assert_eq!( - true, + assert!( filter.matches(&call(address(1), vec![1; 36])), "call with correct address & signature should match" ); - assert_eq!( - true, + assert!( filter.matches(&call(address(1), vec![1; 32])), "call with correct address & signature, but with incorrect input size should match" ); - assert_eq!( - false, - filter.matches(&call(address(1), vec![4u8; 36])), + assert!( + !filter.matches(&call(address(1), vec![4u8; 36])), "call with correct address but incorrect signature for a specific contract filter (i.e. matches some signatures) should be ignored" ); - assert_eq!( - false, - filter.matches(&call(address(0), vec![11u8; 36])), + assert!( + !filter.matches(&call(address(0), vec![11u8; 36])), "this signature should not match filter1, this avoid false passes if someone changes the code" ); - assert_eq!( - false, - filter2.matches(&call(address(1), vec![10u8; 36])), + assert!( + !filter2.matches(&call(address(1), vec![10u8; 36])), "this signature should not match filter2 because the address is not the expected one" ); - assert_eq!( - true, + assert!( filter2.matches(&call(address(0), vec![10u8; 36])), "this signature should match filter2 on the non wildcard clause" ); - assert_eq!( - true, + assert!( filter2.matches(&call(address(0), vec![11u8; 36])), "this signature should match filter2 on the wildcard clause" ); // extend filter1 and test the filter 2 stuff again filter.extend(filter2); - assert_eq!( - true, + assert!( filter.matches(&call(address(0), vec![11u8; 36])), "this signature should not match filter1, this avoid false passes if someone changes the code" ); - assert_eq!( - false, - filter.matches(&call(address(1), vec![10u8; 36])), + assert!( + !filter.matches(&call(address(1), vec![10u8; 36])), "this signature should not match filter2 because the address is not the expected one" ); - assert_eq!( - true, + assert!( filter.matches(&call(address(0), vec![10u8; 36])), "this signature should match filter2 on the non wildcard clause" ); - assert_eq!( - true, + assert!( filter.matches(&call(address(0), vec![11u8; 36])), "this signature should match filter2 on the wildcard clause" ); @@ -1307,11 +1657,13 @@ mod tests { #[test] fn extending_ethereum_block_filter_no_found() { let mut base = EthereumBlockFilter { + polling_intervals: HashSet::new(), contract_addresses: HashSet::new(), trigger_every_block: false, }; let extension = EthereumBlockFilter { + polling_intervals: HashSet::from_iter(vec![(1, 3)]), contract_addresses: HashSet::from_iter(vec![(10, address(1))]), trigger_every_block: false, }; @@ -1322,16 +1674,20 @@ mod tests { HashSet::from_iter(vec![(10, address(1))]), base.contract_addresses, ); + + assert_eq!(HashSet::from_iter(vec![(1, 3)]), base.polling_intervals,); } #[test] - fn extending_ethereum_block_filter_conflict_picks_lowest_block_from_ext() { + fn extending_ethereum_block_filter_conflict_includes_one_copy() { let mut base = EthereumBlockFilter { + polling_intervals: HashSet::from_iter(vec![(3, 3)]), contract_addresses: HashSet::from_iter(vec![(10, address(1))]), trigger_every_block: false, }; let extension = EthereumBlockFilter { + polling_intervals: HashSet::from_iter(vec![(2, 3), (3, 3)]), contract_addresses: HashSet::from_iter(vec![(2, address(1))]), trigger_every_block: false, }; @@ -1342,16 +1698,23 @@ mod tests { HashSet::from_iter(vec![(2, address(1))]), base.contract_addresses, ); + + assert_eq!( + HashSet::from_iter(vec![(2, 3), (3, 3)]), + base.polling_intervals, + ); } #[test] - fn extending_ethereum_block_filter_conflict_picks_lowest_block_from_base() { + fn extending_ethereum_block_filter_conflict_doesnt_include_both_copies() { let mut base = EthereumBlockFilter { + polling_intervals: HashSet::from_iter(vec![(2, 3)]), contract_addresses: HashSet::from_iter(vec![(2, address(1))]), trigger_every_block: false, }; let extension = EthereumBlockFilter { + polling_intervals: HashSet::from_iter(vec![(3, 3), (2, 3)]), contract_addresses: HashSet::from_iter(vec![(10, address(1))]), trigger_every_block: false, }; @@ -1362,116 +1725,118 @@ mod tests { HashSet::from_iter(vec![(2, address(1))]), base.contract_addresses, ); + + assert_eq!( + HashSet::from_iter(vec![(2, 3), (3, 3)]), + base.polling_intervals, + ); } #[test] fn extending_ethereum_block_filter_every_block_in_ext() { let mut base = EthereumBlockFilter { + polling_intervals: HashSet::new(), contract_addresses: HashSet::default(), trigger_every_block: false, }; let extension = EthereumBlockFilter { + polling_intervals: HashSet::new(), contract_addresses: HashSet::default(), trigger_every_block: true, }; base.extend(extension); - assert_eq!(true, base.trigger_every_block); + assert!(base.trigger_every_block); } #[test] - fn extending_ethereum_block_filter_every_block_in_base_and_merge_contract_addresses() { + fn extending_ethereum_block_filter_every_block_in_base_and_merge_contract_addresses_and_polling_intervals() + { let mut base = EthereumBlockFilter { + polling_intervals: HashSet::from_iter(vec![(10, 3)]), contract_addresses: HashSet::from_iter(vec![(10, address(2))]), trigger_every_block: true, }; let extension = EthereumBlockFilter { + polling_intervals: HashSet::new(), contract_addresses: HashSet::from_iter(vec![]), trigger_every_block: false, }; base.extend(extension); - assert_eq!(true, base.trigger_every_block); + assert!(base.trigger_every_block); assert_eq!( HashSet::from_iter(vec![(10, address(2))]), base.contract_addresses, ); + assert_eq!(HashSet::from_iter(vec![(10, 3)]), base.polling_intervals,); } #[test] fn extending_ethereum_block_filter_every_block_in_ext_and_merge_contract_addresses() { let mut base = EthereumBlockFilter { + polling_intervals: HashSet::from_iter(vec![(10, 3)]), contract_addresses: HashSet::from_iter(vec![(10, address(2))]), trigger_every_block: false, }; let extension = EthereumBlockFilter { + polling_intervals: HashSet::from_iter(vec![(10, 3)]), contract_addresses: HashSet::from_iter(vec![(10, address(1))]), trigger_every_block: true, }; base.extend(extension); - assert_eq!(true, base.trigger_every_block); + assert!(base.trigger_every_block); assert_eq!( HashSet::from_iter(vec![(10, address(2)), (10, address(1))]), base.contract_addresses, ); + assert_eq!( + HashSet::from_iter(vec![(10, 3), (10, 3)]), + base.polling_intervals, + ); } #[test] fn extending_ethereum_call_filter() { let mut base = EthereumCallFilter { contract_addresses_function_signatures: HashMap::from_iter(vec![ - ( - Address::from_low_u64_be(0), - (0, HashSet::from_iter(vec![[0u8; 4]])), - ), - ( - Address::from_low_u64_be(1), - (1, HashSet::from_iter(vec![[1u8; 4]])), - ), + (address(0), (0, HashSet::from_iter(vec![[0u8; 4]]))), + (address(1), (1, HashSet::from_iter(vec![[1u8; 4]]))), ]), wildcard_signatures: HashSet::new(), }; let extension = EthereumCallFilter { contract_addresses_function_signatures: HashMap::from_iter(vec![ - ( - Address::from_low_u64_be(0), - (2, HashSet::from_iter(vec![[2u8; 4]])), - ), - ( - Address::from_low_u64_be(3), - (3, HashSet::from_iter(vec![[3u8; 4]])), - ), + (address(0), (2, HashSet::from_iter(vec![[2u8; 4]]))), + (address(3), (3, HashSet::from_iter(vec![[3u8; 4]]))), ]), wildcard_signatures: HashSet::new(), }; base.extend(extension); assert_eq!( - base.contract_addresses_function_signatures - .get(&Address::from_low_u64_be(0)), + base.contract_addresses_function_signatures.get(&address(0)), Some(&(0, HashSet::from_iter(vec![[0u8; 4], [2u8; 4]]))) ); assert_eq!( - base.contract_addresses_function_signatures - .get(&Address::from_low_u64_be(3)), + base.contract_addresses_function_signatures.get(&address(3)), Some(&(3, HashSet::from_iter(vec![[3u8; 4]]))) ); assert_eq!( - base.contract_addresses_function_signatures - .get(&Address::from_low_u64_be(1)), + base.contract_addresses_function_signatures.get(&address(1)), Some(&(1, HashSet::from_iter(vec![[1u8; 4]]))) ); } - fn address(id: u64) -> Address { - Address::from_low_u64_be(id) + fn address(value: u64) -> Address { + Address::left_padding_from(&value.to_be_bytes()) } fn bytes(value: Vec) -> Bytes { @@ -1487,16 +1852,16 @@ fn complete_log_filter() { // Test a few combinations of complete graphs. for i in [1, 2] { - let events: BTreeSet<_> = (0..i).map(H256::from_low_u64_le).collect(); + let events: BTreeSet<_> = (0..i).map(|n| B256::from([n as u8; 32])).collect(); for j in [1, 1000, 2000, 3000] { - let contracts: BTreeSet<_> = (0..j).map(Address::from_low_u64_le).collect(); + let contracts: BTreeSet<_> = (0..j).map(|n| Address::from([n as u8; 20])).collect(); // Construct the complete bipartite graph with i events and j contracts. - let mut contracts_and_events_graph = GraphMap::new(); + let mut filter = EthereumLogFilter::default(); for &contract in &contracts { for &event in &events { - contracts_and_events_graph.add_edge( + filter.contracts_and_events_graph.or_add_edge( LogFilterNode::Contract(contract), LogFilterNode::Event(event), false, @@ -1505,12 +1870,9 @@ fn complete_log_filter() { } // Run `eth_get_logs_filters`, which is what we want to test. - let logs_filters: Vec<_> = EthereumLogFilter { - contracts_and_events_graph, - wildcard_events: HashMap::new(), - } - .eth_get_logs_filters() - .collect(); + let logs_filters: Vec<_> = filter + .eth_get_logs_filters(ENV_VARS.get_logs_max_contracts) + .collect(); // Assert that a contract or event is filtered on iff it was present in the graph. assert_eq!( @@ -1538,27 +1900,46 @@ fn complete_log_filter() { } } +#[test] +fn test_call_filter_first_signature_not_lost() { + use crate::adapter::{EthereumCallFilter, FunctionSelector}; + use alloy::primitives::Address; + + let addr = Address::left_padding_from(&1u64.to_be_bytes()); + let sig1: FunctionSelector = [0xaa, 0xbb, 0xcc, 0xdd]; + let sig2: FunctionSelector = [0x11, 0x22, 0x33, 0x44]; + + let filter: EthereumCallFilter = vec![(100i32, addr, sig1), (100i32, addr, sig2)] + .into_iter() + .collect(); + + let (_, sigs) = filter + .contract_addresses_function_signatures + .get(&addr) + .unwrap(); + assert_eq!(sigs.len(), 2); + assert!(sigs.contains(&sig1)); + assert!(sigs.contains(&sig2)); +} + #[test] fn log_filter_require_transacion_receipt_method() { + let address = |value: u64| Address::left_padding_from(&value.to_be_bytes()); + let b256 = |value: u64| B256::left_padding_from(&value.to_be_bytes()); + // test data - let event_signature_a = H256::zero(); - let event_signature_b = H256::from_low_u64_be(1); - let event_signature_c = H256::from_low_u64_be(2); - let contract_a = Address::from_low_u64_be(3); - let contract_b = Address::from_low_u64_be(4); - let contract_c = Address::from_low_u64_be(5); - - let wildcard_event_with_receipt = H256::from_low_u64_be(6); - let wildcard_event_without_receipt = H256::from_low_u64_be(7); - let wildcard_events = [ - (wildcard_event_with_receipt, true), - (wildcard_event_without_receipt, false), - ] - .into_iter() - .collect(); - - let alien_event_signature = H256::from_low_u64_be(8); // those will not be inserted in the graph - let alien_contract_address = Address::from_low_u64_be(9); + let event_signature_a = b256(0); + let event_signature_b = b256(1); + let event_signature_c = b256(2); + let contract_a = address(3); + let contract_b = address(4); + let contract_c = address(5); + + let wildcard_event_with_receipt = b256(6); + let wildcard_event_without_receipt = b256(7); + + let alien_event_signature = b256(8); // those will not be inserted in the graph + let alien_contract_address = address(9); // test graph nodes let event_a_node = LogFilterNode::Event(event_signature_a); @@ -1583,51 +1964,376 @@ fn log_filter_require_transacion_receipt_method() { // event_b -- contract_a [ receipt=false ] // } // ``` - let mut contracts_and_events_graph = GraphMap::new(); - - let event_a_id = contracts_and_events_graph.add_node(event_a_node); - let event_b_id = contracts_and_events_graph.add_node(event_b_node); - let event_c_id = contracts_and_events_graph.add_node(event_c_node); - let contract_a_id = contracts_and_events_graph.add_node(contract_a_node); - let contract_b_id = contracts_and_events_graph.add_node(contract_b_node); - let contract_c_id = contracts_and_events_graph.add_node(contract_c_node); - contracts_and_events_graph.add_edge(event_a_id, contract_a_id, true); - contracts_and_events_graph.add_edge(event_b_id, contract_b_id, true); - contracts_and_events_graph.add_edge(event_a_id, contract_b_id, false); - contracts_and_events_graph.add_edge(event_b_id, contract_a_id, false); - contracts_and_events_graph.add_edge(event_c_id, contract_c_id, true); - - let filter = EthereumLogFilter { - contracts_and_events_graph, - wildcard_events, - }; + let mut filter = EthereumLogFilter::default(); + filter + .contracts_and_events_graph + .or_add_edge(event_a_node, contract_a_node, true); + filter + .contracts_and_events_graph + .or_add_edge(event_b_node, contract_b_node, true); + filter + .contracts_and_events_graph + .or_add_edge(event_a_node, contract_b_node, false); + filter + .contracts_and_events_graph + .or_add_edge(event_b_node, contract_a_node, false); + filter + .contracts_and_events_graph + .or_add_edge(event_c_node, contract_c_node, true); + filter + .wildcard_events + .or_insert(wildcard_event_with_receipt, true); + filter + .wildcard_events + .or_insert(wildcard_event_without_receipt, false); + + let empty_vec: Vec = vec![]; // connected contracts and events graph - assert!(filter.requires_transaction_receipt(&event_signature_a, Some(&contract_a))); - assert!(filter.requires_transaction_receipt(&event_signature_b, Some(&contract_b))); - assert!(filter.requires_transaction_receipt(&event_signature_c, Some(&contract_c))); - assert!(!filter.requires_transaction_receipt(&event_signature_a, Some(&contract_b))); - assert!(!filter.requires_transaction_receipt(&event_signature_b, Some(&contract_a))); + assert!(filter.requires_transaction_receipt(&event_signature_a, Some(&contract_a), &empty_vec)); + assert!(filter.requires_transaction_receipt(&event_signature_b, Some(&contract_b), &empty_vec)); + assert!(filter.requires_transaction_receipt(&event_signature_c, Some(&contract_c), &empty_vec)); + assert!(!filter.requires_transaction_receipt( + &event_signature_a, + Some(&contract_b), + &empty_vec + )); + assert!(!filter.requires_transaction_receipt( + &event_signature_b, + Some(&contract_a), + &empty_vec + )); // Event C and Contract C are not connected to the other events and contracts - assert!(!filter.requires_transaction_receipt(&event_signature_a, Some(&contract_c))); - assert!(!filter.requires_transaction_receipt(&event_signature_b, Some(&contract_c))); - assert!(!filter.requires_transaction_receipt(&event_signature_c, Some(&contract_a))); - assert!(!filter.requires_transaction_receipt(&event_signature_c, Some(&contract_b))); + assert!(!filter.requires_transaction_receipt( + &event_signature_a, + Some(&contract_c), + &empty_vec + )); + assert!(!filter.requires_transaction_receipt( + &event_signature_b, + Some(&contract_c), + &empty_vec + )); + assert!(!filter.requires_transaction_receipt( + &event_signature_c, + Some(&contract_a), + &empty_vec + )); + assert!(!filter.requires_transaction_receipt( + &event_signature_c, + Some(&contract_b), + &empty_vec + )); // Wildcard events - assert!(filter.requires_transaction_receipt(&wildcard_event_with_receipt, None)); - assert!(!filter.requires_transaction_receipt(&wildcard_event_without_receipt, None)); + assert!(filter.requires_transaction_receipt(&wildcard_event_with_receipt, None, &empty_vec)); + assert!(!filter.requires_transaction_receipt( + &wildcard_event_without_receipt, + None, + &empty_vec + )); // Alien events and contracts always return false - assert!( - !filter.requires_transaction_receipt(&alien_event_signature, Some(&alien_contract_address)) + assert!(!filter.requires_transaction_receipt( + &alien_event_signature, + Some(&alien_contract_address), + &empty_vec + )); + assert!(!filter.requires_transaction_receipt(&alien_event_signature, None, &empty_vec),); + assert!(!filter.requires_transaction_receipt( + &alien_event_signature, + Some(&contract_a), + &empty_vec + )); + assert!(!filter.requires_transaction_receipt( + &alien_event_signature, + Some(&contract_b), + &empty_vec + )); + assert!(!filter.requires_transaction_receipt( + &alien_event_signature, + Some(&contract_c), + &empty_vec + )); + assert!(!filter.requires_transaction_receipt( + &event_signature_a, + Some(&alien_contract_address), + &empty_vec + )); + assert!(!filter.requires_transaction_receipt( + &event_signature_b, + Some(&alien_contract_address), + &empty_vec + )); + assert!(!filter.requires_transaction_receipt( + &event_signature_c, + Some(&alien_contract_address), + &empty_vec + )); +} + +// Tests that `EthereumLogFilter` OR-merges per-handler `receipt` flags across +// every insertion site (`from_data_sources`, `from_mapping`, `extend`). + +#[cfg(test)] +fn receipt_merge_test_addr(n: u64) -> Address { + Address::left_padding_from(&n.to_be_bytes()) +} + +#[cfg(test)] +fn receipt_merge_test_sig(n: u64) -> B256 { + B256::left_padding_from(&n.to_be_bytes()) +} + +#[cfg(test)] +fn receipt_merge_test_mock_abi() -> std::sync::Arc { + std::sync::Arc::new(graph::data_source::common::MappingABI { + name: "mock_abi".to_string(), + contract: abi::JsonAbi::new(), + }) +} + +#[cfg(test)] +fn receipt_merge_test_event_handler( + sig: B256, + topic1: Option>, + topic2: Option>, + topic3: Option>, + receipt: bool, +) -> crate::data_source::MappingEventHandler { + crate::data_source::MappingEventHandler { + event: "Event()".to_string(), + topic0: Some(sig), + topic1, + topic2, + topic3, + handler: "handleEvent".to_string(), + receipt, + calls: graph::data_source::common::CallDecls::default(), + } +} + +#[cfg(test)] +fn receipt_merge_test_mapping( + handlers: Vec, +) -> crate::Mapping { + crate::Mapping { + kind: "ethereum/events".to_string(), + api_version: semver::Version::new(0, 0, 7), + language: "wasm/assemblyscript".to_string(), + entities: vec![], + abis: vec![receipt_merge_test_mock_abi()], + block_handlers: vec![], + call_handlers: vec![], + event_handlers: handlers, + runtime: std::sync::Arc::new(vec![]), + link: graph::prelude::Link { + link: "test".to_string(), + }, + } +} + +#[cfg(test)] +fn receipt_merge_test_data_source( + address: Option
, + handlers: Vec, +) -> crate::data_source::DataSource { + crate::data_source::DataSource { + kind: "ethereum/contract".to_string(), + network: Some("test".to_string()), + name: "Test".to_string(), + manifest_idx: 0, + address, + start_block: 0, + end_block: None, + mapping: receipt_merge_test_mapping(handlers), + context: std::sync::Arc::new(None), + creation_block: None, + contract_abi: receipt_merge_test_mock_abi(), + } +} + +/// Run two data sources with `receipt: true` and `receipt: false` at the +/// same effective filter key through `from_data_sources` and assert the +/// merged filter still requires a transaction receipt. Runs both +/// declaration orders so order-independence is verified per variant. +#[cfg(test)] +fn assert_from_data_sources_or_merges( + label: &str, + address: Option
, + topic1: Option>, + log_topics: &[B256], +) { + let event_sig = receipt_merge_test_sig(100); + let ds_yes = receipt_merge_test_data_source( + address, + vec![receipt_merge_test_event_handler( + event_sig, + topic1.clone(), + None, + None, + true, + )], + ); + let ds_no = receipt_merge_test_data_source( + address, + vec![receipt_merge_test_event_handler( + event_sig, + topic1.clone(), + None, + None, + false, + )], + ); + + for (order, dss) in [("yes,no", [&ds_yes, &ds_no]), ("no,yes", [&ds_no, &ds_yes])] { + let filter = EthereumLogFilter::from_data_sources(dss); + assert!( + filter.requires_transaction_receipt(&event_sig, address.as_ref(), log_topics), + "{label} ({order}): receipt:true must survive a later receipt:false", + ); + } +} + +/// Build two filters via `from_data_sources`, each with one handler at the +/// same effective key but with opposite receipt flags, then merge them via +/// `extend` and assert the merged filter still requires a transaction +/// receipt. Runs both extend directions so order-independence is verified +/// per variant. +#[cfg(test)] +fn assert_extend_or_merges( + label: &str, + address: Option
, + topic1: Option>, + log_topics: &[B256], +) { + let event_sig = receipt_merge_test_sig(105); + let ds_yes = receipt_merge_test_data_source( + address, + vec![receipt_merge_test_event_handler( + event_sig, + topic1.clone(), + None, + None, + true, + )], + ); + let ds_no = receipt_merge_test_data_source( + address, + vec![receipt_merge_test_event_handler( + event_sig, + topic1.clone(), + None, + None, + false, + )], + ); + + for (order, base, ext) in [ + ("yes.extend(no)", &ds_yes, &ds_no), + ("no.extend(yes)", &ds_no, &ds_yes), + ] { + let mut filter = EthereumLogFilter::from_data_sources([base]); + filter.extend(EthereumLogFilter::from_data_sources([ext])); + assert!( + filter.requires_transaction_receipt(&event_sig, address.as_ref(), log_topics), + "{label} ({order}): extend must OR-merge", + ); + } +} + +#[test] +fn from_data_sources_or_merges_at_every_insertion_site() { + let contract = receipt_merge_test_addr(1); + let event_sig = receipt_merge_test_sig(100); + let topic = receipt_merge_test_sig(200); + let with_topic = vec![event_sig, topic]; + + // Each case maps to one arm of the `match ds.address` in `from_data_sources`. + assert_from_data_sources_or_merges("graph edge", Some(contract), None, &[]); + assert_from_data_sources_or_merges( + "addressed topics", + Some(contract), + Some(vec![topic]), + &with_topic, + ); + assert_from_data_sources_or_merges("wildcard", None, None, &[]); + assert_from_data_sources_or_merges("wildcard topics", None, Some(vec![topic]), &with_topic); +} + +#[test] +fn from_mapping_or_merges_via_extend() { + // Two templates handling the same event signature with different receipt + // flags merged via `from_mapping` + `extend`. + let event_sig = receipt_merge_test_sig(104); + + let mapping_yes = receipt_merge_test_mapping(vec![receipt_merge_test_event_handler( + event_sig, None, None, None, true, + )]); + let mapping_no = receipt_merge_test_mapping(vec![receipt_merge_test_event_handler( + event_sig, None, None, None, false, + )]); + + let mut filter = EthereumLogFilter::from_mapping(&mapping_yes); + filter.extend(EthereumLogFilter::from_mapping(&mapping_no)); + + assert!(filter.requires_transaction_receipt(&event_sig, None, &[])); +} + +#[test] +fn extend_or_merges_at_every_collection() { + let contract = receipt_merge_test_addr(5); + let event_sig = receipt_merge_test_sig(105); + let topic = receipt_merge_test_sig(203); + let with_topic = vec![event_sig, topic]; + + assert_extend_or_merges("graph edge", Some(contract), None, &[]); + assert_extend_or_merges( + "addressed topics", + Some(contract), + Some(vec![topic]), + &with_topic, ); - assert!(!filter.requires_transaction_receipt(&alien_event_signature, None)); - assert!(!filter.requires_transaction_receipt(&alien_event_signature, Some(&contract_a))); - assert!(!filter.requires_transaction_receipt(&alien_event_signature, Some(&contract_b))); - assert!(!filter.requires_transaction_receipt(&alien_event_signature, Some(&contract_c))); - assert!(!filter.requires_transaction_receipt(&event_signature_a, Some(&alien_contract_address))); - assert!(!filter.requires_transaction_receipt(&event_signature_b, Some(&alien_contract_address))); - assert!(!filter.requires_transaction_receipt(&event_signature_c, Some(&alien_contract_address))); + assert_extend_or_merges("wildcard", None, None, &[]); + assert_extend_or_merges("wildcard topics", None, Some(vec![topic]), &with_topic); +} + +#[test] +fn requires_transaction_receipt_has_or_semantics_across_handlers() { + // `requires_transaction_receipt` must return true iff at least one handler + // at the key declared `receipt: true`, regardless of declaration order. + let event_sig = receipt_merge_test_sig(108); + + for flags in [ + vec![false], + vec![true], + vec![true, false], + vec![false, true], + vec![false, false, true], + vec![true, false, false], + vec![false, true, false], + vec![true, true, false], + vec![false, false, false], + ] { + let dss: Vec<_> = flags + .iter() + .map(|&r| { + receipt_merge_test_data_source( + None, + vec![receipt_merge_test_event_handler( + event_sig, None, None, None, r, + )], + ) + }) + .collect(); + let filter = EthereumLogFilter::from_data_sources(dss.iter()); + + let any_requires_receipt = flags.iter().any(|&r| r); + assert_eq!( + filter.requires_transaction_receipt(&event_sig, None, &[]), + any_requires_receipt, + "flag sequence {flags:?}", + ); + } } diff --git a/chain/ethereum/src/buffered_call_cache.rs b/chain/ethereum/src/buffered_call_cache.rs new file mode 100644 index 00000000000..29aa84e90ab --- /dev/null +++ b/chain/ethereum/src/buffered_call_cache.rs @@ -0,0 +1,146 @@ +use std::{ + collections::HashMap, + sync::{Arc, Mutex}, +}; + +use async_trait::async_trait; +use graph::{ + cheap_clone::CheapClone, + components::store::EthereumCallCache, + data::store::ethereum::call, + prelude::{BlockPtr, CachedEthereumCall}, + slog::{Logger, error}, +}; + +/// A wrapper around an Ethereum call cache that buffers call results in +/// memory for the duration of a block. If `get_call` or `set_call` are +/// called with a different block pointer than the one used in the previous +/// call, the buffer is cleared. +pub struct BufferedCallCache { + call_cache: Arc, + buffer: Arc>>, + block: Arc>>, +} + +impl BufferedCallCache { + pub fn new(call_cache: Arc) -> Self { + Self { + call_cache, + buffer: Arc::new(Mutex::new(HashMap::new())), + block: Arc::new(Mutex::new(None)), + } + } + + fn check_block(&self, block: &BlockPtr) { + let mut self_block = self.block.lock().unwrap(); + if self_block.as_ref() != Some(block) { + *self_block = Some(block.clone()); + self.buffer.lock().unwrap().clear(); + } + } + + fn get(&self, call: &call::Request) -> Option { + let buffer = self.buffer.lock().unwrap(); + buffer.get(call).map(|retval| { + call.cheap_clone() + .response(retval.clone(), call::Source::Memory) + }) + } +} + +#[async_trait] +impl EthereumCallCache for BufferedCallCache { + async fn get_call( + &self, + call: &call::Request, + block: BlockPtr, + ) -> Result, graph::prelude::Error> { + self.check_block(&block); + + if let Some(value) = self.get(call) { + return Ok(Some(value)); + } + + let result = self.call_cache.get_call(call, block).await?; + + let mut buffer = self.buffer.lock().unwrap(); + if let Some(call::Response { + retval, + req: _, + source: _, + }) = &result + { + buffer.insert(call.cheap_clone(), retval.clone()); + } + Ok(result) + } + + async fn get_calls( + &self, + reqs: &[call::Request], + block: BlockPtr, + ) -> Result<(Vec, Vec), graph::prelude::Error> { + self.check_block(&block); + + let mut missing = Vec::new(); + let mut resps = Vec::new(); + + for call in reqs { + match self.get(call) { + Some(resp) => resps.push(resp), + None => missing.push(call.cheap_clone()), + } + } + + let (stored, calls) = self.call_cache.get_calls(&missing, block).await?; + + { + let mut buffer = self.buffer.lock().unwrap(); + for resp in &stored { + buffer.insert(resp.req.cheap_clone(), resp.retval.clone()); + } + } + + resps.extend(stored); + Ok((resps, calls)) + } + + async fn get_calls_in_block( + &self, + block: BlockPtr, + ) -> Result, graph::prelude::Error> { + self.call_cache.get_calls_in_block(block).await + } + + async fn set_call( + self: Arc, + logger: &Logger, + call: call::Request, + block: BlockPtr, + return_value: call::Retval, + ) -> Result<(), graph::prelude::Error> { + self.check_block(&block); + + // Enter the call into the in-memory cache immediately so that + // handlers will find it, but add it to the underlying cache in the + // background so we do not have to wait for that as it will be a + // cache backed by the database + { + let mut buffer = self.buffer.lock().unwrap(); + buffer.insert(call.cheap_clone(), return_value.clone()); + } + + let cache = self.call_cache.cheap_clone(); + let logger = logger.cheap_clone(); + if let Err(e) = cache + .set_call(&logger, call.cheap_clone(), block, return_value) + .await + { + error!(logger, "BufferedCallCache: call cache set error"; + "contract_address" => format!("{:?}", call.address), + "error" => e.to_string()) + } + + Ok(()) + } +} diff --git a/chain/ethereum/src/call_helper.rs b/chain/ethereum/src/call_helper.rs new file mode 100644 index 00000000000..8a1216b58c5 --- /dev/null +++ b/chain/ethereum/src/call_helper.rs @@ -0,0 +1,156 @@ +use crate::{ContractCallError, ENV_VARS}; +use graph::{ + abi, + data::store::ethereum::call, + prelude::{ + Logger, + alloy::transports::{RpcError, TransportErrorKind}, + serde_json, + }, + slog::info, +}; + +// ------------------------------------------------------------------ +// Constants and helper utilities used across eth_call handling +// ------------------------------------------------------------------ + +// Try to check if the call was reverted. The JSON-RPC response for reverts is +// not standardized, so we have ad-hoc checks for each Ethereum client. + +// 0xfe is the "designated bad instruction" of the EVM, and Solidity uses it for +// asserts. +const PARITY_BAD_INSTRUCTION_FE: &str = "Bad instruction fe"; + +// 0xfd is REVERT, but on some contracts, and only on older blocks, +// this happens. Makes sense to consider it a revert as well. +const PARITY_BAD_INSTRUCTION_FD: &str = "Bad instruction fd"; + +const PARITY_BAD_JUMP_PREFIX: &str = "Bad jump"; +const PARITY_STACK_LIMIT_PREFIX: &str = "Out of stack"; + +// See f0af4ab0-6b7c-4b68-9141-5b79346a5f61. +const PARITY_OUT_OF_GAS: &str = "Out of gas"; + +// Also covers Nethermind reverts +const PARITY_VM_EXECUTION_ERROR: i64 = -32015; +const PARITY_REVERT_PREFIX: &str = "revert"; + +const XDAI_REVERT: &str = "revert"; + +// Deterministic RPC execution errors. We might need to expand this as +// subgraphs come across other errors. See +// https://github.com/ethereum/go-ethereum/blob/cd57d5cd38ef692de8fbedaa56598b4e9fbfbabc/core/vm/errors.go +const RPC_EXECUTION_ERRORS: &[&str] = &[ + // The "revert" substring covers a few known error messages, including: + // Hardhat: "error: transaction reverted", + // Ganache and Moonbeam: "vm exception while processing transaction: revert", + // Geth: "execution reverted" + // And others. + "revert", + "invalid jump destination", + "invalid opcode", + // Ethereum says 1024 is the stack sizes limit, so this is deterministic. + "stack limit reached 1024", + // See f0af4ab0-6b7c-4b68-9141-5b79346a5f61 for why the gas limit is considered deterministic. + "out of gas", + "stack underflow", + "vm execution error", + "invalidjump", + "notactivated", + "invalidfeopcode", + // Reth surfaces EVM halts via `EvmHalt(HaltReason)`, formatted with the + // reason's `Debug` repr (`"EVM error: {0:?}"`). revm's `HaltReason` + // variants are CamelCase with no spaces, so e.g. a stack underflow arrives + // as "EVM error: StackUnderflow", which the space-separated "stack + // underflow" above does not match. "invalidjump"/"invalidfeopcode" already + // cover the matching variants; these add the rest. Reth's OutOfGas is + // handled before EvmHalt and rendered as "out of gas: ...", so it is + // already covered above. See https://github.com/streamingfast/eth-go/pull/10. + "stackunderflow", + "stackoverflow", + "opcodenotfound", +]; + +/// Helper that checks if a RPC error message corresponds to a revert. +fn is_rpc_revert_message(message: &str) -> bool { + let env_rpc_call_errors = ENV_VARS.rpc_eth_call_errors.iter(); + let mut execution_errors = RPC_EXECUTION_ERRORS + .iter() + .copied() + .chain(env_rpc_call_errors.map(|s| s.as_str())); + execution_errors.any(|e| message.to_lowercase().contains(e)) +} + +/// Decode a Solidity revert(reason) payload, returning the reason string when possible. +fn as_solidity_revert_reason(bytes: &[u8]) -> Option { + let selector = &graph::prelude::alloy::primitives::keccak256(b"Error(string)")[..4]; + if bytes.len() >= 4 && &bytes[..4] == selector { + abi::DynSolType::String + .abi_decode(&bytes[4..]) + .ok() + .and_then(|val| val.clone().as_str().map(ToOwned::to_owned)) + } else { + None + } +} + +/// Interpret the error returned by `eth_call`, distinguishing genuine failures from +/// EVM reverts. Returns `Ok(Null)` for reverts or a proper error otherwise. +pub fn interpret_eth_call_error( + logger: &Logger, + err: RpcError, +) -> Result { + fn reverted(logger: &Logger, reason: &str) -> Result { + info!(logger, "Contract call reverted"; "reason" => reason); + Ok(call::Retval::Null) + } + + if let RpcError::ErrorResp(rpc_error) = &err + && is_rpc_revert_message(&rpc_error.message) + { + return reverted(logger, &rpc_error.message); + } + + if let RpcError::ErrorResp(rpc_error) = &err { + let code = rpc_error.code; + let data: Option = rpc_error + .data + .as_ref() + .and_then(|d| serde_json::from_str(d.get()).ok()); + + if code == PARITY_VM_EXECUTION_ERROR + && let Some(data) = data + && is_parity_revert(&data) + { + return reverted(logger, &parity_revert_reason(&data)); + } + } + + Err(ContractCallError::AlloyError(err)) +} + +fn is_parity_revert(data: &str) -> bool { + data.to_lowercase().starts_with(PARITY_REVERT_PREFIX) + || data.starts_with(PARITY_BAD_JUMP_PREFIX) + || data.starts_with(PARITY_STACK_LIMIT_PREFIX) + || data == PARITY_BAD_INSTRUCTION_FE + || data == PARITY_BAD_INSTRUCTION_FD + || data == PARITY_OUT_OF_GAS + || data == XDAI_REVERT +} + +/// Checks if the given data corresponds to a Parity / Nethermind style EVM +/// revert and, if so, tries to extract a human-readable revert reason. Returns `Some` +/// with the reason when the error is identified as a revert, otherwise `None`. +fn parity_revert_reason(data: &str) -> String { + if data == PARITY_BAD_INSTRUCTION_FE { + return PARITY_BAD_INSTRUCTION_FE.to_owned(); + } + + // Otherwise try to decode a Solidity revert reason payload. + let payload = data.trim_start_matches(PARITY_REVERT_PREFIX); + hex::decode(payload) + .ok() + .and_then(|decoded| as_solidity_revert_reason(&decoded)) + .unwrap_or_else(|| "no reason".to_owned()) +} diff --git a/chain/ethereum/src/capabilities.rs b/chain/ethereum/src/capabilities.rs index d1296c4f45c..a036730ad0d 100644 --- a/chain/ethereum/src/capabilities.rs +++ b/chain/ethereum/src/capabilities.rs @@ -1,9 +1,6 @@ -use anyhow::Error; use graph::impl_slog_value; use std::cmp::Ordering; -use std::collections::BTreeSet; use std::fmt; -use std::str::FromStr; use crate::DataSource; @@ -17,7 +14,7 @@ pub struct NodeCapabilities { /// other. No [`Ord`] (i.e. total order) implementation is applicable. impl PartialOrd for NodeCapabilities { fn partial_cmp(&self, other: &Self) -> Option { - product_order([ + product_order(&[ self.archive.cmp(&other.archive), self.traces.cmp(&other.traces), ]) @@ -26,7 +23,7 @@ impl PartialOrd for NodeCapabilities { /// Defines a [product order](https://en.wikipedia.org/wiki/Product_order) over /// an array of [`Ordering`]. -fn product_order(cmps: [Ordering; N]) -> Option { +fn product_order(cmps: &[Ordering]) -> Option { if cmps.iter().all(|c| c.is_eq()) { Some(Ordering::Equal) } else if cmps.iter().all(|c| c.is_le()) { @@ -38,18 +35,6 @@ fn product_order(cmps: [Ordering; N]) -> Option { } } -impl FromStr for NodeCapabilities { - type Err = Error; - - fn from_str(s: &str) -> Result { - let capabilities: BTreeSet<&str> = s.split(',').collect(); - Ok(NodeCapabilities { - archive: capabilities.contains("archive"), - traces: capabilities.contains("traces"), - }) - } -} - impl fmt::Display for NodeCapabilities { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { let NodeCapabilities { archive, traces } = self; diff --git a/chain/ethereum/src/chain.rs b/chain/ethereum/src/chain.rs index 9eeda321ed5..405c3123bdc 100644 --- a/chain/ethereum/src/chain.rs +++ b/chain/ethereum/src/chain.rs @@ -1,43 +1,58 @@ -use anyhow::{anyhow, Result}; use anyhow::{Context, Error}; +use anyhow::{Result, anyhow, bail}; +use async_trait::async_trait; use graph::blockchain::client::ChainClient; -use graph::blockchain::{BlockchainKind, TriggersAdapterSelector}; +use graph::blockchain::firehose_block_ingestor::{FirehoseBlockIngestor, Transforms}; +use graph::blockchain::{ + BlockIngestor, BlockTime, BlockchainKind, ChainIdentifier, ExtendedBlockPtr, + TriggerFilterWrapper, TriggersAdapterSelector, +}; +use graph::components::network_provider::ChainName; +use graph::components::store::{DeploymentCursorTracker, SourceableStore}; use graph::data::subgraph::UnifiedMappingApiVersion; -use graph::firehose::{FirehoseEndpoint, ForkStep}; +use graph::firehose::{FirehoseEndpoint, FirehoseEndpoints, ForkStep}; +use graph::futures03::TryStreamExt; use graph::prelude::{ - BlockHash, EthereumBlock, EthereumCallCache, LightEthereumBlock, LightEthereumBlockExt, + BlockHash, ComponentLoggerConfig, ElasticComponentLoggerConfig, EthereumBlock, + EthereumCallCache, LightEthereumBlock, LightEthereumBlockExt, MetricsRegistry, StoreError, + retry, }; +use graph::slog::{debug, error, trace, warn}; use graph::{ blockchain::{ + Block, BlockPtr, Blockchain, ChainHeadUpdateListener, IngestorError, + RuntimeAdapter as RuntimeAdapterTrait, TriggerFilter as _, block_stream::{ BlockRefetcher, BlockStreamEvent, BlockWithTriggers, FirehoseError, FirehoseMapper as FirehoseMapperTrait, TriggersAdapter as TriggersAdapterTrait, }, firehose_block_stream::FirehoseBlockStream, - polling_block_stream::PollingBlockStream, - Block, BlockPtr, Blockchain, ChainHeadUpdateListener, IngestorError, - RuntimeAdapter as RuntimeAdapterTrait, TriggerFilter as _, }, cheap_clone::CheapClone, components::store::DeploymentLocator, firehose, prelude::{ - async_trait, o, serde_json as json, BlockNumber, ChainStore, EthereumBlockWithCalls, - Future01CompatExt, Logger, LoggerFactory, MetricsRegistry, NodeId, + BlockNumber, ChainStore, EthereumBlockWithCalls, Logger, LoggerFactory, o, + serde_json as json, }, }; use prost::Message; -use std::collections::HashSet; +use std::collections::{BTreeSet, HashSet}; +use std::future::Future; use std::iter::FromIterator; use std::sync::Arc; +use std::time::Duration; use crate::codec::HeaderOnlyBlock; use crate::data_source::DataSourceTemplate; use crate::data_source::UnresolvedDataSourceTemplate; +use crate::ingestor::PollingBlockIngestor; use crate::network::EthereumNetworkAdapters; -use crate::EthereumAdapter; -use crate::NodeCapabilities; +use crate::polling_block_stream::PollingBlockStream; +use crate::runtime::runtime_adapter::eth_call_gas; +use crate::{BufferedCallCache, NodeCapabilities}; use crate::{ + ENV_VARS, SubgraphEthRpcMetrics, TriggerFilter, adapter::EthereumAdapter as _, codec, data_source::{DataSource, UnresolvedDataSource}, @@ -45,13 +60,53 @@ use crate::{ blocks_with_triggers, get_calls, parse_block_triggers, parse_call_triggers, parse_log_triggers, }, - SubgraphEthRpcMetrics, TriggerFilter, ENV_VARS, }; -use graph::blockchain::block_stream::{BlockStream, BlockStreamBuilder, FirehoseCursor}; - +use crate::{EthereumAdapter, RuntimeAdapter}; +use graph::blockchain::block_stream::{ + BlockStream, BlockStreamBuilder, BlockStreamError, BlockStreamMapper, FirehoseCursor, + TriggersAdapterWrapper, +}; /// Celo Mainnet: 42220, Testnet Alfajores: 44787, Testnet Baklava: 62320 const CELO_CHAIN_IDS: [u64; 3] = [42220, 44787, 62320]; +/// Resolved per-chain settings. Populated at chain initialisation from the config file (with +/// ENV_VAR fallbacks) and stored on [`Chain`] and [`crate::EthereumAdapter`]. +#[derive(Clone, Debug)] +pub struct ChainSettings { + pub polling_interval: Duration, + pub json_rpc_timeout: Duration, + pub request_retries: usize, + pub max_block_range_size: BlockNumber, + pub block_batch_size: usize, + pub block_ptr_batch_size: usize, + pub max_event_only_range: BlockNumber, + pub target_triggers_per_block_range: u64, + pub get_logs_max_contracts: usize, + pub block_ingestor_max_concurrent_json_rpc_calls: usize, + pub genesis_block_number: u64, +} + +impl ChainSettings { + /// Constructs a [`ChainSettings`] from environment variable defaults. + /// Used in tests and for firehose-only chains that have no RPC config. + pub fn from_env_defaults() -> Self { + ChainSettings { + polling_interval: graph::env::ENV_VARS.ingestor_polling_interval, + json_rpc_timeout: ENV_VARS.json_rpc_timeout, + request_retries: ENV_VARS.request_retries, + max_block_range_size: ENV_VARS.max_block_range_size, + block_batch_size: ENV_VARS.block_batch_size, + block_ptr_batch_size: ENV_VARS.block_ptr_batch_size, + max_event_only_range: ENV_VARS.max_event_only_range, + target_triggers_per_block_range: ENV_VARS.target_triggers_per_block_range, + get_logs_max_contracts: ENV_VARS.get_logs_max_contracts, + block_ingestor_max_concurrent_json_rpc_calls: ENV_VARS + .block_ingestor_max_concurrent_json_rpc_calls, + genesis_block_number: ENV_VARS.genesis_block_number, + } + } +} + pub struct EthereumStreamBuilder {} #[async_trait] @@ -76,39 +131,124 @@ impl BlockStreamBuilder for EthereumStreamBuilder { ) }); - let firehose_endpoint = chain.chain_client().firehose_endpoint()?; - let logger = chain .logger_factory .subgraph_logger(&deployment) .new(o!("component" => "FirehoseBlockStream")); - let firehose_mapper = Arc::new(FirehoseMapper {}); + let firehose_mapper = Arc::new(FirehoseMapper { adapter, filter }); Ok(Box::new(FirehoseBlockStream::new( deployment.hash, - firehose_endpoint, + chain.chain_client(), subgraph_current_block, block_cursor, firehose_mapper, - adapter, - filter, start_blocks, logger, chain.registry.clone(), ))) } + async fn build_subgraph_block_stream( + &self, + chain: &Chain, + deployment: DeploymentLocator, + start_blocks: Vec, + source_subgraph_stores: Vec>, + subgraph_current_block: Option, + filter: Arc>, + unified_api_version: UnifiedMappingApiVersion, + ) -> Result>> { + self.build_polling( + chain, + deployment, + start_blocks, + source_subgraph_stores, + subgraph_current_block, + filter, + unified_api_version, + ) + .await + } + async fn build_polling( &self, - _chain: Arc, - _deployment: DeploymentLocator, - _start_blocks: Vec, - _subgraph_current_block: Option, - _filter: Arc<::TriggerFilter>, - _unified_api_version: UnifiedMappingApiVersion, + chain: &Chain, + deployment: DeploymentLocator, + start_blocks: Vec, + source_subgraph_stores: Vec>, + subgraph_current_block: Option, + filter: Arc>, + unified_api_version: UnifiedMappingApiVersion, ) -> Result>> { - todo!() + let requirements = filter.chain_filter.node_capabilities(); + let is_using_subgraph_composition = !source_subgraph_stores.is_empty(); + let adapter = TriggersAdapterWrapper::new( + chain + .triggers_adapter(&deployment, &requirements, unified_api_version.clone()) + .unwrap_or_else(|_| { + panic!( + "no adapter for network {} with capabilities {}", + chain.name, requirements + ) + }), + source_subgraph_stores, + ); + + let logger = chain + .logger_factory + .subgraph_logger(&deployment) + .new(o!("component" => "BlockStream")); + let chain_head_update_stream = chain + .chain_head_update_listener + .subscribe(chain.name.to_string(), logger.clone()); + + // Special case: Detect Celo and set the threshold to 0, so that eth_getLogs is always used. + // This is ok because Celo blocks are always final. And we _need_ to do this because + // some events appear only in eth_getLogs but not in transaction receipts. + // See also ca0edc58-0ec5-4c89-a7dd-2241797f5e50. + let reorg_threshold = match chain.chain_client().as_ref() { + ChainClient::Rpc(adapter) => { + let chain_id = adapter + .cheapest() + .await + .ok_or(anyhow!("unable to get eth adapter for chan_id call"))? + .chain_id() + .await?; + + if CELO_CHAIN_IDS.contains(&chain_id) { + 0 + } else { + chain.reorg_threshold + } + } + _ if is_using_subgraph_composition => chain.reorg_threshold, + _ => panic!( + "expected rpc when using polling blockstream : {}", + is_using_subgraph_composition + ), + }; + + let max_block_range_size = if is_using_subgraph_composition { + chain.settings.max_block_range_size * 10 + } else { + chain.settings.max_block_range_size + }; + + Ok(Box::new(PollingBlockStream::new( + chain_head_update_stream, + Arc::new(adapter), + deployment.hash, + filter, + start_blocks, + reorg_threshold, + logger, + max_block_range_size, + chain.settings.target_triggers_per_block_range, + unified_api_version, + subgraph_current_block, + ))) } } @@ -126,7 +266,7 @@ impl BlockRefetcher for EthereumBlockRefetcher { logger: &Logger, cursor: FirehoseCursor, ) -> Result { - let endpoint = chain.chain_client().firehose_endpoint()?; + let endpoint: Arc = chain.chain_client().firehose_endpoint().await?; let block = endpoint.get_block::(cursor, logger).await?; let ethereum_block: EthereumBlockWithCalls = (&block).try_into()?; Ok(BlockFinality::NonFinal(ethereum_block)) @@ -136,22 +276,25 @@ impl BlockRefetcher for EthereumBlockRefetcher { pub struct EthereumAdapterSelector { logger_factory: LoggerFactory, client: Arc>, - registry: Arc, + registry: Arc, chain_store: Arc, + eth_adapters: Arc, } impl EthereumAdapterSelector { pub fn new( logger_factory: LoggerFactory, client: Arc>, - registry: Arc, + registry: Arc, chain_store: Arc, + eth_adapters: Arc, ) -> Self { Self { logger_factory, client, registry, chain_store, + eth_adapters, } } } @@ -177,16 +320,45 @@ impl TriggersAdapterSelector for EthereumAdapterSelector { chain_store: self.chain_store.cheap_clone(), unified_api_version, capabilities: *capabilities, + eth_adapters: self.eth_adapters.cheap_clone(), }; Ok(Arc::new(adapter)) } } +/// We need this so that the runner tests can use a `NoopRuntimeAdapter` +/// instead of the `RuntimeAdapter` from this crate to avoid needing +/// ethereum adapters +pub trait RuntimeAdapterBuilder: Send + Sync + 'static { + fn build( + &self, + eth_adapters: Arc, + call_cache: Arc, + chain_identifier: Arc, + ) -> Arc>; +} + +pub struct EthereumRuntimeAdapterBuilder {} + +impl RuntimeAdapterBuilder for EthereumRuntimeAdapterBuilder { + fn build( + &self, + eth_adapters: Arc, + call_cache: Arc, + chain_identifier: Arc, + ) -> Arc> { + Arc::new(RuntimeAdapter { + eth_adapters, + call_cache, + chain_identifier, + }) + } +} + pub struct Chain { logger_factory: LoggerFactory, - name: String, - node_id: NodeId, - registry: Arc, + pub name: ChainName, + registry: Arc, client: Arc>, chain_store: Arc, call_cache: Arc, @@ -196,7 +368,9 @@ pub struct Chain { block_stream_builder: Arc>, block_refetcher: Arc>, adapter_selector: Arc>, - runtime_adapter: Arc>, + runtime_adapter_builder: Arc, + eth_adapters: Arc, + pub settings: Arc, } impl std::fmt::Debug for Chain { @@ -205,13 +379,44 @@ impl std::fmt::Debug for Chain { } } +/// Walk back from a block pointer by following parent pointers. +/// This is the core logic used as a fallback when the cache doesn't have ancestor block. +/// +async fn walk_back_ancestor( + start_ptr: BlockPtr, + offset: BlockNumber, + root: Option, + mut parent_getter: F, +) -> Result, E> +where + F: FnMut(BlockPtr) -> Fut, + Fut: std::future::Future, E>>, +{ + let mut current_ptr = start_ptr; + + for _ in 0..offset { + match parent_getter(current_ptr.clone()).await? { + Some(parent) => { + if let Some(root_hash) = &root + && parent.hash == *root_hash + { + break; + } + current_ptr = parent; + } + None => return Ok(None), + } + } + + Ok(Some(current_ptr)) +} + impl Chain { /// Creates a new Ethereum [`Chain`]. pub fn new( logger_factory: LoggerFactory, - name: String, - node_id: NodeId, - registry: Arc, + name: ChainName, + registry: Arc, chain_store: Arc, call_cache: Arc, client: Arc>, @@ -219,14 +424,15 @@ impl Chain { block_stream_builder: Arc>, block_refetcher: Arc>, adapter_selector: Arc>, - runtime_adapter: Arc>, + runtime_adapter_builder: Arc, + eth_adapters: Arc, reorg_threshold: BlockNumber, is_ingestible: bool, + settings: Arc, ) -> Self { Chain { logger_factory, name, - node_id, registry, client, chain_store, @@ -235,9 +441,11 @@ impl Chain { block_stream_builder, block_refetcher, adapter_selector, - runtime_adapter, + runtime_adapter_builder, + eth_adapters, reorg_threshold, is_ingestible, + settings, } } @@ -246,15 +454,22 @@ impl Chain { self.call_cache.clone() } + pub async fn block_number( + &self, + hash: &BlockHash, + ) -> Result, Option)>, StoreError> { + self.chain_store.block_number(hash).await + } + // TODO: This is only used to build the block stream which could prolly // be moved to the chain itself and return a block stream future that the // caller can spawn. - pub fn cheapest_adapter(&self) -> Arc { + pub async fn cheapest_adapter(&self) -> Arc { let adapters = match self.client.as_ref() { ChainClient::Firehose(_) => panic!("no adapter with firehose"), ChainClient::Rpc(adapter) => adapter, }; - adapters.cheapest().unwrap() + adapters.cheapest().await.unwrap() } } @@ -282,6 +497,8 @@ impl Blockchain for Chain { type NodeCapabilities = crate::capabilities::NodeCapabilities; + type DecoderHook = crate::data_source::DecoderHook; + fn triggers_adapter( &self, loc: &DeploymentLocator, @@ -292,93 +509,64 @@ impl Blockchain for Chain { .triggers_adapter(loc, capabilities, unified_api_version) } - async fn new_firehose_block_stream( - &self, - deployment: DeploymentLocator, - block_cursor: FirehoseCursor, - start_blocks: Vec, - subgraph_current_block: Option, - filter: Arc, - unified_api_version: UnifiedMappingApiVersion, - ) -> Result>, Error> { - self.block_stream_builder - .build_firehose( - self, - deployment, - block_cursor, - start_blocks, - subgraph_current_block, - filter, - unified_api_version, - ) - .await - } - - async fn new_polling_block_stream( + async fn new_block_stream( &self, deployment: DeploymentLocator, + store: impl DeploymentCursorTracker, start_blocks: Vec, - subgraph_current_block: Option, - filter: Arc, + source_subgraph_stores: Vec>, + filter: Arc>, unified_api_version: UnifiedMappingApiVersion, ) -> Result>, Error> { - let requirements = filter.node_capabilities(); - let adapter = self - .triggers_adapter(&deployment, &requirements, unified_api_version.clone()) - .unwrap_or_else(|_| { - panic!( - "no adapter for network {} with capabilities {}", - self.name, requirements + let current_ptr = store.block_ptr(); + + if !filter.subgraph_filter.is_empty() { + return self + .block_stream_builder + .build_subgraph_block_stream( + self, + deployment, + start_blocks, + source_subgraph_stores, + current_ptr, + filter, + unified_api_version, ) - }); - - let logger = self - .logger_factory - .subgraph_logger(&deployment) - .new(o!("component" => "BlockStream")); - let chain_store = self.chain_store().clone(); - let chain_head_update_stream = self - .chain_head_update_listener - .subscribe(self.name.clone(), logger.clone()); + .await; + } - // Special case: Detect Celo and set the threshold to 0, so that eth_getLogs is always used. - // This is ok because Celo blocks are always final. And we _need_ to do this because - // some events appear only in eth_getLogs but not in transaction receipts. - // See also ca0edc58-0ec5-4c89-a7dd-2241797f5e50. - let chain_id = match self.client.as_ref() { - ChainClient::Rpc(adapter) => { - adapter - .cheapest() - .ok_or(anyhow!("unable to get eth adapter for chan_id call"))? - .chain_id() - .await? + match self.chain_client().as_ref() { + ChainClient::Rpc(_) => { + self.block_stream_builder + .build_polling( + self, + deployment, + start_blocks, + source_subgraph_stores, + current_ptr, + filter, + unified_api_version, + ) + .await } - _ => panic!("expected rpc when using polling blockstream"), - }; - let reorg_threshold = match CELO_CHAIN_IDS.contains(&chain_id) { - false => self.reorg_threshold, - true => 0, - }; - - Ok(Box::new(PollingBlockStream::new( - chain_store, - chain_head_update_stream, - adapter, - self.node_id.clone(), - deployment.hash, - filter, - start_blocks, - reorg_threshold, - logger, - ENV_VARS.max_block_range_size, - ENV_VARS.target_triggers_per_block_range, - unified_api_version, - subgraph_current_block, - ))) + ChainClient::Firehose(_) => { + self.block_stream_builder + .build_firehose( + self, + deployment, + store.firehose_cursor(), + start_blocks, + current_ptr, + filter.chain_filter.clone(), + unified_api_version, + ) + .await + } + } } - fn chain_store(&self) -> Arc { - self.chain_store.clone() + async fn chain_head_ptr(&self) -> Result, Error> { + self.chain_store.cheap_clone().chain_head_ptr().await } async fn block_pointer_from_number( @@ -388,20 +576,34 @@ impl Blockchain for Chain { ) -> Result { match self.client.as_ref() { ChainClient::Firehose(endpoints) => endpoints - .random()? + .endpoint() + .await? .block_ptr_for_number::(logger, number) .await .map_err(IngestorError::Unknown), ChainClient::Rpc(adapters) => { + let cached = self + .chain_store + .cheap_clone() + .block_ptrs_by_numbers(vec![number]) + .await + .unwrap_or_default(); + if let Some(ptrs) = cached.get(&number) + && ptrs.len() == 1 + { + return Ok(BlockPtr::new(ptrs[0].hash.clone(), ptrs[0].number)); + } + let adapter = adapters .cheapest() + .await .with_context(|| format!("no adapter for chain {}", self.name))? .clone(); adapter - .block_pointer_from_number(logger, number) - .compat() + .next_existing_ptr_to_number(logger, number) .await + .map_err(From::from) } } } @@ -418,13 +620,82 @@ impl Blockchain for Chain { self.block_refetcher.get_block(self, logger, cursor).await } - fn runtime_adapter(&self) -> Arc> { - self.runtime_adapter.clone() + async fn runtime( + &self, + ) -> anyhow::Result<(Arc>, Self::DecoderHook)> { + let call_cache = Arc::new(BufferedCallCache::new(self.call_cache.cheap_clone())); + let chain_ident = self.chain_store.chain_identifier().await?; + + let builder = self.runtime_adapter_builder.build( + self.eth_adapters.cheap_clone(), + call_cache.cheap_clone(), + Arc::new(chain_ident.clone()), + ); + let eth_call_gas = eth_call_gas(&chain_ident); + + let decoder_hook = crate::data_source::DecoderHook::new( + self.eth_adapters.cheap_clone(), + call_cache, + eth_call_gas, + ); + + Ok((builder, decoder_hook)) } fn chain_client(&self) -> Arc> { self.client.clone() } + + async fn block_ingestor(&self) -> anyhow::Result> { + let ingestor: Box = match self.chain_client().as_ref() { + ChainClient::Firehose(_) => { + let ingestor = FirehoseBlockIngestor::::new( + self.chain_store.cheap_clone().as_head_store(), + self.chain_client(), + self.logger_factory + .component_logger("EthereumFirehoseBlockIngestor", None), + self.name.clone(), + ); + let ingestor = ingestor.with_transforms(vec![Transforms::EthereumHeaderOnly]); + + Box::new(ingestor) + } + ChainClient::Rpc(_) => { + let logger = self + .logger_factory + .component_logger( + "EthereumPollingBlockIngestor", + Some(ComponentLoggerConfig { + elastic: Some(ElasticComponentLoggerConfig { + index: String::from("block-ingestor-logs"), + }), + }), + ) + .new(o!()); + + if !self.is_ingestible { + bail!( + "Not starting block ingestor (chain is defective), network_name {}", + &self.name + ); + } + + // The block ingestor must be configured to keep at least REORG_THRESHOLD ancestors, + // because the json-rpc BlockStream expects blocks after the reorg threshold to be + // present in the DB. + Box::new(PollingBlockIngestor::new( + logger, + graph::env::ENV_VARS.reorg_threshold(), + self.chain_client(), + self.chain_store.cheap_clone(), + self.settings.polling_interval, + self.name.clone(), + )?) + } + }; + + Ok(ingestor) + } } /// This is used in `EthereumAdapter::triggers_in_block`, called when re-processing a block for @@ -437,6 +708,8 @@ pub enum BlockFinality { // If a block may still be reorged, we need to work with more local data. NonFinal(EthereumBlockWithCalls), + + Ptr(Arc), } impl Default for BlockFinality { @@ -450,15 +723,7 @@ impl BlockFinality { match self { BlockFinality::Final(block) => block, BlockFinality::NonFinal(block) => &block.ethereum_block.block, - } - } -} - -impl<'a> From<&'a BlockFinality> for BlockPtr { - fn from(block: &'a BlockFinality) -> BlockPtr { - match block { - BlockFinality::Final(b) => BlockPtr::from(&**b), - BlockFinality::NonFinal(b) => BlockPtr::from(&b.ethereum_block), + BlockFinality::Ptr(_) => unreachable!("light_block called on HeaderOnly"), } } } @@ -468,6 +733,7 @@ impl Block for BlockFinality { match self { BlockFinality::Final(block) => block.block_ptr(), BlockFinality::NonFinal(block) => block.ethereum_block.block.block_ptr(), + BlockFinality::Ptr(block) => BlockPtr::new(block.hash.clone(), block.number), } } @@ -475,6 +741,9 @@ impl Block for BlockFinality { match self { BlockFinality::Final(block) => block.parent_ptr(), BlockFinality::NonFinal(block) => block.ethereum_block.block.parent_ptr(), + BlockFinality::Ptr(block) => { + Some(BlockPtr::new(block.parent_hash.clone(), block.number - 1)) + } } } @@ -507,6 +776,21 @@ impl Block for BlockFinality { json::to_value(eth_block) } BlockFinality::NonFinal(block) => json::to_value(&block.ethereum_block), + BlockFinality::Ptr(_) => Ok(json::Value::Null), + } + } + + fn timestamp(&self) -> BlockTime { + match self { + BlockFinality::Final(block) => { + let ts = i64::try_from(block.timestamp_u64()).unwrap(); + BlockTime::since_epoch(ts, 0) + } + BlockFinality::NonFinal(block) => { + let ts = i64::try_from(block.ethereum_block.block.timestamp_u64()).unwrap(); + BlockTime::since_epoch(ts, 0) + } + BlockFinality::Ptr(block) => block.timestamp, } } } @@ -520,6 +804,104 @@ pub struct TriggersAdapter { chain_client: Arc>, capabilities: NodeCapabilities, unified_api_version: UnifiedMappingApiVersion, + eth_adapters: Arc, +} + +/// Fetches blocks from the cache based on block numbers, excluding duplicates +/// (i.e., multiple blocks for the same number), and identifying missing blocks that +/// need to be fetched via RPC/Firehose. Returns a tuple of the found blocks and the missing block numbers. +async fn fetch_unique_blocks_from_cache( + logger: &Logger, + chain_store: Arc, + block_numbers: BTreeSet, +) -> (Vec>, Vec) { + // Load blocks from the cache + let blocks_map = chain_store + .cheap_clone() + .block_ptrs_by_numbers(block_numbers.iter().copied().collect::>()) + .await + .map_err(|e| { + error!(logger, "Error accessing block cache {}", e); + e + }) + .unwrap_or_default(); + + // Collect blocks and filter out ones with multiple entries + let blocks: Vec> = blocks_map + .into_values() + .filter_map(|values| { + if values.len() == 1 { + Some(Arc::new(values[0].clone())) + } else { + None + } + }) + .collect(); + + // Identify missing blocks + let missing_blocks: Vec = block_numbers + .into_iter() + .filter(|&number| !blocks.iter().any(|block| block.block_number() == number)) + .collect(); + + if !missing_blocks.is_empty() { + debug!( + logger, + "Loading {} block(s) not in the block cache", + missing_blocks.len() + ); + trace!(logger, "Missing blocks {:?}", missing_blocks.len()); + } + + (blocks, missing_blocks) +} + +// This is used to load blocks from the RPC. +async fn load_blocks_with_rpc( + logger: &Logger, + adapter: Arc, + chain_store: Arc, + block_numbers: BTreeSet, +) -> Result> { + let logger_clone = logger.clone(); + load_blocks( + logger, + chain_store, + block_numbers, + |missing_numbers| async move { + adapter + .load_block_ptrs_by_numbers_rpc(logger_clone, missing_numbers) + .try_collect() + .await + }, + ) + .await +} + +/// Fetches blocks by their numbers, first attempting to load from cache. +/// Missing blocks are retrieved from an external source, with all blocks sorted and converted to `BlockFinality` format. +async fn load_blocks( + logger: &Logger, + chain_store: Arc, + block_numbers: BTreeSet, + fetch_missing: F, +) -> Result> +where + F: FnOnce(Vec) -> Fut, + Fut: Future>>>, +{ + // Fetch cached blocks and identify missing ones + let (mut cached_blocks, missing_block_numbers) = + fetch_unique_blocks_from_cache(logger, chain_store, block_numbers).await; + + // Fetch missing blocks if any + if !missing_block_numbers.is_empty() { + let missing_blocks = fetch_missing(missing_block_numbers).await?; + cached_blocks.extend(missing_blocks); + cached_blocks.sort_by_key(|block| block.number); + } + + Ok(cached_blocks.into_iter().map(BlockFinality::Ptr).collect()) } #[async_trait] @@ -529,9 +911,12 @@ impl TriggersAdapterTrait for TriggersAdapter { from: BlockNumber, to: BlockNumber, filter: &TriggerFilter, - ) -> Result>, Error> { + ) -> Result<(Vec>, BlockNumber), Error> { blocks_with_triggers( - self.chain_client.rpc()?.cheapest_with(&self.capabilities)?, + self.chain_client + .rpc()? + .cheapest_with(&self.capabilities) + .await?, self.logger.clone(), self.chain_store.clone(), self.ethrpc_metrics.clone(), @@ -543,6 +928,100 @@ impl TriggersAdapterTrait for TriggersAdapter { .await } + async fn load_block_ptrs_by_numbers( + &self, + logger: Logger, + block_numbers: BTreeSet, + ) -> Result> { + match &*self.chain_client { + ChainClient::Firehose(endpoints) => { + // If the force_rpc_for_block_ptrs flag is set, we will use the RPC to load the blocks + // even if the firehose is available. If no adapter is available, we will log an error. + // And then fallback to the firehose. + if ENV_VARS.force_rpc_for_block_ptrs { + trace!( + logger, + "Loading blocks from RPC (force_rpc_for_block_ptrs is set)"; + "block_numbers" => format!("{:?}", block_numbers) + ); + match self.eth_adapters.cheapest_with(&self.capabilities).await { + Ok(adapter) => { + match load_blocks_with_rpc( + &logger, + adapter, + self.chain_store.clone(), + block_numbers.clone(), + ) + .await + { + Ok(blocks) => return Ok(blocks), + Err(e) => { + warn!(logger, "Error loading blocks from RPC: {}", e); + } + } + } + Err(e) => { + warn!(logger, "Error getting cheapest adapter: {}", e); + } + } + } + + trace!( + logger, + "Loading blocks from firehose"; + "block_numbers" => format!("{:?}", block_numbers) + ); + + let endpoint = endpoints.endpoint().await?; + let chain_store = self.chain_store.clone(); + let logger_clone = logger.clone(); + + load_blocks( + &logger, + chain_store, + block_numbers, + |missing_numbers| async move { + let blocks = endpoint + .load_blocks_by_numbers::( + missing_numbers.iter().map(|&n| n as u64).collect(), + &logger_clone, + ) + .await? + .into_iter() + .map(|block| { + Arc::new(ExtendedBlockPtr { + hash: block.hash(), + number: block.number(), + parent_hash: block.parent_hash().unwrap_or_default(), + timestamp: block.timestamp(), + }) + }) + .collect::>(); + Ok(blocks) + }, + ) + .await + } + + ChainClient::Rpc(eth_adapters) => { + trace!( + logger, + "Loading blocks from RPC"; + "block_numbers" => format!("{:?}", block_numbers) + ); + + let adapter = eth_adapters.cheapest_with(&self.capabilities).await?; + load_blocks_with_rpc(&logger, adapter, self.chain_store.clone(), block_numbers) + .await + } + } + } + + async fn chain_head_ptr(&self) -> Result, Error> { + let chain_store = self.chain_store.clone(); + chain_store.chain_head_ptr().await + } + async fn triggers_in_block( &self, logger: &Logger, @@ -561,9 +1040,13 @@ impl TriggersAdapterTrait for TriggersAdapter { match &block { BlockFinality::Final(_) => { - let adapter = self.chain_client.rpc()?.cheapest_with(&self.capabilities)?; + let adapter = self + .chain_client + .rpc()? + .cheapest_with(&self.capabilities) + .await?; let block_number = block.number() as BlockNumber; - let blocks = blocks_with_triggers( + let (blocks, _) = blocks_with_triggers( adapter, logger.clone(), self.chain_store.clone(), @@ -587,80 +1070,281 @@ impl TriggersAdapterTrait for TriggersAdapter { triggers.append(&mut parse_block_triggers(&filter.block, full_block)); Ok(BlockWithTriggers::new(block, triggers, logger)) } + BlockFinality::Ptr(_) => unreachable!("triggers_in_block called on HeaderOnly"), } } async fn is_on_main_chain(&self, ptr: BlockPtr) -> Result { - self.chain_client - .rpc()? - .cheapest() - .ok_or(anyhow!("unable to get adapter for is_on_main_chain"))? - .is_on_main_chain(&self.logger, ptr.clone()) - .await + // It is tempting to use the block cache here; but that can go wrong + // when graph-node gets shut down and some of its nonfinal blocks + // then are reorged; when graph-node gets started again when those + // block numbers have become final, it might consider a reorged + // block as canonical; allowing the use of the block cache here + // would require us to also track the finality of blocks. + match &*self.chain_client { + ChainClient::Firehose(endpoints) => { + let endpoint = endpoints.endpoint().await?; + let block = endpoint + .get_block_by_number_with_retry::(ptr.number as u64, &self.logger) + .await + .context(format!( + "Failed to fetch block {} from firehose", + ptr.number + ))?; + Ok(block.hash() == ptr.hash) + } + ChainClient::Rpc(adapter) => { + let adapter = adapter + .cheapest() + .await + .ok_or_else(|| anyhow!("unable to get adapter for is_on_main_chain"))?; + + adapter.is_on_main_chain(&self.logger, ptr).await + } + } } + // Find an ancestor block at the specified offset from the given block pointer. + // Primarily used for reorg detection to verify if the indexed position remains + // on the main chain. + // + // Parameters: + // - ptr: Starting block pointer from which to walk backwards (typically the chain head) + // - offset: Number of blocks to traverse backwards (0 returns ptr, 1 returns parent, etc.) + // - root: Optional block hash that serves as a boundary for traversal. This is ESSENTIAL + // for chains with skipped blocks (e.g., Filecoin EVM) where block numbers are not + // consecutive. When provided, traversal stops upon reaching the child of root, + // ensuring correct ancestor relationships even with gaps in block numbers. + // + // The function attempts to use the database cache first for performance, + // with RPC fallback implemented to handle cases where the cache is unavailable. async fn ancestor_block( &self, ptr: BlockPtr, offset: BlockNumber, + root: Option, ) -> Result, Error> { - let block: Option = self + let ptr_for_log = ptr.clone(); + let cached = self .chain_store .cheap_clone() - .ancestor_block(ptr, offset) - .await? - .map(json::from_value) - .transpose()?; - Ok(block.map(|block| { - BlockFinality::NonFinal(EthereumBlockWithCalls { - ethereum_block: block, - calls: None, - }) - })) + .ancestor_block(ptr.clone(), offset, root.clone()) + .await?; + + // Use full blocks (with receipts) directly from cache. + // Light blocks (no receipts) need to be fetched from Firehose/RPC. + let block_ptr = match cached { + Some((cached_block, ptr)) => match cached_block.into_full_block() { + Some(block) => { + return Ok(Some(BlockFinality::NonFinal(EthereumBlockWithCalls { + ethereum_block: block, + calls: None, + }))); + } + None => { + trace!( + self.logger, + "Cached block #{} {} is light (no receipts). Falling back to Firehose/RPC.", + ptr.number, + ptr.hash_hex(), + ); + ptr + } + }, + None => { + // Cache miss - fall back to walking the chain via parent_ptr() calls. + // This provides resilience when the block cache is empty (e.g., after truncation). + debug!( + self.logger, + "ancestor_block cache miss for {} at offset {}, walking back via parent_ptr", + ptr_for_log.hash_hex(), + offset + ); + + match walk_back_ancestor( + ptr.clone(), + offset, + root.clone(), + |block_ptr| async move { self.parent_ptr(&block_ptr).await }, + ) + .await? + { + Some(ptr) => ptr, + None => return Ok(None), + } + } + }; + + // Fetch the actual block data for the identified block pointer. + // This path is taken for both cache misses and deserialization failures. + match self.chain_client.as_ref() { + ChainClient::Firehose(endpoints) => { + let block = self + .fetch_block_with_firehose(endpoints, &block_ptr) + .await?; + let ethereum_block: EthereumBlockWithCalls = (&block).try_into()?; + Ok(Some(BlockFinality::NonFinal(ethereum_block))) + } + ChainClient::Rpc(adapters) => { + match self.fetch_full_block_with_rpc(adapters, &block_ptr).await? { + Some(ethereum_block) => { + Ok(Some(BlockFinality::NonFinal(EthereumBlockWithCalls { + ethereum_block, + calls: None, + }))) + } + None => Ok(None), + } + } + } } async fn parent_ptr(&self, block: &BlockPtr) -> Result, Error> { - use futures::stream::Stream; use graph::prelude::LightEthereumBlockExt; let block = match self.chain_client.as_ref() { - ChainClient::Firehose(_) => Some(BlockPtr { - hash: BlockHash::from(vec![0xff; 32]), - number: block.number.saturating_sub(1), - }), - ChainClient::Rpc(adapters) => { - let blocks = adapters - .cheapest_with(&self.capabilities)? - .load_blocks( - self.logger.cheap_clone(), - self.chain_store.cheap_clone(), - HashSet::from_iter(Some(block.hash_as_h256())), - ) - .collect() - .compat() - .await?; - assert_eq!(blocks.len(), 1); - - blocks[0].parent_ptr() + ChainClient::Firehose(endpoints) => { + let chain_store = self.chain_store.cheap_clone(); + // First try to get the parent pointer from the store header columns + if let Ok(Some(parent)) = chain_store.block_parent_ptr(&block.hash).await { + return Ok(Some(parent)); + } + + // If not in store, fetch from Firehose + self.fetch_block_with_firehose(endpoints, block) + .await? + .parent_ptr() } + ChainClient::Rpc(adapters) => self + .fetch_light_block_with_rpc(adapters, block) + .await? + .expect("block must exist for parent_ptr") + .parent_ptr(), }; Ok(block) } } -pub struct FirehoseMapper {} +impl TriggersAdapter { + async fn fetch_block_with_firehose( + &self, + endpoints: &FirehoseEndpoints, + block_ptr: &BlockPtr, + ) -> Result { + let endpoint = endpoints.endpoint().await?; + let logger = self.logger.clone(); + let retry_log_message = format!("fetch_block_with_firehose {}", block_ptr); + let block_ptr = block_ptr.clone(); + + let block = retry(retry_log_message, &logger) + .limit(ENV_VARS.request_retries) + .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) + .run(move || { + let endpoint = endpoint.cheap_clone(); + let logger = logger.cheap_clone(); + let block_ptr = block_ptr.clone(); + async move { + endpoint + .get_block_by_ptr::(&block_ptr, &logger) + .await + .context(format!("Failed to fetch block {} from firehose", block_ptr)) + } + }) + .await?; + + Ok(block) + } + + async fn fetch_light_block_with_rpc( + &self, + adapters: &EthereumNetworkAdapters, + block_ptr: &BlockPtr, + ) -> Result>, Error> { + let blocks = adapters + .cheapest_with(&self.capabilities) + .await? + .load_blocks( + self.logger.cheap_clone(), + self.chain_store.cheap_clone(), + HashSet::from_iter(Some(block_ptr.hash.as_b256())), + ) + .await?; + + Ok(blocks.into_iter().next()) + } + + async fn fetch_full_block_with_rpc( + &self, + adapters: &EthereumNetworkAdapters, + block_ptr: &BlockPtr, + ) -> Result, Error> { + // Use the cache-aware light block fetch first; it checks recent_blocks_cache + // and the DB before falling back to eth_getBlockByHash. + let light_block = self.fetch_light_block_with_rpc(adapters, block_ptr).await?; + match light_block { + Some(light_block) => { + let adapter = adapters.cheapest_with(&self.capabilities).await?; + let ethereum_block = adapter + .load_full_block(&self.logger, light_block.inner().clone()) + .await + .map_err(|e| anyhow!("Failed to load full block: {}", e))?; + Ok(Some(ethereum_block)) + } + None => Ok(None), + } + } +} + +pub struct FirehoseMapper { + adapter: Arc>, + filter: Arc, +} + +#[async_trait] +impl BlockStreamMapper for FirehoseMapper { + fn decode_block( + &self, + output: Option<&[u8]>, + ) -> Result, BlockStreamError> { + let block = match output { + Some(block) => codec::Block::decode(block)?, + None => Err(anyhow::anyhow!( + "ethereum mapper is expected to always have a block" + ))?, + }; + + // See comment(437a9f17-67cc-478f-80a3-804fe554b227) ethereum_block.calls is always Some even if calls + // is empty + let ethereum_block: EthereumBlockWithCalls = (&block).try_into()?; + + Ok(Some(BlockFinality::NonFinal(ethereum_block))) + } + + async fn block_with_triggers( + &self, + logger: &Logger, + block: BlockFinality, + ) -> Result, BlockStreamError> { + self.adapter + .triggers_in_block(logger, block, &self.filter) + .await + .map_err(BlockStreamError::from) + } +} #[async_trait] impl FirehoseMapperTrait for FirehoseMapper { + fn trigger_filter(&self) -> &TriggerFilter { + self.filter.as_ref() + } + async fn to_block_stream_event( &self, logger: &Logger, response: &firehose::Response, - adapter: &Arc>, - filter: &TriggerFilter, ) -> Result, FirehoseError> { - let step = ForkStep::from_i32(response.step).unwrap_or_else(|| { + let step = ForkStep::try_from(response.step).unwrap_or_else(|_| { panic!( "unknown step i32 value {}, maybe you forgot update & re-regenerate the protobuf definitions?", response.step @@ -683,15 +1367,9 @@ impl FirehoseMapperTrait for FirehoseMapper { use firehose::ForkStep::*; match step { StepNew => { - // See comment(437a9f17-67cc-478f-80a3-804fe554b227) ethereum_block.calls is always Some even if calls - // is empty - let ethereum_block: EthereumBlockWithCalls = (&block).try_into()?; - - // triggers in block never actually calls the ethereum traces api. - // TODO: Split the trigger parsing from call retrieving. - let block_with_triggers = adapter - .triggers_in_block(logger, BlockFinality::NonFinal(ethereum_block), filter) - .await?; + // unwrap: Input cannot be None so output will be error or block. + let block = self.decode_block(Some(any_block.value.as_ref()))?.unwrap(); + let block_with_triggers = self.block_with_triggers(logger, block).await?; Ok(BlockStreamEvent::ProcessBlock( block_with_triggers, @@ -711,7 +1389,9 @@ impl FirehoseMapperTrait for FirehoseMapper { } StepFinal => { - unreachable!("irreversible step is not handled and should not be requested in the Firehose request") + unreachable!( + "irreversible step is not handled and should not be requested in the Firehose request" + ) } StepUnset => { @@ -749,3 +1429,251 @@ impl FirehoseMapperTrait for FirehoseMapper { .await } } + +#[cfg(test)] +mod tests { + use graph::blockchain::mock::MockChainStore; + use graph::slog; + + use super::*; + use std::sync::Arc; + + // Helper function to create test blocks + fn create_test_block(number: BlockNumber, hash: &str) -> ExtendedBlockPtr { + let hash = BlockHash(hash.as_bytes().to_vec().into_boxed_slice()); + let ptr = BlockPtr::new(hash.clone(), number); + ExtendedBlockPtr { + hash, + number, + parent_hash: BlockHash(vec![0; 32].into_boxed_slice()), + timestamp: BlockTime::for_test(&ptr), + } + } + + #[graph::test] + async fn test_fetch_unique_blocks_single_block() { + let logger = Logger::root(slog::Discard, o!()); + let mut chain_store = MockChainStore::default(); + + // Add a single block + let block = create_test_block(1, "block1"); + chain_store.blocks.insert(1, vec![block.clone()]); + + let block_numbers: BTreeSet<_> = vec![1].into_iter().collect(); + + let (blocks, missing) = + fetch_unique_blocks_from_cache(&logger, Arc::new(chain_store), block_numbers).await; + + assert_eq!(blocks.len(), 1); + assert_eq!(blocks[0].number, 1); + assert!(missing.is_empty()); + } + + #[graph::test] + async fn test_fetch_unique_blocks_duplicate_blocks() { + let logger = Logger::root(slog::Discard, o!()); + let mut chain_store = MockChainStore::default(); + + // Add multiple blocks for the same number + let block1 = create_test_block(1, "block1a"); + let block2 = create_test_block(1, "block1b"); + chain_store + .blocks + .insert(1, vec![block1.clone(), block2.clone()]); + + let block_numbers: BTreeSet<_> = vec![1].into_iter().collect(); + + let (blocks, missing) = + fetch_unique_blocks_from_cache(&logger, Arc::new(chain_store), block_numbers).await; + + // Should filter out the duplicate block + assert!(blocks.is_empty()); + assert_eq!(missing, vec![1]); + assert_eq!(missing[0], 1); + } + + #[graph::test] + async fn test_fetch_unique_blocks_missing_blocks() { + let logger = Logger::root(slog::Discard, o!()); + let mut chain_store = MockChainStore::default(); + + // Add block number 1 but not 2 + let block = create_test_block(1, "block1"); + chain_store.blocks.insert(1, vec![block.clone()]); + + let block_numbers: BTreeSet<_> = vec![1, 2].into_iter().collect(); + + let (blocks, missing) = + fetch_unique_blocks_from_cache(&logger, Arc::new(chain_store), block_numbers).await; + + assert_eq!(blocks.len(), 1); + assert_eq!(blocks[0].number, 1); + assert_eq!(missing, vec![2]); + } + + #[graph::test] + async fn test_fetch_unique_blocks_multiple_valid_blocks() { + let logger = Logger::root(slog::Discard, o!()); + let mut chain_store = MockChainStore::default(); + + // Add multiple valid blocks + let block1 = create_test_block(1, "block1"); + let block2 = create_test_block(2, "block2"); + chain_store.blocks.insert(1, vec![block1.clone()]); + chain_store.blocks.insert(2, vec![block2.clone()]); + + let block_numbers: BTreeSet<_> = vec![1, 2].into_iter().collect(); + + let (blocks, missing) = + fetch_unique_blocks_from_cache(&logger, Arc::new(chain_store), block_numbers).await; + + assert_eq!(blocks.len(), 2); + assert!(blocks.iter().any(|b| b.number == 1)); + assert!(blocks.iter().any(|b| b.number == 2)); + assert!(missing.is_empty()); + } + + #[graph::test] + async fn test_fetch_unique_blocks_mixed_scenario() { + let logger = Logger::root(slog::Discard, o!()); + let mut chain_store = MockChainStore::default(); + + // Add a mix of scenarios: + // - Block 1: Single valid block + // - Block 2: Multiple blocks (duplicate) + // - Block 3: Missing + let block1 = create_test_block(1, "block1"); + let block2a = create_test_block(2, "block2a"); + let block2b = create_test_block(2, "block2b"); + + chain_store.blocks.insert(1, vec![block1.clone()]); + chain_store + .blocks + .insert(2, vec![block2a.clone(), block2b.clone()]); + + let block_numbers: BTreeSet<_> = vec![1, 2, 3].into_iter().collect(); + + let (blocks, missing) = + fetch_unique_blocks_from_cache(&logger, Arc::new(chain_store), block_numbers).await; + + assert_eq!(blocks.len(), 1); + assert_eq!(blocks[0].number, 1); + assert_eq!(missing.len(), 2); + assert!(missing.contains(&2)); + assert!(missing.contains(&3)); + } + + #[tokio::test] + async fn test_walk_back_ancestor() { + use std::collections::HashMap; + + let block_100_hash = BlockHash("block100".as_bytes().to_vec().into_boxed_slice()); + let block_101_hash = BlockHash("block101".as_bytes().to_vec().into_boxed_slice()); + let block_102_hash = BlockHash("block102".as_bytes().to_vec().into_boxed_slice()); + let block_103_hash = BlockHash("block103".as_bytes().to_vec().into_boxed_slice()); + let block_104_hash = BlockHash("block104".as_bytes().to_vec().into_boxed_slice()); + let block_105_hash = BlockHash("block105".as_bytes().to_vec().into_boxed_slice()); + + let block_105 = BlockPtr::new(block_105_hash.clone(), 105); + let block_104 = BlockPtr::new(block_104_hash.clone(), 104); + let block_103 = BlockPtr::new(block_103_hash.clone(), 103); + let block_102 = BlockPtr::new(block_102_hash.clone(), 102); + let block_101 = BlockPtr::new(block_101_hash.clone(), 101); + let block_100 = BlockPtr::new(block_100_hash.clone(), 100); + + let mut parent_map = HashMap::new(); + parent_map.insert(block_105_hash.clone(), block_104.clone()); + parent_map.insert(block_104_hash.clone(), block_103.clone()); + parent_map.insert(block_103_hash.clone(), block_102.clone()); + parent_map.insert(block_102_hash.clone(), block_101.clone()); + parent_map.insert(block_101_hash.clone(), block_100.clone()); + + let result = super::walk_back_ancestor(block_105.clone(), 2, None, |block_ptr| { + let parent = parent_map.get(&block_ptr.hash).cloned(); + async move { Ok::<_, std::convert::Infallible>(parent) } + }) + .await + .unwrap(); + assert_eq!(result, Some(block_103.clone())); + + let result = super::walk_back_ancestor( + block_105.clone(), + 10, + Some(block_102_hash.clone()), + |block_ptr| { + let parent = parent_map.get(&block_ptr.hash).cloned(); + async move { Ok::<_, std::convert::Infallible>(parent) } + }, + ) + .await + .unwrap(); + assert_eq!( + result, + Some(block_103.clone()), + "Should stop at child of root" + ); + } + + #[tokio::test] + async fn test_walk_back_ancestor_skipped_blocks_with_root() { + use std::collections::HashMap; + + let block_100_hash = BlockHash("block100".as_bytes().to_vec().into_boxed_slice()); + let block_101_hash = BlockHash("block101".as_bytes().to_vec().into_boxed_slice()); + let block_102_hash = BlockHash("block102".as_bytes().to_vec().into_boxed_slice()); + let block_110_hash = BlockHash("block110".as_bytes().to_vec().into_boxed_slice()); + let block_111_hash = BlockHash("block111".as_bytes().to_vec().into_boxed_slice()); + let block_112_hash = BlockHash("block112".as_bytes().to_vec().into_boxed_slice()); + let block_120_hash = BlockHash("block120".as_bytes().to_vec().into_boxed_slice()); + + let block_120 = BlockPtr::new(block_120_hash.clone(), 120); + let block_112 = BlockPtr::new(block_112_hash.clone(), 112); + let block_111 = BlockPtr::new(block_111_hash.clone(), 111); + let block_110 = BlockPtr::new(block_110_hash.clone(), 110); + let block_102 = BlockPtr::new(block_102_hash.clone(), 102); + let block_101 = BlockPtr::new(block_101_hash.clone(), 101); + let block_100 = BlockPtr::new(block_100_hash.clone(), 100); + + let mut parent_map = HashMap::new(); + parent_map.insert(block_120_hash.clone(), block_112.clone()); + parent_map.insert(block_112_hash.clone(), block_111.clone()); + parent_map.insert(block_111_hash.clone(), block_110.clone()); + parent_map.insert(block_110_hash.clone(), block_102.clone()); + parent_map.insert(block_102_hash.clone(), block_101.clone()); + parent_map.insert(block_101_hash.clone(), block_100.clone()); + + let result = super::walk_back_ancestor( + block_120.clone(), + 10, + Some(block_110_hash.clone()), + |block_ptr| { + let parent = parent_map.get(&block_ptr.hash).cloned(); + async move { Ok::<_, std::convert::Infallible>(parent) } + }, + ) + .await + .unwrap(); + assert_eq!( + result, + Some(block_111.clone()), + "root=110: should stop at 111 (child of root)" + ); + + let result = super::walk_back_ancestor( + block_120.clone(), + 10, + Some(block_101_hash.clone()), + |block_ptr| { + let parent = parent_map.get(&block_ptr.hash).cloned(); + async move { Ok::<_, std::convert::Infallible>(parent) } + }, + ) + .await + .unwrap(); + assert_eq!( + result, + Some(block_102.clone()), + "root=101: should stop at 102 (child of root, across skip)" + ); + } +} diff --git a/chain/ethereum/src/codec.rs b/chain/ethereum/src/codec.rs index 3f800dff0a5..b40bcd11910 100644 --- a/chain/ethereum/src/codec.rs +++ b/chain/ethereum/src/codec.rs @@ -1,15 +1,26 @@ #[rustfmt::skip] +#[allow(clippy::doc_lazy_continuation, clippy::doc_overindented_list_items)] #[path = "protobuf/sf.ethereum.r#type.v2.rs"] mod pbcodec; use anyhow::format_err; use graph::{ - blockchain::{Block as BlockchainBlock, BlockPtr, ChainStoreBlock, ChainStoreData}, + blockchain::{ + self, Block as BlockchainBlock, BlockPtr, BlockTime, ChainStoreBlock, ChainStoreData, + }, + components::ethereum::{ + AnyBlock, AnyHeader, AnyRpcHeader, AnyTransactionReceiptBare, AnyTxEnvelope, + }, prelude::{ - web3, - web3::types::{Bytes, H160, H2048, H256, H64, U256, U64}, BlockNumber, Error, EthereumBlock, EthereumBlockWithCalls, EthereumCall, LightEthereumBlock, + alloy::{ + self, + consensus::{ReceiptWithBloom, TxEnvelope, TxType}, + network::AnyReceiptEnvelope, + primitives::{Address, B256, Bloom, Bytes, LogData, U256, aliases::B2048}, + rpc::types::{self as alloy_rpc_types, AccessList, AccessListItem, Transaction}, + }, }, }; use std::sync::Arc; @@ -32,13 +43,13 @@ where } } -impl TryDecodeProto<[u8; 256], H2048> for &[u8] {} -impl TryDecodeProto<[u8; 32], H256> for &[u8] {} -impl TryDecodeProto<[u8; 20], H160> for &[u8] {} +impl TryDecodeProto<[u8; 32], B256> for &[u8] {} +impl TryDecodeProto<[u8; 256], B2048> for &[u8] {} +impl TryDecodeProto<[u8; 20], Address> for &[u8] {} -impl From<&BigInt> for web3::types::U256 { +impl From<&BigInt> for U256 { fn from(val: &BigInt) -> Self { - web3::types::U256::from_big_endian(&val.bytes) + U256::from_be_slice(&val.bytes) } } @@ -66,9 +77,9 @@ impl<'a> TryInto for CallAt<'a> { .value .as_ref() .map_or_else(|| U256::from(0), |v| v.into()), - gas_used: U256::from(self.call.gas_consumed), - input: Bytes(self.call.input.clone()), - output: Bytes(self.call.return_data.clone()), + gas_used: self.call.gas_consumed, + input: Bytes::from(self.call.input.clone()), + output: Bytes::from(self.call.return_data.clone()), block_hash: self.block.hash.try_decode_proto("call block hash")?, block_number: self.block.number as i32, transaction_hash: Some(self.trace.hash.try_decode_proto("call transaction hash")?), @@ -77,41 +88,6 @@ impl<'a> TryInto for CallAt<'a> { } } -impl TryInto for Call { - type Error = Error; - - fn try_into(self) -> Result { - Ok(web3::types::Call { - from: self.caller.try_decode_proto("call from address")?, - to: self.address.try_decode_proto("call to address")?, - value: self - .value - .as_ref() - .map_or_else(|| U256::from(0), |v| v.into()), - gas: U256::from(self.gas_limit), - input: Bytes::from(self.input.clone()), - call_type: CallType::from_i32(self.call_type) - .ok_or_else(|| format_err!("invalid call type: {}", self.call_type,))? - .into(), - }) - } -} - -impl From for web3::types::CallType { - fn from(val: CallType) -> Self { - match val { - CallType::Unspecified => web3::types::CallType::None, - CallType::Call => web3::types::CallType::Call, - CallType::Callcode => web3::types::CallType::CallCode, - CallType::Delegate => web3::types::CallType::DelegateCall, - CallType::Static => web3::types::CallType::StaticCall, - - // FIXME (SF): Really not sure what this should map to, we are using None for now, need to revisit - CallType::Create => web3::types::CallType::None, - } - } -} - pub struct LogAt<'a> { log: &'a Log, block: &'a Block, @@ -124,51 +100,38 @@ impl<'a> LogAt<'a> { } } -impl<'a> TryInto for LogAt<'a> { +impl<'a> TryInto for LogAt<'a> { type Error = Error; - fn try_into(self) -> Result { - Ok(web3::types::Log { - address: self.log.address.try_decode_proto("log address")?, - topics: self - .log - .topics - .iter() - .map(|t| t.try_decode_proto("topic")) - .collect::, Error>>()?, - data: Bytes::from(self.log.data.clone()), + fn try_into(self) -> Result { + let topics = self + .log + .topics + .iter() + .map(|t| t.try_decode_proto("topic")) + .collect::, Error>>()?; + + Ok(alloy::rpc::types::Log { + inner: alloy::primitives::Log { + address: self.log.address.try_decode_proto("log address")?, + data: LogData::new(topics, self.log.data.clone().into()) + .ok_or_else(|| format_err!("invalid log data"))?, + }, block_hash: Some(self.block.hash.try_decode_proto("log block hash")?), - block_number: Some(U64::from(self.block.number)), + block_number: Some(self.block.number), transaction_hash: Some(self.trace.hash.try_decode_proto("log transaction hash")?), - transaction_index: Some(U64::from(self.trace.index as u64)), - log_index: Some(U256::from(self.log.block_index)), - transaction_log_index: Some(U256::from(self.log.index)), - log_type: None, - removed: None, + transaction_index: Some(self.trace.index as u64), + log_index: Some(self.log.block_index as u64), + removed: false, + block_timestamp: self + .block + .header + .as_ref() + .and_then(|h| h.timestamp.as_ref().map(|t| t.seconds as u64)), }) } } -impl From for web3::types::U64 { - fn from(val: TransactionTraceStatus) -> Self { - let status: Option = val.into(); - status.unwrap_or_else(|| web3::types::U64::from(0)) - } -} - -impl Into> for TransactionTraceStatus { - fn into(self) -> Option { - match self { - Self::Unknown => { - panic!("Got a transaction trace with status UNKNOWN, datasource is broken") - } - Self::Succeeded => Some(web3::types::U64::from(1)), - Self::Failed => Some(web3::types::U64::from(0)), - Self::Reverted => Some(web3::types::U64::from(0)), - } - } -} - pub struct TransactionTraceAt<'a> { trace: &'a TransactionTrace, block: &'a Block, @@ -180,46 +143,323 @@ impl<'a> TransactionTraceAt<'a> { } } -impl<'a> TryInto for TransactionTraceAt<'a> { +impl<'a> TryInto> for TransactionTraceAt<'a> { type Error = Error; - fn try_into(self) -> Result { - Ok(web3::types::Transaction { - hash: self.trace.hash.try_decode_proto("transaction hash")?, - nonce: U256::from(self.trace.nonce), - block_hash: Some(self.block.hash.try_decode_proto("transaction block hash")?), - block_number: Some(U64::from(self.block.number)), - transaction_index: Some(U64::from(self.trace.index as u64)), - from: Some( - self.trace - .from - .try_decode_proto("transaction from address")?, - ), - to: match self.trace.calls.len() { - 0 => Some(self.trace.to.try_decode_proto("transaction to address")?), - _ => { - match CallType::from_i32(self.trace.calls[0].call_type).ok_or_else(|| { - format_err!("invalid call type: {}", self.trace.calls[0].call_type,) - })? { - CallType::Create => { - None // we don't want the 'to' address on a transaction that creates the contract, to align with RPC behavior - } - _ => Some(self.trace.to.try_decode_proto("transaction to")?), - } - } + fn try_into(self) -> Result, Self::Error> { + use alloy::{ + consensus::transaction::Recovered, + consensus::{ + Signed, TxEip1559, TxEip2930, TxEip4844, TxEip4844Variant, TxEip7702, TxLegacy, }, - value: self.trace.value.as_ref().map_or(U256::zero(), |x| x.into()), - gas_price: self.trace.gas_price.as_ref().map(|x| x.into()), - gas: U256::from(self.trace.gas_limit), - input: Bytes::from(self.trace.input.clone()), - v: None, - r: None, - s: None, - raw: None, - access_list: None, - max_fee_per_gas: None, - max_priority_fee_per_gas: None, - transaction_type: None, + network::{AnyTxEnvelope, AnyTxType, UnknownTxEnvelope, UnknownTypedTransaction}, + primitives::{Bytes, TxKind, U256}, + rpc::types::Transaction as AlloyTransaction, + serde::OtherFields, + }; + use std::collections::BTreeMap; + + // Extract data from trace and block + let block_hash = self.block.hash.try_decode_proto("transaction block hash")?; + let block_number = self.block.number; + let block_timestamp = self + .block + .header + .as_ref() + .and_then(|h| h.timestamp.as_ref().map(|t| t.seconds as u64)); + let transaction_index = Some(self.trace.index as u64); + let from_address = self + .trace + .from + .try_decode_proto("transaction from address")?; + let to = get_to_address(self.trace)?; + let value = self.trace.value.as_ref().map_or(U256::ZERO, |x| x.into()); + let gas_price = self.trace.gas_price.as_ref().map_or(0u128, |x| { + let val: U256 = x.into(); + val.to::() + }); + let gas_limit = self.trace.gas_limit; + let input = Bytes::from(self.trace.input.clone()); + + let tx_type_u64 = u64::try_from(self.trace.r#type).map_err(|_| { + format_err!( + "Invalid transaction type value {} in transaction trace. Transaction type must be a valid u64.", + self.trace.r#type + ) + })?; + + // Try to convert to known Ethereum transaction type + let tx_type_result = TxType::try_from(tx_type_u64); + + // If this is an unknown transaction type, create an UnknownTxEnvelope + if tx_type_result.is_err() { + let mut fields_map = BTreeMap::new(); + + fields_map.insert( + "nonce".to_string(), + jsonrpc_core::serde_json::json!(format!("0x{:x}", self.trace.nonce)), + ); + fields_map.insert( + "from".to_string(), + jsonrpc_core::serde_json::json!(format!("{:?}", from_address)), + ); + if let Some(to_addr) = to { + fields_map.insert( + "to".to_string(), + jsonrpc_core::serde_json::json!(format!("{:?}", to_addr)), + ); + } + fields_map.insert( + "value".to_string(), + jsonrpc_core::serde_json::json!(format!("0x{:x}", value)), + ); + fields_map.insert( + "gas".to_string(), + jsonrpc_core::serde_json::json!(format!("0x{:x}", gas_limit)), + ); + fields_map.insert( + "gasPrice".to_string(), + jsonrpc_core::serde_json::json!(format!("0x{:x}", gas_price)), + ); + fields_map.insert( + "input".to_string(), + jsonrpc_core::serde_json::json!(format!("0x{}", hex::encode(&input))), + ); + + let fields = OtherFields::new(fields_map); + let unknown_tx = UnknownTypedTransaction { + ty: AnyTxType(tx_type_u64 as u8), + fields, + memo: Default::default(), + }; + + let tx_hash = self.trace.hash.try_decode_proto("transaction hash")?; + let unknown_envelope = UnknownTxEnvelope { + hash: tx_hash, + inner: unknown_tx, + }; + + let any_envelope = AnyTxEnvelope::Unknown(unknown_envelope); + let recovered = Recovered::new_unchecked(any_envelope, from_address); + + return Ok(AlloyTransaction { + inner: recovered, + block_hash: Some(block_hash), + block_number: Some(block_number), + block_timestamp, + transaction_index, + effective_gas_price: if gas_price > 0 { Some(gas_price) } else { None }, + }); + } + + let tx_type = tx_type_result.unwrap(); + let nonce = self.trace.nonce; + + // Extract EIP-1559 fee fields from trace + let max_fee_per_gas_u128 = self.trace.max_fee_per_gas.as_ref().map_or(gas_price, |x| { + let val: U256 = x.into(); + val.to::() + }); + + let max_priority_fee_per_gas_u128 = + self.trace + .max_priority_fee_per_gas + .as_ref() + .map_or(0u128, |x| { + let val: U256 = x.into(); + val.to::() + }); + + // Extract access list from trace + let access_list: AccessList = self + .trace + .access_list + .iter() + .map(|access_tuple| -> Result<_, Error> { + let address = access_tuple + .address + .try_decode_proto("access tuple address")?; + let storage_keys = access_tuple + .storage_keys + .iter() + .map(|key| key.try_decode_proto("storage key")) + .collect::, _>>()?; + Ok(AccessListItem { + address, + storage_keys, + }) + }) + .collect::, Error>>()? + .into(); + + // Extract actual signature components from trace + let signature = extract_signature_from_trace(self.trace, tx_type)?; + + let to_kind = match to { + Some(addr) => TxKind::Call(addr), + None => TxKind::Create, + }; + + let envelope = match tx_type { + TxType::Legacy => { + let tx = TxLegacy { + chain_id: None, + nonce, + gas_price, + gas_limit, + to: to_kind, + value, + input: input.clone(), + }; + let signed_tx = Signed::new_unchecked( + tx, + signature, + self.trace.hash.try_decode_proto("transaction hash")?, + ); + TxEnvelope::Legacy(signed_tx) + } + TxType::Eip2930 => { + let tx = TxEip2930 { + // Firehose protobuf doesn't provide chain_id for transactions. + // Using 0 as placeholder since the transaction has already been validated on-chain. + chain_id: 0, + nonce, + gas_price, + gas_limit, + to: to_kind, + value, + access_list: access_list.clone(), // Use actual access list from trace + input: input.clone(), + }; + let signed_tx = Signed::new_unchecked( + tx, + signature, + self.trace.hash.try_decode_proto("transaction hash")?, + ); + TxEnvelope::Eip2930(signed_tx) + } + TxType::Eip1559 => { + let tx = TxEip1559 { + // Firehose protobuf doesn't provide chain_id for transactions. + // Using 0 as placeholder since the transaction has already been validated on-chain. + chain_id: 0, + nonce, + gas_limit, + max_fee_per_gas: max_fee_per_gas_u128, + max_priority_fee_per_gas: max_priority_fee_per_gas_u128, + to: to_kind, + value, + access_list: access_list.clone(), // Use actual access list from trace + input: input.clone(), + }; + let signed_tx = Signed::new_unchecked( + tx, + signature, + self.trace.hash.try_decode_proto("transaction hash")?, + ); + TxEnvelope::Eip1559(signed_tx) + } + TxType::Eip4844 => { + let to_address = to.ok_or_else(|| { + format_err!("EIP-4844 transactions cannot be contract creation transactions. The 'to' field must contain a valid address.") + })?; + + let blob_versioned_hashes: Vec = self + .trace + .blob_hashes + .iter() + .map(|hash| hash.try_decode_proto("blob hash")) + .collect::, _>>()?; + + let max_fee_per_blob_gas_u128 = + self.trace.blob_gas_fee_cap.as_ref().map_or(0u128, |x| { + let val: U256 = x.into(); + val.to::() + }); + + let tx_eip4844 = TxEip4844 { + // Firehose protobuf doesn't provide chain_id for transactions. + // Using 0 as placeholder since the transaction has already been validated on-chain. + chain_id: 0, + nonce, + gas_limit, + max_fee_per_gas: max_fee_per_gas_u128, + max_priority_fee_per_gas: max_priority_fee_per_gas_u128, + to: to_address, + value, + access_list: access_list.clone(), // Use actual access list from trace + blob_versioned_hashes, + max_fee_per_blob_gas: max_fee_per_blob_gas_u128, + input: input.clone(), + }; + let tx = TxEip4844Variant::TxEip4844(tx_eip4844); + let signed_tx = Signed::new_unchecked( + tx, + signature, + self.trace.hash.try_decode_proto("transaction hash")?, + ); + TxEnvelope::Eip4844(signed_tx) + } + TxType::Eip7702 => { + let to_address = to.ok_or_else(|| { + format_err!("EIP-7702 transactions cannot be contract creation transactions. The 'to' field must contain a valid address.") + })?; + + // Convert set_code_authorizations to alloy authorization list + let authorization_list: Vec = self + .trace + .set_code_authorizations + .iter() + .map(|auth| -> Result<_, Error> { + let inner = alloy::eips::eip7702::Authorization { + chain_id: U256::from_be_slice(&auth.chain_id), + address: auth.address.try_decode_proto("authorization address")?, + nonce: auth.nonce, + }; + + let r = U256::from_be_slice(&auth.r); + let s = U256::from_be_slice(&auth.s); + let y_parity = auth.v as u8; + + Ok(alloy::eips::eip7702::SignedAuthorization::new_unchecked( + inner, y_parity, r, s, + )) + }) + .collect::, Error>>()?; + + let tx = TxEip7702 { + // Firehose protobuf doesn't provide chain_id for transactions. + // Using 0 as placeholder since the transaction has already been validated on-chain. + chain_id: 0, + nonce, + gas_limit, + max_fee_per_gas: max_fee_per_gas_u128, + max_priority_fee_per_gas: max_priority_fee_per_gas_u128, + to: to_address, + value, + access_list: access_list.clone(), // Use actual access list from trace + authorization_list, + input: input.clone(), + }; + let signed_tx = Signed::new_unchecked( + tx, + signature, + self.trace.hash.try_decode_proto("transaction hash")?, + ); + TxEnvelope::Eip7702(signed_tx) + } + }; + + let any_envelope = AnyTxEnvelope::Ethereum(envelope); + let recovered = Recovered::new_unchecked(any_envelope, from_address); + + Ok(AlloyTransaction { + inner: recovered, + block_hash: Some(block_hash), + block_number: Some(block_number), + block_timestamp, + transaction_index, + effective_gas_price: if gas_price > 0 { Some(gas_price) } else { None }, // gas_price already contains effective gas price per protobuf spec }) } } @@ -232,143 +472,131 @@ impl TryInto for &Block { } } +impl TryInto for &Block { + type Error = Error; + + fn try_into(self) -> Result { + let header = self.header(); + + let block_hash = self.hash.try_decode_proto("block hash")?; + let consensus_header = alloy::consensus::Header { + number: header.number, + beneficiary: header.coinbase.try_decode_proto("author / coinbase")?, + parent_hash: header.parent_hash.try_decode_proto("parent hash")?, + ommers_hash: header.uncle_hash.try_decode_proto("uncle hash")?, + state_root: header.state_root.try_decode_proto("state root")?, + transactions_root: header + .transactions_root + .try_decode_proto("transactions root")?, + receipts_root: header.receipt_root.try_decode_proto("receipt root")?, + gas_used: header.gas_used, + gas_limit: header.gas_limit, + base_fee_per_gas: header.base_fee_per_gas.as_ref().map(|v| { + let val: U256 = v.into(); + val.to::() + }), + extra_data: Bytes::from(header.extra_data.clone()), + logs_bloom: if header.logs_bloom.is_empty() { + Bloom::ZERO + } else { + Bloom::try_from(header.logs_bloom.as_slice())? + }, + timestamp: header.timestamp.as_ref().map_or(0, |v| v.seconds as u64), + difficulty: header + .difficulty + .as_ref() + .map_or_else(|| U256::ZERO, |v| v.into()), + + mix_hash: header.mix_hash.try_decode_proto("mix hash")?, + nonce: header.nonce.into(), + + withdrawals_root: if header.withdrawals_root.is_empty() { + None + } else { + Some( + header + .withdrawals_root + .try_decode_proto("withdrawals root")?, + ) + }, + blob_gas_used: header.blob_gas_used, + excess_blob_gas: header.excess_blob_gas, + parent_beacon_block_root: if header.parent_beacon_root.is_empty() { + None + } else { + Some( + header + .parent_beacon_root + .try_decode_proto("parent beacon root")?, + ) + }, + requests_hash: if header.requests_hash.is_empty() { + None + } else { + Some(header.requests_hash.try_decode_proto("requests hash")?) + }, + block_access_list_hash: None, + slot_number: None, + }; + + let rpc_header = alloy::rpc::types::Header { + hash: block_hash, + inner: consensus_header, + total_difficulty: { + #[allow(deprecated)] + let total_difficulty = &header.total_difficulty; + total_difficulty.as_ref().map(|v| v.into()) + }, + size: Some(U256::from(self.size)), + }; + + let transactions = self + .transaction_traces + .iter() + .map(|t| TransactionTraceAt::new(t, self).try_into()) + .collect::>, Error>>()?; + + let uncles = self + .uncles + .iter() + .map(|u| u.hash.try_decode_proto("uncle hash")) + .collect::, _>>()?; + + use alloy::rpc::types::Block; + + let any_header: AnyRpcHeader = rpc_header.map(AnyHeader::from); + + Ok(Block { + header: any_header, + transactions: alloy::rpc::types::BlockTransactions::Full(transactions), + uncles, + withdrawals: None, + }) + } +} + impl TryInto for &Block { type Error = Error; fn try_into(self) -> Result { - let header = self - .header - .as_ref() - .expect("block header should always be present from gRPC Firehose"); + let alloy_block: AnyBlock = self.try_into()?; + + let transaction_receipts = self + .transaction_traces + .iter() + .filter_map(|t| transaction_trace_to_alloy_txn_reciept(t, self).transpose()) + .collect::, Error>>()? + .into_iter() + // Transaction receipts will be shared along the code, so we put them into an + // Arc here to avoid excessive cloning. + .map(Arc::new) + .collect(); + #[allow(unreachable_code)] let block = EthereumBlockWithCalls { ethereum_block: EthereumBlock { - block: Arc::new(LightEthereumBlock { - hash: Some(self.hash.try_decode_proto("block hash")?), - number: Some(U64::from(self.number)), - author: header.coinbase.try_decode_proto("author / coinbase")?, - parent_hash: header.parent_hash.try_decode_proto("parent hash")?, - uncles_hash: header.uncle_hash.try_decode_proto("uncle hash")?, - state_root: header.state_root.try_decode_proto("state root")?, - transactions_root: header - .transactions_root - .try_decode_proto("transactions root")?, - receipts_root: header.receipt_root.try_decode_proto("receipt root")?, - gas_used: U256::from(header.gas_used), - gas_limit: U256::from(header.gas_limit), - base_fee_per_gas: Some( - header - .base_fee_per_gas - .as_ref() - .map_or_else(U256::default, |v| v.into()), - ), - extra_data: Bytes::from(header.extra_data.clone()), - logs_bloom: match &header.logs_bloom.len() { - 0 => None, - _ => Some(header.logs_bloom.try_decode_proto("logs bloom")?), - }, - timestamp: header - .timestamp - .as_ref() - .map_or_else(U256::default, |v| U256::from(v.seconds)), - difficulty: header - .difficulty - .as_ref() - .map_or_else(U256::default, |v| v.into()), - total_difficulty: Some( - header - .total_difficulty - .as_ref() - .map_or_else(U256::default, |v| v.into()), - ), - // FIXME (SF): Firehose does not have seal fields, are they really used? Might be required for POA chains only also, I've seen that stuff on xDai (is this important?) - seal_fields: vec![], - uncles: self - .uncles - .iter() - .map(|u| u.hash.try_decode_proto("uncle hash")) - .collect::, _>>()?, - transactions: self - .transaction_traces - .iter() - .map(|t| TransactionTraceAt::new(t, self).try_into()) - .collect::, Error>>()?, - size: Some(U256::from(self.size)), - mix_hash: Some(header.mix_hash.try_decode_proto("mix hash")?), - nonce: Some(H64::from_low_u64_be(header.nonce)), - }), - transaction_receipts: self - .transaction_traces - .iter() - .filter_map(|t| { - t.receipt.as_ref().map(|r| { - Ok(web3::types::TransactionReceipt { - transaction_hash: t.hash.try_decode_proto("transaction hash")?, - transaction_index: U64::from(t.index), - block_hash: Some( - self.hash.try_decode_proto("transaction block hash")?, - ), - block_number: Some(U64::from(self.number)), - cumulative_gas_used: U256::from(r.cumulative_gas_used), - // FIXME (SF): What is the rule here about gas_used being None, when it's 0? - gas_used: Some(U256::from(t.gas_used)), - contract_address: { - match t.calls.len() { - 0 => None, - _ => { - match CallType::from_i32(t.calls[0].call_type) - .ok_or_else(|| { - format_err!( - "invalid call type: {}", - t.calls[0].call_type, - ) - })? { - CallType::Create => { - Some(t.calls[0].address.try_decode_proto( - "transaction contract address", - )?) - } - _ => None, - } - } - } - }, - logs: r - .logs - .iter() - .map(|l| LogAt::new(l, self, t).try_into()) - .collect::, Error>>()?, - status: TransactionTraceStatus::from_i32(t.status) - .ok_or_else(|| { - format_err!( - "invalid transaction trace status: {}", - t.status - ) - })? - .into(), - root: match r.state_root.len() { - 0 => None, // FIXME (SF): should this instead map to [0;32]? - // FIXME (SF): if len < 32, what do we do? - _ => Some( - r.state_root.try_decode_proto("transaction state root")?, - ), - }, - logs_bloom: r - .logs_bloom - .try_decode_proto("transaction logs bloom")?, - from: t.from.try_decode_proto("transaction from")?, - to: Some(t.to.try_decode_proto("transaction to")?), - transaction_type: None, - effective_gas_price: None, - }) - }) - }) - .collect::, Error>>()? - .into_iter() - // Transaction receipts will be shared along the code, so we put them into an - // Arc here to avoid excessive cloning. - .map(Arc::new) - .collect(), + block: Arc::new(LightEthereumBlock::new(alloy_block)), + transaction_receipts, }, // Comment (437a9f17-67cc-478f-80a3-804fe554b227): This Some() will avoid calls in the triggers_in_block // TODO: Refactor in a way that this is no longer needed. @@ -391,12 +619,118 @@ impl TryInto for &Block { } } +fn transaction_trace_to_alloy_txn_reciept( + t: &TransactionTrace, + block: &Block, +) -> Result, Error> { + use alloy::consensus::{Eip658Value, Receipt}; + let r = t.receipt.as_ref(); + + if r.is_none() { + return Ok(None); + } + + let r = r.unwrap(); + + let contract_address = match t.calls.len() { + 0 => None, + _ => { + match CallType::try_from(t.calls[0].call_type).map_err(|_| { + graph::anyhow::anyhow!("invalid call type: {}", t.calls[0].call_type) + })? { + CallType::Create => Some( + t.calls[0] + .address + .try_decode_proto("transaction contract address")?, + ), + _ => None, + } + } + }; + + let state_root = match &r.state_root { + b if b.is_empty() => None, + _ => Some(r.state_root.try_decode_proto("transaction state root")?), + }; + + let status = match TransactionTraceStatus::try_from(t.status) + .map_err(|_| format_err!("invalid transaction trace status: {}", t.status))? + { + TransactionTraceStatus::Unknown => { + return Err(format_err!( + "Transaction trace has UNKNOWN status; datasource is broken" + )); + } + TransactionTraceStatus::Succeeded => true, + TransactionTraceStatus::Failed | TransactionTraceStatus::Reverted => false, + }; + + // [EIP-658]: https://eips.ethereum.org/EIPS/eip-658 + // Before EIP-658, the state root field was used to indicate the status of the transaction. + // After EIP-658, the status field is used to indicate the status of the transaction. + let status = match state_root { + Some(root) => Eip658Value::PostState(root), + None => Eip658Value::Eip658(status), + }; + + let logs: Vec = r + .logs + .iter() + .map(|l| LogAt::new(l, block, t).try_into()) + .collect::, Error>>()?; + + let core_receipt = Receipt { + status, + cumulative_gas_used: r.cumulative_gas_used, + logs, + }; + + let logs_bloom = Bloom::try_from(r.logs_bloom.as_slice())?; + + let receipt_with_bloom = ReceiptWithBloom::new(core_receipt, logs_bloom); + + let tx_type_u64 = u64::try_from(t.r#type).map_err(|_| { + format_err!( + "Invalid transaction type value {} in transaction receipt. Transaction type must be a valid u64.", + t.r#type + ) + })?; + + let any_envelope = AnyReceiptEnvelope { + inner: receipt_with_bloom, + r#type: tx_type_u64 as u8, + }; + + let receipt = alloy_rpc_types::TransactionReceipt { + transaction_hash: t.hash.try_decode_proto("transaction hash")?, + transaction_index: Some(t.index as u64), + block_hash: Some(block.hash.try_decode_proto("transaction block hash")?), + block_number: Some(block.number), + gas_used: t.gas_used, + contract_address, + from: t.from.try_decode_proto("transaction from")?, + to: get_to_address(t)?, + effective_gas_price: t.gas_price.as_ref().map_or(0u128, |x| { + let val: U256 = x.into(); + val.to::() + }), // gas_price already contains effective gas price per protobuf spec + blob_gas_used: r.blob_gas_used, + blob_gas_price: r.blob_gas_price.as_ref().map(|x| { + let val: U256 = x.into(); + val.to::() + }), + inner: any_envelope, + }; + + Ok(Some(receipt)) +} + impl BlockHeader { pub fn parent_ptr(&self) -> Option { match self.parent_hash.len() { 0 => None, _ => Some(BlockPtr::from(( - H256::from_slice(self.parent_hash.as_ref()), + B256::from_slice(self.parent_hash.as_ref()), self.number - 1, ))), } @@ -405,13 +739,13 @@ impl BlockHeader { impl<'a> From<&'a BlockHeader> for BlockPtr { fn from(b: &'a BlockHeader) -> BlockPtr { - BlockPtr::from((H256::from_slice(b.hash.as_ref()), b.number)) + BlockPtr::from((B256::from_slice(b.hash.as_ref()), b.number)) } } impl<'a> From<&'a Block> for BlockPtr { fn from(b: &'a Block) -> BlockPtr { - BlockPtr::from((H256::from_slice(b.hash.as_ref()), b.number)) + BlockPtr::from((B256::from_slice(b.hash.as_ref()), b.number)) } } @@ -449,6 +783,11 @@ impl BlockchainBlock for Block { fn data(&self) -> Result { self.header().to_json() } + + fn timestamp(&self) -> BlockTime { + let ts = self.header().timestamp.as_ref().unwrap(); + BlockTime::since_epoch(ts.seconds, ts.nanos as u32) + } } impl HeaderOnlyBlock { @@ -502,6 +841,38 @@ impl BlockchainBlock for HeaderOnlyBlock { fn data(&self) -> Result { self.header().to_json() } + + fn timestamp(&self) -> blockchain::BlockTime { + let ts = self.header().timestamp.as_ref().unwrap(); + blockchain::BlockTime::since_epoch(ts.seconds, ts.nanos as u32) + } +} + +fn extract_signature_from_trace( + _trace: &TransactionTrace, + _tx_type: TxType, +) -> Result { + use alloy::primitives::{Signature as PrimitiveSignature, U256}; + + // Create a dummy signature with r = 0, s = 0 and even y-parity (false) + let dummy = PrimitiveSignature::new(U256::ZERO, U256::ZERO, false); + + Ok(dummy) +} + +fn get_to_address(trace: &TransactionTrace) -> Result, Error> { + // Try to detect contract creation transactions, which have no 'to' address + let is_contract_creation = trace.to.is_empty() + || trace + .calls + .first() + .is_some_and(|call| CallType::try_from(call.call_type) == Ok(CallType::Create)); + + if is_contract_creation { + Ok(None) + } else { + Ok(Some(trace.to.try_decode_proto("transaction to address")?)) + } } #[cfg(test)] @@ -516,14 +887,19 @@ mod test { #[test] fn ensure_block_serialization() { let now = Utc::now().timestamp(); - let mut block = Block::default(); - let mut header = BlockHeader::default(); - header.timestamp = Some(Timestamp { - seconds: now, - nanos: 0, - }); - block.header = Some(header); + let header = BlockHeader { + timestamp: Some(Timestamp { + seconds: now, + nanos: 0, + }), + ..Default::default() + }; + + let block = Block { + header: Some(header.clone()), + ..Default::default() + }; let str_block = block.data().unwrap().to_string(); @@ -533,4 +909,116 @@ mod test { format!(r#"{{"block":{{"data":null,"timestamp":"{}"}}}}"#, now) ); } + + #[test] + fn test_unknown_transaction_type_conversion() { + use super::TransactionTraceAt; + use crate::codec::TransactionTrace; + use graph::prelude::alloy::network::AnyTxEnvelope; + use graph::prelude::alloy::primitives::B256; + + let header = BlockHeader { + number: 123456, + timestamp: Some(Timestamp { + seconds: 1234567890, + nanos: 0, + }), + ..Default::default() + }; + let block = Block { + header: Some(header), + number: 123456, + hash: vec![0u8; 32], + ..Default::default() + }; + + let trace = TransactionTrace { + r#type: 126, // 0x7e Optimism deposit transaction + hash: vec![1u8; 32], + from: vec![2u8; 20], + to: vec![3u8; 20], + nonce: 42, + gas_limit: 21000, + index: 0, + ..Default::default() + }; + + let trace_at = TransactionTraceAt::new(&trace, &block); + let result: Result< + graph::prelude::alloy::rpc::types::Transaction, + graph::prelude::Error, + > = trace_at.try_into(); + + assert!( + result.is_ok(), + "Should successfully convert unknown transaction type" + ); + + let tx = result.unwrap(); + + match tx.inner.inner() { + AnyTxEnvelope::Unknown(unknown_envelope) => { + assert_eq!(unknown_envelope.inner.ty.0, 126); + assert_eq!(unknown_envelope.hash, B256::from_slice(&trace.hash)); + assert!( + !unknown_envelope.inner.fields.is_empty(), + "OtherFields should contain transaction data" + ); + } + _ => panic!("Expected AnyTxEnvelope::Unknown, got Ethereum variant"), + } + + assert_eq!(tx.block_number, Some(123456)); + assert_eq!(tx.transaction_index, Some(0)); + assert_eq!(tx.block_hash, Some(B256::from_slice(&block.hash))); + } + + #[test] + fn test_unknown_receipt_type_conversion() { + use super::transaction_trace_to_alloy_txn_reciept; + use crate::codec::TransactionTrace; + + let header = BlockHeader { + number: 123456, + ..Default::default() + }; + let block = Block { + header: Some(header), + hash: vec![0u8; 32], + ..Default::default() + }; + + let receipt = super::TransactionReceipt { + cumulative_gas_used: 21000, + logs_bloom: vec![0u8; 256], + ..Default::default() + }; + let trace = TransactionTrace { + r#type: 126, // 0x7e Optimism deposit transaction + hash: vec![1u8; 32], + from: vec![2u8; 20], + to: vec![3u8; 20], + index: 0, + gas_used: 21000, + status: 1, + receipt: Some(receipt), + ..Default::default() + }; + + let result = transaction_trace_to_alloy_txn_reciept(&trace, &block); + + assert!( + result.is_ok(), + "Should successfully convert receipt with unknown transaction type" + ); + + let receipt_opt = result.unwrap(); + assert!(receipt_opt.is_some(), "Receipt should be present"); + + let receipt = receipt_opt.unwrap(); + + assert_eq!(receipt.inner.r#type, 126); + assert_eq!(receipt.gas_used, 21000); + assert_eq!(receipt.transaction_index, Some(0)); + } } diff --git a/chain/ethereum/src/data_source.rs b/chain/ethereum/src/data_source.rs index 13c9f6fe62b..709884496e8 100644 --- a/chain/ethereum/src/data_source.rs +++ b/chain/ethereum/src/data_source.rs @@ -1,37 +1,68 @@ -use anyhow::{anyhow, Error}; -use anyhow::{ensure, Context}; -use graph::blockchain::TriggerWithHandler; -use graph::components::store::StoredDynamicDataSource; -use graph::data_source::CausalityRegion; -use graph::prelude::ethabi::ethereum_types::H160; -use graph::prelude::ethabi::StateMutability; -use graph::prelude::futures03::future::try_join; -use graph::prelude::futures03::stream::FuturesOrdered; -use graph::prelude::{Link, SubgraphManifestValidationError}; -use graph::slog::{o, trace}; +use anyhow::{Context, ensure}; +use anyhow::{Error, anyhow}; +use async_trait::async_trait; +use graph::abi; +use graph::abi::EventExt; +use graph::abi::FunctionExt; +use graph::blockchain::{BlockPtr, TriggerWithHandler}; +use graph::components::ethereum::AnyTransaction; +use graph::components::link_resolver::LinkResolverContext; +use graph::components::metrics::subgraph::SubgraphInstanceMetrics; +use graph::components::store::{EthereumCallCache, StoredDynamicDataSource}; +use graph::components::subgraph::{HostMetrics, InstanceDSTemplateInfo, MappingError}; +use graph::components::trigger_processor::RunnableTriggers; +use graph::data::subgraph::DeploymentHash; +use graph::data_source::common::{ + AbiJson, CallDecls, DeclaredCall, FindMappingABI, MappingABI, UnresolvedCallDecls, + UnresolvedMappingABI, +}; +use graph::data_source::{CausalityRegion, MappingTrigger as MappingTriggerType}; +use graph::env::ENV_VARS; +use graph::futures03::TryStreamExt; +use graph::futures03::future::try_join; +use graph::futures03::stream::FuturesOrdered; +use graph::prelude::alloy::primitives::keccak256; +use graph::prelude::alloy::{ + consensus::{TxEnvelope, TxLegacy}, + network::TransactionResponse, + primitives::{Address, B256, U256}, + rpc::types::Log, +}; +use graph::prelude::{Link, SubgraphManifestValidationError, alloy}; +use graph::slog::{debug, error, o, trace}; +use itertools::Itertools; +use serde::de::Error as ErrorD; +use serde::{Deserialize, Deserializer}; +use std::collections::HashSet; +use std::num::NonZeroU32; use std::str::FromStr; use std::sync::Arc; -use tiny_keccak::{keccak256, Keccak}; +use std::time::{Duration, Instant}; use graph::{ blockchain::{self, Blockchain}, prelude::{ - async_trait, - ethabi::{Address, Contract, Event, Function, LogParam, ParamType, RawLog}, - info, serde_json, warn, - web3::types::{Log, Transaction, H256}, - BlockNumber, CheapClone, DataSourceTemplateInfo, Deserialize, EthereumCall, - LightEthereumBlock, LightEthereumBlockExt, LinkResolver, Logger, TryStreamExt, + BlockNumber, CheapClone, EthereumCall, LightEthereumBlock, LightEthereumBlockExt, + LinkResolver, Logger, serde_json, warn, }, }; -use graph::data::subgraph::{calls_host_fn, DataSourceContext, Source}; +use graph::data::subgraph::{ + DataSourceContext, MIN_SPEC_VERSION, SPEC_VERSION_0_0_8, SPEC_VERSION_1_2_0, Source, + calls_host_fn, +}; +use crate::NodeCapabilities; +use crate::adapter::EthereumAdapter as _; use crate::chain::Chain; +use crate::network::EthereumNetworkAdapters; use crate::trigger::{EthereumBlockTriggerType, EthereumTrigger, MappingTrigger}; // The recommended kind is `ethereum`, `ethereum/contract` is accepted for backwards compatibility. const ETHEREUM_KINDS: &[&str] = &["ethereum/contract", "ethereum"]; +const EVENT_HANDLER_KIND: &str = "event"; +const CALL_HANDLER_KIND: &str = "call"; +const BLOCK_HANDLER_KIND: &str = "block"; /// Runtime representation of a data source. // Note: Not great for memory usage that this needs to be `Clone`, considering how there may be tens @@ -44,6 +75,7 @@ pub struct DataSource { pub manifest_idx: u32, pub address: Option
, pub start_block: BlockNumber, + pub end_block: Option, pub mapping: Mapping, pub context: Arc>, pub creation_block: Option, @@ -51,20 +83,26 @@ pub struct DataSource { } impl blockchain::DataSource for DataSource { - fn from_template_info(info: DataSourceTemplateInfo) -> Result { - let DataSourceTemplateInfo { - template, + fn from_template_info( + info: InstanceDSTemplateInfo, + ds_template: &graph::data_source::DataSourceTemplate, + ) -> Result { + // Note: There clearly is duplication between the data in `ds_template and the `template` + // field here. Both represent a template definition, would be good to unify them. + let InstanceDSTemplateInfo { + template: _, params, context, creation_block, } = info; - let template = template.into_onchain().ok_or(anyhow!( + + let template = ds_template.as_onchain().ok_or(anyhow!( "Cannot create onchain data source from offchain template" ))?; // Obtain the address from the parameters let string = params - .get(0) + .first() .with_context(|| { format!( "Failed to create data source from template `{}`: address parameter is missing", @@ -86,13 +124,14 @@ impl blockchain::DataSource for DataSource { .with_context(|| format!("template `{}`", template.name))?; Ok(DataSource { - kind: template.kind, - network: template.network, - name: template.name, + kind: template.kind.clone(), + network: template.network.clone(), + name: template.name.clone(), manifest_idx: template.manifest_idx, address: Some(address), - start_block: 0, - mapping: template.mapping, + start_block: creation_block, + end_block: None, + mapping: template.mapping.clone(), context: Arc::new(context), creation_block: Some(creation_block), contract_abi, @@ -100,13 +139,47 @@ impl blockchain::DataSource for DataSource { } fn address(&self) -> Option<&[u8]> { - self.address.as_ref().map(|x| x.as_bytes()) + self.address.as_ref().map(|x| x.as_slice()) + } + + fn has_declared_calls(&self) -> bool { + self.mapping + .event_handlers + .iter() + .any(|handler| !handler.calls.decls.is_empty()) + } + + fn handler_kinds(&self) -> HashSet<&str> { + let mut kinds = HashSet::new(); + + let Mapping { + event_handlers, + call_handlers, + block_handlers, + .. + } = &self.mapping; + + if !event_handlers.is_empty() { + kinds.insert(EVENT_HANDLER_KIND); + } + if !call_handlers.is_empty() { + kinds.insert(CALL_HANDLER_KIND); + } + for handler in block_handlers.iter() { + kinds.insert(handler.kind()); + } + + kinds } fn start_block(&self) -> BlockNumber { self.start_block } + fn end_block(&self) -> Option { + self.end_block + } + fn match_and_decode( &self, trigger: &::TriggerData, @@ -146,12 +219,12 @@ impl blockchain::DataSource for DataSource { address, mapping, context, - // The creation block is ignored for detection duplicate data sources. // Contract ABI equality is implicit in `mapping.abis` equality. creation_block: _, contract_abi: _, start_block: _, + end_block: _, } = self; // mapping_request_sender, host_metrics, and (most of) host_exports are operational structs @@ -170,7 +243,7 @@ impl blockchain::DataSource for DataSource { } fn as_stored_dynamic_data_source(&self) -> StoredDynamicDataSource { - let param = self.address.map(|addr| addr.0.into()); + let param = self.address.map(|addr| addr.as_slice().into()); StoredDynamicDataSource { manifest_idx: self.manifest_idx, param, @@ -209,14 +282,15 @@ impl blockchain::DataSource for DataSource { let contract_abi = template.mapping.find_abi(&template.source.abi)?; - let address = param.map(|x| H160::from_slice(&x)); + let address = param.map(|x| Address::from_slice(&x)); Ok(DataSource { kind: template.kind.to_string(), network: template.network.as_ref().map(|s| s.to_string()), name: template.name.clone(), manifest_idx, address, - start_block: 0, + start_block: creation_block.unwrap_or(0), + end_block: None, mapping: template.mapping.clone(), context: Arc::new(context), creation_block, @@ -224,7 +298,7 @@ impl blockchain::DataSource for DataSource { }) } - fn validate(&self) -> Vec { + fn validate(&self, spec_version: &semver::Version) -> Vec { let mut errors = vec![]; if !ETHEREUM_KINDS.contains(&self.kind.as_str()) { @@ -242,23 +316,50 @@ impl blockchain::DataSource for DataSource { errors.push(SubgraphManifestValidationError::SourceAddressRequired.into()); }; - // Validate that there are no more than one of each type of block_handler - let has_too_many_block_handlers = { - let mut non_filtered_block_handler_count = 0; - let mut call_filtered_block_handler_count = 0; - self.mapping - .block_handlers - .iter() - .for_each(|block_handler| { - if block_handler.filter.is_none() { - non_filtered_block_handler_count += 1 - } else { - call_filtered_block_handler_count += 1 - } - }); - non_filtered_block_handler_count > 1 || call_filtered_block_handler_count > 1 - }; - if has_too_many_block_handlers { + // Ensure that there is at most one instance of each type of block handler + // and that a combination of a non-filtered block handler and a filtered block handler is not allowed. + + let mut non_filtered_block_handler_count = 0; + let mut call_filtered_block_handler_count = 0; + let mut polling_filtered_block_handler_count = 0; + let mut initialization_handler_count = 0; + self.mapping + .block_handlers + .iter() + .for_each(|block_handler| { + match block_handler.filter { + None => non_filtered_block_handler_count += 1, + Some(ref filter) => match filter { + BlockHandlerFilter::Call => call_filtered_block_handler_count += 1, + BlockHandlerFilter::Once => initialization_handler_count += 1, + BlockHandlerFilter::Polling { every: _ } => { + polling_filtered_block_handler_count += 1 + } + }, + }; + }); + + let has_non_filtered_block_handler = non_filtered_block_handler_count > 0; + // If there is a non-filtered block handler, we need to check if there are any + // filtered block handlers except for the ones with call filter + // If there are, we do not allow that combination + let has_restricted_filtered_and_non_filtered_combination = has_non_filtered_block_handler + && (polling_filtered_block_handler_count > 0 || initialization_handler_count > 0); + + if has_restricted_filtered_and_non_filtered_combination { + errors.push(anyhow!( + "data source has a combination of filtered and non-filtered block handlers that is not allowed" + )); + } + + // Check the number of handlers for each type + // If there is more than one of any type, we have too many handlers + let has_too_many = non_filtered_block_handler_count > 1 + || call_filtered_block_handler_count > 1 + || initialization_handler_count > 1 + || polling_filtered_block_handler_count > 1; + + if has_too_many { errors.push(anyhow!("data source has duplicated block handlers")); } @@ -276,6 +377,33 @@ impl blockchain::DataSource for DataSource { } } + if spec_version < &SPEC_VERSION_1_2_0 { + for handler in &self.mapping.event_handlers { + if !handler.calls.decls.is_empty() { + errors.push(anyhow!( + "handler {}: declaring eth calls on handlers is only supported for specVersion >= 1.2.0", handler.event + )); + break; + } + } + } + + for handler in &self.mapping.event_handlers { + for call in handler.calls.decls.as_ref() { + match self.mapping.find_abi(&call.expr.abi) { + // TODO: Handle overloaded functions by passing a signature + Ok(abi) => match abi.function(&call.expr.abi, &call.expr.func, None) { + Ok(_) => {} + Err(e) => { + errors.push(e); + } + }, + Err(e) => { + errors.push(e); + } + } + } + } errors } @@ -283,11 +411,69 @@ impl blockchain::DataSource for DataSource { self.mapping.api_version.clone() } + fn min_spec_version(&self) -> semver::Version { + let mut min_version = MIN_SPEC_VERSION; + + for handler in &self.mapping.block_handlers { + match handler.filter { + Some(BlockHandlerFilter::Polling { every: _ }) | Some(BlockHandlerFilter::Once) => { + min_version = std::cmp::max(min_version, SPEC_VERSION_0_0_8); + } + _ => {} + } + } + + for handler in &self.mapping.event_handlers { + if handler.has_additional_topics() { + min_version = std::cmp::max(min_version, SPEC_VERSION_1_2_0); + } + } + + min_version + } + fn runtime(&self) -> Option>> { Some(self.mapping.runtime.cheap_clone()) } } +/// Generic function that creates a mock legacy Transaction from ANY log +fn create_dummy_transaction( + block_number: u64, + block_hash: B256, + transaction_index: Option, + transaction_hash: Option, +) -> Result { + use graph::components::ethereum::AnyTxEnvelope; + use graph::prelude::alloy::{ + consensus::Signed, consensus::transaction::Recovered, primitives::Signature, + rpc::types::Transaction, + }; + + let tx = TxLegacy::default(); + + // Create a dummy signature + let signature = Signature::new(U256::ZERO, U256::ZERO, false); + + let tx_hash = transaction_hash.ok_or(anyhow!("Log has no transaction hash"))?; + let signed_tx = Signed::new_unchecked(tx, signature, tx_hash); + let eth_envelope = TxEnvelope::Legacy(signed_tx); + + // Wrap in AnyTxEnvelope + let any_envelope = AnyTxEnvelope::Ethereum(eth_envelope); + + let recovered = Recovered::new_unchecked(any_envelope, Address::ZERO); + + Ok(Transaction { + inner: recovered, + block_hash: Some(block_hash), + block_number: Some(block_number), + block_timestamp: None, + transaction_index, + effective_gas_price: None, + }) +} + impl DataSource { fn from_manifest( kind: String, @@ -311,6 +497,7 @@ impl DataSource { manifest_idx, address: source.address, start_block: source.start_block, + end_block: source.end_block, mapping, context: Arc::new(context), creation_block, @@ -318,22 +505,16 @@ impl DataSource { }) } - fn handlers_for_log(&self, log: &Log) -> Result, Error> { - // Get signature from the log - let topic0 = log.topics.get(0).context("Ethereum event has no topics")?; - - let handlers = self - .mapping + fn handlers_for_log(&self, log: &alloy::rpc::types::Log) -> Vec { + self.mapping .event_handlers .iter() - .filter(|handler| *topic0 == handler.topic0()) + .filter(|handler| handler.matches(log)) .cloned() - .collect::>(); - - Ok(handlers) + .collect::>() } - fn handler_for_call(&self, call: &EthereumCall) -> Result, Error> { + fn handler_for_call(&self, call: &EthereumCall) -> Result, Error> { // First four bytes of the input for the call are the first four // bytes of hash of the function signature ensure!( @@ -343,60 +524,73 @@ impl DataSource { let target_method_id = &call.input.0[..4]; - Ok(self - .mapping - .call_handlers - .iter() - .find(move |handler| { - let fhash = keccak256(handler.function.as_bytes()); - let actual_method_id = [fhash[0], fhash[1], fhash[2], fhash[3]]; - target_method_id == actual_method_id - }) - .cloned()) + Ok(self.mapping.call_handlers.iter().find(move |handler| { + let fhash = keccak256(handler.function.as_bytes()); + let actual_method_id = [fhash[0], fhash[1], fhash[2], fhash[3]]; + target_method_id == actual_method_id + })) } fn handler_for_block( &self, trigger_type: &EthereumBlockTriggerType, - ) -> Option { + block: BlockNumber, + ) -> Option<&MappingBlockHandler> { match trigger_type { - EthereumBlockTriggerType::Every => self - .mapping - .block_handlers - .iter() - .find(move |handler| handler.filter.is_none()) - .cloned(), + // Start matches only initialization handlers with a `once` filter + EthereumBlockTriggerType::Start => { + self.mapping + .block_handlers + .iter() + .find(move |handler| match handler.filter { + Some(BlockHandlerFilter::Once) => block == self.start_block, + _ => false, + }) + } + // End matches all handlers without a filter or with a `polling` filter + EthereumBlockTriggerType::End => { + self.mapping + .block_handlers + .iter() + .find(move |handler| match handler.filter { + Some(BlockHandlerFilter::Polling { every }) => { + let start_block = self.start_block; + (block - start_block) % every.get() as i32 == 0 + } + None => true, + _ => false, + }) + } EthereumBlockTriggerType::WithCallTo(_address) => self .mapping .block_handlers .iter() - .find(move |handler| handler.filter == Some(BlockHandlerFilter::Call)) - .cloned(), + .find(move |handler| handler.filter == Some(BlockHandlerFilter::Call)), } } - /// Returns the contract event with the given signature, if it exists. A an event from the ABI + /// Returns the contract event with the given signature, if it exists. An event from the ABI /// will be matched if: /// 1. An event signature is equal to `signature`. /// 2. There are no equal matches, but there is exactly one event that equals `signature` if all /// `indexed` modifiers are removed from the parameters. - fn contract_event_with_signature(&self, signature: &str) -> Option<&Event> { + fn contract_event_with_signature(&self, signature: &str) -> Option<&abi::Event> { // Returns an `Event(uint256,address)` signature for an event, without `indexed` hints. - fn ambiguous_event_signature(event: &Event) -> String { + fn ambiguous_event_signature(event: &abi::Event) -> String { format!( "{}({})", event.name, event .inputs .iter() - .map(|input| event_param_type_signature(&input.kind)) + .map(|input| input.selector_type().into_owned()) .collect::>() .join(",") ) } // Returns an `Event(indexed uint256,address)` type signature for an event. - fn event_signature(event: &Event) -> String { + fn event_signature(event: &abi::Event) -> String { format!( "{}({})", event.name, @@ -406,40 +600,13 @@ impl DataSource { .map(|input| format!( "{}{}", if input.indexed { "indexed " } else { "" }, - event_param_type_signature(&input.kind) + input.selector_type() )) .collect::>() .join(",") ) } - // Returns the signature of an event parameter type (e.g. `uint256`). - fn event_param_type_signature(kind: &ParamType) -> String { - use ParamType::*; - - match kind { - Address => "address".into(), - Bytes => "bytes".into(), - Int(size) => format!("int{}", size), - Uint(size) => format!("uint{}", size), - Bool => "bool".into(), - String => "string".into(), - Array(inner) => format!("{}[]", event_param_type_signature(inner)), - FixedBytes(size) => format!("bytes{}", size), - FixedArray(inner, size) => { - format!("{}[{}]", event_param_type_signature(inner), size) - } - Tuple(components) => format!( - "({})", - components - .iter() - .map(event_param_type_signature) - .collect::>() - .join(",") - ), - } - } - self.contract_abi .contract .events() @@ -478,7 +645,9 @@ impl DataSource { }) } - fn contract_function_with_signature(&self, target_signature: &str) -> Option<&Function> { + fn contract_function_with_signature(&self, target_signature: &str) -> Option<&abi::Function> { + use abi::StateMutability; + self.contract_abi .contract .functions() @@ -492,32 +661,25 @@ impl DataSource { let mut arguments = function .inputs .iter() - .map(|input| format!("{}", input.kind)) + .map(|input| input.selector_type().into_owned()) .collect::>() .join(","); // `address,uint256,bool) arguments.push(')'); // `operation(address,uint256,bool)` - let actual_signature = vec![function.name.clone(), arguments].join("("); + let actual_signature = [function.name.clone(), arguments].join("("); target_signature == actual_signature }) } fn matches_trigger_address(&self, trigger: &EthereumTrigger) -> bool { - let ds_address = match self.address { - Some(addr) => addr, - + let Some(ds_address) = self.address else { // 'wildcard' data sources match any trigger address. - None => return true, + return true; }; - let trigger_address = match trigger { - EthereumTrigger::Block(_, EthereumBlockTriggerType::WithCallTo(address)) => address, - EthereumTrigger::Call(call) => &call.to, - EthereumTrigger::Log(log, _) => &log.address, - - // Unfiltered block triggers match any data source address. - EthereumTrigger::Block(_, EthereumBlockTriggerType::Every) => return true, + let Some(trigger_address) = trigger.address() else { + return true; }; ds_address == *trigger_address @@ -541,7 +703,7 @@ impl DataSource { match trigger { EthereumTrigger::Block(_, trigger_type) => { - let handler = match self.handler_for_block(trigger_type) { + let handler = match self.handler_for_block(trigger_type, block.number()) { Some(handler) => handler, None => return Ok(None), }; @@ -549,12 +711,15 @@ impl DataSource { MappingTrigger::Block { block: block.cheap_clone(), }, - handler.handler, + handler.handler.clone(), block.block_ptr(), + block.timestamp(), ))) } - EthereumTrigger::Log(log, receipt) => { - let potential_handlers = self.handlers_for_log(log)?; + EthereumTrigger::Log(log_ref) => { + let log = Arc::new(log_ref.log().clone()); + let receipt = log_ref.receipt(); + let potential_handlers = self.handlers_for_log(&log); // Map event handlers to (event handler, event ABI) pairs; fail if there are // handlers that don't exist in the contract ABI @@ -586,11 +751,7 @@ impl DataSource { .into_iter() .filter_map(|(event_handler, event_abi)| { event_abi - .parse_log(RawLog { - topics: log.topics.clone(), - data: log.data.clone().0, - }) - .map(|log| log.params) + .decode_log(&log) .map_err(|e| { trace!( logger, @@ -627,37 +788,49 @@ impl DataSource { // associated transaction and instead have `transaction_hash == block.hash`, // in which case we pass a dummy transaction to the mappings. // See also ca0edc58-0ec5-4c89-a7dd-2241797f5e50. - let transaction = if log.transaction_hash != block.hash { + // There is another special case in zkSync-era, where the transaction hash in this case would be zero + // See https://docs.zksync.io/zk-stack/concepts/blocks.html#fictive-l2-block-finalizing-the-batch + let transaction = if log.transaction_hash == Some(block.hash()) + || log.transaction_hash == Some(B256::ZERO) + { + create_dummy_transaction( + block.number_u64(), + block.hash(), + log.transaction_index, + log.transaction_hash, + )? + } else { + // This is the general case where the log's transaction hash does not match the block's hash + // and is not a special zero hash, implying a real transaction associated with this log. block - .transaction_for_log(log) + .transaction_for_log(&log) .context("Found no transaction for event")? - } else { - // Infer some fields from the log and fill the rest with zeros. - Transaction { - hash: log.transaction_hash.unwrap(), - block_hash: block.hash, - block_number: block.number, - transaction_index: log.transaction_index, - from: Some(H160::zero()), - ..Transaction::default() - } }; let logging_extras = Arc::new(o! { "signature" => event_handler.event.to_string(), - "address" => format!("{}", &log.address), - "transaction" => format!("{}", &transaction.hash), + "address" => format!("{}", &log.address()), + "transaction" => format!("{}", &transaction.tx_hash()), }); + let handler = event_handler.handler.clone(); + let calls = DeclaredCall::from_log_trigger_with_event( + &self.mapping, + &event_handler.calls, + &log, + ¶ms, + )?; Ok(Some(TriggerWithHandler::::new_with_logging_extras( MappingTrigger::Log { block: block.cheap_clone(), transaction: Arc::new(transaction), - log: log.cheap_clone(), + log, params, - receipt: receipt.clone(), + receipt: receipt.map(|r| r.cheap_clone()), + calls, }, - event_handler.handler, + handler, block.block_ptr(), + block.timestamp(), logging_extras, ))) } @@ -681,20 +854,15 @@ impl DataSource { ) })?; - // Parse the inputs - // - // Take the input for the call, chop off the first 4 bytes, then call - // `function.decode_input` to get a vector of `Token`s. Match the `Token`s - // with the `Param`s in `function.inputs` to create a `Vec`. - let tokens = match function_abi.decode_input(&call.input.0[4..]).with_context( - || { + let values = match function_abi + .abi_decode_input(&call.input.0[4..]) + .with_context(|| { format!( "Generating function inputs for the call {:?} failed, raw input: {}", &function_abi, hex::encode(&call.input.0) ) - }, - ) { + }) { Ok(val) => val, // See also 280b0108-a96e-4738-bb37-60ce11eeb5bf Err(err) => { @@ -704,27 +872,22 @@ impl DataSource { }; ensure!( - tokens.len() == function_abi.inputs.len(), + values.len() == function_abi.inputs.len(), "Number of arguments in call does not match \ number of inputs in function signature." ); - let inputs = tokens + let inputs = values .into_iter() .enumerate() - .map(|(i, token)| LogParam { + .map(|(i, value)| abi::DynSolParam { name: function_abi.inputs[i].name.clone(), - value: token, + value, }) .collect::>(); - // Parse the outputs - // - // Take the output for the call, then call `function.decode_output` to - // get a vector of `Token`s. Match the `Token`s with the `Param`s in - // `function.outputs` to create a `Vec`. - let tokens = function_abi - .decode_output(&call.output.0) + let values = function_abi + .abi_decode_output(&call.output.0) .with_context(|| { format!( "Decoding function outputs for the call {:?} failed, raw output: {}", @@ -734,17 +897,17 @@ impl DataSource { })?; ensure!( - tokens.len() == function_abi.outputs.len(), + values.len() == function_abi.outputs.len(), "Number of parameters in the call output does not match \ number of outputs in the function signature." ); - let outputs = tokens + let outputs = values .into_iter() .enumerate() - .map(|(i, token)| LogParam { + .map(|(i, value)| abi::DynSolParam { name: function_abi.outputs[i].name.clone(), - value: token, + value, }) .collect::>(); @@ -756,7 +919,7 @@ impl DataSource { let logging_extras = Arc::new(o! { "function" => handler.function.to_string(), "to" => format!("{}", &call.to), - "transaction" => format!("{}", &transaction.hash), + "transaction" => format!("{}", &transaction.tx_hash()), }); Ok(Some(TriggerWithHandler::::new_with_logging_extras( MappingTrigger::Call { @@ -766,8 +929,9 @@ impl DataSource { inputs, outputs, }, - handler.handler, + handler.handler.clone(), block.block_ptr(), + block.timestamp(), logging_extras, ))) } @@ -775,6 +939,256 @@ impl DataSource { } } +pub struct DecoderHook { + eth_adapters: Arc, + call_cache: Arc, + eth_call_gas: Option, +} + +impl DecoderHook { + pub fn new( + eth_adapters: Arc, + call_cache: Arc, + eth_call_gas: Option, + ) -> Self { + Self { + eth_adapters, + call_cache, + eth_call_gas, + } + } +} + +impl DecoderHook { + /// Perform a batch of eth_calls, observing the execution time of each + /// call. Returns a list of the call labels for which we received a + /// `None` response, indicating a revert + async fn eth_calls( + &self, + logger: &Logger, + block_ptr: &BlockPtr, + calls_and_metrics: Vec<(Arc, DeclaredCall)>, + ) -> Result, MappingError> { + // This check is not just to speed things up, but is also needed to + // make sure the runner tests don't fail; they don't have declared + // eth calls, but without this check we try to get an eth adapter + // even when there are no calls, which fails in the runner test + // setup + if calls_and_metrics.is_empty() { + return Ok(vec![]); + } + + let start = Instant::now(); + + let (metrics, calls): (Vec<_>, Vec<_>) = calls_and_metrics.into_iter().unzip(); + + let (calls, labels): (Vec<_>, Vec<_>) = calls + .into_iter() + .map(|call| call.as_eth_call(block_ptr.clone(), self.eth_call_gas)) + .unzip(); + + let eth_adapter = self.eth_adapters.call_or_cheapest(Some(&NodeCapabilities { + archive: true, + traces: false, + }))?; + + let call_refs = calls.iter().collect::>(); + let results = eth_adapter + .contract_calls(logger, &call_refs, self.call_cache.cheap_clone()) + .await + .map_err(|e| { + // An error happened, everybody gets charged + let elapsed = start.elapsed().as_secs_f64() / call_refs.len() as f64; + for (metrics, call) in metrics.iter().zip(call_refs) { + metrics.observe_eth_call_execution_time( + elapsed, + &call.contract_name, + &call.function.name, + ); + } + MappingError::from(e) + })?; + + // We don't have time measurements for each call (though that would be nice) + // Use the average time of all calls that we want to observe as the time for + // each call + let to_observe = results + .iter() + .filter(|(_, source)| source.observe()) + .count() as f64; + let elapsed = start.elapsed().as_secs_f64() / to_observe; + + results + .iter() + .zip(metrics) + .zip(calls) + .for_each(|(((_, source), metrics), call)| { + if source.observe() { + metrics.observe_eth_call_execution_time( + elapsed, + &call.contract_name, + &call.function.name, + ); + } + }); + + let labels = results + .iter() + .zip(labels) + .filter_map(|((res, _), label)| if res.is_none() { Some(label) } else { None }) + .map(|s| s.to_string()) + .collect(); + Ok(labels) + } + + fn collect_declared_calls<'a>( + &self, + runnables: &Vec>, + ) -> Vec<(Arc, DeclaredCall)> { + // Extract all hosted triggers from runnables + let all_triggers = runnables + .iter() + .flat_map(|runnable| &runnable.hosted_triggers); + + // Collect calls from both onchain and subgraph triggers + let mut all_calls = Vec::new(); + + for trigger in all_triggers { + let host_metrics = trigger.host.host_metrics(); + + match &trigger.mapping_trigger.trigger { + MappingTriggerType::Onchain(t) => { + if let MappingTrigger::Log { calls, .. } = t { + for call in calls.clone() { + all_calls.push((host_metrics.cheap_clone(), call)); + } + } + } + MappingTriggerType::Subgraph(t) => { + for call in t.calls.clone() { + // Convert subgraph call to the expected DeclaredCall type if needed + // or handle differently based on the types + all_calls.push((host_metrics.cheap_clone(), call)); + } + } + MappingTriggerType::Offchain(_) => {} + } + } + + all_calls + } + + /// Deduplicate calls. Unfortunately, we can't get `DeclaredCall` to + /// implement `Hash` or `Ord` easily, so we can only deduplicate by + /// comparing the whole call not with a `HashSet` or `BTreeSet`. + /// Since that can be inefficient, we don't deduplicate if we have an + /// enormous amount of calls; in that case though, things will likely + /// blow up because of the amount of I/O that many calls cause. + /// Cutting off at 1000 is fairly arbitrary + fn deduplicate_calls( + &self, + calls: Vec<(Arc, DeclaredCall)>, + ) -> Vec<(Arc, DeclaredCall)> { + if calls.len() >= 1000 { + return calls; + } + + let mut uniq_calls = Vec::new(); + for (metrics, call) in calls { + if !uniq_calls.iter().any(|(_, c)| c == &call) { + uniq_calls.push((metrics, call)); + } + } + uniq_calls + } + + /// Log information about failed eth calls. 'Failure' here simply + /// means that the call was reverted; outright errors lead to a real + /// error. For reverted calls, `self.eth_calls` returns the label + /// from the manifest for that call. + /// + /// One reason why declared calls can fail is if they are attached + /// to the wrong handler, or if arguments are specified incorrectly. + /// Calls that revert every once in a while might be ok and what the + /// user intended, but we want to clearly log so that users can spot + /// mistakes in their manifest, which will lead to unnecessary eth + /// calls + fn log_declared_call_results( + logger: &Logger, + failures: &[String], + calls_count: usize, + trigger_count: usize, + elapsed: Duration, + ) { + let fail_count = failures.len(); + + if fail_count > 0 { + let mut counts: Vec<_> = failures.iter().counts().into_iter().collect(); + counts.sort_by_key(|(label, _)| *label); + + let failure_summary = counts + .into_iter() + .map(|(label, count)| { + let times = if count == 1 { "time" } else { "times" }; + format!("{label} ({count} {times})") + }) + .join(", "); + + error!(logger, "Declared calls failed"; + "triggers" => trigger_count, + "calls_count" => calls_count, + "fail_count" => fail_count, + "calls_ms" => elapsed.as_millis(), + "failures" => format!("[{}]", failure_summary) + ); + } else { + debug!(logger, "Declared calls"; + "triggers" => trigger_count, + "calls_count" => calls_count, + "calls_ms" => elapsed.as_millis() + ); + } + } +} + +#[async_trait] +impl blockchain::DecoderHook for DecoderHook { + async fn after_decode<'a>( + &self, + logger: &Logger, + block_ptr: &BlockPtr, + runnables: Vec>, + metrics: &Arc, + ) -> Result>, MappingError> { + if ENV_VARS.mappings.disable_declared_calls { + return Ok(runnables); + } + + let _section = metrics.stopwatch.start_section("declared_ethereum_call"); + + let start = Instant::now(); + // Collect and process declared calls + let calls = self.collect_declared_calls(&runnables); + let deduplicated_calls = self.deduplicate_calls(calls); + + // Execute calls and log results + let calls_count = deduplicated_calls.len(); + let results = self + .eth_calls(logger, block_ptr, deduplicated_calls) + .await?; + + Self::log_declared_call_results( + logger, + &results, + calls_count, + runnables.len(), + start.elapsed(), + ); + + Ok(runnables) + } +} + #[derive(Clone, Debug, Eq, PartialEq, Deserialize)] pub struct UnresolvedDataSource { pub kind: String, @@ -789,9 +1203,11 @@ pub struct UnresolvedDataSource { impl blockchain::UnresolvedDataSource for UnresolvedDataSource { async fn resolve( self, + deployment_hash: &DeploymentHash, resolver: &Arc, logger: &Logger, manifest_idx: u32, + spec_version: &semver::Version, ) -> Result { let UnresolvedDataSource { kind, @@ -802,15 +1218,18 @@ impl blockchain::UnresolvedDataSource for UnresolvedDataSource { context, } = self; - info!(logger, "Resolve data source"; "name" => &name, "source_address" => format_args!("{:?}", source.address), "source_start_block" => source.start_block); - - let mapping = mapping.resolve(resolver, logger).await?; + let mapping = mapping.resolve(deployment_hash, resolver, logger, spec_version).await.with_context(|| { + format!( + "failed to resolve data source {} with source_address {:?} and source_start_block {}", + name, source.address, source.start_block + ) + })?; DataSource::from_manifest(kind, network, name, source, mapping, context, manifest_idx) } } -#[derive(Clone, Debug, Default, Hash, Eq, PartialEq, Deserialize)] +#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize)] pub struct UnresolvedDataSourceTemplate { pub kind: String, pub network: Option, @@ -833,9 +1252,11 @@ pub struct DataSourceTemplate { impl blockchain::UnresolvedDataSourceTemplate for UnresolvedDataSourceTemplate { async fn resolve( self, + deployment_hash: &DeploymentHash, resolver: &Arc, logger: &Logger, manifest_idx: u32, + spec_version: &semver::Version, ) -> Result { let UnresolvedDataSourceTemplate { kind, @@ -845,7 +1266,10 @@ impl blockchain::UnresolvedDataSourceTemplate for UnresolvedDataSourceTem mapping, } = self; - info!(logger, "Resolve data source template"; "name" => &name); + let mapping = mapping + .resolve(deployment_hash, resolver, logger, spec_version) + .await + .with_context(|| format!("failed to resolve data source template {}", name))?; Ok(DataSourceTemplate { kind, @@ -853,7 +1277,7 @@ impl blockchain::UnresolvedDataSourceTemplate for UnresolvedDataSourceTem name, manifest_idx, source, - mapping: mapping.resolve(resolver, logger).await?, + mapping, }) } } @@ -874,9 +1298,13 @@ impl blockchain::DataSourceTemplate for DataSourceTemplate { fn manifest_idx(&self) -> u32 { self.manifest_idx } + + fn kind(&self) -> &str { + &self.kind + } } -#[derive(Clone, Debug, Default, Hash, Eq, PartialEq, Deserialize)] +#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize)] #[serde(rename_all = "camelCase")] pub struct UnresolvedMapping { pub kind: String, @@ -889,7 +1317,7 @@ pub struct UnresolvedMapping { #[serde(default)] pub call_handlers: Vec, #[serde(default)] - pub event_handlers: Vec, + pub event_handlers: Vec, pub file: Link, } @@ -921,8 +1349,10 @@ impl Mapping { .iter() .any(|handler| matches!(handler.filter, Some(BlockHandlerFilter::Call))) } +} - pub fn find_abi(&self, abi_name: &str) -> Result, Error> { +impl FindMappingABI for Mapping { + fn find_abi(&self, abi_name: &str) -> Result, Error> { Ok(self .abis .iter() @@ -935,8 +1365,10 @@ impl Mapping { impl UnresolvedMapping { pub async fn resolve( self, + deployment_hash: &DeploymentHash, resolver: &Arc, logger: &Logger, + spec_version: &semver::Version, ) -> Result { let UnresolvedMapping { kind, @@ -950,88 +1382,92 @@ impl UnresolvedMapping { file: link, } = self; - info!(logger, "Resolve mapping"; "link" => &link.link); - let api_version = semver::Version::parse(&api_version)?; let (abis, runtime) = try_join( // resolve each abi abis.into_iter() .map(|unresolved_abi| async { - Result::<_, Error>::Ok(Arc::new( - unresolved_abi.resolve(resolver, logger).await?, - )) + unresolved_abi + .resolve(deployment_hash, resolver, logger) + .await }) .collect::>() .try_collect::>(), async { - let module_bytes = resolver.cat(logger, &link).await?; + let module_bytes = resolver + .cat(&LinkResolverContext::new(deployment_hash, logger), &link) + .await?; Ok(Arc::new(module_bytes)) }, ) - .await?; + .await + .with_context(|| format!("failed to resolve mapping {}", link.link))?; + + // Resolve event handlers with ABI context + let resolved_event_handlers = event_handlers + .into_iter() + .map(|unresolved_handler| { + // Find the ABI for this event handler + let (_, abi_json) = abis.first().ok_or_else(|| { + anyhow!( + "No ABI found for event '{}' in event handler '{}'", + unresolved_handler.event, + unresolved_handler.handler + ) + })?; + + unresolved_handler.resolve(abi_json, spec_version) + }) + .collect::, anyhow::Error>>()?; + + // Extract just the MappingABIs for the final Mapping struct + let mapping_abis = abis.into_iter().map(|(abi, _)| Arc::new(abi)).collect(); Ok(Mapping { kind, api_version, language, entities, - abis, + abis: mapping_abis, block_handlers: block_handlers.clone(), call_handlers: call_handlers.clone(), - event_handlers: event_handlers.clone(), + event_handlers: resolved_event_handlers, runtime, link, }) } } -#[derive(Clone, Debug, Hash, Eq, PartialEq, Deserialize)] -pub struct UnresolvedMappingABI { - pub name: String, - pub file: Link, -} - -#[derive(Clone, Debug, PartialEq)] -pub struct MappingABI { - pub name: String, - pub contract: Contract, -} - -impl UnresolvedMappingABI { - pub async fn resolve( - self, - resolver: &Arc, - logger: &Logger, - ) -> Result { - info!( - logger, - "Resolve ABI"; - "name" => &self.name, - "link" => &self.file.link - ); - - let contract_bytes = resolver.cat(logger, &self.file).await?; - let contract = Contract::load(&*contract_bytes)?; - Ok(MappingABI { - name: self.name, - contract, - }) - } -} - #[derive(Clone, Debug, Hash, Eq, PartialEq, Deserialize)] pub struct MappingBlockHandler { pub handler: String, pub filter: Option, } +impl MappingBlockHandler { + pub fn kind(&self) -> &str { + match &self.filter { + Some(filter) => match filter { + BlockHandlerFilter::Call => "block_filter_call", + BlockHandlerFilter::Once => "block_filter_once", + BlockHandlerFilter::Polling { .. } => "block_filter_polling", + }, + None => BLOCK_HANDLER_KIND, + } + } +} + #[derive(Clone, Debug, Hash, Eq, PartialEq, Deserialize)] #[serde(tag = "kind", rename_all = "lowercase")] pub enum BlockHandlerFilter { // Call filter will trigger on all blocks where the data source contract // address has been called Call, + // This filter will trigger once at the startBlock + Once, + // This filter will trigger in a recurring interval set by the `every` field. + Polling { every: NonZeroU32 }, } #[derive(Clone, Debug, Hash, Eq, PartialEq, Deserialize)] @@ -1040,34 +1476,121 @@ pub struct MappingCallHandler { pub handler: String, } -#[derive(Clone, Debug, Hash, Eq, PartialEq, Deserialize)] -pub struct MappingEventHandler { +#[derive(Clone, Debug, Eq, PartialEq, Deserialize)] +pub struct UnresolvedMappingEventHandler { pub event: String, - pub topic0: Option, + pub topic0: Option, + #[serde(deserialize_with = "deserialize_b256_vec", default)] + pub topic1: Option>, + #[serde(deserialize_with = "deserialize_b256_vec", default)] + pub topic2: Option>, + #[serde(deserialize_with = "deserialize_b256_vec", default)] + pub topic3: Option>, pub handler: String, #[serde(default)] pub receipt: bool, + #[serde(default)] + pub calls: UnresolvedCallDecls, +} + +impl UnresolvedMappingEventHandler { + pub fn resolve( + self, + abi_json: &AbiJson, + spec_version: &semver::Version, + ) -> Result { + let resolved_calls = self + .calls + .resolve(abi_json, Some(&self.event), spec_version)?; + + Ok(MappingEventHandler { + event: self.event, + topic0: self.topic0, + topic1: self.topic1, + topic2: self.topic2, + topic3: self.topic3, + handler: self.handler, + receipt: self.receipt, + calls: resolved_calls, + }) + } +} + +#[derive(Clone, Debug, Hash, Eq, PartialEq)] +pub struct MappingEventHandler { + pub event: String, + pub topic0: Option, + pub topic1: Option>, + pub topic2: Option>, + pub topic3: Option>, + pub handler: String, + pub receipt: bool, + pub calls: CallDecls, +} + +// Custom deserializer for B256 fields that removes the '0x' prefix before parsing +fn deserialize_b256_vec<'de, D>(deserializer: D) -> Result>, D::Error> +where + D: Deserializer<'de>, +{ + let s: Option> = Option::deserialize(deserializer)?; + + match s { + Some(vec) => { + let mut b256_vec = Vec::new(); + for hex_str in vec { + // Remove '0x' prefix if present + let clean_hex_str = hex_str.trim_start_matches("0x"); + // Ensure the hex string is 64 characters long, after removing '0x' + let padded_hex_str = format!("{:0>64}", clean_hex_str); + // Parse the padded string into H256, handling potential errors + b256_vec.push( + B256::from_str(&padded_hex_str) + .map_err(|e| D::Error::custom(format!("Failed to parse B256: {}", e)))?, + ); + } + Ok(Some(b256_vec)) + } + None => Ok(None), + } } impl MappingEventHandler { - pub fn topic0(&self) -> H256 { + pub fn topic0(&self) -> B256 { self.topic0 - .unwrap_or_else(|| string_to_h256(&self.event.replace("indexed ", ""))) + .unwrap_or_else(|| string_to_b256(&self.event.replace("indexed ", ""))) + } + + pub fn matches(&self, log: &Log) -> bool { + let matches_topic = |index: usize, topic_opt: &Option>| -> bool { + topic_opt.as_ref().is_none_or(|topic_vec| { + log.topics() + .get(index) + .is_some_and(|log_topic| topic_vec.contains(log_topic)) + }) + }; + + if let Some(topic0) = log.topics().first() { + return self.topic0() == *topic0 + && matches_topic(1, &self.topic1) + && matches_topic(2, &self.topic2) + && matches_topic(3, &self.topic3); + } + + // Logs without topic0 should simply be skipped + false + } + + pub fn has_additional_topics(&self) -> bool { + self.topic1.as_ref().is_some_and(|v| !v.is_empty()) + || self.topic2.as_ref().is_some_and(|v| !v.is_empty()) + || self.topic3.as_ref().is_some_and(|v| !v.is_empty()) } } -/// Hashes a string to a H256 hash. -fn string_to_h256(s: &str) -> H256 { - let mut result = [0u8; 32]; - let data = s.replace(' ', "").into_bytes(); - let mut sponge = Keccak::new_keccak256(); - sponge.update(&data); - sponge.finalize(&mut result); - - // This was deprecated but the replacement seems to not be available in the - // version web3 uses. - #[allow(deprecated)] - H256::from_slice(&result) +/// Hashes a string to a B256 hash. +fn string_to_b256(s: &str) -> B256 { + keccak256(s.replace(' ', "").as_bytes()) } #[derive(Clone, Debug, Default, Hash, Eq, PartialEq, Deserialize)] diff --git a/chain/ethereum/src/env.rs b/chain/ethereum/src/env.rs index 46a256cd4e8..b15ee0ee018 100644 --- a/chain/ethereum/src/env.rs +++ b/chain/ethereum/src/env.rs @@ -1,6 +1,6 @@ use envconfig::Envconfig; use graph::env::EnvVarBoolean; -use graph::prelude::{envconfig, lazy_static, BlockNumber}; +use graph::prelude::{BlockNumber, envconfig, lazy_static}; use std::fmt; use std::time::Duration; @@ -13,16 +13,13 @@ lazy_static! { pub struct EnvVars { /// Additional deterministic errors that have not yet been hardcoded. /// - /// Set by the environment variable `GRAPH_GETH_ETH_CALL_ERRORS`, separated + /// Set by the environment variable `GRAPH_RPC_ETH_CALL_ERRORS`, separated /// by `;`. - pub geth_eth_call_errors: Vec, + pub rpc_eth_call_errors: Vec, /// Set by the environment variable `GRAPH_ETH_GET_LOGS_MAX_CONTRACTS`. The /// default value is 2000. pub get_logs_max_contracts: usize, - /// Set by the environment variable `ETHEREUM_REORG_THRESHOLD`. The default - /// value is 250 blocks. - pub reorg_threshold: BlockNumber, /// Set by the environment variable `ETHEREUM_TRACE_STREAM_STEP_SIZE`. The /// default value is 50 blocks. pub trace_stream_step_size: BlockNumber, @@ -36,6 +33,9 @@ pub struct EnvVars { /// Set by the environment variable `ETHEREUM_BLOCK_BATCH_SIZE`. The /// default value is 10 blocks. pub block_batch_size: usize, + /// Set by the environment variable `ETHEREUM_BLOCK_PTR_BATCH_SIZE`. The + /// default value is 10 blocks. + pub block_ptr_batch_size: usize, /// Maximum number of blocks to request in each chunk. /// /// Set by the environment variable `GRAPH_ETHEREUM_MAX_BLOCK_RANGE_SIZE`. @@ -49,6 +49,10 @@ pub struct EnvVars { /// Set by the environment variable `GRAPH_ETHEREUM_JSON_RPC_TIMEOUT` /// (expressed in seconds). The default value is 180s. pub json_rpc_timeout: Duration, + + /// Set by the environment variable `GRAPH_ETHEREUM_BLOCK_RECEIPTS_CHECK_TIMEOUT` + /// (expressed in seconds). The default value is 10s. + pub block_receipts_check_timeout: Duration, /// This is used for requests that will not fail the subgraph if the limit /// is reached, but will simply restart the syncing step, so it can be low. /// This limit guards against scenarios such as requesting a block hash that @@ -83,6 +87,14 @@ pub struct EnvVars { /// Set by the flag `GRAPH_ETHEREUM_GENESIS_BLOCK_NUMBER`. The default value /// is 0. pub genesis_block_number: u64, + /// Set by the flag `GRAPH_ETH_CALL_NO_GAS`. + /// This is a comma separated list of chain ids for which the gas field will not be set + /// when calling `eth_call`. + pub eth_call_no_gas: Vec, + /// Set by the flag `GRAPH_ETHEREUM_FORCE_RPC_FOR_BLOCK_PTRS`. On by default. + /// When enabled, forces the use of RPC instead of Firehose for loading block pointers by numbers. + /// This is used in composable subgraphs. Firehose can be slow for loading block pointers by numbers. + pub force_rpc_for_block_ptrs: bool, } // This does not print any values avoid accidentally leaking any sensitive env vars @@ -102,18 +114,21 @@ impl From for EnvVars { fn from(x: Inner) -> Self { Self { get_logs_max_contracts: x.get_logs_max_contracts, - geth_eth_call_errors: x - .geth_eth_call_errors + rpc_eth_call_errors: x + .rpc_eth_call_errors .split(';') .filter(|s| !s.is_empty()) .map(str::to_string) .collect(), - reorg_threshold: x.reorg_threshold, trace_stream_step_size: x.trace_stream_step_size, max_event_only_range: x.max_event_only_range, block_batch_size: x.block_batch_size, + block_ptr_batch_size: x.block_ptr_batch_size, max_block_range_size: x.max_block_range_size, json_rpc_timeout: Duration::from_secs(x.json_rpc_timeout_in_secs), + block_receipts_check_timeout: Duration::from_secs( + x.block_receipts_check_timeout_in_seccs, + ), request_retries: x.request_retries, block_ingestor_max_concurrent_json_rpc_calls: x .block_ingestor_max_concurrent_json_rpc_calls, @@ -124,6 +139,13 @@ impl From for EnvVars { cleanup_blocks: x.cleanup_blocks.0, target_triggers_per_block_range: x.target_triggers_per_block_range, genesis_block_number: x.genesis_block_number, + eth_call_no_gas: x + .eth_call_no_gas + .split(',') + .filter(|s| !s.is_empty()) + .map(str::to_string) + .collect(), + force_rpc_for_block_ptrs: x.force_rpc_for_block_ptrs.0, } } } @@ -136,24 +158,25 @@ impl Default for EnvVars { #[derive(Clone, Debug, Envconfig)] struct Inner { - #[envconfig(from = "GRAPH_GETH_ETH_CALL_ERRORS", default = "")] - geth_eth_call_errors: String, + #[envconfig(from = "GRAPH_RPC_ETH_CALL_ERRORS", default = "")] + rpc_eth_call_errors: String, #[envconfig(from = "GRAPH_ETH_GET_LOGS_MAX_CONTRACTS", default = "2000")] get_logs_max_contracts: usize, - // JSON-RPC specific. - #[envconfig(from = "ETHEREUM_REORG_THRESHOLD", default = "250")] - reorg_threshold: BlockNumber, #[envconfig(from = "ETHEREUM_TRACE_STREAM_STEP_SIZE", default = "50")] trace_stream_step_size: BlockNumber, #[envconfig(from = "GRAPH_ETHEREUM_MAX_EVENT_ONLY_RANGE", default = "500")] max_event_only_range: BlockNumber, #[envconfig(from = "ETHEREUM_BLOCK_BATCH_SIZE", default = "10")] block_batch_size: usize, + #[envconfig(from = "ETHEREUM_BLOCK_PTR_BATCH_SIZE", default = "100")] + block_ptr_batch_size: usize, #[envconfig(from = "GRAPH_ETHEREUM_MAX_BLOCK_RANGE_SIZE", default = "2000")] max_block_range_size: BlockNumber, #[envconfig(from = "GRAPH_ETHEREUM_JSON_RPC_TIMEOUT", default = "180")] json_rpc_timeout_in_secs: u64, + #[envconfig(from = "GRAPH_ETHEREUM_BLOCK_RECEIPTS_CHECK_TIMEOUT", default = "10")] + block_receipts_check_timeout_in_seccs: u64, #[envconfig(from = "GRAPH_ETHEREUM_REQUEST_RETRIES", default = "10")] request_retries: usize, #[envconfig( @@ -172,4 +195,8 @@ struct Inner { target_triggers_per_block_range: u64, #[envconfig(from = "GRAPH_ETHEREUM_GENESIS_BLOCK_NUMBER", default = "0")] genesis_block_number: u64, + #[envconfig(from = "GRAPH_ETH_CALL_NO_GAS", default = "421613,421614")] + eth_call_no_gas: String, + #[envconfig(from = "GRAPH_ETHEREUM_FORCE_RPC_FOR_BLOCK_PTRS", default = "true")] + force_rpc_for_block_ptrs: EnvVarBoolean, } diff --git a/chain/ethereum/src/ethereum_adapter.rs b/chain/ethereum/src/ethereum_adapter.rs index 384daacdb12..9c85f0ac551 100644 --- a/chain/ethereum/src/ethereum_adapter.rs +++ b/chain/ethereum/src/ethereum_adapter.rs @@ -1,38 +1,56 @@ -use futures::future; -use futures::prelude::*; +use async_trait::async_trait; use futures03::{future::BoxFuture, stream::FuturesUnordered}; -use graph::blockchain::client::ChainClient; +use graph::abi; +use graph::abi::DynSolValueExt; +use graph::abi::FunctionExt; use graph::blockchain::BlockHash; use graph::blockchain::ChainIdentifier; +use graph::blockchain::ExtendedBlockPtr; +use graph::blockchain::client::ChainClient; +use graph::components::ethereum::*; use graph::components::transaction_receipt::LightTransactionReceipt; -use graph::data::subgraph::UnifiedMappingApiVersion; +use graph::data::store::ethereum::call; +use graph::data::store::scalar; use graph::data::subgraph::API_VERSION_0_0_7; -use graph::prelude::ethabi::ParamType; -use graph::prelude::ethabi::Token; -use graph::prelude::tokio::try_join; -use graph::{ - blockchain::{block_stream::BlockWithTriggers, BlockPtr, IngestorError}, - prelude::{ - anyhow::{self, anyhow, bail, ensure, Context}, - async_trait, debug, error, ethabi, - futures03::{self, compat::Future01CompatExt, FutureExt, StreamExt, TryStreamExt}, - hex, info, retry, serde_json as json, stream, tiny_keccak, trace, warn, - web3::{ - self, - types::{ - Address, BlockId, BlockNumber as Web3BlockNumber, Bytes, CallRequest, Filter, - FilterBuilder, Log, Transaction, TransactionReceipt, H256, +use graph::data::subgraph::UnifiedMappingApiVersion; +use graph::data_source::common::ContractCall; +use graph::derive::CheapClone; +use graph::futures01::Future; +use graph::futures01::Stream; +use graph::futures01::stream; +use graph::futures03::future::try_join_all; +use graph::futures03::{ + self, FutureExt, StreamExt, TryFutureExt, TryStreamExt, compat::Future01CompatExt, +}; +use graph::prelude::{ + alloy::{ + self, + network::TransactionResponse, + primitives::{Address, B256}, + providers::{ + Identity, Provider, RootProvider, + ext::TraceApi, + fillers::{ + BlobGasFiller, ChainIdFiller, FillProvider, GasFiller, JoinFill, NonceFiller, }, }, - BlockNumber, ChainStore, CheapClone, DynTryFuture, Error, EthereumCallCache, Logger, - TimeoutError, TryFutureExt, + rpc::types::{ + TransactionInput, TransactionRequest, + trace::{filter::TraceFilter as AlloyTraceFilter, parity::LocalizedTransactionTrace}, + }, + transports::{RpcError, TransportErrorKind}, }, + tokio::try_join, }; +use graph::slog::o; use graph::{ - components::ethereum::*, - prelude::web3::api::Web3, - prelude::web3::transports::Batch, - prelude::web3::types::{Trace, TraceFilter, TraceFilterBuilder, H160}, + blockchain::{BlockPtr, IngestorError, block_stream::BlockWithTriggers}, + prelude::{ + BlockNumber, ChainStore, CheapClone, DynTryFuture, Error, EthereumCallCache, Logger, + TimeoutError, + anyhow::{self, Context, anyhow, bail, ensure}, + debug, error, hex, info, retry, trace, warn, + }, }; use itertools::Itertools; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; @@ -40,180 +58,178 @@ use std::convert::TryFrom; use std::iter::FromIterator; use std::pin::Pin; use std::sync::Arc; -use std::time::Instant; +use std::time::{Duration, Instant}; +use tokio::sync::RwLock; +use tokio::time::timeout; -use crate::adapter::ProviderStatus; -use crate::chain::BlockFinality; use crate::Chain; use crate::NodeCapabilities; +use crate::TriggerFilter; +use crate::adapter::EthGetLogsFilter; +use crate::adapter::EthereumRpcError; +use crate::adapter::ProviderStatus; +use crate::call_helper::interpret_eth_call_error; +use crate::chain::BlockFinality; +use crate::chain::ChainSettings; +use crate::trigger::{LogPosition, LogRef}; use crate::{ + ENV_VARS, adapter::{ - EthGetLogsFilter, EthereumAdapter as EthereumAdapterTrait, EthereumBlockFilter, - EthereumCallFilter, EthereumContractCall, EthereumContractCallError, EthereumLogFilter, - ProviderEthRpcMetrics, SubgraphEthRpcMetrics, + ContractCallError, EthereumAdapter as EthereumAdapterTrait, EthereumBlockFilter, + EthereumCallFilter, EthereumLogFilter, ProviderEthRpcMetrics, SubgraphEthRpcMetrics, }, transport::Transport, trigger::{EthereumBlockTriggerType, EthereumTrigger}, - TriggerFilter, ENV_VARS, }; -#[derive(Debug, Clone)] +type AlloyProvider = FillProvider< + JoinFill< + Identity, + JoinFill>>, + >, + RootProvider, + AnyNetworkBare, +>; + +#[derive(Clone)] pub struct EthereumAdapter { logger: Logger, - url_hostname: Arc, - /// The label for the provider from the configuration provider: String, - web3: Arc>, + alloy: Arc, metrics: Arc, supports_eip_1898: bool, call_only: bool, + supports_block_receipts: Arc>>, + pub(crate) settings: Arc, } -/// Gas limit for `eth_call`. The value of 50_000_000 is a protocol-wide parameter so this -/// should be changed only for debugging purposes and never on an indexer in the network. This -/// value was chosen because it is the Geth default -/// https://github.com/ethereum/go-ethereum/blob/e4b687cf462870538743b3218906940ae590e7fd/eth/ethconfig/config.go#L91. -/// It is not safe to set something higher because Geth will silently override the gas limit -/// with the default. This means that we do not support indexing against a Geth node with -/// `RPCGasCap` set below 50 million. -// See also f0af4ab0-6b7c-4b68-9141-5b79346a5f61. -const ETH_CALL_GAS: u32 = 50_000_000; +impl std::fmt::Debug for EthereumAdapter { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("EthereumAdapter") + .field("logger", &self.logger) + .field("provider", &self.provider) + .field("alloy", &"") + .field("metrics", &self.metrics) + .field("supports_eip_1898", &self.supports_eip_1898) + .field("call_only", &self.call_only) + .field("supports_block_receipts", &self.supports_block_receipts) + .field("settings", &self.settings) + .finish() + } +} impl CheapClone for EthereumAdapter { fn cheap_clone(&self) -> Self { Self { logger: self.logger.clone(), provider: self.provider.clone(), - url_hostname: self.url_hostname.cheap_clone(), - web3: self.web3.cheap_clone(), + alloy: self.alloy.clone(), metrics: self.metrics.cheap_clone(), supports_eip_1898: self.supports_eip_1898, call_only: self.call_only, + supports_block_receipts: self.supports_block_receipts.cheap_clone(), + settings: self.settings.clone(), } } } +/// Logger with provider context. Internal methods should take this type +/// to ensure provider info is always included in logs. +#[derive(Clone, CheapClone)] +struct ProviderLogger(Logger); + +impl ProviderLogger { + fn new(logger: &Logger, provider: &str) -> Self { + Self(Logger::new(logger, o!("provider" => provider.to_string()))) + } +} + +impl std::ops::Deref for ProviderLogger { + type Target = Logger; + fn deref(&self) -> &Logger { + &self.0 + } +} + impl EthereumAdapter { pub fn is_call_only(&self) -> bool { self.call_only } + /// Returns a logger with provider context added + fn provider_logger(&self, logger: &Logger) -> ProviderLogger { + ProviderLogger::new(logger, &self.provider) + } + pub async fn new( logger: Logger, provider: String, - url: &str, transport: Transport, provider_metrics: Arc, supports_eip_1898: bool, call_only: bool, + settings: Arc, ) -> Self { - // Unwrap: The transport was constructed with this url, so it is valid and has a host. - let hostname = graph::url::Url::parse(url) - .unwrap() - .host_str() - .unwrap() - .to_string(); - - let web3 = Arc::new(Web3::new(transport)); - - // Use the client version to check if it is ganache. For compatibility with unit tests, be - // are lenient with errors, defaulting to false. - let is_ganache = web3 - .web3() - .client_version() - .await - .map(|s| s.contains("TestRPC")) - .unwrap_or(false); + let alloy = match &transport { + Transport::RPC(client) => Arc::new( + alloy::providers::ProviderBuilder::<_, _, AnyNetworkBare>::default() + .network::() + .connect_client(client.clone()), + ), + Transport::IPC(ipc_connect) => Arc::new( + alloy::providers::ProviderBuilder::<_, _, AnyNetworkBare>::default() + .network::() + .connect_ipc(ipc_connect.clone()) + .await + .expect("Failed to connect to Ethereum IPC"), + ), + Transport::WS(ws_connect) => Arc::new( + alloy::providers::ProviderBuilder::<_, _, AnyNetworkBare>::default() + .network::() + .connect_ws(ws_connect.clone()) + .await + .expect("Failed to connect to Ethereum WS"), + ), + }; EthereumAdapter { logger, provider, - url_hostname: Arc::new(hostname), - web3, + alloy, metrics: provider_metrics, - supports_eip_1898: supports_eip_1898 && !is_ganache, + supports_eip_1898, call_only, + supports_block_receipts: Arc::new(RwLock::new(None)), + settings, } } async fn traces( self, - logger: Logger, + logger: ProviderLogger, subgraph_metrics: Arc, from: BlockNumber, to: BlockNumber, - addresses: Vec, - ) -> Result, Error> { + addresses: Vec
, + ) -> Result, Error> { assert!(!self.call_only); - let eth = self.clone(); let retry_log_message = format!("trace_filter RPC call for block range: [{}..{}]", from, to); + let eth = self.clone(); + retry(retry_log_message, &logger) - .limit(ENV_VARS.request_retries) - .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) + .redact_log_urls(true) + .limit(self.settings.request_retries) + .timeout_secs(self.settings.json_rpc_timeout.as_secs()) .run(move || { - let trace_filter: TraceFilter = match addresses.len() { - 0 => TraceFilterBuilder::default() - .from_block(from.into()) - .to_block(to.into()) - .build(), - _ => TraceFilterBuilder::default() - .from_block(from.into()) - .to_block(to.into()) - .to_address(addresses.clone()) - .build(), - }; - - let eth = eth.cheap_clone(); - let logger_for_triggers = logger.clone(); - let logger_for_error = logger.clone(); - let start = Instant::now(); + let eth = eth.clone(); + let logger = logger.clone(); let subgraph_metrics = subgraph_metrics.clone(); - let provider_metrics = eth.metrics.clone(); - let provider = self.provider.clone(); - + let addresses = addresses.clone(); async move { - let result = eth - .web3 - .trace() - .filter(trace_filter) + eth.execute_trace_filter_request(logger, subgraph_metrics, from, to, addresses) .await - .map(move |traces| { - if !traces.is_empty() { - if to == from { - debug!( - logger_for_triggers, - "Received {} traces for block {}", - traces.len(), - to - ); - } else { - debug!( - logger_for_triggers, - "Received {} traces for blocks [{}, {}]", - traces.len(), - from, - to - ); - } - } - traces - }) - .map_err(Error::from); - - let elapsed = start.elapsed().as_secs_f64(); - provider_metrics.observe_request(elapsed, "trace_filter", &provider); - subgraph_metrics.observe_request(elapsed, "trace_filter", &provider); - if let Err(e) = &result { - provider_metrics.add_error("trace_filter", &provider); - subgraph_metrics.add_error("trace_filter", &provider); - debug!( - logger_for_error, - "Error querying traces error = {:#} from = {} to = {}", e, from, to - ); - } - result } }) .map_err(move |e| { @@ -229,28 +245,177 @@ impl EthereumAdapter { .await } + async fn execute_trace_filter_request( + &self, + logger: ProviderLogger, + subgraph_metrics: Arc, + from: BlockNumber, + to: BlockNumber, + addresses: Vec
, + ) -> Result, Error> { + let alloy_trace_filter = Self::build_trace_filter(from, to, &addresses); + let start = Instant::now(); + + let result = self.alloy.trace_filter(&alloy_trace_filter).await; + + if let Ok(traces) = &result { + self.log_trace_results(&logger, from, to, traces.len()); + } + + self.record_trace_metrics( + &subgraph_metrics, + start.elapsed().as_secs_f64(), + &result, + from, + to, + &logger, + ); + + result.map_err(Error::from) + } + + fn build_trace_filter( + from: BlockNumber, + to: BlockNumber, + addresses: &[Address], + ) -> AlloyTraceFilter { + let filter = AlloyTraceFilter::default() + .from_block(from as u64) + .to_block(to as u64); + + if !addresses.is_empty() { + filter.to_address(addresses.to_vec()) + } else { + filter + } + } + + fn log_trace_results( + &self, + logger: &ProviderLogger, + from: BlockNumber, + to: BlockNumber, + trace_len: usize, + ) { + if trace_len > 0 { + if to == from { + debug!(logger, "Received {} traces for block {}", trace_len, to); + } else { + debug!( + logger, + "Received {} traces for blocks [{}, {}]", trace_len, from, to + ); + } + } + } + + fn record_trace_metrics( + &self, + subgraph_metrics: &Arc, + elapsed: f64, + result: &Result, RpcError>, + from: BlockNumber, + to: BlockNumber, + logger: &ProviderLogger, + ) { + self.metrics + .observe_request(elapsed, "trace_filter", &self.provider); + subgraph_metrics.observe_request(elapsed, "trace_filter", &self.provider); + + if let Err(e) = result { + self.metrics.add_error("trace_filter", &self.provider); + subgraph_metrics.add_error("trace_filter", &self.provider); + debug!( + logger, + "Error querying traces error = {:#} from = {} to = {}", e, from, to + ); + } + } + + // This is a lazy check for block receipt support. It is only called once and then the result is + // cached. The result is not used for anything critical, so it is fine to be lazy. + async fn check_block_receipt_support_and_update_cache( + &self, + alloy: Arc, + block_hash: B256, + supports_eip_1898: bool, + call_only: bool, + logger: ProviderLogger, + ) -> bool { + // This is the lazy part. If the result is already in `supports_block_receipts`, we don't need + // to check again. + { + let supports_block_receipts = self.supports_block_receipts.read().await; + if let Some(supports_block_receipts) = *supports_block_receipts { + return supports_block_receipts; + } + } + + info!(logger, "Checking eth_getBlockReceipts support"); + let result = timeout( + ENV_VARS.block_receipts_check_timeout, + check_block_receipt_support(alloy, block_hash, supports_eip_1898, call_only), + ) + .await; + + let result = match result { + Ok(Ok(_)) => { + info!(logger, "Provider supports block receipts"); + true + } + Ok(Err(err)) => { + warn!(logger, "Skipping use of block receipts, reason: {}", err); + false + } + Err(_) => { + warn!( + logger, + "Skipping use of block receipts, reason: Timeout after {} seconds", + ENV_VARS.block_receipts_check_timeout.as_secs() + ); + false + } + }; + + // We set the result in `self.supports_block_receipts` so that the next time this function is called, we don't + // need to check again. + let mut supports_block_receipts = self.supports_block_receipts.write().await; + if supports_block_receipts.is_none() { + *supports_block_receipts = Some(result); + } + + result + } + + /// Alloy-exclusive version of logs_with_sigs using alloy types and methods async fn logs_with_sigs( &self, - logger: Logger, + logger: ProviderLogger, subgraph_metrics: Arc, from: BlockNumber, to: BlockNumber, filter: Arc, too_many_logs_fingerprints: &'static [&'static str], - ) -> Result, TimeoutError> { + ) -> Result< + Vec, + TimeoutError>, + > { assert!(!self.call_only); let eth_adapter = self.clone(); let retry_log_message = format!("eth_getLogs RPC call for block range: [{}..{}]", from, to); retry(retry_log_message, &logger) - .when(move |res: &Result<_, web3::error::Error>| match res { - Ok(_) => false, - Err(e) => !too_many_logs_fingerprints - .iter() - .any(|f| e.to_string().contains(f)), - }) - .limit(ENV_VARS.request_retries) - .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) + .redact_log_urls(true) + .when( + move |res: &Result<_, RpcError>| match res { + Ok(_) => false, + Err(e) => !too_many_logs_fingerprints + .iter() + .any(|f| e.to_string().contains(f)), + }, + ) + .limit(self.settings.request_retries) + .timeout_secs(self.settings.json_rpc_timeout.as_secs()) .run(move || { let eth_adapter = eth_adapter.cheap_clone(); let subgraph_metrics = subgraph_metrics.clone(); @@ -261,16 +426,9 @@ impl EthereumAdapter { async move { let start = Instant::now(); - // Create a log filter - let log_filter: Filter = FilterBuilder::default() - .from_block(from.into()) - .to_block(to.into()) - .address(filter.contracts.clone()) - .topics(Some(filter.event_signatures.clone()), None, None, None) - .build(); + let alloy_filter = filter.to_alloy_filter(from, to); - // Request logs from client - let result = eth_adapter.web3.eth().logs(log_filter).boxed().await; + let result = eth_adapter.alloy.get_logs(&alloy_filter).await; let elapsed = start.elapsed().as_secs_f64(); provider_metrics.observe_request(elapsed, "eth_getLogs", &provider); subgraph_metrics.observe_request(elapsed, "eth_getLogs", &provider); @@ -286,12 +444,12 @@ impl EthereumAdapter { fn trace_stream( self, - logger: &Logger, + logger: ProviderLogger, subgraph_metrics: Arc, from: BlockNumber, to: BlockNumber, - addresses: Vec, - ) -> impl Stream + Send { + addresses: Vec
, + ) -> impl futures03::Stream> + Send { if from > to { panic!( "Can not produce a call stream on a backwards block range: from = {}, to = {}", @@ -305,52 +463,57 @@ impl EthereumAdapter { true => 1, }; - let eth = self; - let logger = logger.clone(); - stream::unfold(from, move |start| { - if start > to { - return None; + let ranges: Vec<(BlockNumber, BlockNumber)> = { + let mut ranges = Vec::new(); + let mut start = from; + while start <= to { + let end = (start + step_size - 1).min(to); + ranges.push((start, end)); + start = end + 1; } - let end = (start + step_size - 1).min(to); - let new_start = end + 1; - if start == end { - debug!(logger, "Requesting traces for block {}", start); - } else { - debug!(logger, "Requesting traces for blocks [{}, {}]", start, end); + ranges + }; + + let block_batch_size = self.settings.block_batch_size; + let eth = self; + + futures03::stream::iter(ranges.into_iter().map(move |(start, end)| { + let eth = eth.clone(); + let logger = logger.clone(); + let subgraph_metrics = subgraph_metrics.clone(); + let addresses = addresses.clone(); + + async move { + if start == end { + debug!(logger, "Requesting traces for block {}", start); + } else { + debug!(logger, "Requesting traces for blocks [{}, {}]", start, end); + } + + eth.traces(logger, subgraph_metrics, start, end, addresses) + .await } - Some(futures::future::ok(( - eth.clone() - .traces( - logger.cheap_clone(), - subgraph_metrics.clone(), - start, - end, - addresses.clone(), - ) - .boxed() - .compat(), - new_start, - ))) - }) - .buffered(ENV_VARS.block_batch_size) - .map(stream::iter_ok) - .flatten() + })) + .buffered(block_batch_size) + .map_ok(|traces| futures03::stream::iter(traces.into_iter().map(Ok))) + .try_flatten() } fn log_stream( &self, - logger: Logger, + logger: ProviderLogger, subgraph_metrics: Arc, from: BlockNumber, to: BlockNumber, filter: EthGetLogsFilter, - ) -> DynTryFuture<'static, Vec, Error> { + ) -> DynTryFuture<'static, Vec, Error> { // Codes returned by Ethereum node providers if an eth_getLogs request is too heavy. - // The first one is for Infura when it hits the log limit, the rest for Alchemy timeouts. const TOO_MANY_LOGS_FINGERPRINTS: &[&str] = &[ - "ServerError(-32005)", - "503 Service Unavailable", - "ServerError(-32000)", + "ServerError(-32005)", // Infura + "503 Service Unavailable", // Alchemy + "ServerError(-32000)", // Alchemy + "Try with this block range", // zKSync era + "block range too large", // Monad ]; if from > to { @@ -367,7 +530,7 @@ impl EthereumAdapter { let step = match filter.contracts.is_empty() { // `to - from + 1` blocks will be scanned. false => to - from, - true => (to - from).min(ENV_VARS.max_event_only_range - 1), + true => (to - from).min(self.settings.max_event_only_range - 1), }; // Typically this will loop only once and fetch the entire range in one request. But if the @@ -430,192 +593,257 @@ impl EthereumAdapter { .boxed() } - fn call( + fn block_ptr_to_id(&self, block_ptr: &BlockPtr) -> alloy::rpc::types::BlockId { + if !self.supports_eip_1898 { + alloy::rpc::types::BlockId::number(block_ptr.number as u64) + } else { + alloy::rpc::types::BlockId::hash(block_ptr.hash.as_b256()) + } + } + + async fn code( &self, - logger: Logger, - contract_address: Address, - call_data: Bytes, + logger: ProviderLogger, + address: Address, block_ptr: BlockPtr, - ) -> impl Future + Send { - let web3 = self.web3.clone(); + ) -> Result { + let alloy = self.alloy.clone(); + + let block_id = self.block_ptr_to_id(&block_ptr); + let retry_log_message = format!("eth_getCode RPC call for block {}", block_ptr); - // Ganache does not support calls by block hash. - // See https://github.com/trufflesuite/ganache-cli/issues/973 - let block_id = if !self.supports_eip_1898 { - BlockId::Number(block_ptr.number.into()) - } else { - BlockId::Hash(block_ptr.hash_as_h256()) - }; - let retry_log_message = format!("eth_call RPC call for block {}", block_ptr); retry(retry_log_message, &logger) - .when(|result| match result { - Ok(_) | Err(EthereumContractCallError::Revert(_)) => false, - Err(_) => true, - }) - .limit(ENV_VARS.request_retries) - .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) + .redact_log_urls(true) + .when(|result| result.is_err()) + .limit(self.settings.request_retries) + .timeout_secs(self.settings.json_rpc_timeout.as_secs()) .run(move || { - let call_data = call_data.clone(); - let web3 = web3.cheap_clone(); - + let alloy = alloy.cheap_clone(); async move { - let req = CallRequest { - to: Some(contract_address), - gas: Some(web3::types::U256::from(ETH_CALL_GAS)), - data: Some(call_data.clone()), - from: None, - gas_price: None, - value: None, - access_list: None, - max_fee_per_gas: None, - max_priority_fee_per_gas: None, - transaction_type: None, - }; - let result = web3.eth().call(req, Some(block_id)).boxed().await; - - // Try to check if the call was reverted. The JSON-RPC response for reverts is - // not standardized, so we have ad-hoc checks for each Ethereum client. - - // 0xfe is the "designated bad instruction" of the EVM, and Solidity uses it for - // asserts. - const PARITY_BAD_INSTRUCTION_FE: &str = "Bad instruction fe"; - - // 0xfd is REVERT, but on some contracts, and only on older blocks, - // this happens. Makes sense to consider it a revert as well. - const PARITY_BAD_INSTRUCTION_FD: &str = "Bad instruction fd"; - - const PARITY_BAD_JUMP_PREFIX: &str = "Bad jump"; - const PARITY_STACK_LIMIT_PREFIX: &str = "Out of stack"; - - // See f0af4ab0-6b7c-4b68-9141-5b79346a5f61. - const PARITY_OUT_OF_GAS: &str = "Out of gas"; - - const PARITY_VM_EXECUTION_ERROR: i64 = -32015; - const PARITY_REVERT_PREFIX: &str = "Reverted 0x"; - const XDAI_REVERT: &str = "revert"; - - // Deterministic Geth execution errors. We might need to expand this as - // subgraphs come across other errors. See - // https://github.com/ethereum/go-ethereum/blob/cd57d5cd38ef692de8fbedaa56598b4e9fbfbabc/core/vm/errors.go - const GETH_EXECUTION_ERRORS: &[&str] = &[ - // The "revert" substring covers a few known error messages, including: - // Hardhat: "error: transaction reverted", - // Ganache and Moonbeam: "vm exception while processing transaction: revert", - // Geth: "execution reverted" - // And others. - "revert", - "invalid jump destination", - "invalid opcode", - // Ethereum says 1024 is the stack sizes limit, so this is deterministic. - "stack limit reached 1024", - // See f0af4ab0-6b7c-4b68-9141-5b79346a5f61 for why the gas limit is considered deterministic. - "out of gas", - ]; - - let env_geth_call_errors = ENV_VARS.geth_eth_call_errors.iter(); - let mut geth_execution_errors = GETH_EXECUTION_ERRORS - .iter() - .copied() - .chain(env_geth_call_errors.map(|s| s.as_str())); - - let as_solidity_revert_with_reason = |bytes: &[u8]| { - let solidity_revert_function_selector = - &tiny_keccak::keccak256(b"Error(string)")[..4]; - - match bytes.len() >= 4 && &bytes[..4] == solidity_revert_function_selector { - false => None, - true => ethabi::decode(&[ParamType::String], &bytes[4..]) - .ok() - .and_then(|tokens| tokens[0].clone().into_string()), - } - }; + let result = alloy.get_code_at(address).block_id(block_id).await; + match result { + Ok(code) => Ok(code), + Err(err) => Err(EthereumRpcError::AlloyError(err)), + } + } + }) + .await + .map_err(|e| e.into_inner().unwrap_or(EthereumRpcError::Timeout)) + } + async fn balance( + &self, + logger: ProviderLogger, + address: Address, + block_ptr: BlockPtr, + ) -> Result { + let alloy = self.alloy.clone(); + + let block_id = self.block_ptr_to_id(&block_ptr); + let retry_log_message = format!("eth_getBalance RPC call for block {}", block_ptr); + + retry(retry_log_message, &logger) + .redact_log_urls(true) + .when(|result| result.is_err()) + .limit(self.settings.request_retries) + .timeout_secs(self.settings.json_rpc_timeout.as_secs()) + .run(move || { + let alloy = alloy.cheap_clone(); + async move { + let result = alloy.get_balance(address).block_id(block_id).await; match result { - // A successful response. - Ok(bytes) => Ok(bytes), + Ok(balance) => Ok(balance), + Err(err) => Err(EthereumRpcError::AlloyError(err)), + } + } + }) + .await + .map_err(|e| e.into_inner().unwrap_or(EthereumRpcError::Timeout)) + } - // Check for Geth revert. - Err(web3::Error::Rpc(rpc_error)) - if geth_execution_errors - .any(|e| rpc_error.message.to_lowercase().contains(e)) => - { - Err(EthereumContractCallError::Revert(rpc_error.message)) - } + async fn call( + &self, + logger: ProviderLogger, + call_data: call::Request, + block_ptr: BlockPtr, + gas: Option, + ) -> Result { + let alloy = self.alloy.clone(); - // Check for Parity revert. - Err(web3::Error::Rpc(ref rpc_error)) - if rpc_error.code.code() == PARITY_VM_EXECUTION_ERROR => - { - match rpc_error.data.as_ref().and_then(|d| d.as_str()) { - Some(data) - if data.starts_with(PARITY_REVERT_PREFIX) - || data.starts_with(PARITY_BAD_JUMP_PREFIX) - || data.starts_with(PARITY_STACK_LIMIT_PREFIX) - || data == PARITY_BAD_INSTRUCTION_FE - || data == PARITY_BAD_INSTRUCTION_FD - || data == PARITY_OUT_OF_GAS - || data == XDAI_REVERT => - { - let reason = if data == PARITY_BAD_INSTRUCTION_FE { - PARITY_BAD_INSTRUCTION_FE.to_owned() - } else { - let payload = data.trim_start_matches(PARITY_REVERT_PREFIX); - hex::decode(payload) - .ok() - .and_then(|payload| { - as_solidity_revert_with_reason(&payload) - }) - .unwrap_or("no reason".to_owned()) - }; - Err(EthereumContractCallError::Revert(reason)) - } + let alloy_block_id = self.block_ptr_to_id(&block_ptr); + let retry_log_message = format!("eth_call RPC call for block {}", block_ptr); + retry(retry_log_message, &logger) + .redact_log_urls(true) + .limit(self.settings.request_retries) + .timeout_secs(self.settings.json_rpc_timeout.as_secs()) + .run(move || { + let call_data = call_data.clone(); + let alloy = alloy.cheap_clone(); + let logger = logger.cheap_clone(); + async move { + let mut req = TransactionRequest::default() + .input(TransactionInput::both(alloy::primitives::Bytes::from( + call_data.encoded_call.to_vec(), + ))) + .to(call_data.address); - // The VM execution error was not identified as a revert. - _ => Err(EthereumContractCallError::Web3Error(web3::Error::Rpc( - rpc_error.clone(), - ))), - } - } + if let Some(gas) = gas { + req = req.gas_limit(gas as u64); + } - // The error was not identified as a revert. - Err(err) => Err(EthereumContractCallError::Web3Error(err)), + let result = alloy.call(req.into()).block(alloy_block_id).await; + + match result { + Ok(bytes) => Ok(call::Retval::Value(scalar::Bytes::from(bytes))), + Err(err) => interpret_eth_call_error(&logger, err), } } }) - .map_err(|e| e.into_inner().unwrap_or(EthereumContractCallError::Timeout)) - .boxed() - .compat() + .await + .map_err(|e| e.into_inner().unwrap_or(ContractCallError::Timeout)) } + async fn call_and_cache( + &self, + logger: &ProviderLogger, + call: &ContractCall, + req: call::Request, + cache: Arc, + ) -> Result { + let result = self + .call( + logger.clone(), + req.cheap_clone(), + call.block_ptr.clone(), + call.gas, + ) + .await?; + if let Err(e) = cache + .set_call( + logger, + req.cheap_clone(), + call.block_ptr.cheap_clone(), + result.clone(), + ) + .await + { + error!(logger, "EthereumAdapter: call cache set error"; + "contract_address" => format!("{:?}", req.address), + "error" => e.to_string()); + } + + Ok(req.response(result, call::Source::Rpc)) + } /// Request blocks by hash through JSON-RPC. fn load_blocks_rpc( &self, logger: Logger, - ids: Vec, - ) -> impl Stream, Error = Error> + Send { - let web3 = self.web3.clone(); - - stream::iter_ok::<_, Error>(ids.into_iter().map(move |hash| { - let web3 = web3.clone(); - retry(format!("load block {}", hash), &logger) - .limit(ENV_VARS.request_retries) - .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) - .run(move || { - Box::pin(web3.eth().block_with_txs(BlockId::Hash(hash))) - .compat() - .from_err::() - .and_then(move |block| { - block.map(Arc::new).ok_or_else(|| { - anyhow::anyhow!("Ethereum node did not find block {:?}", hash) - }) + ids: Vec, + ) -> impl futures03::Stream, Error>> + Send { + let alloy = self.alloy.clone(); + let request_retries = self.settings.request_retries; + let json_rpc_timeout_secs = self.settings.json_rpc_timeout.as_secs(); + + futures03::stream::iter(ids.into_iter().map(move |hash| { + let alloy = alloy.clone(); + let logger = logger.clone(); + + async move { + retry(format!("load block {}", hash), &logger) + .redact_log_urls(true) + .limit(request_retries) + .timeout_secs(json_rpc_timeout_secs) + .run(move || { + let alloy = alloy.cheap_clone(); + async move { + alloy + .get_block_by_hash(hash) + .full() + .await + .map_err(Error::from) + .and_then(|block| { + block + .map(|b| Arc::new(LightEthereumBlock::new(b))) + .ok_or_else(|| { + anyhow::anyhow!( + "Ethereum node did not find block {:?}", + hash + ) + }) + }) + } + }) + .await + .map_err(|e| { + e.into_inner().unwrap_or_else(|| { + anyhow::anyhow!("Ethereum node took too long to return block {}", hash) }) - .compat() - }) - .boxed() - .compat() - .from_err() + }) + } })) - .buffered(ENV_VARS.block_batch_size) + .buffered(self.settings.block_batch_size) + } + + /// Request blocks by number through JSON-RPC. + pub fn load_block_ptrs_by_numbers_rpc( + &self, + logger: Logger, + numbers: Vec, + ) -> impl futures03::Stream, Error>> + Send { + let alloy = self.alloy.clone(); + let request_retries = self.settings.request_retries; + let json_rpc_timeout_secs = self.settings.json_rpc_timeout.as_secs(); + + futures03::stream::iter(numbers.into_iter().map(move |number| { + let alloy = alloy.clone(); + let logger = logger.clone(); + + async move { + retry(format!("load block {}", number), &logger) + .redact_log_urls(true) + .limit(request_retries) + .timeout_secs(json_rpc_timeout_secs) + .run(move || { + let alloy = alloy.cheap_clone(); + + async move { + let block_result = alloy + .get_block_by_number(alloy::rpc::types::BlockNumberOrTag::Number( + number as u64, + )) + .await; + + match block_result { + Ok(Some(block)) => { + let ptr = ExtendedBlockPtr::try_from(( + block.header.hash, + i32::try_from(block.header.number).unwrap(), + block.header.parent_hash, + block.header.timestamp, + )) + .map_err(|e| { + anyhow::anyhow!("Failed to convert block: {}", e) + })?; + Ok(Arc::new(ptr)) + } + Ok(None) => Err(anyhow::anyhow!( + "Ethereum node did not find block with number {:?}", + number + )), + Err(e) => Err(anyhow::anyhow!("Failed to fetch block: {}", e)), + } + } + }) + .await + .map_err(|e| match e { + TimeoutError::Elapsed => { + anyhow::anyhow!("Timeout while fetching block {}", number) + } + TimeoutError::Inner(e) => e, + }) + } + })) + .buffered(self.settings.block_ptr_batch_size) } /// Request blocks ptrs for numbers through JSON-RPC. @@ -626,20 +854,23 @@ impl EthereumAdapter { logger: Logger, block_nums: Vec, ) -> impl Stream + Send { - let web3 = self.web3.clone(); + let alloy = self.alloy.clone(); + let json_rpc_timeout_secs = self.settings.json_rpc_timeout.as_secs(); stream::iter_ok::<_, Error>(block_nums.into_iter().map(move |block_num| { - let web3 = web3.clone(); + let alloy = alloy.clone(); retry(format!("load block ptr {}", block_num), &logger) + .redact_log_urls(true) + .when(|res| !res.is_ok() && !detect_null_block(res)) .no_limit() - .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) + .timeout_secs(json_rpc_timeout_secs) .run(move || { - let web3 = web3.clone(); + let alloy = alloy.cheap_clone(); async move { - let block = web3 - .eth() - .block(BlockId::Number(Web3BlockNumber::Number(block_num.into()))) - .boxed() + let block = alloy + .get_block_by_number(alloy::rpc::types::BlockNumberOrTag::Number( + block_num as u64, + )) .await?; block.ok_or_else(|| { @@ -650,9 +881,17 @@ impl EthereumAdapter { .boxed() .compat() .from_err() + .then(|res| { + if detect_null_block(&res) { + Ok(None) + } else { + Some(res).transpose() + } + }) })) - .buffered(ENV_VARS.block_batch_size) - .map(|b| b.into()) + .buffered(self.settings.block_batch_size) + .filter_map(|b| b) + .map(|b| BlockPtr::from((b.header.hash, b.header.number))) } /// Check if `block_ptr` refers to a block that is on the main chain, according to the Ethereum @@ -670,13 +909,12 @@ impl EthereumAdapter { logger: &Logger, block_ptr: BlockPtr, ) -> Result { - let block_hash = self - .block_hash_by_block_number(logger, block_ptr.number) - .compat() + // TODO: This considers null blocks, but we could instead bail if we encounter one as a + // small optimization. + let canonical_block = self + .next_existing_ptr_to_number(logger, block_ptr.number) .await?; - block_hash - .ok_or_else(|| anyhow!("Ethereum node is missing block #{}", block_ptr.number)) - .map(|block_hash| block_hash == block_ptr.hash_as_h256()) + Ok(canonical_block == block_ptr) } pub(crate) fn logs_in_block_range( @@ -686,21 +924,26 @@ impl EthereumAdapter { from: BlockNumber, to: BlockNumber, log_filter: EthereumLogFilter, - ) -> DynTryFuture<'static, Vec, Error> { + ) -> DynTryFuture<'static, Vec, Error> { let eth: Self = self.cheap_clone(); - let logger = logger.clone(); - - futures03::stream::iter(log_filter.eth_get_logs_filters().map(move |filter| { - eth.cheap_clone().log_stream( - logger.cheap_clone(), - subgraph_metrics.cheap_clone(), - from, - to, - filter, - ) - })) + let logger = self.provider_logger(logger); + + let max_contracts = eth.settings.get_logs_max_contracts; + futures03::stream::iter( + log_filter + .eth_get_logs_filters(max_contracts) + .map(move |filter| { + eth.cheap_clone().log_stream( + logger.cheap_clone(), + subgraph_metrics.cheap_clone(), + from, + to, + filter, + ) + }), + ) // Real limits on the number of parallel requests are imposed within the adapter. - .buffered(ENV_VARS.block_ingestor_max_concurrent_json_rpc_calls) + .buffered(self.settings.block_ingestor_max_concurrent_json_rpc_calls) .try_concat() .boxed() } @@ -714,19 +957,20 @@ impl EthereumAdapter { call_filter: &'a EthereumCallFilter, ) -> Box + Send + 'a> { let eth = self.clone(); + let logger = self.provider_logger(logger); let EthereumCallFilter { contract_addresses_function_signatures, wildcard_signatures, } = call_filter; - let mut addresses: Vec = contract_addresses_function_signatures + let mut addresses: Vec
= contract_addresses_function_signatures .iter() .filter(|(_addr, (start_block, _fsigs))| start_block <= &to) .map(|(addr, (_start_block, _fsigs))| *addr) - .collect::>() + .collect::>() .into_iter() - .collect::>(); + .collect::>(); if addresses.is_empty() && wildcard_signatures.is_empty() { // The filter has no started data sources in the requested range, nothing to do. @@ -743,27 +987,81 @@ impl EthereumAdapter { Box::new( eth.trace_stream(logger, subgraph_metrics, from, to, addresses) - .filter_map(|trace| EthereumCall::try_from_trace(&trace)) - .filter(move |call| { - // `trace_filter` can only filter by calls `to` an address and - // a block range. Since subgraphs are subscribing to calls - // for a specific contract function an additional filter needs - // to be applied - call_filter.matches(call) - }), + .try_filter_map(move |trace| { + let maybe_call = EthereumCall::try_from_trace(&trace) + .filter(|call| call_filter.matches(call)); + futures03::future::ready(Ok(maybe_call)) + }) + .boxed() + .compat(), ) } + // Used to get the block triggers with a `polling` or `once` filter + /// `polling_filter_type` is used to differentiate between `polling` and `once` filters + /// A `polling_filter_type` value of `BlockPollingFilterType::Once` is the case for + /// intialization triggers + /// A `polling_filter_type` value of `BlockPollingFilterType::Polling` is the case for + /// polling triggers + pub(crate) fn blocks_matching_polling_intervals( + &self, + logger: Logger, + from: i32, + to: i32, + filter: &EthereumBlockFilter, + ) -> Pin< + Box< + dyn std::future::Future, anyhow::Error>> + + std::marker::Send + + '_, + >, + > { + // Create a HashMap of block numbers to Vec. + let matching_blocks = (from..=to) + .filter_map(|block_number| { + let triggers = + block_trigger_types_from_intervals(block_number, &filter.polling_intervals); + if triggers.is_empty() { + None + } else { + Some((block_number, triggers)) + } + }) + .collect::>(); + + let blocks_matching_polling_filter = self.load_ptrs_for_blocks( + logger.clone(), + matching_blocks.keys().cloned().collect_vec(), + ); + + let block_futures = blocks_matching_polling_filter.map(move |ptrs| { + ptrs.into_iter() + .flat_map(|ptr| { + matching_blocks + .get(&ptr.number) + // Safe to unwrap since we are iterating over ptrs which was created from + // the keys of matching_blocks + .unwrap() + .iter() + .map(move |trigger| EthereumTrigger::Block(ptr.clone(), trigger.clone())) + }) + .collect::>() + }); + + block_futures.compat().boxed() + } + pub(crate) async fn calls_in_block( &self, logger: &Logger, subgraph_metrics: Arc, block_number: BlockNumber, - block_hash: H256, + block_hash: alloy::primitives::B256, ) -> Result, Error> { let eth = self.clone(); + let logger = self.provider_logger(logger); let addresses = Vec::new(); - let traces = eth + let traces: Vec = eth .trace_stream( logger, subgraph_metrics.clone(), @@ -771,8 +1069,7 @@ impl EthereumAdapter { block_number, addresses, ) - .collect() - .compat() + .try_collect() .await?; // `trace_stream` returns all of the traces for the block, and this @@ -790,7 +1087,7 @@ impl EthereumAdapter { // all the traces for the block, we need to ensure that the // block hash for the traces is equal to the desired block hash. // Assume all traces are for the same block. - if traces.iter().nth(0).unwrap().block_hash != block_hash { + if traces.first().unwrap().block_hash != Some(block_hash) { return Err(anyhow!( "Trace stream returned traces for an unexpected block: \ number = `{}`, hash = `{}`", @@ -811,7 +1108,7 @@ impl EthereumAdapter { logger: Logger, from: BlockNumber, to: BlockNumber, - ) -> Box, Error = Error> + Send> { + ) -> Box, Error = Error> + Send + '_> { // Currently we can't go to the DB for this because there might be duplicate entries for // the same block number. debug!(&logger, "Requesting hashes for blocks [{}, {}]", from, to); @@ -821,29 +1118,45 @@ impl EthereumAdapter { ) } + pub(crate) fn load_ptrs_for_blocks( + &self, + logger: Logger, + blocks: Vec, + ) -> Box, Error = Error> + Send + '_> { + // Currently we can't go to the DB for this because there might be duplicate entries for + // the same block number. + debug!(&logger, "Requesting hashes for blocks {:?}", blocks); + Box::new(self.load_block_ptrs_rpc(logger, blocks).collect()) + } + pub async fn chain_id(&self) -> Result { let logger = self.logger.clone(); - let web3 = self.web3.clone(); - u64::try_from( - retry("chain_id RPC call", &logger) - .no_limit() - .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) - .run(move || { - let web3 = web3.cheap_clone(); - async move { web3.eth().chain_id().await } - }) - .await?, - ) - .map_err(Error::msg) + let alloy = self.alloy.clone(); + retry("chain_id RPC call", &logger) + .redact_log_urls(true) + .no_limit() + .timeout_secs(self.settings.json_rpc_timeout.as_secs()) + .run(move || { + let alloy = alloy.cheap_clone(); + async move { alloy.get_chain_id().await.map_err(Error::from) } + }) + .await + .map_err(|e| e.into_inner().unwrap_or(EthereumRpcError::Timeout.into())) } } -#[async_trait] -impl EthereumAdapterTrait for EthereumAdapter { - fn url_hostname(&self) -> &str { - &self.url_hostname +// Detects null blocks as can occur on Filecoin EVM chains, by checking for the FEVM-specific +// error returned when requesting such a null round. Ideally there should be a defined reponse or +// message for this case, or a check that is less dependent on the Filecoin implementation. +fn detect_null_block(res: &Result) -> bool { + match res { + Ok(_) => false, + Err(e) => e.to_string().contains("requested epoch was a null round"), } +} +#[async_trait] +impl EthereumAdapterTrait for EthereumAdapter { fn provider(&self) -> &str { &self.provider } @@ -851,18 +1164,19 @@ impl EthereumAdapterTrait for EthereumAdapter { async fn net_identifiers(&self) -> Result { let logger = self.logger.clone(); - let web3 = self.web3.clone(); + let alloy = self.alloy.clone(); let metrics = self.metrics.clone(); let provider = self.provider().to_string(); let net_version_future = retry("net_version RPC call", &logger) + .redact_log_urls(true) .no_limit() .timeout_secs(20) .run(move || { - let web3 = web3.cheap_clone(); + let alloy = alloy.cheap_clone(); let metrics = metrics.cheap_clone(); let provider = provider.clone(); async move { - web3.net().version().await.map_err(|e| { + alloy.get_net_version().await.map_err(|e| { metrics.set_status(ProviderStatus::VersionFail, &provider); e.into() }) @@ -875,31 +1189,32 @@ impl EthereumAdapterTrait for EthereumAdapter { }) .boxed(); - let web3 = self.web3.clone(); + let alloy_provider = self.alloy.clone(); let metrics = self.metrics.clone(); let provider = self.provider().to_string(); + let genesis_block_number = self.settings.genesis_block_number; let retry_log_message = format!( "eth_getBlockByNumber({}, false) RPC call", - ENV_VARS.genesis_block_number + genesis_block_number ); let gen_block_hash_future = retry(retry_log_message, &logger) + .redact_log_urls(true) .no_limit() .timeout_secs(30) .run(move || { - let web3 = web3.cheap_clone(); + let alloy_genesis = alloy_provider.cheap_clone(); let metrics = metrics.cheap_clone(); let provider = provider.clone(); async move { - web3.eth() - .block(BlockId::Number(Web3BlockNumber::Number( - ENV_VARS.genesis_block_number.into(), - ))) + alloy_genesis + .get_block_by_number(alloy::rpc::types::BlockNumberOrTag::Number( + genesis_block_number, + )) .await - .map_err(|e| { + .inspect_err(|_| { metrics.set_status(ProviderStatus::GenesisFail, &provider); - e })? - .and_then(|gen_block| gen_block.hash.map(BlockHash::from)) + .map(|gen_block| BlockHash::from(gen_block.header.hash)) .ok_or_else(|| anyhow!("Ethereum node could not find genesis block")) } }) @@ -918,7 +1233,7 @@ impl EthereumAdapterTrait for EthereumAdapter { })?; let ident = ChainIdentifier { - net_version, + net_version: net_version.to_string(), genesis_block_hash, }; @@ -927,407 +1242,466 @@ impl EthereumAdapterTrait for EthereumAdapter { Ok(ident) } - fn latest_block_header( - &self, - logger: &Logger, - ) -> Box, Error = IngestorError> + Send> { - let web3 = self.web3.clone(); - Box::new( - retry("eth_getBlockByNumber(latest) no txs RPC call", logger) - .no_limit() - .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) - .run(move || { - let web3 = web3.cheap_clone(); - async move { - let block_opt = web3 - .eth() - .block(Web3BlockNumber::Latest.into()) - .await - .map_err(|e| { - anyhow!("could not get latest block from Ethereum: {}", e) - })?; + async fn latest_block_ptr(&self, logger: &Logger) -> Result { + let alloy = self.alloy.clone(); + retry("eth_getBlockByNumber(latest) no txs RPC call", logger) + .redact_log_urls(true) + .limit(ENV_VARS.request_retries) + .timeout_secs(self.settings.json_rpc_timeout.as_secs()) + .run(move || { + let alloy = alloy.cheap_clone(); + async move { + let block_opt = alloy + .get_block_by_number(alloy::rpc::types::BlockNumberOrTag::Latest) + .await + .map_err(|e| anyhow!("could not get latest block from Ethereum: {}", e))?; - block_opt - .ok_or_else(|| anyhow!("no latest block returned from Ethereum").into()) - } - }) - .map_err(move |e| { - e.into_inner().unwrap_or_else(move || { - anyhow!("Ethereum node took too long to return latest block").into() - }) - }) - .boxed() - .compat(), - ) - } + let block = block_opt + .ok_or_else(|| anyhow!("no latest block returned from Ethereum"))?; - fn latest_block( - &self, - logger: &Logger, - ) -> Box + Send + Unpin> { - let web3 = self.web3.clone(); - Box::new( - retry("eth_getBlockByNumber(latest) with txs RPC call", logger) - .no_limit() - .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) - .run(move || { - let web3 = web3.cheap_clone(); - async move { - let block_opt = web3 - .eth() - .block_with_txs(Web3BlockNumber::Latest.into()) - .await - .map_err(|e| { - anyhow!("could not get latest block from Ethereum: {}", e) - })?; - block_opt - .ok_or_else(|| anyhow!("no latest block returned from Ethereum").into()) - } - }) - .map_err(move |e| { - e.into_inner().unwrap_or_else(move || { - anyhow!("Ethereum node took too long to return latest block").into() - }) + Ok(BlockPtr::from((block.header.hash, block.header.number))) + } + }) + .map_err(move |e| { + e.into_inner().unwrap_or_else(move || { + anyhow!("Ethereum node took too long to return latest block").into() }) - .boxed() - .compat(), - ) + }) + .await } - fn load_block( - &self, - logger: &Logger, - block_hash: H256, - ) -> Box + Send> { - Box::new( - self.block_by_hash(logger, block_hash) - .and_then(move |block_opt| { - block_opt.ok_or_else(move || { - anyhow!( - "Ethereum node could not find block with hash {}", - block_hash - ) - }) - }), - ) + async fn is_reachable(&self) -> bool { + let alloy = self.alloy.clone(); + tokio::time::timeout(Duration::from_secs(10), alloy.get_block_number()) + .await + .map(|r| r.is_ok()) + .unwrap_or(false) } - fn block_by_hash( + async fn block_by_hash( &self, logger: &Logger, - block_hash: H256, - ) -> Box, Error = Error> + Send> { - let web3 = self.web3.clone(); + block_hash: B256, + ) -> Result, Error> { + let alloy = self.alloy.clone(); let logger = logger.clone(); let retry_log_message = format!( "eth_getBlockByHash RPC call for block hash {:?}", block_hash ); - Box::new( - retry(retry_log_message, &logger) - .limit(ENV_VARS.request_retries) - .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) - .run(move || { - Box::pin(web3.eth().block_with_txs(BlockId::Hash(block_hash))) - .compat() - .from_err() - .compat() - }) - .map_err(move |e| { - e.into_inner().unwrap_or_else(move || { - anyhow!("Ethereum node took too long to return block {}", block_hash) - }) + + retry(retry_log_message, &logger) + .redact_log_urls(true) + .limit(self.settings.request_retries) + .timeout_secs(self.settings.json_rpc_timeout.as_secs()) + .run(move || { + let alloy = alloy.cheap_clone(); + async move { + alloy + .get_block_by_hash(block_hash) + .full() + .await + .map_err(Error::from) + } + }) + .map_err(move |e| { + e.into_inner().unwrap_or_else(move || { + anyhow!("Ethereum node took too long to return block {}", block_hash) }) - .boxed() - .compat(), - ) + }) + .await } - fn block_by_number( + async fn block_by_number( &self, logger: &Logger, block_number: BlockNumber, - ) -> Box, Error = Error> + Send> { - let web3 = self.web3.clone(); + ) -> Result, Error> { + let alloy = self.alloy.clone(); let logger = logger.clone(); let retry_log_message = format!( "eth_getBlockByNumber RPC call for block number {}", block_number ); - Box::new( - retry(retry_log_message, &logger) - .no_limit() - .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) - .run(move || { - let web3 = web3.cheap_clone(); - async move { - web3.eth() - .block_with_txs(BlockId::Number(block_number.into())) - .await - .map_err(Error::from) - } - }) - .map_err(move |e| { - e.into_inner().unwrap_or_else(move || { - anyhow!( - "Ethereum node took too long to return block {}", - block_number - ) - }) + retry(retry_log_message, &logger) + .redact_log_urls(true) + .no_limit() + .timeout_secs(self.settings.json_rpc_timeout.as_secs()) + .run(move || { + let alloy = alloy.clone(); + async move { + alloy + .get_block_by_number(alloy::rpc::types::BlockNumberOrTag::Number( + block_number as u64, + )) + .full() + .await + .map_err(Error::from) + } + }) + .map_err(move |e| { + e.into_inner().unwrap_or_else(move || { + anyhow!( + "Ethereum node took too long to return block {}", + block_number + ) }) - .boxed() - .compat(), - ) + }) + .await } - fn load_full_block( + async fn load_full_block( &self, logger: &Logger, - block: LightEthereumBlock, - ) -> Pin> + Send>> - { - let web3 = Arc::clone(&self.web3); - let logger = logger.clone(); - let block_hash = block.hash.expect("block is missing block hash"); + block: AnyBlock, + ) -> Result { + let alloy = self.alloy.clone(); + let logger = self.provider_logger(logger); + let block_hash = block.header.hash; // The early return is necessary for correctness, otherwise we'll // request an empty batch which is not valid in JSON-RPC. if block.transactions.is_empty() { trace!(logger, "Block {} contains no transactions", block_hash); - return Box::pin(std::future::ready(Ok(EthereumBlock { - block: Arc::new(block), + return Ok(EthereumBlock { + block: Arc::new(LightEthereumBlock::new(block)), transaction_receipts: Vec::new(), - }))); + }); } - let hashes: Vec<_> = block.transactions.iter().map(|txn| txn.hash).collect(); - let receipts_future = if ENV_VARS.fetch_receipts_in_batches { - // Deprecated batching retrieval of transaction receipts. - fetch_transaction_receipts_in_batch_with_retry(web3, hashes, block_hash, logger).boxed() - } else { - let hash_stream = graph::tokio_stream::iter(hashes); - let receipt_stream = graph::tokio_stream::StreamExt::map(hash_stream, move |tx_hash| { - fetch_transaction_receipt_with_retry( - web3.cheap_clone(), - tx_hash, - block_hash, - logger.cheap_clone(), - ) - }) - .buffered(ENV_VARS.block_ingestor_max_concurrent_json_rpc_calls); - graph::tokio_stream::StreamExt::collect::< - Result>, IngestorError>, - >(receipt_stream) - .boxed() - }; + let hashes: Vec<_> = block.transactions.hashes().collect(); - let block_future = - futures03::TryFutureExt::map_ok(receipts_future, move |transaction_receipts| { - EthereumBlock { - block: Arc::new(block), - transaction_receipts, - } - }); + let supports_block_receipts = self + .check_block_receipt_support_and_update_cache( + alloy.clone(), + block_hash, + self.supports_eip_1898, + self.call_only, + logger.clone(), + ) + .await; - Box::pin(block_future) + fetch_receipts_with_retry( + alloy, + hashes, + block_hash, + logger, + supports_block_receipts, + &self.settings, + ) + .await + .map(|transaction_receipts| EthereumBlock { + block: Arc::new(LightEthereumBlock::new(block)), + transaction_receipts, + }) } - fn block_pointer_from_number( + async fn get_balance( &self, logger: &Logger, - block_number: BlockNumber, - ) -> Box + Send> { - Box::new( - self.block_hash_by_block_number(logger, block_number) - .and_then(move |block_hash_opt| { - block_hash_opt.ok_or_else(|| { - anyhow!( - "Ethereum node could not find start block hash by block number {}", - &block_number - ) - }) - }) - .from_err() - .map(move |block_hash| BlockPtr::from((block_hash, block_number))), - ) + address: Address, + block_ptr: BlockPtr, + ) -> Result { + let logger = self.provider_logger(logger); + debug!( + logger, "eth_getBalance"; + "address" => format!("{}", address), + "block" => format!("{}", block_ptr) + ); + self.balance(logger, address, block_ptr).await } - fn block_hash_by_block_number( + async fn get_code( &self, logger: &Logger, - block_number: BlockNumber, - ) -> Box, Error = Error> + Send> { - let web3 = self.web3.clone(); - let retry_log_message = format!( - "eth_getBlockByNumber RPC call for block number {}", - block_number + address: Address, + block_ptr: BlockPtr, + ) -> Result { + let logger = self.provider_logger(logger); + debug!( + logger, "eth_getCode"; + "address" => format!("{}", address), + "block" => format!("{}", block_ptr) ); - Box::new( - retry(retry_log_message, logger) + self.code(logger, address, block_ptr).await + } + + async fn next_existing_ptr_to_number( + &self, + logger: &Logger, + block_number: BlockNumber, + ) -> Result { + let mut next_number = block_number; + loop { + let retry_log_message = format!( + "eth_getBlockByNumber RPC call for block number {}", + next_number + ); + let alloy = self.alloy.clone(); + let logger = logger.clone(); + let res = retry(retry_log_message, &logger) + .redact_log_urls(true) + .when(|res| !res.is_ok() && !detect_null_block(res)) .no_limit() - .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) + .timeout_secs(self.settings.json_rpc_timeout.as_secs()) .run(move || { - let web3 = web3.cheap_clone(); + let alloy = alloy.cheap_clone(); async move { - web3.eth() - .block(BlockId::Number(block_number.into())) + alloy + .get_block_by_number(alloy::rpc::types::BlockNumberOrTag::Number( + next_number as u64, + )) .await - .map(|block_opt| block_opt.and_then(|block| block.hash)) + .map(|block_opt| { + block_opt.map(|block| BlockHash::from(block.header.hash.0.to_vec())) + }) .map_err(Error::from) } }) - .boxed() - .compat() + .await .map_err(move |e| { e.into_inner().unwrap_or_else(move || { anyhow!( "Ethereum node took too long to return data for block #{}", - block_number + next_number ) }) - }), - ) + }); + if detect_null_block(&res) { + next_number += 1; + continue; + } + return match res { + Ok(Some(hash)) => Ok(BlockPtr::new(hash, next_number)), + Ok(None) => Err(anyhow!("Block {} does not contain hash", next_number)), + Err(e) => Err(e), + }; + } } - fn contract_call( + async fn contract_call( &self, logger: &Logger, - call: EthereumContractCall, + inp_call: &ContractCall, cache: Arc, - ) -> Box, Error = EthereumContractCallError> + Send> { - // Emit custom error for type mismatches. - for (token, kind) in call - .args - .iter() - .zip(call.function.inputs.iter().map(|p| &p.kind)) - { - if !token.type_check(kind) { - return Box::new(future::err(EthereumContractCallError::TypeError( - token.clone(), - kind.clone(), - ))); + ) -> Result<(Option>, call::Source), ContractCallError> { + let mut result = self.contract_calls(logger, &[inp_call], cache).await?; + // unwrap: self.contract_calls returns as many results as there were calls + Ok(result.pop().unwrap()) + } + + async fn contract_calls( + &self, + logger: &Logger, + calls: &[&ContractCall], + cache: Arc, + ) -> Result>, call::Source)>, ContractCallError> { + fn as_req( + logger: &ProviderLogger, + call: &ContractCall, + index: u32, + ) -> Result { + // Emit custom error for type mismatches. + for (val, kind) in call + .args + .iter() + .zip(call.function.inputs.iter().map(|p| p.selector_type())) + { + let kind: abi::DynSolType = kind.parse().map_err(|err| { + ContractCallError::ABIError(anyhow!( + "failed to parse function input type '{kind}': {err}" + )) + })?; + + if !val.type_check(&kind) { + return Err(ContractCallError::TypeError(val.clone(), kind.clone())); + } } + + // Encode the call parameters according to the ABI + let req = { + let encoded_call = call + .function + .abi_encode_input(&call.args) + .map_err(ContractCallError::EncodingError)?; + call::Request::new(call.address, encoded_call, index) + }; + + trace!(logger, "eth_call"; + "fn" => &call.function.name, + "address" => hex::encode(call.address), + "data" => hex::encode(req.encoded_call.as_ref()), + "block_hash" => call.block_ptr.hash_hex(), + "block_number" => call.block_ptr.block_number() + ); + Ok(req) + } + + fn decode( + logger: &ProviderLogger, + resp: call::Response, + call: &ContractCall, + ) -> (Option>, call::Source) { + let call::Response { + retval, + source, + req: _, + } = resp; + match retval { + call::Retval::Value(output) => match call.function.abi_decode_output(&output) { + Ok(tokens) => (Some(tokens), source), + Err(e) => { + // Decode failures are reverts. The reasoning is that if Solidity fails to + // decode an argument, that's a revert, so the same goes for the output. + let reason = format!("failed to decode output: {}", e); + info!(logger, "Contract call reverted"; "reason" => reason); + (None, call::Source::Rpc) + } + }, + call::Retval::Null => { + // We got a `0x` response. For old Geth, this can mean a revert. It can also be + // that the contract actually returned an empty response. A view call is meant + // to return something, so we treat empty responses the same as reverts. + info!(logger, "Contract call reverted"; "reason" => "empty response"); + (None, call::Source::Rpc) + } + } + } + + fn log_call_error(logger: &ProviderLogger, e: &ContractCallError, call: &ContractCall) { + match e { + ContractCallError::AlloyError(e) => error!( + logger, + "Ethereum node returned an error when calling function \"{}\" of contract \"{}\": {}", + call.function.name, + call.contract_name, + e + ), + ContractCallError::Timeout => error!( + logger, + "Ethereum node did not respond when calling function \"{}\" of contract \"{}\"", + call.function.name, + call.contract_name + ), + _ => error!( + logger, + "Failed to call function \"{}\" of contract \"{}\": {}", + call.function.name, + call.contract_name, + e + ), + } + } + + let logger = self.provider_logger(logger); + + if calls.is_empty() { + return Ok(Vec::new()); } - // Encode the call parameters according to the ABI - let call_data = match call.function.encode_input(&call.args) { - Ok(data) => data, - Err(e) => return Box::new(future::err(EthereumContractCallError::EncodingError(e))), - }; + let block_ptr = calls.first().unwrap().block_ptr.clone(); + if calls.iter().any(|call| call.block_ptr != block_ptr) { + return Err(ContractCallError::Internal( + "all calls must have the same block pointer".to_string(), + )); + } - debug!(logger, "eth_call"; - "address" => hex::encode(call.address), - "data" => hex::encode(&call_data) - ); + let reqs: Vec<_> = calls + .iter() + .enumerate() + .map(|(index, call)| as_req(&logger, call, index as u32)) + .collect::>()?; - // Check if we have it cached, if not do the call and cache. - Box::new( - match cache - .get_call(call.address, &call_data, call.block_ptr.clone()) - .map_err(|e| error!(logger, "call cache get error"; "error" => e.to_string())) - .ok() - .flatten() - { - Some(result) => { - Box::new(future::ok(result)) as Box + Send> - } - None => { - let cache = cache.clone(); - let call = call.clone(); - let logger = logger.clone(); - Box::new( - self.call( - logger.clone(), - call.address, - Bytes(call_data.clone()), - call.block_ptr.clone(), - ) - .map(move |result| { - // Don't block handler execution on writing to the cache. - let for_cache = result.0.clone(); - let _ = graph::spawn_blocking_allow_panic(move || { - cache - .set_call(call.address, &call_data, call.block_ptr, &for_cache) - .map_err(|e| { - error!(logger, "call cache set error"; - "error" => e.to_string()) - }) - }); - result.0 - }), - ) + let (mut resps, missing) = cache + .get_calls(&reqs, block_ptr) + .await + .map_err(|e| error!(logger, "call cache get error"; "error" => e.to_string())) + .unwrap_or_else(|_| (Vec::new(), reqs)); + + let futs = missing.into_iter().map(|req| { + let cache = cache.clone(); + let logger = logger.clone(); + async move { + let call = calls[req.index as usize]; + match self.call_and_cache(&logger, call, req, cache.clone()).await { + Ok(resp) => Ok(resp), + Err(e) => { + log_call_error(&logger, &e, call); + Err(e) + } } } - // Decode the return values according to the ABI - .and_then(move |output| { - if output.is_empty() { - // We got a `0x` response. For old Geth, this can mean a revert. It can also be - // that the contract actually returned an empty response. A view call is meant - // to return something, so we treat empty responses the same as reverts. - Err(EthereumContractCallError::Revert("empty response".into())) - } else { - // Decode failures are reverts. The reasoning is that if Solidity fails to - // decode an argument, that's a revert, so the same goes for the output. - call.function.decode_output(&output).map_err(|e| { - EthereumContractCallError::Revert(format!("failed to decode output: {}", e)) - }) - } - }), - ) + }); + resps.extend(try_join_all(futs).await?); + + // If we make it here, we have a response for every call. + debug_assert_eq!(resps.len(), calls.len()); + + // Bring the responses into the same order as the calls + resps.sort_by_key(|resp| resp.req.index); + + let decoded: Vec<_> = resps + .into_iter() + .map(|res| { + let call = &calls[res.req.index as usize]; + decode(&logger, res, call) + }) + .collect(); + + Ok(decoded) } /// Load Ethereum blocks in bulk, returning results as they come back as a Stream. - fn load_blocks( + async fn load_blocks( &self, logger: Logger, chain_store: Arc, - block_hashes: HashSet, - ) -> Box, Error = Error> + Send> { + block_hashes: HashSet, + ) -> Result>, Error> { let block_hashes: Vec<_> = block_hashes.iter().cloned().collect(); // Search for the block in the store first then use json-rpc as a backup. - let mut blocks: Vec> = chain_store - .blocks(&block_hashes.iter().map(|&b| b.into()).collect::>()) + let mut blocks: Vec<_> = chain_store + .cheap_clone() + .blocks(block_hashes.iter().map(|&b| b.into()).collect::>()) + .await .map_err(|e| error!(&logger, "Error accessing block cache {}", e)) .unwrap_or_default() - .into_iter() - .filter_map(|value| json::from_value(value).ok()) - .map(Arc::new) + .iter() + .map(|b| b.to_light_block()) .collect(); let missing_blocks = Vec::from_iter( block_hashes .into_iter() - .filter(|hash| !blocks.iter().any(|b| b.hash == Some(*hash))), + .filter(|hash| !blocks.iter().any(|b| b.hash() == *hash)), ); // Return a stream that lazily loads batches of blocks. debug!(logger, "Requesting {} block(s)", missing_blocks.len()); - Box::new( - self.load_blocks_rpc(logger.clone(), missing_blocks) - .collect() - .map(move |new_blocks| { - let upsert_blocks: Vec<_> = new_blocks - .iter() - .map(|block| BlockFinality::Final(block.clone())) - .collect(); - let block_refs: Vec<_> = upsert_blocks - .iter() - .map(|block| block as &dyn graph::blockchain::Block) - .collect(); - if let Err(e) = chain_store.upsert_light_blocks(block_refs.as_slice()) { - error!(logger, "Error writing to block cache {}", e); - } - blocks.extend(new_blocks); - blocks.sort_by_key(|block| block.number); - stream::iter_ok(blocks) - }) - .flatten_stream(), - ) + let new_blocks: Vec<_> = self + .load_blocks_rpc(logger.clone(), missing_blocks) + .try_collect() + .await?; + let upsert_blocks: Vec<_> = new_blocks + .iter() + .map(|block| BlockFinality::Final(block.clone())) + .collect(); + let block_refs: Vec<_> = upsert_blocks + .iter() + .map(|block| block as &dyn graph::blockchain::Block) + .collect(); + if let Err(e) = chain_store.upsert_light_blocks(block_refs.as_slice()).await { + error!(logger, "Error writing to block cache {}", e); + } + blocks.extend(new_blocks); + blocks.sort_by_key(|block| block.number()); + Ok(blocks) } } -/// Returns blocks with triggers, corresponding to the specified range and filters. +/// Returns blocks with triggers, corresponding to the specified range and filters; and the resolved +/// `to` block, which is the nearest non-null block greater than or equal to the passed `to` block. /// If a block contains no triggers, there may be no corresponding item in the stream. -/// However the `to` block will always be present, even if triggers are empty. +/// However the (resolved) `to` block will always be present, even if triggers are empty. /// /// Careful: don't use this function without considering race conditions. /// Chain reorgs could happen at any time, and could affect the answer received. @@ -1347,22 +1721,55 @@ pub(crate) async fn blocks_with_triggers( to: BlockNumber, filter: &TriggerFilter, unified_api_version: UnifiedMappingApiVersion, -) -> Result>, Error> { +) -> Result<(Vec>, BlockNumber), Error> { // Each trigger filter needs to be queried for the same block range // and the blocks yielded need to be deduped. If any error occurs // while searching for a trigger type, the entire operation fails. let eth = adapter.clone(); let call_filter = EthereumCallFilter::from(&filter.block); + let logger = ProviderLogger::new(&logger, eth.provider()); // Scan the block range to find relevant triggers let trigger_futs: FuturesUnordered, anyhow::Error>>> = FuturesUnordered::new(); + // Resolve the nearest non-null "to" block + debug!(logger, "Finding nearest valid `to` block to {}", to); + + let to_ptr = eth.next_existing_ptr_to_number(&logger, to).await?; + let to_hash = to_ptr.hash.as_b256(); + let to = to_ptr.block_number(); + + // This is for `start` triggers which can be initialization handlers which needs to be run + // before all other triggers + if filter.block.trigger_every_block { + let logger = logger.clone(); + let block_future = eth + .block_range_to_ptrs((*logger).clone(), from, to) + .map(move |ptrs| { + ptrs.into_iter() + .flat_map(|ptr| { + vec![ + EthereumTrigger::Block(ptr.clone(), EthereumBlockTriggerType::Start), + EthereumTrigger::Block(ptr, EthereumBlockTriggerType::End), + ] + }) + .collect() + }) + .compat() + .boxed(); + trigger_futs.push(block_future) + } else if !filter.block.polling_intervals.is_empty() { + let block_futures_matching_once_filter = + eth.blocks_matching_polling_intervals((*logger).clone(), from, to, &filter.block); + trigger_futs.push(block_futures_matching_once_filter); + } + // Scan for Logs if !filter.log.is_empty() { let logs_future = get_logs_and_transactions( ð, - &logger, + logger.clone(), subgraph_metrics.clone(), from, to, @@ -1384,19 +1791,7 @@ pub(crate) async fn blocks_with_triggers( trigger_futs.push(calls_future) } - // Scan for Blocks - if filter.block.trigger_every_block { - let block_future = eth - .block_range_to_ptrs(logger.clone(), from, to) - .map(move |ptrs| { - ptrs.into_iter() - .map(|ptr| EthereumTrigger::Block(ptr, EthereumBlockTriggerType::Every)) - .collect() - }) - .compat() - .boxed(); - trigger_futs.push(block_future) - } else if !filter.block.contract_addresses.is_empty() { + if !filter.block.contract_addresses.is_empty() { // To determine which blocks include a call to addresses // in the block filter, transform the `block_filter` into // a `call_filter` and run `blocks_with_calls` @@ -1414,31 +1809,16 @@ pub(crate) async fn blocks_with_triggers( trigger_futs.push(block_future) } - // Get hash for "to" block - let to_hash_fut = eth - .block_hash_by_block_number(&logger, to) - .and_then(|hash| match hash { - Some(hash) => Ok(hash), - None => { - warn!(logger, - "Ethereum endpoint is behind"; - "url" => eth.url_hostname() - ); - bail!("Block {} not found in the chain", to) - } - }) - .compat(); - - // Join on triggers and block hash resolution - let (triggers, to_hash) = futures03::join!(trigger_futs.try_concat(), to_hash_fut); - - // Unpack and handle possible errors in the previously joined futures - let triggers = - triggers.with_context(|| format!("Failed to obtain triggers for block {}", to))?; - let to_hash = to_hash.with_context(|| format!("Failed to infer hash for block {}", to))?; + // Join on triggers, unpack and handle possible errors + let triggers = trigger_futs + .try_concat() + .await + .with_context(|| format!("Failed to obtain triggers for block {}", to))?; - let mut block_hashes: HashSet = - triggers.iter().map(EthereumTrigger::block_hash).collect(); + let mut block_hashes: HashSet = triggers + .iter() + .map(|trigger| trigger.block_hash()) + .collect(); let mut triggers_by_block: HashMap> = triggers.into_iter().fold(HashMap::new(), |mut map, t| { map.entry(t.block_number()).or_default().push(t); @@ -1453,10 +1833,12 @@ pub(crate) async fn blocks_with_triggers( let logger2 = logger.cheap_clone(); - let blocks = eth - .load_blocks(logger.cheap_clone(), chain_store.clone(), block_hashes) - .and_then( - move |block| match triggers_by_block.remove(&(block.number() as BlockNumber)) { + let blocks: Vec<_> = eth + .load_blocks((*logger).cheap_clone(), chain_store.clone(), block_hashes) + .await? + .into_iter() + .map( + move |block| match triggers_by_block.remove(&(block.number())) { Some(triggers) => Ok(BlockWithTriggers::new( BlockFinality::Final(block), triggers, @@ -1468,9 +1850,7 @@ pub(crate) async fn blocks_with_triggers( )), }, ) - .collect() - .compat() - .await?; + .collect::>()?; // Filter out call triggers that come from unsuccessful transactions let futures = blocks.into_iter().map(|block| { @@ -1499,7 +1879,7 @@ pub(crate) async fn blocks_with_triggers( )); } - Ok(blocks) + Ok((blocks, to)) } pub(crate) async fn get_calls( @@ -1528,13 +1908,13 @@ pub(crate) async fn get_calls( } else { client .rpc()? - .cheapest_with(capabilities)? + .cheapest_with(capabilities) + .await? .calls_in_block( &logger, subgraph_metrics.clone(), - BlockNumber::try_from(ethereum_block.block.number.unwrap().as_u64()) - .unwrap(), - ethereum_block.block.hash.unwrap(), + ethereum_block.block.number(), + ethereum_block.block.hash(), ) .await? }; @@ -1543,6 +1923,9 @@ pub(crate) async fn get_calls( calls: Some(calls), })) } + BlockFinality::Ptr(_) => { + unreachable!("get_calls called with BlockFinality::Ptr") + } } } @@ -1554,19 +1937,36 @@ pub(crate) fn parse_log_triggers( return vec![]; } - block + let total_logs: usize = block .transaction_receipts .iter() - .flat_map(move |receipt| { - receipt - .logs - .iter() - .filter(move |log| log_filter.matches(log)) - .map(move |log| { - EthereumTrigger::Log(Arc::new(log.clone()), Some(receipt.cheap_clone())) + .map(|r| r.logs().len()) + .sum(); + let mut triggers = Vec::with_capacity(total_logs); + + for receipt in &block.transaction_receipts { + for (index, log) in receipt.logs().iter().enumerate() { + let requires_transaction_receipt = log + .topics() + .first() + .map(|signature| { + log_filter.requires_transaction_receipt( + signature, + Some(&log.address()), + log.topics(), + ) }) - }) - .collect() + .unwrap_or(false); + + triggers.push(EthereumTrigger::Log(LogRef::LogPosition(LogPosition { + index, + receipt: receipt.cheap_clone(), + requires_transaction_receipt, + }))); + } + } + + triggers } pub(crate) fn parse_call_triggers( @@ -1594,6 +1994,39 @@ pub(crate) fn parse_call_triggers( } } +/// For a given `block_number`, return the block trigger types that fire +/// based on the rules in `polling_intervals`. Each entry is `(start_block, +/// interval)` where `interval == 0` encodes a `once` rule and `interval > 0` +/// encodes a `polling every interval` rule. Both rule kinds can fire at the +/// same block (e.g. a once and polling rule sharing a `start_block`), so the +/// returned Vec may contain `Start`, `End`, both, or neither. +pub(crate) fn block_trigger_types_from_intervals( + block_number: i32, + polling_intervals: &HashSet<(i32, i32)>, +) -> Vec { + let has_once_trigger = polling_intervals + .iter() + .any(|(start_block, interval)| *interval == 0 && block_number == *start_block); + + let has_polling_trigger = polling_intervals.iter().any(|(start_block, interval)| { + *interval > 0 + && block_number >= *start_block + && (block_number - start_block) % *interval == 0 + }); + + let mut triggers = Vec::new(); + if has_once_trigger { + triggers.push(EthereumBlockTriggerType::Start); + } + if has_polling_trigger { + triggers.push(EthereumBlockTriggerType::End); + } + triggers +} + +/// This method does not parse block triggers with `once` filters. +/// This is because it is to be run before any other triggers are run. +/// So we have `parse_initialization_triggers` for that. pub(crate) fn parse_block_triggers( block_filter: &EthereumBlockFilter, block: &EthereumBlockWithCalls, @@ -1602,10 +2035,13 @@ pub(crate) fn parse_block_triggers( return vec![]; } - let block_ptr = BlockPtr::from(&block.ethereum_block); + let block_ptr = block.ethereum_block.block.block_ptr(); let trigger_every_block = block_filter.trigger_every_block; let call_filter = EthereumCallFilter::from(block_filter); let block_ptr2 = block_ptr.cheap_clone(); + let block_ptr3 = block_ptr.cheap_clone(); + let block_number = block_ptr.number; + let mut triggers = match &block.calls { Some(calls) => calls .iter() @@ -1620,19 +2056,32 @@ pub(crate) fn parse_block_triggers( None => vec![], }; if trigger_every_block { + triggers.push(EthereumTrigger::Block( + block_ptr.clone(), + EthereumBlockTriggerType::Start, + )); triggers.push(EthereumTrigger::Block( block_ptr, - EthereumBlockTriggerType::Every, + EthereumBlockTriggerType::End, )); + } else if !block_filter.polling_intervals.is_empty() { + for trigger_type in + block_trigger_types_from_intervals(block_number, &block_filter.polling_intervals) + { + triggers.push(EthereumTrigger::Block( + block_ptr3.cheap_clone(), + trigger_type, + )); + } } triggers } async fn fetch_receipt_from_ethereum_client( eth: &EthereumAdapter, - transaction_hash: &H256, -) -> anyhow::Result { - match eth.web3.eth().transaction_receipt(*transaction_hash).await { + transaction_hash: B256, +) -> anyhow::Result { + match eth.alloy.get_transaction_receipt(transaction_hash).await { Ok(Some(receipt)) => Ok(receipt), Ok(None) => bail!("Could not find transaction receipt"), Err(error) => bail!("Failed to fetch transaction receipt: {}", error), @@ -1643,7 +2092,7 @@ async fn filter_call_triggers_from_unsuccessful_transactions( mut block: BlockWithTriggers, eth: &EthereumAdapter, chain_store: &Arc, - logger: &Logger, + logger: &ProviderLogger, ) -> anyhow::Result> { // Return early if there is no trigger data if block.trigger_data.is_empty() { @@ -1653,14 +2102,14 @@ async fn filter_call_triggers_from_unsuccessful_transactions( let initial_number_of_triggers = block.trigger_data.len(); // Get the transaction hash from each call trigger - let transaction_hashes: BTreeSet = block + let transaction_hashes: BTreeSet = block .trigger_data .iter() - .filter_map(|trigger| match trigger { - EthereumTrigger::Call(call_trigger) => Some(call_trigger.transaction_hash), + .filter_map(|trigger| match trigger.as_chain() { + Some(EthereumTrigger::Call(call_trigger)) => Some(call_trigger.transaction_hash), _ => None, }) - .collect::>>() + .collect::>>() .ok_or(anyhow!( "failed to obtain transaction hash from call triggers" ))?; @@ -1671,18 +2120,24 @@ async fn filter_call_triggers_from_unsuccessful_transactions( } // And obtain all Transaction values for the calls in this block. - let transactions: Vec<&Transaction> = { + let transactions: Vec<&AnyTransaction> = { match &block.block { - BlockFinality::Final(ref block) => block - .transactions + BlockFinality::Final(block) => block + .transactions() + .ok_or_else(|| anyhow!("Block transactions not available"))? .iter() - .filter(|transaction| transaction_hashes.contains(&transaction.hash)) + .filter(|transaction| transaction_hashes.contains(&transaction.tx_hash())) .collect(), BlockFinality::NonFinal(_block_with_calls) => { unreachable!( "this function should not be called when dealing with non-final blocks" ) } + BlockFinality::Ptr(_block) => { + unreachable!( + "this function should not be called when dealing with header-only blocks" + ) + } } }; @@ -1693,21 +2148,21 @@ async fn filter_call_triggers_from_unsuccessful_transactions( // We'll also need the receipts for those transactions. In this step we collect all receipts // we have in store for the current block. - let mut receipts = chain_store - .transaction_receipts_in_block(&block.ptr().hash_as_h256()) + let mut receipts: BTreeMap = chain_store + .transaction_receipts_in_block(&block.ptr().hash.as_b256()) .await? .into_iter() .map(|receipt| (receipt.transaction_hash, receipt)) - .collect::>(); + .collect::>(); // Do we have a receipt for each transaction under analysis? - let mut receipts_and_transactions: Vec<(&Transaction, LightTransactionReceipt)> = Vec::new(); - let mut transactions_without_receipt: Vec<&Transaction> = Vec::new(); + let mut receipts_and_transactions: Vec<(&AnyTransaction, LightTransactionReceipt)> = Vec::new(); + let mut transactions_without_receipt: Vec<&AnyTransaction> = Vec::new(); for transaction in transactions.iter() { - if let Some(receipt) = receipts.remove(&transaction.hash) { - receipts_and_transactions.push((transaction, receipt)); + if let Some(receipt) = receipts.remove(&transaction.tx_hash()) { + receipts_and_transactions.push((*transaction, receipt)); } else { - transactions_without_receipt.push(transaction); + transactions_without_receipt.push(*transaction); } } @@ -1715,7 +2170,7 @@ async fn filter_call_triggers_from_unsuccessful_transactions( let futures = transactions_without_receipt .iter() .map(|transaction| async move { - fetch_receipt_from_ethereum_client(eth, &transaction.hash) + fetch_receipt_from_ethereum_client(eth, transaction.tx_hash()) .await .map(|receipt| (transaction, receipt)) }); @@ -1730,12 +2185,9 @@ async fn filter_call_triggers_from_unsuccessful_transactions( // additional Ethereum API calls for future scans on this block. // With all transactions and receipts in hand, we can evaluate the success of each transaction - let mut transaction_success: BTreeMap<&H256, bool> = BTreeMap::new(); + let mut transaction_success: BTreeMap = BTreeMap::new(); for (transaction, receipt) in receipts_and_transactions.into_iter() { - transaction_success.insert( - &transaction.hash, - evaluate_transaction_status(receipt.status), - ); + transaction_success.insert(transaction.tx_hash(), receipt.status); } // Confidence check: Did we inspect the status of all transactions? @@ -1748,7 +2200,7 @@ async fn filter_call_triggers_from_unsuccessful_transactions( // Filter call triggers from unsuccessful transactions block.trigger_data.retain(|trigger| { - if let EthereumTrigger::Call(call_trigger) = trigger { + if let Some(EthereumTrigger::Call(call_trigger)) = trigger.as_chain() { // Unwrap: We already checked that those values exist transaction_success[&call_trigger.transaction_hash.unwrap()] } else { @@ -1777,75 +2229,242 @@ async fn filter_call_triggers_from_unsuccessful_transactions( /// Deprecated. Wraps the [`fetch_transaction_receipts_in_batch`] in a retry loop. async fn fetch_transaction_receipts_in_batch_with_retry( - web3: Arc>, - hashes: Vec, - block_hash: H256, - logger: Logger, -) -> Result>, IngestorError> { + alloy: Arc, + hashes: Vec, + block_hash: B256, + logger: ProviderLogger, + settings: &ChainSettings, +) -> Result>, IngestorError> { let retry_log_message = format!( "batch eth_getTransactionReceipt RPC call for block {:?}", block_hash ); retry(retry_log_message, &logger) - .limit(ENV_VARS.request_retries) + .redact_log_urls(true) + .limit(settings.request_retries) .no_logging() - .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) + .timeout_secs(settings.json_rpc_timeout.as_secs()) .run(move || { - let web3 = web3.cheap_clone(); + let alloy = alloy.cheap_clone(); let hashes = hashes.clone(); let logger = logger.cheap_clone(); - fetch_transaction_receipts_in_batch(web3, hashes, block_hash, logger).boxed() + fetch_transaction_receipts_in_batch(alloy, hashes, block_hash, logger).boxed() }) .await .map_err(|_timeout| anyhow!(block_hash).into()) } -/// Deprecated. Attempts to fetch multiple transaction receipts in a batching contex. +/// Deprecated. Attempts to fetch multiple transaction receipts in a batching context. async fn fetch_transaction_receipts_in_batch( - web3: Arc>, - hashes: Vec, - block_hash: H256, - logger: Logger, -) -> Result>, IngestorError> { - let batching_web3 = Web3::new(Batch::new(web3.transport().clone())); - let eth = batching_web3.eth(); - let receipt_futures = hashes - .into_iter() - .map(move |hash| { - let logger = logger.cheap_clone(); - eth.transaction_receipt(hash) - .map_err(IngestorError::from) - .and_then(move |some_receipt| async move { - resolve_transaction_receipt(some_receipt, hash, block_hash, logger) - }) + alloy: Arc, + hashes: Vec, + block_hash: B256, + logger: ProviderLogger, +) -> Result>, IngestorError> { + // Use the batch method to get all receipts at once + let receipts = batch_get_transaction_receipts(alloy, hashes.clone()) + .await + .map_err(|e| { + IngestorError::Unknown(anyhow::anyhow!("Batch receipt fetch failed: {}", e)) + })?; + + let mut result = Vec::new(); + for (receipt, hash) in receipts.into_iter().zip(hashes.iter()) { + if let Some(receipt) = receipt { + let validated_receipt = resolve_transaction_receipt( + Some(receipt), + *hash, + block_hash, + logger.cheap_clone(), + )?; + result.push(Arc::new(validated_receipt)); + } else { + return Err(IngestorError::ReceiptUnavailable(block_hash, *hash)); + } + } + + Ok(result) +} + +async fn batch_get_transaction_receipts( + provider: Arc, + tx_hashes: Vec, +) -> Result>, Box> { + let mut batch = alloy::rpc::client::BatchRequest::new(provider.client()); + let mut receipt_futures = Vec::new(); + + // Add all receipt requests to batch + for tx_hash in &tx_hashes { + let receipt_future = batch.add_call::<(B256,), Option>( + "eth_getTransactionReceipt", + &(*tx_hash,), + )?; + receipt_futures.push(receipt_future); + } + + // Execute batch + batch.send().await?; + + // Collect results in order + let mut results = Vec::new(); + for receipt_future in receipt_futures { + let receipt = receipt_future.await?; + results.push(receipt); + } + + Ok(results) +} + +pub(crate) async fn check_block_receipt_support( + alloy: Arc, + block_hash: B256, + supports_eip_1898: bool, + call_only: bool, +) -> Result<(), Error> { + use alloy::rpc::types::BlockId; + if call_only { + return Err(anyhow!("Provider is call-only")); + } + + if !supports_eip_1898 { + return Err(anyhow!("Provider does not support EIP 1898")); + } + + // Fetch block receipts from the provider for the latest block. + let block_receipts_result = alloy.get_block_receipts(BlockId::from(block_hash)).await; + + // Determine if the provider supports block receipts based on the fetched result. + match block_receipts_result { + Ok(Some(receipts)) if !receipts.is_empty() => Ok(()), + Ok(_) => Err(anyhow!("Block receipts are empty")), + Err(err) => Err(anyhow!("Error fetching block receipts: {}", err)), + } +} + +// Fetches transaction receipts with retries. This function acts as a dispatcher +// based on whether block receipts are supported or individual transaction receipts +// need to be fetched. +async fn fetch_receipts_with_retry( + alloy: Arc, + hashes: Vec, + block_hash: B256, + logger: ProviderLogger, + supports_block_receipts: bool, + settings: &ChainSettings, +) -> Result>, IngestorError> { + if supports_block_receipts { + return fetch_block_receipts_with_retry(alloy, hashes, block_hash, logger, settings).await; + } + fetch_individual_receipts_with_retry(alloy, hashes, block_hash, logger, settings).await +} + +// Fetches receipts for each transaction in the block individually. +async fn fetch_individual_receipts_with_retry( + alloy: Arc, + hashes: Vec, + block_hash: B256, + logger: ProviderLogger, + settings: &ChainSettings, +) -> Result>, IngestorError> { + if ENV_VARS.fetch_receipts_in_batches { + return fetch_transaction_receipts_in_batch_with_retry( + alloy, hashes, block_hash, logger, settings, + ) + .await; + } + + let request_retries = settings.request_retries; + let json_rpc_timeout = settings.json_rpc_timeout; + let concurrent_requests = settings.block_ingestor_max_concurrent_json_rpc_calls; + + // Use a stream to fetch receipts individually + let hash_stream = tokio_stream::iter(hashes); + let receipt_stream = hash_stream + .map(move |tx_hash| { + fetch_transaction_receipt_with_retry( + alloy.cheap_clone(), + tx_hash, + block_hash, + logger.cheap_clone(), + request_retries, + json_rpc_timeout, + ) }) - .collect::>(); + .buffered(concurrent_requests); - batching_web3.transport().submit_batch().await?; + tokio_stream::StreamExt::collect::>, IngestorError>>( + receipt_stream, + ) + .await +} - let mut collected = vec![]; - for receipt in receipt_futures.into_iter() { - collected.push(Arc::new(receipt.await?)) +/// Fetches transaction receipts of all transactions in a block with `eth_getBlockReceipts` call. +async fn fetch_block_receipts_with_retry( + alloy: Arc, + hashes: Vec, + block_hash: B256, + logger: ProviderLogger, + settings: &ChainSettings, +) -> Result>, IngestorError> { + use graph::prelude::alloy::rpc::types::BlockId; + let retry_log_message = format!("eth_getBlockReceipts RPC call for block {:?}", block_hash); + + // Perform the retry operation + let receipts_option = retry(retry_log_message, &logger) + .redact_log_urls(true) + .limit(settings.request_retries) + .timeout_secs(settings.json_rpc_timeout.as_secs()) + .run(move || alloy.get_block_receipts(BlockId::from(block_hash)).boxed()) + .await + .map_err(|_timeout| -> IngestorError { anyhow!(block_hash).into() })?; + + // Check if receipts are available, and transform them if they are + match receipts_option { + Some(receipts) => { + // Create a HashSet from the transaction hashes of the receipts + let receipt_hashes_set: HashSet<_> = + receipts.iter().map(|r| r.transaction_hash).collect(); + + // Check if the set contains all the hashes and has the same length as the hashes vec + if hashes.len() == receipt_hashes_set.len() + && hashes.iter().all(|hash| receipt_hashes_set.contains(hash)) + { + let transformed_receipts = receipts.into_iter().map(Arc::new).collect(); + Ok(transformed_receipts) + } else { + // If there's a mismatch in numbers or a missing hash, return an error + Err(IngestorError::BlockReceiptsMismatched(block_hash)) + } + } + None => { + // If no receipts are found, return an error + Err(IngestorError::BlockReceiptsUnavailable(block_hash)) + } } - Ok(collected) } -/// Retries fetching a single transaction receipt. +/// Retries fetching a single transaction receipt using alloy, then converts to web3 format. async fn fetch_transaction_receipt_with_retry( - web3: Arc>, - transaction_hash: H256, - block_hash: H256, - logger: Logger, -) -> Result, IngestorError> { - let logger = logger.cheap_clone(); + alloy: Arc, + transaction_hash: B256, + block_hash: B256, + logger: ProviderLogger, + request_retries: usize, + json_rpc_timeout: Duration, +) -> Result, IngestorError> { let retry_log_message = format!( "eth_getTransactionReceipt RPC call for transaction {:?}", transaction_hash ); + retry(retry_log_message, &logger) - .limit(ENV_VARS.request_retries) - .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) - .run(move || web3.eth().transaction_receipt(transaction_hash).boxed()) + .redact_log_urls(true) + .limit(request_retries) + .timeout_secs(json_rpc_timeout.as_secs()) + .run(move || { + let alloy_clone = alloy.clone(); + async move { alloy_clone.get_transaction_receipt(transaction_hash).await }.boxed() + }) .await .map_err(|_timeout| anyhow!(block_hash).into()) .and_then(move |some_receipt| { @@ -1855,11 +2474,11 @@ async fn fetch_transaction_receipt_with_retry( } fn resolve_transaction_receipt( - transaction_receipt: Option, - transaction_hash: H256, - block_hash: H256, - logger: Logger, -) -> Result { + transaction_receipt: Option, + transaction_hash: B256, + block_hash: B256, + logger: ProviderLogger, +) -> Result { match transaction_receipt { // A receipt might be missing because the block was uncled, and the transaction never // made it back into the main chain. @@ -1912,7 +2531,7 @@ fn resolve_transaction_receipt( /// Retrieves logs and the associated transaction receipts, if required by the [`EthereumLogFilter`]. async fn get_logs_and_transactions( adapter: &Arc, - logger: &Logger, + logger: ProviderLogger, subgraph_metrics: Arc, from: BlockNumber, to: BlockNumber, @@ -1922,7 +2541,7 @@ async fn get_logs_and_transactions( // Obtain logs externally let logs = adapter .logs_in_block_range( - logger, + &logger, subgraph_metrics.cheap_clone(), from, to, @@ -1932,12 +2551,16 @@ async fn get_logs_and_transactions( // Not all logs have associated transaction hashes, nor do all triggers require them. // We also restrict receipts retrieval for some api versions. - let transaction_hashes_by_block: HashMap> = logs + let transaction_hashes_by_block: HashMap> = logs .iter() .filter(|_| unified_api_version.equal_or_greater_than(&API_VERSION_0_0_7)) .filter(|log| { - if let Some(signature) = log.topics.first() { - log_filter.requires_transaction_receipt(signature, Some(&log.address)) + if let Some(signature) = log.topics().first() { + log_filter.requires_transaction_receipt( + signature, + Some(&log.address()), + log.topics(), + ) } else { false } @@ -1952,7 +2575,7 @@ async fn get_logs_and_transactions( } }) .fold( - HashMap::>::new(), + HashMap::>::new(), |mut acc, (block_hash, txn_hash)| { acc.entry(block_hash).or_default().insert(txn_hash); acc @@ -1974,7 +2597,8 @@ async fn get_logs_and_transactions( let optional_receipt = log .transaction_hash .and_then(|txn| transaction_receipts_by_hash.get(&txn).cloned()); - let value = EthereumTrigger::Log(Arc::new(log), optional_receipt); + + let value = EthereumTrigger::Log(LogRef::FullLog(Arc::new(log), optional_receipt)); log_triggers.push(value); } @@ -1984,13 +2608,13 @@ async fn get_logs_and_transactions( /// Tries to retrive all transaction receipts for a set of transaction hashes. async fn get_transaction_receipts_for_transaction_hashes( adapter: &EthereumAdapter, - transaction_hashes_by_block: &HashMap>, + transaction_hashes_by_block: &HashMap>, subgraph_metrics: Arc, - logger: Logger, -) -> Result>, anyhow::Error> { + logger: ProviderLogger, +) -> Result>, anyhow::Error> { use std::collections::hash_map::Entry::Vacant; - let mut receipts_by_hash: HashMap> = HashMap::new(); + let mut receipts_by_hash: HashMap> = HashMap::new(); // Return early if input set is empty if transaction_hashes_by_block.is_empty() { @@ -1999,20 +2623,22 @@ async fn get_transaction_receipts_for_transaction_hashes( // Keep a record of all unique transaction hashes for which we'll request receipts. We will // later use this to check if we have collected the receipts from all required transactions. - let mut unique_transaction_hashes: HashSet<&H256> = HashSet::new(); + let mut unique_transaction_hashes: HashSet<&B256> = HashSet::new(); // Request transaction receipts concurrently let receipt_futures = FuturesUnordered::new(); - let web3 = Arc::clone(&adapter.web3); + let alloy = Arc::clone(&adapter.alloy); for (block_hash, transaction_hashes) in transaction_hashes_by_block { for transaction_hash in transaction_hashes { unique_transaction_hashes.insert(transaction_hash); let receipt_future = fetch_transaction_receipt_with_retry( - web3.cheap_clone(), + alloy.cheap_clone(), *transaction_hash, *block_hash, logger.cheap_clone(), + adapter.settings.request_retries, + adapter.settings.json_rpc_timeout, ); receipt_futures.push(receipt_future) } @@ -2066,24 +2692,28 @@ mod tests { use crate::trigger::{EthereumBlockTriggerType, EthereumTrigger}; - use super::{parse_block_triggers, EthereumBlock, EthereumBlockFilter, EthereumBlockWithCalls}; + use super::{ + EthereumBlock, EthereumBlockFilter, EthereumBlockWithCalls, + block_trigger_types_from_intervals, check_block_receipt_support, parse_block_triggers, + }; use graph::blockchain::BlockPtr; - use graph::prelude::ethabi::ethereum_types::U64; - use graph::prelude::web3::types::{Address, Block, Bytes, H256}; - use graph::prelude::EthereumCall; + use graph::components::ethereum::AnyNetworkBare; + use graph::prelude::alloy::primitives::{Address, B256, Bytes}; + use graph::prelude::alloy::providers::ProviderBuilder; + use graph::prelude::alloy::providers::mock::Asserter; + use graph::prelude::{EthereumCall, LightEthereumBlock, create_minimal_block_for_test}; + use jsonrpc_core::serde_json::{self, Value}; use std::collections::HashSet; use std::iter::FromIterator; use std::sync::Arc; #[test] fn parse_block_triggers_every_block() { + let block = create_minimal_block_for_test(2, hash(2)); + let block = EthereumBlockWithCalls { ethereum_block: EthereumBlock { - block: Arc::new(Block { - hash: Some(hash(2)), - number: Some(U64::from(2)), - ..Default::default() - }), + block: Arc::new(LightEthereumBlock::new(block)), ..Default::default() }, calls: Some(vec![EthereumCall { @@ -2094,12 +2724,16 @@ mod tests { }; assert_eq!( - vec![EthereumTrigger::Block( - BlockPtr::from((hash(2), 2)), - EthereumBlockTriggerType::Every - )], + vec![ + EthereumTrigger::Block( + BlockPtr::from((hash(2), 2)), + EthereumBlockTriggerType::Start + ), + EthereumTrigger::Block(BlockPtr::from((hash(2), 2)), EthereumBlockTriggerType::End) + ], parse_block_triggers( &EthereumBlockFilter { + polling_intervals: HashSet::new(), contract_addresses: HashSet::from_iter(vec![(10, address(1))]), trigger_every_block: true, }, @@ -2109,15 +2743,121 @@ mod tests { ); } + #[graph::test] + async fn test_check_block_receipts_support() { + let json_receipts = r#"[{ + "blockHash": "0x23f785604642e91613881fc3c9d16740ee416e340fd36f3fa2239f203d68fd33", + "blockNumber": "0x12f7f81", + "contractAddress": null, + "cumulativeGasUsed": "0x26f66", + "effectiveGasPrice": "0x140a1bd03", + "from": "0x56fc0708725a65ebb633efdaec931c0600a9face", + "gasUsed": "0x26f66", + "logs": [], + "logsBloom": "0x00000000010000000000000000000000000000000000000000000000040000000000000000000000000008000000000002000000080020000000040000000000000000000000000808000008000000000000000000040000000000000000000000000000000000000000000000000000000000000000000000000010000800000000000000000000000000000000000000000000010000000000000000000000000000000000200000000000000000000000000000000000002000000008000000000002000000000000000000000000000000000400000000000000000000000000200000000000000010000000000000000000000000000000000000000000", + "status": "0x1", + "to": "0x51c72848c68a965f66fa7a88855f9f7784502a7f", + "transactionHash": "0xabfe9e82d71c843a91251fd1272b0dd80bc0b8d94661e3a42c7bb9e7f55789cf", + "transactionIndex": "0x0", + "type": "0x2" + }]"#; + + let json_empty = r#"[]"#; + + // Helper function to run a single test case + async fn run_test_case( + json_response: &str, + expected_err: Option<&str>, + supports_eip_1898: bool, + call_only: bool, + ) -> Result<(), anyhow::Error> { + let json_value: Value = serde_json::from_str(json_response).unwrap(); + + let asserter = Asserter::new(); + let provider = ProviderBuilder::<_, _, AnyNetworkBare>::default() + .network::() + .connect_mocked_client(asserter.clone()); + + asserter.push_success(&json_value); + + let result = check_block_receipt_support( + Arc::new(provider), + B256::ZERO, + supports_eip_1898, + call_only, + ) + .await; + + match expected_err { + Some(err_msg) => match result { + Ok(_) => panic!("Expected error but got Ok"), + Err(e) => { + assert!(e.to_string().contains(err_msg)); + } + }, + None => match result { + Ok(_) => (), + Err(e) => { + eprintln!("Error: {}", e); + panic!("Unexpected error: {}", e); + } + }, + } + Ok(()) + } + + // Test case 1: Valid block receipts + run_test_case(json_receipts, None, true, false) + .await + .unwrap(); + + // Test case 2: Empty block receipts + run_test_case(json_empty, Some("Block receipts are empty"), true, false) + .await + .unwrap(); + + // Test case 3: Null response + run_test_case("null", Some("Block receipts are empty"), true, false) + .await + .unwrap(); + + // Test case 3: Simulating an RPC error + // Note: In the context of this test, we cannot directly simulate an RPC error. + // Instead, we simulate a response that would cause a decoding error, such as an unexpected key("error"). + // The function should handle this as an error case. + run_test_case( + r#"{"error":"RPC Error"}"#, + Some("Error fetching block receipts:"), + true, + false, + ) + .await + .unwrap(); + + // Test case 5: Does not support EIP-1898 + run_test_case( + json_receipts, + Some("Provider does not support EIP 1898"), + false, + false, + ) + .await + .unwrap(); + + // Test case 5: Does not support Call only adapters + run_test_case(json_receipts, Some("Provider is call-only"), true, true) + .await + .unwrap(); + } + #[test] fn parse_block_triggers_specific_call_not_found() { + let block = create_minimal_block_for_test(2, hash(2)); + + #[allow(unreachable_code)] let block = EthereumBlockWithCalls { ethereum_block: EthereumBlock { - block: Arc::new(Block { - hash: Some(hash(2)), - number: Some(U64::from(2)), - ..Default::default() - }), + block: Arc::new(LightEthereumBlock::new(block)), ..Default::default() }, calls: Some(vec![EthereumCall { @@ -2131,6 +2871,7 @@ mod tests { Vec::::new(), parse_block_triggers( &EthereumBlockFilter { + polling_intervals: HashSet::new(), contract_addresses: HashSet::from_iter(vec![(1, address(1))]), trigger_every_block: false, }, @@ -2142,13 +2883,12 @@ mod tests { #[test] fn parse_block_triggers_specific_call_found() { + let block = create_minimal_block_for_test(2, hash(2)); + + #[allow(unreachable_code)] let block = EthereumBlockWithCalls { ethereum_block: EthereumBlock { - block: Arc::new(Block { - hash: Some(hash(2)), - number: Some(U64::from(2)), - ..Default::default() - }), + block: Arc::new(LightEthereumBlock::new(block)), ..Default::default() }, calls: Some(vec![EthereumCall { @@ -2165,6 +2905,7 @@ mod tests { )], parse_block_triggers( &EthereumBlockFilter { + polling_intervals: HashSet::new(), contract_addresses: HashSet::from_iter(vec![(1, address(4))]), trigger_every_block: false, }, @@ -2174,12 +2915,121 @@ mod tests { ); } + #[test] + fn block_trigger_types_once_only() { + let intervals = HashSet::from_iter(vec![(100, 0)]); + + assert_eq!( + vec![EthereumBlockTriggerType::Start], + block_trigger_types_from_intervals(100, &intervals), + "once rule fires Start at start_block" + ); + assert_eq!( + Vec::::new(), + block_trigger_types_from_intervals(99, &intervals), + "once rule does not fire before start_block" + ); + assert_eq!( + Vec::::new(), + block_trigger_types_from_intervals(101, &intervals), + "once rule does not fire after start_block" + ); + } + + #[test] + fn block_trigger_types_polling_only() { + let intervals = HashSet::from_iter(vec![(100, 10)]); + + assert_eq!( + vec![EthereumBlockTriggerType::End], + block_trigger_types_from_intervals(100, &intervals), + "polling rule fires End at start_block" + ); + assert_eq!( + vec![EthereumBlockTriggerType::End], + block_trigger_types_from_intervals(110, &intervals), + "polling rule fires End at start_block + interval" + ); + assert_eq!( + Vec::::new(), + block_trigger_types_from_intervals(105, &intervals), + "polling rule does not fire off-interval" + ); + assert_eq!( + Vec::::new(), + block_trigger_types_from_intervals(90, &intervals), + "polling rule does not fire before start_block" + ); + } + + #[test] + fn block_trigger_types_once_and_polling_same_start_block() { + // A single data source with both a `once` handler and a `polling` handler + // contributes both entries at its start_block. Both must fire at start_block. + let intervals = HashSet::from_iter(vec![(100, 0), (100, 10)]); + + assert_eq!( + vec![ + EthereumBlockTriggerType::Start, + EthereumBlockTriggerType::End, + ], + block_trigger_types_from_intervals(100, &intervals), + "both Start and End should fire when once and polling rules share start_block" + ); + assert_eq!( + vec![EthereumBlockTriggerType::End], + block_trigger_types_from_intervals(110, &intervals), + "only polling fires at later interval matches" + ); + } + + #[test] + fn block_trigger_types_cross_datasource_collision() { + // Two data sources: DS-A with once at 100, DS-B with polling every 10 from 50. + // Block 100 satisfies DS-A's once rule and also (100-50) % 10 == 0, so both fire. + let intervals = HashSet::from_iter(vec![(100, 0), (50, 10)]); + + assert_eq!( + vec![ + EthereumBlockTriggerType::Start, + EthereumBlockTriggerType::End, + ], + block_trigger_types_from_intervals(100, &intervals), + "both triggers fire when a once rule and an unrelated polling rule collide" + ); + assert_eq!( + vec![EthereumBlockTriggerType::End], + block_trigger_types_from_intervals(60, &intervals), + "only polling fires at a block where only the polling rule matches" + ); + } + + #[test] + fn block_trigger_types_no_match() { + let intervals = HashSet::from_iter(vec![(100, 0), (100, 10)]); + assert_eq!( + Vec::::new(), + block_trigger_types_from_intervals(99, &intervals), + "no triggers when block matches neither rule" + ); + } + + #[test] + fn block_trigger_types_empty_intervals() { + let intervals: HashSet<(i32, i32)> = HashSet::new(); + assert_eq!( + Vec::::new(), + block_trigger_types_from_intervals(100, &intervals), + "empty intervals yields no triggers" + ); + } + fn address(id: u64) -> Address { - Address::from_low_u64_be(id) + Address::left_padding_from(&id.to_be_bytes()) } - fn hash(id: u8) -> H256 { - H256::from([id; 32]) + fn hash(id: u8) -> B256 { + B256::from_slice(&[id; 32]) } fn bytes(value: Vec) -> Bytes { diff --git a/chain/ethereum/src/ingestor.rs b/chain/ethereum/src/ingestor.rs index a0d1f9a8247..dcbcc42c6f8 100644 --- a/chain/ethereum/src/ingestor.rs +++ b/chain/ethereum/src/ingestor.rs @@ -1,74 +1,57 @@ -use crate::{chain::BlockFinality, EthereumAdapter, EthereumAdapterTrait, ENV_VARS}; +use crate::{ENV_VARS, chain::BlockFinality}; +use crate::{EthereumAdapter, EthereumAdapterTrait as _}; +use async_trait::async_trait; +use futures::future::select_ok; +use graph::blockchain::BlockchainKind; +use graph::blockchain::client::ChainClient; +use graph::components::network_provider::ChainName; +use graph::prelude::alloy::primitives::B256; +use graph::slog::o; +use graph::util::backoff::ExponentialBackoff; use graph::{ - blockchain::{BlockHash, BlockPtr, IngestorError}, + blockchain::{BlockHash, BlockIngestor, BlockPtr, IngestorError}, cheap_clone::CheapClone, prelude::{ - error, ethabi::ethereum_types::H256, info, tokio, trace, warn, ChainStore, Error, - EthereumBlockWithCalls, Future01CompatExt, LogCode, Logger, + ChainStore, Error, EthereumBlockWithCalls, LogCode, Logger, debug, error, info, tokio, + trace, warn, }, }; use std::{sync::Arc, time::Duration}; -pub struct BlockIngestor { +pub struct PollingBlockIngestor { logger: Logger, ancestor_count: i32, - eth_adapter: Arc, + chain_client: Arc>, chain_store: Arc, polling_interval: Duration, + network_name: ChainName, } -impl BlockIngestor { +impl PollingBlockIngestor { pub fn new( logger: Logger, ancestor_count: i32, - eth_adapter: Arc, + chain_client: Arc>, chain_store: Arc, polling_interval: Duration, - ) -> Result { - Ok(BlockIngestor { + network_name: ChainName, + ) -> Result { + Ok(PollingBlockIngestor { logger, ancestor_count, - eth_adapter, + chain_client, chain_store, polling_interval, + network_name, }) } - pub async fn into_polling_stream(self) { - loop { - match self.do_poll().await { - // Some polls will fail due to transient issues - Err(err @ IngestorError::BlockUnavailable(_)) => { - info!( - self.logger, - "Trying again after block polling failed: {}", err - ); - } - Err(err @ IngestorError::ReceiptUnavailable(_, _)) => { - info!( - self.logger, - "Trying again after block polling failed: {}", err - ); - } - Err(IngestorError::Unknown(inner_err)) => { - warn!( - self.logger, - "Trying again after block polling failed: {}", inner_err - ); - } - Ok(()) => (), - } - - if ENV_VARS.cleanup_blocks { - self.cleanup_cached_blocks() - } - - tokio::time::sleep(self.polling_interval).await; - } - } - - fn cleanup_cached_blocks(&self) { - match self.chain_store.cleanup_cached_blocks(self.ancestor_count) { + async fn cleanup_cached_blocks(&self) { + match self + .chain_store + .cleanup_cached_blocks(self.ancestor_count) + .await + { Ok(Some((min_block, count))) => { if count > 0 { info!( @@ -88,8 +71,12 @@ impl BlockIngestor { } } - async fn do_poll(&self) -> Result<(), IngestorError> { - trace!(self.logger, "BlockIngestor::do_poll"); + async fn do_poll( + &self, + logger: &Logger, + eth_adapter: Arc, + ) -> Result<(), IngestorError> { + trace!(&logger, "BlockIngestor::do_poll"); // Get chain head ptr from store let head_block_ptr_opt = self.chain_store.cheap_clone().chain_head_ptr().await?; @@ -97,7 +84,7 @@ impl BlockIngestor { // To check if there is a new block or not, fetch only the block header since that's cheaper // than the full block. This is worthwhile because most of the time there won't be a new // block, as we expect the poll interval to be much shorter than the block time. - let latest_block = self.latest_block().await?; + let latest_block = self.latest_block(logger, ð_adapter).await?; if let Some(head_block) = head_block_ptr_opt.as_ref() { // If latest block matches head block in store, nothing needs to be done @@ -109,7 +96,7 @@ impl BlockIngestor { // An ingestor might wait or move forward, but it never // wavers and goes back. More seriously, this keeps us from // later trying to ingest a block with the same number again - warn!(self.logger, + warn!(&logger, "Provider went backwards - ignoring this latest block"; "current_block_head" => head_block.number, "latest_block_head" => latest_block.number); @@ -121,7 +108,7 @@ impl BlockIngestor { match head_block_ptr_opt { None => { info!( - self.logger, + &logger, "Downloading latest blocks from Ethereum, this may take a few minutes..." ); } @@ -137,7 +124,7 @@ impl BlockIngestor { }; if distance > 0 { info!( - self.logger, + &logger, "Syncing {} blocks from Ethereum", blocks_needed; "current_block_head" => head_number, @@ -154,7 +141,9 @@ impl BlockIngestor { // Might be a no-op if latest block is one that we have seen. // ingest_blocks will return a (potentially incomplete) list of blocks that are // missing. - let mut missing_block_hash = self.ingest_block(&latest_block.hash).await?; + let mut missing_block_hash = self + .ingest_block(logger, ð_adapter, &latest_block.hash) + .await?; // Repeatedly fetch missing parent blocks, and ingest them. // ingest_blocks will continue to tell us about more missing parent @@ -175,29 +164,25 @@ impl BlockIngestor { // iteration will have at most block number N-1. // - Therefore, the loop will iterate at most ancestor_count times. while let Some(hash) = missing_block_hash { - missing_block_hash = self.ingest_block(&hash).await?; + missing_block_hash = self.ingest_block(logger, ð_adapter, &hash).await?; } Ok(()) } async fn ingest_block( &self, + logger: &Logger, + eth_adapter: &Arc, block_hash: &BlockHash, ) -> Result, IngestorError> { - // TODO: H256::from_slice can panic - let block_hash = H256::from_slice(block_hash.as_slice()); + let block_hash = B256::from_slice(block_hash.as_slice()); // Get the fully populated block - let block = self - .eth_adapter - .block_by_hash(&self.logger, block_hash) - .compat() + let block = eth_adapter + .block_by_hash(logger, block_hash) .await? .ok_or(IngestorError::BlockUnavailable(block_hash))?; - let ethereum_block = self - .eth_adapter - .load_full_block(&self.logger, block) - .await?; + let ethereum_block = eth_adapter.load_full_block(logger, block).await?; // We need something that implements `Block` to store the block; the // store does not care whether the block is final or not @@ -217,16 +202,428 @@ impl BlockIngestor { .await .map(|missing| missing.map(|h256| h256.into())) .map_err(|e| { - error!(self.logger, "failed to update chain head"); + error!(logger, "failed to update chain head"); IngestorError::Unknown(e) }) } - async fn latest_block(&self) -> Result { - self.eth_adapter - .latest_block_header(&self.logger) - .compat() - .await - .map(|block| block.into()) + async fn latest_block( + &self, + logger: &Logger, + eth_adapter: &Arc, + ) -> Result { + eth_adapter.latest_block_ptr(logger).await + } + + /// Executes one polling iteration. On failure delegates to `on_poll_failure` for + /// probe+switch logic. + async fn poll_once( + &self, + providers: &[Arc], + current_provider: &mut Option, + ) { + // Resolve by name; resets to first provider if the tracked one left the list. + let eth_adapter = resolve_provider(providers, current_provider, &self.logger).clone(); + // Pin the name so the next iteration knows which provider is active. + let provider_name = eth_adapter.provider().to_string(); + let logger = self.logger.new(o!("provider" => provider_name.clone())); + *current_provider = Some(provider_name); + + if let Err(err) = self.do_poll(&logger, eth_adapter).await { + error!(logger, "Trying again after block polling failed: {}", err); + on_poll_failure(providers, current_provider, &self.logger).await; + } + } +} + +/// Returns the currently-tracked provider from `providers`. +/// +/// If the tracked provider is no longer in the list (it became invalid and was removed by +/// `ProviderManager`), logs a warning, resets the state, and returns the first available +/// provider. +fn resolve_provider<'a, A: crate::EthereumAdapterTrait>( + providers: &'a [Arc], + current_provider: &mut Option, + logger: &Logger, +) -> &'a Arc { + if let Some(name) = current_provider.as_ref() { + if let Some(found) = providers.iter().find(|p| p.provider() == name) { + return found; + } + warn!( + logger, + "Current RPC provider is no longer available, resetting to first provider"; + "provider" => name, + ); + *current_provider = None; + } + &providers[0] +} + +async fn on_poll_failure( + providers: &[Arc], + current_provider: &mut Option, + logger: &Logger, +) { + if providers.len() <= 1 { + return; + } + + let current_name = match current_provider.as_ref() { + Some(name) => name.clone(), + None => return, + }; + + // Probe the current provider before trying alternatives. do_poll() can fail for + // reasons unrelated to RPC availability (e.g. DB errors from chain_head_ptr or + // attempt_chain_head_update, or a BlockUnavailable from a chain reorg). All of + // these surface as IngestorError::Unknown, indistinguishable from an RPC failure + // at the match level. If the current provider responds to eth_blockNumber, the + // failure was not caused by provider unavailability — switching cannot help. + let current = providers.iter().find(|p| p.provider() == current_name); + if let Some(current) = current + && current.is_reachable().await + { + return; + } + + // Probe all alternatives in parallel; switch to the first that responds. + let futs = providers + .iter() + .filter(|p| p.provider() != current_name) + .map(|p| { + let name = p.provider().to_string(); + debug!(logger, "Trying RPC provider"; "provider" => &name); + Box::pin(async move { + if p.is_reachable().await { + Ok(name) + } else { + Err(()) + } + }) + }); + + match select_ok(futs).await { + Ok((next_name, _)) => { + warn!( + logger, + "Switching RPC provider for block ingestor"; + "from" => ¤t_name, + "to" => &next_name, + ); + *current_provider = Some(next_name); + } + Err(_) => { + warn!( + logger, + "All RPC providers unreachable, continuing on current provider"; + "provider" => ¤t_name, + ); + } + } +} + +#[async_trait] +impl BlockIngestor for PollingBlockIngestor { + async fn run(self: Box) { + let mut backoff = + ExponentialBackoff::new(Duration::from_millis(250), Duration::from_secs(30)); + // Name of the provider currently in use. `None` until the first poll. + let mut current_provider: Option = None; + + loop { + let providers = self + .chain_client + .rpc() + .expect("PollingBlockIngestor is only created for RPC chains") + .all_cheapest() + .await; + + if providers.is_empty() { + error!(self.logger, "No RPC providers available for block ingestor"); + backoff.sleep_async().await; + continue; + } + backoff.reset(); + + self.poll_once(&providers, &mut current_provider).await; + + if ENV_VARS.cleanup_blocks { + self.cleanup_cached_blocks().await; + } + + tokio::time::sleep(self.polling_interval).await; + } + } + + fn network_name(&self) -> ChainName { + self.network_name.clone() + } + + fn kind(&self) -> BlockchainKind { + BlockchainKind::Ethereum + } +} + +#[cfg(test)] +mod tests { + use super::on_poll_failure; + use super::*; + use crate::adapter::{ + ContractCallError, EthereumAdapter as EthereumAdapterTrait, EthereumRpcError, + }; + use async_trait::async_trait; + use graph::blockchain::{BlockPtr, ChainIdentifier}; + use graph::components::ethereum::AnyBlock; + use graph::components::ethereum::LightEthereumBlock; + use graph::data::store::ethereum::call; + use graph::data_source::common::ContractCall; + use graph::prelude::alloy::primitives::{Address, B256, Bytes, U256}; + use graph::prelude::{BlockNumber, Error, EthereumCallCache, Logger}; + use graph::slog::Discard; + use std::collections::HashSet; + use std::sync::Arc; + + struct MockEthAdapter { + provider_name: String, + reachable: bool, + } + + impl MockEthAdapter { + fn new(name: &str, reachable: bool) -> Arc { + Arc::new(Self { + provider_name: name.to_string(), + reachable, + }) + } + } + + #[async_trait] + impl EthereumAdapterTrait for MockEthAdapter { + fn provider(&self) -> &str { + &self.provider_name + } + + async fn is_reachable(&self) -> bool { + self.reachable + } + + async fn net_identifiers(&self) -> Result { + unimplemented!() + } + async fn latest_block_ptr( + &self, + _: &Logger, + ) -> Result { + unimplemented!() + } + async fn load_blocks( + &self, + _: Logger, + _: Arc, + _: HashSet, + ) -> Result>, Error> { + unimplemented!() + } + async fn block_by_hash(&self, _: &Logger, _: B256) -> Result, Error> { + unimplemented!() + } + async fn block_by_number( + &self, + _: &Logger, + _: BlockNumber, + ) -> Result, Error> { + unimplemented!() + } + async fn load_full_block( + &self, + _: &Logger, + _: AnyBlock, + ) -> Result { + unimplemented!() + } + async fn next_existing_ptr_to_number( + &self, + _: &Logger, + _: BlockNumber, + ) -> Result { + unimplemented!() + } + async fn contract_call( + &self, + _: &Logger, + _: &ContractCall, + _: Arc, + ) -> Result<(Option>, call::Source), ContractCallError> + { + unimplemented!() + } + async fn contract_calls( + &self, + _: &Logger, + _: &[&ContractCall], + _: Arc, + ) -> Result>, call::Source)>, ContractCallError> + { + unimplemented!() + } + async fn get_balance( + &self, + _: &Logger, + _: Address, + _: BlockPtr, + ) -> Result { + unimplemented!() + } + async fn get_code( + &self, + _: &Logger, + _: Address, + _: BlockPtr, + ) -> Result { + unimplemented!() + } + } + + fn discard_logger() -> Logger { + Logger::root(Discard, o!()) + } + + #[test] + fn test_current_provider_unavailable_resets_to_first() { + // p0 left the validated list; only p1 and p2 remain (p1 is now at index 0). + let providers: Vec> = vec![ + MockEthAdapter::new("p1", true), + MockEthAdapter::new("p2", true), + ]; + let mut current_provider = Some("p0".to_string()); + let resolved = resolve_provider(&providers, &mut current_provider, &discard_logger()); + assert_eq!(resolved.provider(), "p1"); + assert_eq!(current_provider, None); + } + + #[tokio::test] + async fn test_current_reachable_no_switch() { + // Current provider is reachable: on_poll_failure should return early without + // switching, even if alternatives are also reachable. + let providers: Vec> = vec![ + MockEthAdapter::new("p0", true), + MockEthAdapter::new("p1", true), + ]; + let mut current_provider = Some("p0".to_string()); + on_poll_failure(&providers, &mut current_provider, &discard_logger()).await; + assert_eq!(current_provider, Some("p0".to_string())); + } + + #[tokio::test] + async fn test_successful_probe_switches_provider() { + let providers: Vec> = vec![ + MockEthAdapter::new("p0", false), + MockEthAdapter::new("p1", true), + ]; + let mut current_provider = Some("p0".to_string()); + on_poll_failure(&providers, &mut current_provider, &discard_logger()).await; + assert_eq!(current_provider, Some("p1".to_string())); + } + + #[tokio::test] + async fn test_all_unreachable_stays_on_current() { + let providers: Vec> = vec![ + MockEthAdapter::new("p0", false), + MockEthAdapter::new("p1", false), + ]; + let mut current_provider = Some("p0".to_string()); + on_poll_failure(&providers, &mut current_provider, &discard_logger()).await; + assert_eq!(current_provider, Some("p0".to_string())); + } + + #[tokio::test] + async fn test_single_provider_no_switch() { + let providers: Vec> = vec![MockEthAdapter::new("p0", true)]; + let mut current_provider = Some("p0".to_string()); + on_poll_failure(&providers, &mut current_provider, &discard_logger()).await; + assert_eq!(current_provider, Some("p0".to_string())); + } + + // --- Provider-list churn tests --- + // + // These tests simulate the state carried across loop iterations where + // all_cheapest() returns a different subset each time. + + /// Current provider survives a list shrink; its index shifts but identity is preserved. + /// + /// Before: [p0, p1, p2], active = p1 (index 1) + /// After: [p1, p2], active = p1 (index 0) + /// + /// With index-based tracking this would silently move to p2. Name-based tracking + /// must keep us on p1. + #[test] + fn test_list_shrinks_current_provider_index_remaps() { + let providers: Vec> = vec![ + MockEthAdapter::new("p1", true), + MockEthAdapter::new("p2", true), + ]; + let mut current_provider = Some("p1".to_string()); + let resolved = resolve_provider(&providers, &mut current_provider, &discard_logger()); + // p1 is now at index 0 — must not drift to p2 + assert_eq!(resolved.provider(), "p1"); + assert_eq!(current_provider, Some("p1".to_string())); + } + + /// After failing over to p1, p1 itself disappears from the validated list in the + /// next iteration. The ingestor must fall back to the first available provider (p0). + /// + /// Iteration 1: [p0(unreachable), p1(reachable)] — p0 fails, switch to p1 + /// Iteration 2: [p0(reachable), p2(reachable)] — p1 gone, reset to p0 + #[tokio::test] + async fn test_failover_target_then_disappears_resets_to_first() { + let logger = discard_logger(); + + // Iteration 1: p0 fails, switch to p1. + let providers_iter1: Vec> = vec![ + MockEthAdapter::new("p0", false), + MockEthAdapter::new("p1", true), + ]; + let mut current_provider = Some("p0".to_string()); + on_poll_failure(&providers_iter1, &mut current_provider, &logger).await; + assert_eq!(current_provider, Some("p1".to_string())); + + // Iteration 2: p1 has dropped from the validated list. + let providers_iter2: Vec> = vec![ + MockEthAdapter::new("p0", true), + MockEthAdapter::new("p2", true), + ]; + let resolved = resolve_provider(&providers_iter2, &mut current_provider, &logger); + assert_eq!(resolved.provider(), "p0"); // reset to first + assert_eq!(current_provider, None); + } + + /// After failing over to p1, the original provider p0 reappears in the list. + /// The ingestor must stay on p1 — there is no proactive return to p0. + /// + /// Iteration 1: [p0(unreachable), p1(reachable)] — switch to p1 + /// Iteration 2: [p0(reachable), p1(reachable)] — p0 is back; must stay on p1 + #[tokio::test] + async fn test_original_provider_reappears_no_involuntary_return() { + let logger = discard_logger(); + + // Iteration 1: switch to p1. + let providers_iter1: Vec> = vec![ + MockEthAdapter::new("p0", false), + MockEthAdapter::new("p1", true), + ]; + let mut current_provider = Some("p0".to_string()); + on_poll_failure(&providers_iter1, &mut current_provider, &logger).await; + assert_eq!(current_provider, Some("p1".to_string())); + + // Iteration 2: p0 is back in the validated list alongside p1. + let providers_iter2: Vec> = vec![ + MockEthAdapter::new("p0", true), + MockEthAdapter::new("p1", true), + ]; + let resolved = resolve_provider(&providers_iter2, &mut current_provider, &logger); + // Must resolve to p1, not drift back to p0. + assert_eq!(resolved.provider(), "p1"); + assert_eq!(current_provider, Some("p1".to_string())); } } diff --git a/chain/ethereum/src/lib.rs b/chain/ethereum/src/lib.rs index eeb207bf2d7..b21eb4717ae 100644 --- a/chain/ethereum/src/lib.rs +++ b/chain/ethereum/src/lib.rs @@ -1,21 +1,30 @@ mod adapter; +mod buffered_call_cache; +mod call_helper; mod capabilities; pub mod codec; mod data_source; mod env; mod ethereum_adapter; mod ingestor; +mod polling_block_stream; pub mod runtime; mod transport; pub use self::capabilities::NodeCapabilities; pub use self::ethereum_adapter::EthereumAdapter; pub use self::runtime::RuntimeAdapter; -pub use self::transport::Transport; +pub use self::transport::{Compression, Transport}; pub use env::ENV_VARS; +pub use buffered_call_cache::BufferedCallCache; + // ETHDEP: These concrete types should probably not be exposed. -pub use data_source::{DataSource, DataSourceTemplate, Mapping, MappingABI, TemplateSource}; +pub use data_source::{ + BlockHandlerFilter, DataSource, DataSourceTemplate, Mapping, MappingBlockHandler, + MappingCallHandler, TemplateSource, UnresolvedDataSource, UnresolvedDataSourceTemplate, + UnresolvedMapping, UnresolvedMappingEventHandler, +}; pub mod chain; @@ -23,12 +32,11 @@ pub mod network; pub mod trigger; pub use crate::adapter::{ - EthereumAdapter as EthereumAdapterTrait, EthereumContractCall, EthereumContractCallError, - ProviderEthRpcMetrics, SubgraphEthRpcMetrics, TriggerFilter, + ContractCallError, EthereumAdapter as EthereumAdapterTrait, ProviderEthRpcMetrics, + SubgraphEthRpcMetrics, TriggerFilter, }; pub use crate::chain::Chain; -pub use crate::network::EthereumNetworks; -pub use ingestor::BlockIngestor; +pub use graph::blockchain::BlockIngestor; #[cfg(test)] mod tests; diff --git a/chain/ethereum/src/network.rs b/chain/ethereum/src/network.rs index 2779f76a799..ad9cd0a83c0 100644 --- a/chain/ethereum/src/network.rs +++ b/chain/ethereum/src/network.rs @@ -1,20 +1,29 @@ -use anyhow::{anyhow, bail, Context}; -use graph::cheap_clone::CheapClone; -use graph::firehose::SubgraphLimit; -use graph::prelude::rand::{self, seq::IteratorRandom}; -use std::cmp::Ordering; -use std::collections::HashMap; +use anyhow::{anyhow, bail}; +use async_trait::async_trait; +use graph::blockchain::ChainIdentifier; +use graph::components::network_provider::ChainName; +use graph::components::network_provider::NetworkDetails; +use graph::components::network_provider::ProviderManager; +use graph::components::network_provider::ProviderName; +use graph::endpoint::EndpointMetrics; +use graph::firehose::{AvailableCapacity, SubgraphLimit}; +use graph::prelude::rand::seq::IteratorRandom; +use graph::prelude::rand::{self, Rng}; +use itertools::Itertools; use std::sync::Arc; pub use graph::impl_slog_value; use graph::prelude::Error; +use crate::EthereumAdapter; use crate::adapter::EthereumAdapter as _; use crate::capabilities::NodeCapabilities; -use crate::EthereumAdapter; + +pub const DEFAULT_ADAPTER_ERROR_RETEST_PERCENT: f64 = 0.2; #[derive(Debug, Clone)] pub struct EthereumNetworkAdapter { + endpoint_metrics: Arc, pub capabilities: NodeCapabilities, adapter: Arc, /// The maximum number of times this adapter can be used. We use the @@ -24,75 +33,244 @@ pub struct EthereumNetworkAdapter { limit: SubgraphLimit, } +#[async_trait] +impl NetworkDetails for EthereumNetworkAdapter { + fn provider_name(&self) -> ProviderName { + self.adapter.provider().into() + } + + async fn chain_identifier(&self) -> Result { + self.adapter.net_identifiers().await + } + + async fn provides_extended_blocks(&self) -> Result { + Ok(true) + } +} + impl EthereumNetworkAdapter { + pub fn new( + endpoint_metrics: Arc, + capabilities: NodeCapabilities, + adapter: Arc, + limit: SubgraphLimit, + ) -> Self { + Self { + endpoint_metrics, + capabilities, + adapter, + limit, + } + } + + #[cfg(debug_assertions)] fn is_call_only(&self) -> bool { self.adapter.is_call_only() } + + pub fn get_capacity(&self) -> AvailableCapacity { + self.limit.get_capacity(Arc::strong_count(&self.adapter)) + } + + pub fn current_error_count(&self) -> u64 { + self.endpoint_metrics.get_count(&self.provider().into()) + } + pub fn provider(&self) -> &str { + self.adapter.provider() + } } -#[derive(Debug, Clone, Default)] +#[derive(Debug, Clone)] pub struct EthereumNetworkAdapters { - pub adapters: Vec, - pub call_only_adapters: Vec, + chain_id: ChainName, + manager: ProviderManager, + call_only_adapters: Vec, + // Percentage of request that should be used to retest errored adapters. + retest_percent: f64, } impl EthereumNetworkAdapters { - pub fn push_adapter(&mut self, adapter: EthereumNetworkAdapter) { - if adapter.is_call_only() { - self.call_only_adapters.push(adapter); - } else { - self.adapters.push(adapter); + pub fn empty_for_testing() -> Self { + Self { + chain_id: "".into(), + manager: ProviderManager::default(), + call_only_adapters: vec![], + retest_percent: DEFAULT_ADAPTER_ERROR_RETEST_PERCENT, } } - pub fn all_cheapest_with( - &self, + + #[cfg(debug_assertions)] + pub async fn for_testing( + mut adapters: Vec, + call_only: Vec, + ) -> Self { + use std::cmp::Ordering; + + use graph::components::network_provider::ProviderCheckStrategy; + use graph::slog::{Discard, Logger, o}; + + let chain_id: ChainName = "testing".into(); + adapters.sort_by(|a, b| { + a.capabilities + .partial_cmp(&b.capabilities) + .unwrap_or(Ordering::Equal) + }); + + let provider = ProviderManager::new( + Logger::root(Discard, o!()), + vec![(chain_id.clone(), adapters)], + ProviderCheckStrategy::MarkAsValid, + ); + + Self::new(chain_id, provider, call_only, None) + } + + pub fn new( + chain_id: ChainName, + manager: ProviderManager, + call_only_adapters: Vec, + retest_percent: Option, + ) -> Self { + #[cfg(debug_assertions)] + call_only_adapters.iter().for_each(|a| { + a.is_call_only(); + }); + + Self { + chain_id, + manager, + call_only_adapters, + retest_percent: retest_percent.unwrap_or(DEFAULT_ADAPTER_ERROR_RETEST_PERCENT), + } + } + + fn available_with_capabilities<'a>( + input: Vec<&'a EthereumNetworkAdapter>, required_capabilities: &NodeCapabilities, - ) -> impl Iterator> + '_ { - let cheapest_sufficient_capability = self - .adapters + ) -> impl Iterator + 'a { + let cheapest_sufficient_capability = input .iter() .find(|adapter| &adapter.capabilities >= required_capabilities) .map(|adapter| &adapter.capabilities); - self.adapters - .iter() + input + .into_iter() .filter(move |adapter| Some(&adapter.capabilities) == cheapest_sufficient_capability) - .filter(|adapter| { - adapter - .limit - .has_capacity(Arc::strong_count(&adapter.adapter)) - }) - .map(|adapter| adapter.adapter.cheap_clone()) + .filter(|adapter| adapter.get_capacity() > AvailableCapacity::Unavailable) } - pub fn cheapest_with( + /// returns all the available adapters that meet the required capabilities + /// if no adapters are available at the time or none that meet the capabilities then + /// an empty iterator is returned. + pub async fn all_cheapest_with( &self, required_capabilities: &NodeCapabilities, + ) -> impl Iterator + '_ { + let all = self + .manager + .providers(&self.chain_id) + .await + .map(|adapters| adapters.collect_vec()) + .unwrap_or_default(); + + Self::available_with_capabilities(all, required_capabilities) + } + + // get all the adapters, don't trigger the ProviderManager's validations because we want + // this function to remain sync. If no adapters are available an empty iterator is returned. + pub(crate) fn all_unverified_cheapest_with( + &self, + required_capabilities: &NodeCapabilities, + ) -> impl Iterator + '_ { + let all = self + .manager + .providers_unchecked(&self.chain_id) + .collect_vec(); + + Self::available_with_capabilities(all, required_capabilities) + } + + // handle adapter selection from a list, implements the availability checking with an abstracted + // source of the adapter list. + fn cheapest_from( + input: Vec<&EthereumNetworkAdapter>, + required_capabilities: &NodeCapabilities, + retest_percent: f64, ) -> Result, Error> { - // Select randomly from the cheapest adapters that have sufficent capabilities. - self.all_cheapest_with(required_capabilities) - .choose(&mut rand::thread_rng()) - .with_context(|| { - anyhow!( - "A matching Ethereum network with {:?} was not found.", - required_capabilities - ) - }) + let retest_rng: f64 = rand::rng().random(); + + let cheapest = input.into_iter().choose_multiple(&mut rand::rng(), 3); + let cheapest = cheapest.iter(); + + // If request falls below the retest threshold, use this request to try and + // reset the failed adapter. If a request succeeds the adapter will be more + // likely to be selected afterwards. + if retest_rng < retest_percent { + cheapest.max_by_key(|adapter| adapter.current_error_count()) + } else { + // The assumption here is that most RPC endpoints will not have limits + // which makes the check for low/high available capacity less relevant. + // So we essentially assume if it had available capacity when calling + // `all_cheapest_with` then it prolly maintains that state and so we + // just select whichever adapter is working better according to + // the number of errors. + cheapest.min_by_key(|adapter| adapter.current_error_count()) + } + .map(|adapter| adapter.adapter.clone()) + .ok_or(anyhow!( + "A matching Ethereum network with {:?} was not found.", + required_capabilities + )) + } + + pub(crate) fn unverified_cheapest_with( + &self, + required_capabilities: &NodeCapabilities, + ) -> Result, Error> { + let cheapest = self.all_unverified_cheapest_with(required_capabilities); + + Self::cheapest_from( + cheapest.choose_multiple(&mut rand::rng(), 3), + required_capabilities, + self.retest_percent, + ) + } + + /// This is the public entry point and should always use verified adapters + pub async fn cheapest_with( + &self, + required_capabilities: &NodeCapabilities, + ) -> Result, Error> { + let cheapest = self + .all_cheapest_with(required_capabilities) + .await + .choose_multiple(&mut rand::rng(), 3); + + Self::cheapest_from(cheapest, required_capabilities, self.retest_percent) } - pub fn cheapest(&self) -> Option> { + /// Returns all validated providers. Unvalidated providers are excluded. + pub async fn all_cheapest(&self) -> Vec> { + self.manager + .providers(&self.chain_id) + .await + .map(|adapters| adapters.map(|a| a.adapter.clone()).collect()) + .unwrap_or_default() + } + + pub async fn cheapest(&self) -> Option> { // EthereumAdapters are sorted by their NodeCapabilities when the EthereumNetworks // struct is instantiated so they do not need to be sorted here - self.adapters - .first() + self.manager + .providers(&self.chain_id) + .await + .map(|mut adapters| adapters.next()) + .unwrap_or_default() .map(|ethereum_network_adapter| ethereum_network_adapter.adapter.clone()) } - pub fn remove(&mut self, provider: &str) { - self.adapters - .retain(|adapter| adapter.adapter.provider() != provider); - } - + /// call_or_cheapest will bypass ProviderManagers' validation in order to remain non async. + /// ideally this should only be called for already validated providers. pub fn call_or_cheapest( &self, capabilities: Option<&NodeCapabilities>, @@ -102,11 +280,13 @@ impl EthereumNetworkAdapters { // so we will ignore this error and return whatever comes out of `cheapest_with` match self.call_only_adapter() { Ok(Some(adapter)) => Ok(adapter), - _ => self.cheapest_with(capabilities.unwrap_or(&NodeCapabilities { - // Archive is required for call_only - archive: true, - traces: false, - })), + _ => { + self.unverified_cheapest_with(capabilities.unwrap_or(&NodeCapabilities { + // Archive is required for call_only + archive: true, + traces: false, + })) + } } } @@ -136,100 +316,33 @@ impl EthereumNetworkAdapters { } } -#[derive(Clone)] -pub struct EthereumNetworks { - pub networks: HashMap, -} - -impl EthereumNetworks { - pub fn new() -> EthereumNetworks { - EthereumNetworks { - networks: HashMap::new(), - } - } - - pub fn insert( - &mut self, - name: String, - capabilities: NodeCapabilities, - adapter: Arc, - limit: SubgraphLimit, - ) { - let network_adapters = self.networks.entry(name).or_default(); - - network_adapters.push_adapter(EthereumNetworkAdapter { - capabilities, - adapter, - limit, - }); - } - - pub fn remove(&mut self, name: &str, provider: &str) { - if let Some(adapters) = self.networks.get_mut(name) { - adapters.remove(provider); - } - } - - pub fn extend(&mut self, other_networks: EthereumNetworks) { - self.networks.extend(other_networks.networks); - } - - pub fn flatten(&self) -> Vec<(String, NodeCapabilities, Arc)> { - self.networks - .iter() - .flat_map(|(network_name, network_adapters)| { - network_adapters - .adapters - .iter() - .map(move |network_adapter| { - ( - network_name.clone(), - network_adapter.capabilities, - network_adapter.adapter.clone(), - ) - }) - }) - .collect() - } - - pub fn sort(&mut self) { - for adapters in self.networks.values_mut() { - adapters.adapters.sort_by(|a, b| { - a.capabilities - .partial_cmp(&b.capabilities) - // We can't define a total ordering over node capabilities, - // so incomparable items are considered equal and end up - // near each other. - .unwrap_or(Ordering::Equal) - }) - } - } - - pub fn adapter_with_capabilities( - &self, - network_name: String, - requirements: &NodeCapabilities, - ) -> Result, Error> { - self.networks - .get(&network_name) - .ok_or(anyhow!("network not supported: {}", &network_name)) - .and_then(|adapters| adapters.cheapest_with(requirements)) - } -} - #[cfg(test)] mod tests { + use graph::cheap_clone::CheapClone; + use graph::components::network_provider::ProviderCheckStrategy; + use graph::components::network_provider::ProviderManager; + use graph::components::network_provider::ProviderName; + use graph::data::value::Word; + + use graph::http::HeaderMap; + use graph::{ + endpoint::EndpointMetrics, + firehose::SubgraphLimit, + prelude::MetricsRegistry, + slog::{Discard, Logger, o}, + url::Url, + }; use std::sync::Arc; - use graph::{firehose::SubgraphLimit, prelude::MetricsRegistry, tokio, url::Url}; - use graph_mock::MockMetricsRegistry; - use http::HeaderMap; - - use crate::{EthereumAdapter, EthereumNetworks, ProviderEthRpcMetrics, Transport}; + use crate::{ + Compression, EthereumAdapter, EthereumAdapterTrait, ProviderEthRpcMetrics, Transport, + chain::ChainSettings, + }; - use super::NodeCapabilities; + use super::{EthereumNetworkAdapter, EthereumNetworkAdapters, NodeCapabilities}; #[test] + #[allow(clippy::neg_cmp_op_on_partial_ord)] fn ethereum_capabilities_comparison() { let archive = NodeCapabilities { archive: true, @@ -253,55 +366,61 @@ mod tests { }; // Test all real combinations of capability comparisons - assert_eq!(false, &full >= &archive); - assert_eq!(false, &full >= &traces); - assert_eq!(false, &full >= &archive_traces); - assert_eq!(true, &full >= &full); - assert_eq!(false, &full >= &full_traces); - - assert_eq!(true, &archive >= &archive); - assert_eq!(false, &archive >= &traces); - assert_eq!(false, &archive >= &archive_traces); - assert_eq!(true, &archive >= &full); - assert_eq!(false, &archive >= &full_traces); - - assert_eq!(false, &traces >= &archive); - assert_eq!(true, &traces >= &traces); - assert_eq!(false, &traces >= &archive_traces); - assert_eq!(true, &traces >= &full); - assert_eq!(true, &traces >= &full_traces); - - assert_eq!(true, &archive_traces >= &archive); - assert_eq!(true, &archive_traces >= &traces); - assert_eq!(true, &archive_traces >= &archive_traces); - assert_eq!(true, &archive_traces >= &full); - assert_eq!(true, &archive_traces >= &full_traces); - - assert_eq!(false, &full_traces >= &archive); - assert_eq!(true, &full_traces >= &traces); - assert_eq!(false, &full_traces >= &archive_traces); - assert_eq!(true, &full_traces >= &full); - assert_eq!(true, &full_traces >= &full_traces); + assert!(!(full >= archive)); + assert!(!(full >= traces)); + assert!(!(full >= archive_traces)); + assert!(full >= full); + assert!(!(full >= full_traces)); + + assert!(archive >= archive); + assert!(!(archive >= traces)); + assert!(!(archive >= archive_traces)); + assert!(archive >= full); + assert!(!(archive >= full_traces)); + + assert!(!(traces >= archive)); + assert!(traces >= traces); + assert!(!(traces >= archive_traces)); + assert!(traces >= full); + assert!(traces >= full_traces); + + assert!(archive_traces >= archive); + assert!(archive_traces >= traces); + assert!(archive_traces >= archive_traces); + assert!(archive_traces >= full); + assert!(archive_traces >= full_traces); + + assert!(!(full_traces >= archive)); + assert!(full_traces >= traces); + assert!(!(full_traces >= archive_traces)); + assert!(full_traces >= full); + assert!(full_traces >= full_traces); } - #[tokio::test] + #[graph::test] async fn adapter_selector_selects_eth_call() { - let chain = "mainnet".to_string(); + let metrics = Arc::new(EndpointMetrics::mock()); let logger = graph::log::logger(true); - let mock_registry: Arc = Arc::new(MockMetricsRegistry::new()); - let transport = - Transport::new_rpc(Url::parse("http://127.0.0.1").unwrap(), HeaderMap::new()); + let mock_registry = Arc::new(MetricsRegistry::mock()); + let transport = Transport::new_rpc( + Url::parse("http://127.0.0.1").unwrap(), + HeaderMap::new(), + metrics.clone(), + "", + false, + Compression::None, + ); let provider_metrics = Arc::new(ProviderEthRpcMetrics::new(mock_registry.clone())); let eth_call_adapter = Arc::new( EthereumAdapter::new( logger.clone(), String::new(), - "http://127.0.0.1", transport.clone(), provider_metrics.clone(), true, true, + Arc::new(ChainSettings::from_env_defaults()), ) .await, ); @@ -310,49 +429,51 @@ mod tests { EthereumAdapter::new( logger.clone(), String::new(), - "http://127.0.0.1", transport.clone(), provider_metrics.clone(), true, false, + Arc::new(ChainSettings::from_env_defaults()), ) .await, ); - let mut adapters = { - let mut ethereum_networks = EthereumNetworks::new(); - ethereum_networks.insert( - chain.clone(), + let mut adapters: EthereumNetworkAdapters = EthereumNetworkAdapters::for_testing( + vec![EthereumNetworkAdapter::new( + metrics.cheap_clone(), NodeCapabilities { archive: true, traces: false, }, - eth_call_adapter.clone(), + eth_adapter.clone(), SubgraphLimit::Limit(3), - ); - ethereum_networks.insert( - chain.clone(), + )], + vec![EthereumNetworkAdapter::new( + metrics.cheap_clone(), NodeCapabilities { archive: true, traces: false, }, - eth_adapter.clone(), + eth_call_adapter.clone(), SubgraphLimit::Limit(3), - ); - ethereum_networks.networks.get(&chain).unwrap().clone() - }; + )], + ) + .await; // one reference above and one inside adapters struct assert_eq!(Arc::strong_count(ð_call_adapter), 2); assert_eq!(Arc::strong_count(ð_adapter), 2); { // Not Found - assert!(adapters - .cheapest_with(&NodeCapabilities { - archive: false, - traces: true, - }) - .is_err()); + assert!( + adapters + .cheapest_with(&NodeCapabilities { + archive: false, + traces: true, + }) + .await + .is_err() + ); // Check cheapest is not call only let adapter = adapters @@ -360,18 +481,16 @@ mod tests { archive: true, traces: false, }) + .await .unwrap(); - assert_eq!(adapter.is_call_only(), false); + assert!(!adapter.is_call_only()); } // Check limits { let adapter = adapters.call_or_cheapest(None).unwrap(); assert!(adapter.is_call_only()); - assert_eq!( - adapters.call_or_cheapest(None).unwrap().is_call_only(), - false - ); + assert!(!adapters.call_or_cheapest(None).unwrap().is_call_only()); } // Check empty falls back to call only @@ -383,28 +502,34 @@ mod tests { traces: false, })) .unwrap(); - assert_eq!(adapter.is_call_only(), false); + assert!(!adapter.is_call_only()); } } - #[tokio::test] + #[graph::test] async fn adapter_selector_unlimited() { - let chain = "mainnet".to_string(); + let metrics = Arc::new(EndpointMetrics::mock()); let logger = graph::log::logger(true); - let mock_registry: Arc = Arc::new(MockMetricsRegistry::new()); - let transport = - Transport::new_rpc(Url::parse("http://127.0.0.1").unwrap(), HeaderMap::new()); + let mock_registry = Arc::new(MetricsRegistry::mock()); + let transport = Transport::new_rpc( + Url::parse("http://127.0.0.1").unwrap(), + HeaderMap::new(), + metrics.clone(), + "", + false, + Compression::None, + ); let provider_metrics = Arc::new(ProviderEthRpcMetrics::new(mock_registry.clone())); let eth_call_adapter = Arc::new( EthereumAdapter::new( logger.clone(), String::new(), - "http://127.0.0.1", transport.clone(), provider_metrics.clone(), true, true, + Arc::new(ChainSettings::from_env_defaults()), ) .await, ); @@ -413,66 +538,73 @@ mod tests { EthereumAdapter::new( logger.clone(), String::new(), - "http://127.0.0.1", transport.clone(), provider_metrics.clone(), true, false, + Arc::new(ChainSettings::from_env_defaults()), ) .await, ); - let adapters = { - let mut ethereum_networks = EthereumNetworks::new(); - ethereum_networks.insert( - chain.clone(), + let adapters: EthereumNetworkAdapters = EthereumNetworkAdapters::for_testing( + vec![EthereumNetworkAdapter::new( + metrics.cheap_clone(), NodeCapabilities { archive: true, traces: false, }, eth_call_adapter.clone(), SubgraphLimit::Unlimited, - ); - ethereum_networks.insert( - chain.clone(), + )], + vec![EthereumNetworkAdapter::new( + metrics.cheap_clone(), NodeCapabilities { archive: true, traces: false, }, eth_adapter.clone(), - SubgraphLimit::Limit(3), - ); - ethereum_networks.networks.get(&chain).unwrap().clone() - }; + SubgraphLimit::Limit(2), + )], + ) + .await; // one reference above and one inside adapters struct assert_eq!(Arc::strong_count(ð_call_adapter), 2); assert_eq!(Arc::strong_count(ð_adapter), 2); - let keep: Vec> = vec![0; 10] + // verify that after all call_only were exhausted, we can still + // get normal adapters + let keep: Vec> = [0; 10] .iter() .map(|_| adapters.call_or_cheapest(None).unwrap()) .collect(); - assert_eq!(keep.iter().any(|a| !a.is_call_only()), false); + assert!(!keep.iter().any(|a| !a.is_call_only())); } - #[tokio::test] + #[graph::test] async fn adapter_selector_disable_call_only_fallback() { - let chain = "mainnet".to_string(); + let metrics = Arc::new(EndpointMetrics::mock()); let logger = graph::log::logger(true); - let mock_registry: Arc = Arc::new(MockMetricsRegistry::new()); - let transport = - Transport::new_rpc(Url::parse("http://127.0.0.1").unwrap(), HeaderMap::new()); + let mock_registry = Arc::new(MetricsRegistry::mock()); + let transport = Transport::new_rpc( + Url::parse("http://127.0.0.1").unwrap(), + HeaderMap::new(), + metrics.clone(), + "", + false, + Compression::None, + ); let provider_metrics = Arc::new(ProviderEthRpcMetrics::new(mock_registry.clone())); let eth_call_adapter = Arc::new( EthereumAdapter::new( logger.clone(), String::new(), - "http://127.0.0.1", transport.clone(), provider_metrics.clone(), true, true, + Arc::new(ChainSettings::from_env_defaults()), ) .await, ); @@ -481,86 +613,357 @@ mod tests { EthereumAdapter::new( logger.clone(), String::new(), - "http://127.0.0.1", transport.clone(), provider_metrics.clone(), true, false, + Arc::new(ChainSettings::from_env_defaults()), ) .await, ); - let adapters = { - let mut ethereum_networks = EthereumNetworks::new(); - ethereum_networks.insert( - chain.clone(), + let adapters: EthereumNetworkAdapters = EthereumNetworkAdapters::for_testing( + vec![EthereumNetworkAdapter::new( + metrics.cheap_clone(), NodeCapabilities { archive: true, traces: false, }, eth_call_adapter.clone(), SubgraphLimit::Disabled, - ); - ethereum_networks.insert( - chain.clone(), + )], + vec![EthereumNetworkAdapter::new( + metrics.cheap_clone(), NodeCapabilities { archive: true, traces: false, }, eth_adapter.clone(), SubgraphLimit::Limit(3), - ); - ethereum_networks.networks.get(&chain).unwrap().clone() - }; + )], + ) + .await; // one reference above and one inside adapters struct assert_eq!(Arc::strong_count(ð_call_adapter), 2); assert_eq!(Arc::strong_count(ð_adapter), 2); - assert_eq!( - adapters.call_or_cheapest(None).unwrap().is_call_only(), - false - ); + assert!(!adapters.call_or_cheapest(None).unwrap().is_call_only()); } - #[tokio::test] + #[graph::test] async fn adapter_selector_no_call_only_fallback() { - let chain = "mainnet".to_string(); + let metrics = Arc::new(EndpointMetrics::mock()); let logger = graph::log::logger(true); - let mock_registry: Arc = Arc::new(MockMetricsRegistry::new()); - let transport = - Transport::new_rpc(Url::parse("http://127.0.0.1").unwrap(), HeaderMap::new()); + let mock_registry = Arc::new(MetricsRegistry::mock()); + let transport = Transport::new_rpc( + Url::parse("http://127.0.0.1").unwrap(), + HeaderMap::new(), + metrics.clone(), + "", + false, + Compression::None, + ); let provider_metrics = Arc::new(ProviderEthRpcMetrics::new(mock_registry.clone())); let eth_adapter = Arc::new( EthereumAdapter::new( logger.clone(), String::new(), - "http://127.0.0.1", transport.clone(), provider_metrics.clone(), true, false, + Arc::new(ChainSettings::from_env_defaults()), ) .await, ); - let adapters = { - let mut ethereum_networks = EthereumNetworks::new(); - ethereum_networks.insert( - chain.clone(), + let adapters: EthereumNetworkAdapters = EthereumNetworkAdapters::for_testing( + vec![EthereumNetworkAdapter::new( + metrics.cheap_clone(), NodeCapabilities { archive: true, traces: false, }, eth_adapter.clone(), SubgraphLimit::Limit(3), - ); - ethereum_networks.networks.get(&chain).unwrap().clone() - }; + )], + vec![], + ) + .await; // one reference above and one inside adapters struct assert_eq!(Arc::strong_count(ð_adapter), 2); + assert!(!adapters.call_or_cheapest(None).unwrap().is_call_only()); + } + + #[graph::test] + async fn eth_adapter_selection_multiple_adapters() { + let logger = Logger::root(Discard, o!()); + let unavailable_provider = "unavailable-provider"; + let error_provider = "error-provider"; + let no_error_provider = "no-error-provider"; + + let mock_registry = Arc::new(MetricsRegistry::mock()); + let metrics = Arc::new(EndpointMetrics::new( + logger, + &[unavailable_provider, error_provider, no_error_provider], + mock_registry.clone(), + )); + let logger = graph::log::logger(true); + let provider_metrics = Arc::new(ProviderEthRpcMetrics::new(mock_registry.clone())); + let chain_id: Word = "chain_id".into(); + + let adapters = [ + fake_adapter( + &logger, + unavailable_provider, + &provider_metrics, + &metrics, + false, + ) + .await, + fake_adapter(&logger, error_provider, &provider_metrics, &metrics, false).await, + fake_adapter( + &logger, + no_error_provider, + &provider_metrics, + &metrics, + false, + ) + .await, + ]; + + // Set errors + metrics.report_for_test(&ProviderName::from(error_provider), false); + + let mut no_retest_adapters = vec![]; + let mut always_retest_adapters = vec![]; + + adapters.iter().cloned().for_each(|adapter| { + let limit = if adapter.provider() == unavailable_provider { + SubgraphLimit::Disabled + } else { + SubgraphLimit::Unlimited + }; + + no_retest_adapters.push(EthereumNetworkAdapter { + endpoint_metrics: metrics.clone(), + capabilities: NodeCapabilities { + archive: true, + traces: false, + }, + adapter: adapter.clone(), + limit: limit.clone(), + }); + always_retest_adapters.push(EthereumNetworkAdapter { + endpoint_metrics: metrics.clone(), + capabilities: NodeCapabilities { + archive: true, + traces: false, + }, + adapter, + limit, + }); + }); + let manager = ProviderManager::::new( + logger, + vec![( + chain_id.clone(), + no_retest_adapters + .iter() + .cloned() + .chain(always_retest_adapters.iter().cloned()) + .collect(), + )] + .into_iter(), + ProviderCheckStrategy::MarkAsValid, + ); + + let no_retest_adapters = + EthereumNetworkAdapters::new(chain_id.clone(), manager.clone(), vec![], Some(0f64)); + + let always_retest_adapters = + EthereumNetworkAdapters::new(chain_id, manager.clone(), vec![], Some(1f64)); + + assert_eq!( + no_retest_adapters + .cheapest_with(&NodeCapabilities { + archive: true, + traces: false, + }) + .await + .unwrap() + .provider(), + no_error_provider + ); assert_eq!( - adapters.call_or_cheapest(None).unwrap().is_call_only(), - false + always_retest_adapters + .cheapest_with(&NodeCapabilities { + archive: true, + traces: false, + }) + .await + .unwrap() + .provider(), + error_provider ); } + + #[graph::test] + async fn eth_adapter_selection_single_adapter() { + let logger = Logger::root(Discard, o!()); + let unavailable_provider = "unavailable-provider"; + let error_provider = "error-provider"; + let no_error_provider = "no-error-provider"; + + let mock_registry = Arc::new(MetricsRegistry::mock()); + let metrics = Arc::new(EndpointMetrics::new( + logger, + &[unavailable_provider, error_provider, no_error_provider], + mock_registry.clone(), + )); + let chain_id: Word = "chain_id".into(); + let logger = graph::log::logger(true); + let provider_metrics = Arc::new(ProviderEthRpcMetrics::new(mock_registry.clone())); + + // Set errors + metrics.report_for_test(&ProviderName::from(error_provider), false); + + let mut no_retest_adapters = vec![]; + no_retest_adapters.push(EthereumNetworkAdapter { + endpoint_metrics: metrics.clone(), + capabilities: NodeCapabilities { + archive: true, + traces: false, + }, + adapter: fake_adapter(&logger, error_provider, &provider_metrics, &metrics, false) + .await, + limit: SubgraphLimit::Unlimited, + }); + + let mut always_retest_adapters = vec![]; + always_retest_adapters.push(EthereumNetworkAdapter { + endpoint_metrics: metrics.clone(), + capabilities: NodeCapabilities { + archive: true, + traces: false, + }, + adapter: fake_adapter( + &logger, + no_error_provider, + &provider_metrics, + &metrics, + false, + ) + .await, + limit: SubgraphLimit::Unlimited, + }); + let manager = ProviderManager::::new( + logger.clone(), + always_retest_adapters + .iter() + .cloned() + .map(|a| (chain_id.clone(), vec![a])), + ProviderCheckStrategy::MarkAsValid, + ); + + let always_retest_adapters = + EthereumNetworkAdapters::new(chain_id.clone(), manager.clone(), vec![], Some(1f64)); + + assert_eq!( + always_retest_adapters + .cheapest_with(&NodeCapabilities { + archive: true, + traces: false, + }) + .await + .unwrap() + .provider(), + no_error_provider + ); + + let manager = ProviderManager::::new( + logger.clone(), + no_retest_adapters + .iter() + .cloned() + .map(|a| (chain_id.clone(), vec![a])), + ProviderCheckStrategy::MarkAsValid, + ); + + let no_retest_adapters = + EthereumNetworkAdapters::new(chain_id.clone(), manager, vec![], Some(0f64)); + assert_eq!( + no_retest_adapters + .cheapest_with(&NodeCapabilities { + archive: true, + traces: false, + }) + .await + .unwrap() + .provider(), + error_provider + ); + + let mut no_available_adapter = vec![]; + no_available_adapter.push(EthereumNetworkAdapter { + endpoint_metrics: metrics.clone(), + capabilities: NodeCapabilities { + archive: true, + traces: false, + }, + adapter: fake_adapter( + &logger, + no_error_provider, + &provider_metrics, + &metrics, + false, + ) + .await, + limit: SubgraphLimit::Disabled, + }); + let manager = ProviderManager::new( + logger, + vec![(chain_id.clone(), no_available_adapter.to_vec())].into_iter(), + ProviderCheckStrategy::MarkAsValid, + ); + + let no_available_adapter = EthereumNetworkAdapters::new(chain_id, manager, vec![], None); + let res = no_available_adapter + .cheapest_with(&NodeCapabilities { + archive: true, + traces: false, + }) + .await; + assert!(res.is_err(), "{:?}", res); + } + + async fn fake_adapter( + logger: &Logger, + provider: &str, + provider_metrics: &Arc, + endpoint_metrics: &Arc, + call_only: bool, + ) -> Arc { + let transport = Transport::new_rpc( + Url::parse("http://127.0.0.1").unwrap(), + HeaderMap::new(), + endpoint_metrics.clone(), + "", + false, + Compression::None, + ); + + Arc::new( + EthereumAdapter::new( + logger.clone(), + provider.to_string(), + transport.clone(), + provider_metrics.clone(), + true, + call_only, + Arc::new(ChainSettings::from_env_defaults()), + ) + .await, + ) + } } diff --git a/graph/src/blockchain/polling_block_stream.rs b/chain/ethereum/src/polling_block_stream.rs similarity index 84% rename from graph/src/blockchain/polling_block_stream.rs rename to chain/ethereum/src/polling_block_stream.rs index daebeef2bd4..339386207f1 100644 --- a/graph/src/blockchain/polling_block_stream.rs +++ b/chain/ethereum/src/polling_block_stream.rs @@ -1,5 +1,4 @@ -use anyhow::Error; -use futures03::{stream::Stream, Future, FutureExt}; +use anyhow::{Error, anyhow}; use std::cmp; use std::collections::VecDeque; use std::pin::Pin; @@ -7,23 +6,24 @@ use std::sync::Arc; use std::task::{Context, Poll}; use std::time::Duration; -use super::block_stream::{ - BlockStream, BlockStreamEvent, BlockWithTriggers, ChainHeadUpdateStream, FirehoseCursor, - TriggersAdapter, +use graph::blockchain::block_stream::{ + BUFFERED_BLOCK_STREAM_SIZE, BlockStream, BlockStreamError, BlockStreamEvent, BlockWithTriggers, + ChainHeadUpdateStream, FirehoseCursor, TriggersAdapterWrapper, }; -use super::{Block, BlockPtr, Blockchain}; +use graph::blockchain::{Block, BlockPtr, TriggerFilterWrapper}; +use graph::futures03::{Future, FutureExt, stream::Stream}; +use graph::prelude::{BLOCK_NUMBER_MAX, DeploymentHash}; +use graph::slog::{Logger, info, trace, warn}; -use crate::components::store::BlockNumber; -use crate::data::subgraph::UnifiedMappingApiVersion; -use crate::prelude::*; +use graph::components::store::BlockNumber; +use graph::data::subgraph::UnifiedMappingApiVersion; + +use crate::Chain; // A high number here forces a slow start. const STARTING_PREVIOUS_TRIGGERS_PER_BLOCK: f64 = 1_000_000.0; -enum BlockStreamState -where - C: Blockchain, -{ +enum BlockStreamState { /// Starting or restarting reconciliation. /// /// Valid next states: Reconciliation @@ -32,13 +32,13 @@ where /// The BlockStream is reconciling the subgraph store state with the chain store state. /// /// Valid next states: YieldingBlocks, Idle, BeginReconciliation (in case of revert) - Reconciliation(Pin, Error>> + Send>>), + Reconciliation(Pin> + Send>>), /// The BlockStream is emitting blocks that must be processed in order to bring the subgraph /// store up to date with the chain store. /// /// Valid next states: BeginReconciliation - YieldingBlocks(Box>>), + YieldingBlocks(VecDeque>), /// The BlockStream experienced an error and is pausing before attempting to produce /// blocks again. @@ -55,16 +55,13 @@ where /// A single next step to take in reconciling the state of the subgraph store with the state of the /// chain store. -enum ReconciliationStep -where - C: Blockchain, -{ +enum ReconciliationStep { /// Revert(to) the block the subgraph should be reverted to, so it becomes the new subgraph /// head. Revert(BlockPtr), /// Move forwards, processing one or more blocks. Second element is the block range size. - ProcessDescendantBlocks(Vec>, BlockNumber), + ProcessDescendantBlocks(Vec>, BlockNumber), /// This step is a no-op, but we need to check again for a next step. Retry, @@ -74,18 +71,13 @@ where Done, } -struct PollingBlockStreamContext -where - C: Blockchain, -{ - chain_store: Arc, - adapter: Arc>, - node_id: NodeId, +struct PollingBlockStreamContext { + adapter: Arc>, subgraph_id: DeploymentHash, // This is not really a block number, but the (unsigned) difference // between two block numbers reorg_threshold: BlockNumber, - filter: Arc, + filter: Arc>, start_blocks: Vec, logger: Logger, previous_triggers_per_block: f64, @@ -98,12 +90,10 @@ where current_block: Option, } -impl Clone for PollingBlockStreamContext { +impl Clone for PollingBlockStreamContext { fn clone(&self) -> Self { Self { - chain_store: self.chain_store.cheap_clone(), adapter: self.adapter.clone(), - node_id: self.node_id.clone(), subgraph_id: self.subgraph_id.clone(), reorg_threshold: self.reorg_threshold, filter: self.filter.clone(), @@ -119,37 +109,29 @@ impl Clone for PollingBlockStreamContext { } } -pub struct PollingBlockStream { - state: BlockStreamState, +pub struct PollingBlockStream { + state: BlockStreamState, consecutive_err_count: u32, chain_head_update_stream: ChainHeadUpdateStream, - ctx: PollingBlockStreamContext, + ctx: PollingBlockStreamContext, } // This is the same as `ReconciliationStep` but without retries. -enum NextBlocks -where - C: Blockchain, -{ +enum NextBlocks { /// Blocks and range size - Blocks(VecDeque>, BlockNumber), + Blocks(VecDeque>, BlockNumber), // The payload is block the subgraph should be reverted to, so it becomes the new subgraph head. Revert(BlockPtr), Done, } -impl PollingBlockStream -where - C: Blockchain, -{ +impl PollingBlockStream { pub fn new( - chain_store: Arc, chain_head_update_stream: ChainHeadUpdateStream, - adapter: Arc>, - node_id: NodeId, + adapter: Arc>, subgraph_id: DeploymentHash, - filter: Arc, + filter: Arc>, start_blocks: Vec, reorg_threshold: BlockNumber, logger: Logger, @@ -164,9 +146,7 @@ where chain_head_update_stream, ctx: PollingBlockStreamContext { current_block: start_block, - chain_store, adapter, - node_id, subgraph_id, reorg_threshold, logger, @@ -182,12 +162,9 @@ where } } -impl PollingBlockStreamContext -where - C: Blockchain, -{ +impl PollingBlockStreamContext { /// Perform reconciliation steps until there are blocks to yield or we are up-to-date. - async fn next_blocks(&self) -> Result, Error> { + async fn next_blocks(&self) -> Result { let ctx = self.clone(); loop { @@ -205,20 +182,20 @@ where return Ok(NextBlocks::Done); } ReconciliationStep::Revert(parent_ptr) => { - return Ok(NextBlocks::Revert(parent_ptr)) + return Ok(NextBlocks::Revert(parent_ptr)); } } } } /// Determine the next reconciliation step. Does not modify Store or ChainStore. - async fn get_next_step(&self) -> Result, Error> { + async fn get_next_step(&self) -> Result { let ctx = self.clone(); let start_blocks = self.start_blocks.clone(); let max_block_range_size = self.max_block_range_size; // Get pointers from database for comparison - let head_ptr_opt = ctx.chain_store.chain_head_ptr().await?; + let head_ptr_opt = ctx.adapter.chain_head_ptr().await?; let subgraph_ptr = self.current_block.clone(); // If chain head ptr is not set yet @@ -247,10 +224,10 @@ where // Only continue if the subgraph block ptr is behind the head block ptr. // subgraph_ptr > head_ptr shouldn't happen, but if it does, it's safest to just stop. - if let Some(ptr) = &subgraph_ptr { - if ptr.number >= head_ptr.number { - return Ok(ReconciliationStep::Done); - } + if let Some(ptr) = &subgraph_ptr + && ptr.number >= head_ptr.number + { + return Ok(ReconciliationStep::Done); } // Subgraph ptr is behind head ptr. @@ -363,22 +340,41 @@ where // 1000 triggers found, 2 per block, range_size = 1000 / 2 = 500 let range_size_upper_limit = max_block_range_size.min(ctx.previous_block_range_size * 10); - let range_size = if ctx.previous_triggers_per_block == 0.0 { + let target_range_size = if ctx.previous_triggers_per_block == 0.0 { range_size_upper_limit } else { (self.target_triggers_per_block_range as f64 / ctx.previous_triggers_per_block) .max(1.0) .min(range_size_upper_limit as f64) as BlockNumber }; - let to = cmp::min(from + range_size - 1, to_limit); + let to = cmp::min(from + target_range_size - 1, to_limit); info!( ctx.logger, "Scanning blocks [{}, {}]", from, to; - "range_size" => range_size + "target_range_size" => target_range_size ); - let blocks = self.adapter.scan_triggers(from, to, &self.filter).await?; + // Update with actually scanned range, to account for any skipped null blocks. + let (blocks, to) = self + .adapter + .scan_triggers(&self.logger, from, to, &self.filter) + .await?; + let range_size = to - from + 1; + + // If the target block (`to`) is within the reorg threshold, indicating no non-null finalized blocks are + // greater than or equal to `to`, we retry later. This deferment allows the chain head to advance, + // ensuring the target block range becomes finalized. It effectively minimizes the risk of chain reorg + // affecting the processing by waiting for a more stable set of blocks. + if to > head_ptr.number - reorg_threshold { + return Ok(ReconciliationStep::Retry); + } + + info!( + ctx.logger, + "Scanned blocks [{}, {}]", from, to; + "range_size" => range_size + ); Ok(ReconciliationStep::ProcessDescendantBlocks( blocks, range_size, @@ -406,6 +402,7 @@ where // block number, and checking to see if the block we found matches the // subgraph_ptr. + #[allow(clippy::unnecessary_unwrap)] let subgraph_ptr = subgraph_ptr.expect("subgraph block pointer should not be `None` here"); @@ -415,7 +412,10 @@ where // In principle this block should be in the store, but we have seen this error for deep // reorgs in ropsten. - let head_ancestor_opt = self.adapter.ancestor_block(head_ptr, offset).await?; + let head_ancestor_opt = self + .adapter + .ancestor_block(head_ptr, offset, Some(subgraph_ptr.hash.clone())) + .await?; match head_ancestor_opt { None => { @@ -427,6 +427,15 @@ where Ok(ReconciliationStep::Retry) } Some(head_ancestor) => { + // Check if there was an interceding skipped (null) block. + if head_ancestor.number() != subgraph_ptr.number + 1 { + warn!( + ctx.logger, + "skipped block detected: {}", + subgraph_ptr.number + 1 + ); + } + // We stopped one block short, so we'll compare the parent hash to the // subgraph ptr. if head_ancestor.parent_hash().as_ref() == Some(&subgraph_ptr.hash) { @@ -463,10 +472,14 @@ where } } -impl BlockStream for PollingBlockStream {} +impl BlockStream for PollingBlockStream { + fn buffer_size_hint(&self) -> usize { + BUFFERED_BLOCK_STREAM_SIZE + } +} -impl Stream for PollingBlockStream { - type Item = Result, Error>; +impl Stream for PollingBlockStream { + type Item = Result, BlockStreamError>; fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { let result = loop { @@ -499,15 +512,17 @@ impl Stream for PollingBlockStream { total_triggers as f64 / block_range_size as f64; self.ctx.previous_block_range_size = block_range_size; if total_triggers > 0 { - debug!( + info!( self.ctx.logger, - "Processing {} triggers", total_triggers + "Found {} triggers in {} blocks with a block range of {}", + total_triggers, + next_blocks.len(), + block_range_size ); } // Switch to yielding state until next_blocks is depleted - self.state = - BlockStreamState::YieldingBlocks(Box::new(next_blocks)); + self.state = BlockStreamState::YieldingBlocks(next_blocks); // Yield the first block in next_blocks continue; @@ -554,7 +569,7 @@ impl Stream for PollingBlockStream { } // Yielding blocks from reconciliation process - BlockStreamState::YieldingBlocks(ref mut next_blocks) => { + BlockStreamState::YieldingBlocks(next_blocks) => { match next_blocks.pop_front() { // Yield one block Some(next_block) => { @@ -574,7 +589,7 @@ impl Stream for PollingBlockStream { } // Pausing after an error, before looking for more blocks - BlockStreamState::RetryAfterDelay(ref mut delay) => match delay.as_mut().poll(cx) { + BlockStreamState::RetryAfterDelay(delay) => match delay.as_mut().poll(cx) { Poll::Ready(Ok(..)) | Poll::Ready(Err(_)) => { self.state = BlockStreamState::BeginReconciliation; } @@ -595,8 +610,8 @@ impl Stream for PollingBlockStream { // Chain head update stream ended Poll::Ready(None) => { // Should not happen - return Poll::Ready(Some(Err(anyhow::anyhow!( - "chain head update stream ended unexpectedly" + return Poll::Ready(Some(Err(BlockStreamError::from( + anyhow::anyhow!("chain head update stream ended unexpectedly"), )))); } @@ -606,6 +621,6 @@ impl Stream for PollingBlockStream { } }; - result + result.map_err(BlockStreamError::from) } } diff --git a/chain/ethereum/src/protobuf/sf.ethereum.r#type.v2.rs b/chain/ethereum/src/protobuf/sf.ethereum.r#type.v2.rs index 1e6b7841c8d..4c4d8be6c90 100644 --- a/chain/ethereum/src/protobuf/sf.ethereum.r#type.v2.rs +++ b/chain/ethereum/src/protobuf/sf.ethereum.r#type.v2.rs @@ -1,84 +1,179 @@ -#[allow(clippy::derive_partial_eq_without_eq)] +// This file is @generated by prost-build. +/// Block is the representation of the tracing of a block in the Ethereum +/// blockchain. A block is a collection of \[TransactionTrace\] that are grouped +/// together and processed as an atomic unit. Each \[TransactionTrace\] is composed +/// of a series of \[Call\] (a.k.a internal transactions) and there is also at +/// least one call per transaction a.k.a the root call which essentially has the +/// same parameters as the transaction itself (e.g. `from`, `to`, `gas`, `value`, +/// etc.). +/// +/// The exact tracing method used to build the block must be checked against +/// \[DetailLevel\] field. There is two levels of details available, `BASE` and +/// `EXTENDED`. The `BASE` level has been extracted using archive node RPC calls +/// and will contain only the block header, transaction receipts and event logs. +/// Refers to the Firehose service provider to know which blocks are offered on +/// each network. +/// +/// The `EXTENDED` level has been extracted using the Firehose tracer and all +/// fields are available in this Protobuf. +/// +/// The Ethereum block model is used across many chains which means that it +/// happen that certain fields are not available in one chain but are available +/// in another. Each field should be documented when necesssary if it's available +/// on a subset of chains. +/// +/// One major concept to get about the Block is the concept of 'ordinal'. The +/// ordinal is a number that is used to globally order every element of execution +/// that happened throughout the processing of the block like +/// \[TransactionTracer\], \[Call\], \[Log\], \[BalanceChange\], \[StateChange\], etc. +/// Element that have a start and end interval, \[Transaction\] and \[Call\], will +/// have two ordinals: `begin_ordinal` and `end_ordinal`. Element that are +/// executed as "point in time" \[Log\], \[BalanceChange\], \[StateChange\], etc. will +/// have only one ordinal named `ordinal`. If you take all of the message in the +/// Block that have an 'ordinal' field in an array and you sort each element +/// against the `ordinal` field, you will get the exact order of execution of +/// each element in the block. +/// +/// All the 'ordinal' fields in a block are globally unique for the given block, +/// it is **not** a chain-wide global ordering. Furthermore, caution must be take +/// with reverted elements due to execution failure. For anything attached to a +/// \[Call\] that has a `state_reverted` field set to `true`, the `ordinal` field +/// is not reliable and should not be used to order the element against other +/// elements in the block as those element might have 0 as the ordinal. Only +/// successful calls have a reliable `ordinal` field. #[derive(Clone, PartialEq, ::prost::Message)] pub struct Block { - #[prost(int32, tag = "1")] - pub ver: i32, + /// Hash is the block's hash. #[prost(bytes = "vec", tag = "2")] pub hash: ::prost::alloc::vec::Vec, + /// Number is the block's height at which this block was mined. #[prost(uint64, tag = "3")] pub number: u64, + /// Size is the size in bytes of the RLP encoding of the block according to Ethereum + /// rules. #[prost(uint64, tag = "4")] pub size: u64, + /// Header contain's the block's header information like its parent hash, the merkel root hash + /// and all other information the form a block. #[prost(message, optional, tag = "5")] pub header: ::core::option::Option, - /// Uncles represents block produced with a valid solution but were not actually choosen + /// Uncles represents block produced with a valid solution but were not actually chosen /// as the canonical block for the given height so they are mostly "forked" blocks. /// /// If the Block has been produced using the Proof of Stake consensus algorithm, this /// field will actually be always empty. #[prost(message, repeated, tag = "6")] pub uncles: ::prost::alloc::vec::Vec, + /// TransactionTraces hold the execute trace of all the transactions that were executed + /// in this block. In in there that you will find most of the Ethereum data model. + /// + /// They are ordered by the order of execution of the transaction in the block. #[prost(message, repeated, tag = "10")] pub transaction_traces: ::prost::alloc::vec::Vec, + /// BalanceChanges here is the array of ETH transfer that happened at the block level + /// outside of the normal transaction flow of a block. The best example of this is mining + /// reward for the block mined, the transfer of ETH to the miner happens outside the normal + /// transaction flow of the chain and is recorded as a `BalanceChange` here since we cannot + /// attached it to any transaction. + /// + /// Only available in DetailLevel: EXTENDED #[prost(message, repeated, tag = "11")] pub balance_changes: ::prost::alloc::vec::Vec, + /// DetailLevel affects the data available in this block. + /// + /// ## DetailLevel_EXTENDED + /// + /// Describes the most complete block, with traces, balance changes, storage + /// changes. It is extracted during the execution of the block. + /// + /// ## DetailLevel_BASE + /// + /// Describes a block that contains only the block header, transaction receipts + /// and event logs: everything that can be extracted using the base JSON-RPC + /// interface + /// () + /// Furthermore, the eth_getTransactionReceipt call has been avoided because it + /// brings only minimal improvements at the cost of requiring an archive node + /// or a full node with complete transaction index. + #[prost(enumeration = "block::DetailLevel", tag = "12")] + pub detail_level: i32, + /// CodeChanges here is the array of smart code change that happened that happened at the block level + /// outside of the normal transaction flow of a block. Some Ethereum's fork like BSC and Polygon + /// has some capabilities to upgrade internal smart contracts used usually to track the validator + /// list. + /// + /// On hard fork, some procedure runs to upgrade the smart contract code to a new version. In those + /// network, a `CodeChange` for each modified smart contract on upgrade would be present here. Note + /// that this happen rarely, so the vast majority of block will have an empty list here. + /// + /// Only available in DetailLevel: EXTENDED #[prost(message, repeated, tag = "20")] pub code_changes: ::prost::alloc::vec::Vec, + /// System calls are introduced in Cancun, along with blobs. They are executed outside of transactions but affect the state. + /// + /// Only available in DetailLevel: EXTENDED + #[prost(message, repeated, tag = "21")] + pub system_calls: ::prost::alloc::vec::Vec, + /// Withdrawals represents the list of validator balance withdrawals processed in this block. + /// Introduced in the Shanghai hard fork (EIP-4895). + /// + /// This field has been added because Geth blocks include withdrawals after Shanghai fork, + /// but our previous Firehose model didn't capture this data. Currently experimental - + /// NOT ready for production use yet as we validate the tracing implementation. + /// + /// Only available when Shanghai fork is active on the chain. + #[prost(message, repeated, tag = "22")] + pub withdrawals: ::prost::alloc::vec::Vec, + /// Ver represents that data model version of the block, it is used internally by Firehose on Ethereum + /// as a validation that we are reading the correct version. + #[prost(int32, tag = "1")] + pub ver: i32, } -/// HeaderOnlyBlock is used to optimally unpack the \[Block\] structure (note the -/// corresponding message number for the `header` field) while consuming less -/// memory, when only the `header` is desired. -/// -/// WARN: this is a client-side optimization pattern and should be moved in the -/// consuming code. -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct HeaderOnlyBlock { - #[prost(message, optional, tag = "5")] - pub header: ::core::option::Option, -} -/// BlockWithRefs is a lightweight block, with traces and transactions -/// purged from the `block` within, and only. It is used in transports -/// to pass block data around. -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct BlockWithRefs { - #[prost(string, tag = "1")] - pub id: ::prost::alloc::string::String, - #[prost(message, optional, tag = "2")] - pub block: ::core::option::Option, - #[prost(message, optional, tag = "3")] - pub transaction_trace_refs: ::core::option::Option, - #[prost(bool, tag = "4")] - pub irreversible: bool, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct TransactionRefs { - #[prost(bytes = "vec", repeated, tag = "1")] - pub hashes: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec>, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct UnclesHeaders { - #[prost(message, repeated, tag = "1")] - pub uncles: ::prost::alloc::vec::Vec, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct BlockRef { - #[prost(bytes = "vec", tag = "1")] - pub hash: ::prost::alloc::vec::Vec, - #[prost(uint64, tag = "2")] - pub number: u64, +/// Nested message and enum types in `Block`. +pub mod block { + #[derive( + Clone, + Copy, + Debug, + PartialEq, + Eq, + Hash, + PartialOrd, + Ord, + ::prost::Enumeration + )] + #[repr(i32)] + pub enum DetailLevel { + DetaillevelExtended = 0, + /// DETAILLEVEL_TRACE = 1; // TBD + DetaillevelBase = 2, + } + impl DetailLevel { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + Self::DetaillevelExtended => "DETAILLEVEL_EXTENDED", + Self::DetaillevelBase => "DETAILLEVEL_BASE", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "DETAILLEVEL_EXTENDED" => Some(Self::DetaillevelExtended), + "DETAILLEVEL_BASE" => Some(Self::DetaillevelBase), + _ => None, + } + } + } } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct BlockHeader { #[prost(bytes = "vec", tag = "1")] pub parent_hash: ::prost::alloc::vec::Vec, - /// Uncle hash of the block, some reference it as `sha3Uncles`, but `sha3`` is badly worded, so we prefer `uncle_hash`, also - /// referred as `ommers` in EIP specification. + /// Uncle hash of the block, some reference it as `sha3Uncles`, but ```sha3`` is badly worded, so we prefer ```uncle_hash`, also referred as `ommers\` in EIP specification. /// /// If the Block containing this `BlockHeader` has been produced using the Proof of Stake /// consensus algorithm, this field will actually be constant and set to `0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347`. @@ -100,12 +195,10 @@ pub struct BlockHeader { /// consensus algorithm, this field will actually be constant and set to `0x00`. #[prost(message, optional, tag = "8")] pub difficulty: ::core::option::Option, - /// TotalDifficulty is the sum of all previous blocks difficulty including this block difficulty. + /// TotalDifficulty used to be the sum of all previous blocks difficulty including this block difficulty. /// - /// If the Block containing this `BlockHeader` has been produced using the Proof of Stake - /// consensus algorithm, this field will actually be constant and set to the terminal total difficulty - /// that was required to transition to Proof of Stake algorithm, which varies per network. It is set to - /// 58 750 000 000 000 000 000 000 on Ethereum Mainnet and to 10 790 000 on Ethereum Testnet Goerli. + /// It has been deprecated in geth v1.15.0 but was already removed from the JSON-RPC interface for a while + #[deprecated] #[prost(message, optional, tag = "17")] pub total_difficulty: ::core::option::Option, #[prost(uint64, tag = "9")] @@ -121,7 +214,7 @@ pub struct BlockHeader { /// forks are using bigger values to carry special consensus data. /// /// If the Block containing this `BlockHeader` has been produced using the Proof of Stake - /// consensus algorithm, this field is strictly enforced to be <= 32 bytes. + /// consensus algorithm, this field is strictly enforced to be \<= 32 bytes. #[prost(bytes = "vec", tag = "13")] pub extra_data: ::prost::alloc::vec::Vec, /// MixHash is used to prove, when combined with the `nonce` that sufficient amount of computation has been @@ -137,38 +230,102 @@ pub struct BlockHeader { pub nonce: u64, /// Hash is the hash of the block which is actually the computation: /// - /// Keccak256(rlp([ - /// parent_hash, - /// uncle_hash, - /// coinbase, - /// state_root, - /// transactions_root, - /// receipt_root, - /// logs_bloom, - /// difficulty, - /// number, - /// gas_limit, - /// gas_used, - /// timestamp, - /// extra_data, - /// mix_hash, - /// nonce, - /// base_fee_per_gas - /// ])) - /// + /// Keccak256(rlp(\[ + /// parent_hash, + /// uncle_hash, + /// coinbase, + /// state_root, + /// transactions_root, + /// receipt_root, + /// logs_bloom, + /// difficulty, + /// number, + /// gas_limit, + /// gas_used, + /// timestamp, + /// extra_data, + /// mix_hash, + /// nonce, + /// base_fee_per_gas (to be included only if London fork is active) + /// withdrawals_root (to be included only if Shangai fork is active) + /// blob_gas_used (to be included only if Cancun fork is active) + /// excess_blob_gas (to be included only if Cancun fork is active) + /// parent_beacon_root (to be included only if Cancun fork is active) + /// requests_hash (to be included only if Prague fork is active) + /// \])) #[prost(bytes = "vec", tag = "16")] pub hash: ::prost::alloc::vec::Vec, /// Base fee per gas according to EIP-1559 (e.g. London Fork) rules, only set if London is present/active on the chain. #[prost(message, optional, tag = "18")] pub base_fee_per_gas: ::core::option::Option, + /// Withdrawals root hash according to EIP-4895 (e.g. Shangai Fork) rules, only set if Shangai is present/active on the chain. + /// + /// Only available in DetailLevel: EXTENDED + #[prost(bytes = "vec", tag = "19")] + pub withdrawals_root: ::prost::alloc::vec::Vec, + /// TxDependency is list of transaction indexes that are dependent on each other in the block + /// header. This is metadata only that was used by the internal Polygon parallel execution engine. + /// + /// This field was available in a few versions on Polygon Mainnet and Polygon Mumbai chains. It was actually + /// removed and is not populated anymore. It's now embedded in the `extraData` field, refer to Polygon source + /// code to determine how to extract it if you need it. + /// + /// Only available in DetailLevel: EXTENDED + #[prost(message, optional, tag = "20")] + pub tx_dependency: ::core::option::Option, + /// BlobGasUsed was added by EIP-4844 and is ignored in legacy headers. + #[prost(uint64, optional, tag = "22")] + pub blob_gas_used: ::core::option::Option, + /// ExcessBlobGas was added by EIP-4844 and is ignored in legacy headers. + #[prost(uint64, optional, tag = "23")] + pub excess_blob_gas: ::core::option::Option, + /// ParentBeaconRoot was added by EIP-4788 and is ignored in legacy headers. + #[prost(bytes = "vec", tag = "24")] + pub parent_beacon_root: ::prost::alloc::vec::Vec, + /// RequestsHash was added by EIP-7685 and is ignored in legacy headers. + #[prost(bytes = "vec", tag = "25")] + pub requests_hash: ::prost::alloc::vec::Vec, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] +pub struct Uint64NestedArray { + #[prost(message, repeated, tag = "1")] + pub val: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct Uint64Array { + #[prost(uint64, repeated, tag = "1")] + pub val: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct BigInt { #[prost(bytes = "vec", tag = "1")] pub bytes: ::prost::alloc::vec::Vec, } -#[allow(clippy::derive_partial_eq_without_eq)] +/// TransactionTrace is full trace of execution of the transaction when the +/// it actually executed on chain. +/// +/// It contains all the transaction details like `from`, `to`, `gas`, etc. +/// as well as all the internal calls that were made during the transaction. +/// +/// The `calls` vector contains Call objects which have balance changes, events +/// storage changes, etc. +/// +/// If ordering is important between elements, almost each message like `Log`, +/// `Call`, `StorageChange`, etc. have an ordinal field that is represents "execution" +/// order of the said element against all other elements in this block. +/// +/// Due to how the call tree works doing "naively", looping through all calls then +/// through a Call's element like `logs` while not yielding the elements in the order +/// they were executed on chain. A log in call could have been done before or after +/// another in another call depending on the actual call tree. +/// +/// The `calls` are ordered by creation order and the call tree can be re-computing +/// using fields found in `Call` object (parent/child relationship). +/// +/// Another important thing to note is that even if a transaction succeed, some calls +/// within it could have been reverted internally, if this is important to you, you must +/// check the field `state_reverted` on the `Call` to determine if it was fully committed +/// to the chain or not. #[derive(Clone, PartialEq, ::prost::Message)] pub struct TransactionTrace { /// consensus @@ -213,7 +370,7 @@ pub struct TransactionTrace { /// The value is always set even for transaction before Berlin fork because those before the fork are still legacy transactions. #[prost(enumeration = "transaction_trace::Type", tag = "12")] pub r#type: i32, - /// AcccessList represents the storage access this transaction has agreed to do in which case those storage + /// AccessList represents the storage access this transaction has agreed to do in which case those storage /// access cost less gas unit per access. /// /// This will is populated only if `TransactionTrace.Type == TRX_TYPE_ACCESS_LIST || TRX_TYPE_DYNAMIC_FEE` which @@ -224,6 +381,8 @@ pub struct TransactionTrace { /// /// This will is populated only if `TransactionTrace.Type == TRX_TYPE_DYNAMIC_FEE` which is possible only /// if London fork is active on the chain. + /// + /// Only available in DetailLevel: EXTENDED #[prost(message, optional, tag = "11")] pub max_fee_per_gas: ::core::option::Option, /// MaxPriorityFeePerGas is priority fee per gas the user to pay in extra to the miner on top of the block's @@ -231,6 +390,8 @@ pub struct TransactionTrace { /// /// This will is populated only if `TransactionTrace.Type == TRX_TYPE_DYNAMIC_FEE` which is possible only /// if London fork is active on the chain. + /// + /// Only available in DetailLevel: EXTENDED #[prost(message, optional, tag = "13")] pub max_priority_fee_per_gas: ::core::option::Option, /// meta @@ -240,20 +401,98 @@ pub struct TransactionTrace { pub hash: ::prost::alloc::vec::Vec, #[prost(bytes = "vec", tag = "22")] pub from: ::prost::alloc::vec::Vec, + /// Only available in DetailLevel: EXTENDED + /// Known Issues + /// + /// * Version 3: + /// Field not populated. It will be empty. + /// + /// Fixed in `Version 4`, see for information about block versions. #[prost(bytes = "vec", tag = "23")] pub return_data: ::prost::alloc::vec::Vec, + /// Only available in DetailLevel: EXTENDED #[prost(bytes = "vec", tag = "24")] pub public_key: ::prost::alloc::vec::Vec, + /// The block's global ordinal when the transaction started executing, refer to + /// \[Block\] documentation for further information about ordinals and total ordering. #[prost(uint64, tag = "25")] pub begin_ordinal: u64, + /// The block's global ordinal when the transaction finished executing, refer to + /// \[Block\] documentation for further information about ordinals and total ordering. #[prost(uint64, tag = "26")] pub end_ordinal: u64, + /// TransactionTraceStatus is the status of the transaction execution and will let you know if the transaction + /// was successful or not. + /// + /// ## Explanation relevant only for blocks with `DetailLevel: EXTENDED` + /// + /// A successful transaction has been recorded to the blockchain's state for calls in it that were successful. + /// This means it's possible only a subset of the calls were properly recorded, refer to \[calls\[\].state_reverted\] field + /// to determine which calls were reverted. + /// + /// A quirks of the Ethereum protocol is that a transaction `FAILED` or `REVERTED` still affects the blockchain's + /// state for **some** of the state changes. Indeed, in those cases, the transactions fees are still paid to the miner + /// which means there is a balance change for the transaction's emitter (e.g. `from`) to pay the gas fees, an optional + /// balance change for gas refunded to the transaction's emitter (e.g. `from`) and a balance change for the miner who + /// received the transaction fees. There is also a nonce change for the transaction's emitter (e.g. `from`). + /// + /// This means that to properly record the state changes for a transaction, you need to conditionally procees the + /// transaction's status. + /// + /// For a `SUCCEEDED` transaction, you iterate over the `calls` array and record the state changes for each call for + /// which `state_reverted == false` (if a transaction succeeded, the call at #0 will always `state_reverted == false` + /// because it aligns with the transaction). + /// + /// For a `FAILED` or `REVERTED` transaction, you iterate over the root call (e.g. at #0, will always exist) for + /// balance changes you process those where `reason` is either `REASON_GAS_BUY`, `REASON_GAS_REFUND` or + /// `REASON_REWARD_TRANSACTION_FEE` and for nonce change, still on the root call, you pick the nonce change which the + /// smallest ordinal (if more than one). #[prost(enumeration = "TransactionTraceStatus", tag = "30")] pub status: i32, #[prost(message, optional, tag = "31")] pub receipt: ::core::option::Option, + /// Only available in DetailLevel: EXTENDED #[prost(message, repeated, tag = "32")] pub calls: ::prost::alloc::vec::Vec, + /// BlobGas is the amount of gas the transaction is going to pay for the blobs, this is a computed value + /// equivalent to `self.blob_gas_fee_cap * len(self.blob_hashes)` and provided in the model for convenience. + /// + /// This is specified by + /// + /// This will is populated only if `TransactionTrace.Type == TRX_TYPE_BLOB` which is possible only + /// if Cancun fork is active on the chain. + #[prost(uint64, optional, tag = "33")] + pub blob_gas: ::core::option::Option, + /// BlobGasFeeCap is the maximum fee per data gas the user is willing to pay for the data gas used. + /// + /// This is specified by + /// + /// This will is populated only if `TransactionTrace.Type == TRX_TYPE_BLOB` which is possible only + /// if Cancun fork is active on the chain. + #[prost(message, optional, tag = "34")] + pub blob_gas_fee_cap: ::core::option::Option, + /// BlobHashes field represents a list of hash outputs from 'kzg_to_versioned_hash' which + /// essentially is a version byte + the sha256 hash of the blob commitment (e.g. + /// `BLOB_COMMITMENT_VERSION_KZG + sha256(commitment)\[1:\]`. + /// + /// This is specified by + /// + /// This will is populated only if `TransactionTrace.Type == TRX_TYPE_BLOB` which is possible only + /// if Cancun fork is active on the chain. + #[prost(bytes = "vec", repeated, tag = "35")] + pub blob_hashes: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec>, + /// SetCodeAuthorizations represents the authorizations of a transaction to set code to an EOA (Externally Owned Accounts) + /// as defined in EIP-7702. The list will contain all the authorizations as they were specified in the + /// transaction itself regardless of their validity. If you need to determined if a given authorization was + /// correctly applied on chain's state, refer to \[SetCodeAuthorization.discarded\] field that records + /// if the authorization was discarded or not by the chain due to invalidity. + /// + /// This is specified by + /// + /// This will is populated only if `TransactionTrace.Type == TRX_TYPE_SET_CODE` which is possible only + /// if Prague fork is active on the chain. + #[prost(message, repeated, tag = "36")] + pub set_code_authorizations: ::prost::alloc::vec::Vec, } /// Nested message and enum types in `TransactionTrace`. pub mod transaction_trace { @@ -272,15 +511,38 @@ pub mod transaction_trace { pub enum Type { /// All transactions that ever existed prior Berlin fork before EIP-2718 was implemented. TrxTypeLegacy = 0, - /// Field that specifies an access list of contract/storage_keys that is going to be used + /// Transaction that specicy an access list of contract/storage_keys that is going to be used /// in this transaction. /// /// Added in Berlin fork (EIP-2930). TrxTypeAccessList = 1, - /// Transaction that specifies an access list just like TRX_TYPE_ACCESS_LIST but in addition defines the + /// Transaction that specifis an access list just like TRX_TYPE_ACCESS_LIST but in addition defines the /// max base gas gee and max priority gas fee to pay for this transaction. Transaction's of those type are /// executed against EIP-1559 rules which dictates a dynamic gas cost based on the congestion of the network. TrxTypeDynamicFee = 2, + /// Transaction which contain a large amount of data that cannot be accessed by EVM execution, but whose commitment + /// can be accessed. The format is intended to be fully compatible with the format that will be used in full sharding. + /// + /// Transaction that defines an access list just like TRX_TYPE_ACCESS_LIST and enables dynamic fee just like + /// TRX_TYPE_DYNAMIC_FEE but in addition defines the fields 'max_fee_per_data_gas' of type 'uint256' and the fields + /// 'blob_versioned_hashes' which represents a list of hash outputs from 'kzg_to_versioned_hash'. + /// + /// Activated in Cancun fork (EIP-4844) + TrxTypeBlob = 3, + /// Transaction that sets code to an EOA (Externally Owned Accounts) + /// + /// Activated in Prague (EIP-7702) + TrxTypeSetCode = 4, + /// Arbitrum-specific transactions + TrxTypeArbitrumDeposit = 100, + TrxTypeArbitrumUnsigned = 101, + TrxTypeArbitrumContract = 102, + TrxTypeArbitrumRetry = 104, + TrxTypeArbitrumSubmitRetryable = 105, + TrxTypeArbitrumInternal = 106, + TrxTypeArbitrumLegacy = 120, + /// OPTIMISM-specific transactions + TrxTypeOptimismDeposit = 126, } impl Type { /// String value of the enum field names used in the ProtoBuf definition. @@ -289,9 +551,21 @@ pub mod transaction_trace { /// (if the ProtoBuf definition does not change) and safe for programmatic use. pub fn as_str_name(&self) -> &'static str { match self { - Type::TrxTypeLegacy => "TRX_TYPE_LEGACY", - Type::TrxTypeAccessList => "TRX_TYPE_ACCESS_LIST", - Type::TrxTypeDynamicFee => "TRX_TYPE_DYNAMIC_FEE", + Self::TrxTypeLegacy => "TRX_TYPE_LEGACY", + Self::TrxTypeAccessList => "TRX_TYPE_ACCESS_LIST", + Self::TrxTypeDynamicFee => "TRX_TYPE_DYNAMIC_FEE", + Self::TrxTypeBlob => "TRX_TYPE_BLOB", + Self::TrxTypeSetCode => "TRX_TYPE_SET_CODE", + Self::TrxTypeArbitrumDeposit => "TRX_TYPE_ARBITRUM_DEPOSIT", + Self::TrxTypeArbitrumUnsigned => "TRX_TYPE_ARBITRUM_UNSIGNED", + Self::TrxTypeArbitrumContract => "TRX_TYPE_ARBITRUM_CONTRACT", + Self::TrxTypeArbitrumRetry => "TRX_TYPE_ARBITRUM_RETRY", + Self::TrxTypeArbitrumSubmitRetryable => { + "TRX_TYPE_ARBITRUM_SUBMIT_RETRYABLE" + } + Self::TrxTypeArbitrumInternal => "TRX_TYPE_ARBITRUM_INTERNAL", + Self::TrxTypeArbitrumLegacy => "TRX_TYPE_ARBITRUM_LEGACY", + Self::TrxTypeOptimismDeposit => "TRX_TYPE_OPTIMISM_DEPOSIT", } } /// Creates an enum from field names used in the ProtoBuf definition. @@ -300,6 +574,18 @@ pub mod transaction_trace { "TRX_TYPE_LEGACY" => Some(Self::TrxTypeLegacy), "TRX_TYPE_ACCESS_LIST" => Some(Self::TrxTypeAccessList), "TRX_TYPE_DYNAMIC_FEE" => Some(Self::TrxTypeDynamicFee), + "TRX_TYPE_BLOB" => Some(Self::TrxTypeBlob), + "TRX_TYPE_SET_CODE" => Some(Self::TrxTypeSetCode), + "TRX_TYPE_ARBITRUM_DEPOSIT" => Some(Self::TrxTypeArbitrumDeposit), + "TRX_TYPE_ARBITRUM_UNSIGNED" => Some(Self::TrxTypeArbitrumUnsigned), + "TRX_TYPE_ARBITRUM_CONTRACT" => Some(Self::TrxTypeArbitrumContract), + "TRX_TYPE_ARBITRUM_RETRY" => Some(Self::TrxTypeArbitrumRetry), + "TRX_TYPE_ARBITRUM_SUBMIT_RETRYABLE" => { + Some(Self::TrxTypeArbitrumSubmitRetryable) + } + "TRX_TYPE_ARBITRUM_INTERNAL" => Some(Self::TrxTypeArbitrumInternal), + "TRX_TYPE_ARBITRUM_LEGACY" => Some(Self::TrxTypeArbitrumLegacy), + "TRX_TYPE_OPTIMISM_DEPOSIT" => Some(Self::TrxTypeOptimismDeposit), _ => None, } } @@ -307,36 +593,84 @@ pub mod transaction_trace { } /// AccessTuple represents a list of storage keys for a given contract's address and is used /// for AccessList construction. -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct AccessTuple { #[prost(bytes = "vec", tag = "1")] pub address: ::prost::alloc::vec::Vec, #[prost(bytes = "vec", repeated, tag = "2")] pub storage_keys: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec>, } -/// TransactionTraceWithBlockRef -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct TransactionTraceWithBlockRef { - #[prost(message, optional, tag = "1")] - pub trace: ::core::option::Option, - #[prost(message, optional, tag = "2")] - pub block_ref: ::core::option::Option, +/// SetCodeAuthorization represents the authorization of a transaction to set code of an EOA (Externally Owned Account) +/// as defined in EIP-7702. +/// +/// The 'authority' field is the address that is authorizing the delegation mechanism. The 'authority' value is computed +/// from the signature contained in the message using the computation +/// `authority = ecrecover(keccak(MAGIC || rlp(\[chain_id, address, nonce\])), y_parity, r, s)` +/// where `MAGIC` is `0x5`, `||` is the bytes concatenation operator, `ecrecover` is the Ethereum signature recovery +/// and `y_parity` is the recovery ID value denoted `v` in the message below. Checking the go-ethereum implementation +/// at might prove easier to "read". +/// +/// We do extract the 'authority' value from the signature in the message and store it in the 'authority' field for +/// convenience so you don't need to perform the computation yourself. +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct SetCodeAuthorization { + /// Discarded determines if this authorization was skipped due to being invalid. As EIP-7702 states, + /// if the authorization is invalid (invalid signature, nonce mismatch, etc.) it must be simply + /// discarded and the transaction is processed as if the authorization was not present in the + /// authorization list. + /// + /// This boolean records if the authorization was discarded or not by the chain due to invalidity. + #[prost(bool, tag = "1")] + pub discarded: bool, + /// ChainID is the chain ID of the chain where the transaction was executed, used + /// to recover the authority from the signature. + #[prost(bytes = "vec", tag = "2")] + pub chain_id: ::prost::alloc::vec::Vec, + /// Address contains the address this account is delegating to. This address usually + /// contain code that this account essentially "delegates" to. + /// + /// Note: This was missing when EIP-7702 was first activated on Holesky, Sepolia, BSC Chapel, + /// BSC Mainnet and Arbitrum Sepolia but was ready for Ethereum Mainnet hard fork. We will backfill + /// those missing values in the near future at which point we will remove this note. + #[prost(bytes = "vec", tag = "8")] + pub address: ::prost::alloc::vec::Vec, + /// Nonce is the nonce of the account that is authorizing delegation mechanism, EIP-7702 rules + /// states that nonce should be verified using this rule: + /// + /// * Verify the nonce of authority is equal to nonce. In case authority does not exist in the trie, + /// verify that nonce is equal to 0. + /// + /// Read SetCodeAuthorization to know how to recover the `authority` value. + #[prost(uint64, tag = "3")] + pub nonce: u64, + /// V is the recovery ID value for the signature Y point. While it's defined as a + /// `uint32`, it's actually bounded by a `uint8` data type withing the Ethereum protocol. + #[prost(uint32, tag = "4")] + pub v: u32, + /// R is the signature's X point on the elliptic curve (32 bytes). + #[prost(bytes = "vec", tag = "5")] + pub r: ::prost::alloc::vec::Vec, + /// S is the signature's Y point on the elliptic curve (32 bytes). + #[prost(bytes = "vec", tag = "6")] + pub s: ::prost::alloc::vec::Vec, + /// Authority is the address of the account that is authorizing delegation mechanism, it + /// is computed from the signature contained in the message and stored for convenience. + /// + /// If the authority cannot be recovered from the signature, this field will be empty and + /// the `discarded` field will be set to `true`. + #[prost(bytes = "vec", optional, tag = "7")] + pub authority: ::core::option::Option<::prost::alloc::vec::Vec>, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct TransactionReceipt { /// State root is an intermediate state_root hash, computed in-between transactions to make /// **sure** you could build a proof and point to state in the middle of a block. Geth client - /// uses `PostState + root + PostStateOrStatus`` while Parity used `status_code, root...`` this piles - /// hardforks, see (read the EIPs first): - /// - - /// - - /// - - /// - /// Moreover, the notion of `Outcome`` in parity, which segregates the two concepts, which are - /// stored in the same field `status_code`` can be computed based on such a hack of the `state_root` + /// uses ```PostState + root + PostStateOrStatus`` while Parity used ```status_code, root...\`` this piles + /// hard forks, see (read the EIPs first): + /// + /// * + /// + /// Moreover, the notion of ```Outcome`` in parity, which segregates the two concepts, which are stored in the same field ```status_code\`` can be computed based on such a hack of the `state_root` /// field, following `EIP-658`. /// /// Before Byzantinium hard fork, this field is always empty. @@ -348,9 +682,25 @@ pub struct TransactionReceipt { pub logs_bloom: ::prost::alloc::vec::Vec, #[prost(message, repeated, tag = "4")] pub logs: ::prost::alloc::vec::Vec, + /// BlobGasUsed is the amount of blob gas that has been used within this transaction. At time + /// of writing, this is equal to `self.blob_gas_fee_cap * len(self.blob_hashes)`. + /// + /// This is specified by + /// + /// This will is populated only if `TransactionTrace.Type == TRX_TYPE_BLOB` which is possible only + /// if Cancun fork is active on the chain. + #[prost(uint64, optional, tag = "5")] + pub blob_gas_used: ::core::option::Option, + /// BlobGasPrice is the amount to pay per blob item in the transaction. + /// + /// This is specified by + /// + /// This will is populated only if `TransactionTrace.Type == TRX_TYPE_BLOB` which is possible only + /// if Cancun fork is active on the chain. + #[prost(message, optional, tag = "6")] + pub blob_gas_price: ::core::option::Option, } -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct Log { #[prost(bytes = "vec", tag = "1")] pub address: ::prost::alloc::vec::Vec, @@ -359,8 +709,10 @@ pub struct Log { #[prost(bytes = "vec", tag = "3")] pub data: ::prost::alloc::vec::Vec, /// Index is the index of the log relative to the transaction. This index - /// is always populated regardless of the state revertion of the the call + /// is always populated regardless of the state reversion of the the call /// that emitted this log. + /// + /// Only available in DetailLevel: EXTENDED #[prost(uint32, tag = "4")] pub index: u32, /// BlockIndex represents the index of the log relative to the Block. @@ -369,8 +721,9 @@ pub struct Log { /// that emitted the log has been reverted by the chain. /// /// Currently, there is two locations where a Log can be obtained: - /// - block.transaction_traces\[].receipt.logs[\] - /// - block.transaction_traces\[].calls[].logs[\] + /// + /// * block.transaction_traces\[\].receipt.logs\[\] + /// * block.transaction_traces\[\].calls\[\].logs\[\] /// /// In the `receipt` case, the logs will be populated only when the call /// that emitted them has not been reverted by the chain and when in this @@ -380,10 +733,11 @@ pub struct Log { /// the `blockIndex` value will always be 0. #[prost(uint32, tag = "6")] pub block_index: u32, + /// The block's global ordinal when the log was recorded, refer to \[Block\] + /// documentation for further information about ordinals and total ordering. #[prost(uint64, tag = "7")] pub ordinal: u64, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Call { #[prost(uint32, tag = "1")] @@ -398,6 +752,25 @@ pub struct Call { pub caller: ::prost::alloc::vec::Vec, #[prost(bytes = "vec", tag = "6")] pub address: ::prost::alloc::vec::Vec, + /// AddressDelegatesTo contains the address from which the actual code to execute will be loaded + /// as defined per EIP-7702 rules. If the Call's address value resolves to a code + /// that delegates to another address, this field will be populated with the address + /// that the call is delegated to. It will be empty in all other situations. + /// + /// Assumes that a 'SetCode' transaction set address `0xA` to delegates to address `0xB`, + /// then when a call is made to `0xA`, the Call object would have: + /// + /// * caller = + /// * address = 0xA + /// * address_delegates_to = 0xB + /// + /// Again, it's important to emphasize that this field relates to EIP-7702, if the call is + /// a DELEGATE or CALLCODE type, this field will not be populated and will remain empty. + /// + /// It will be populated only if EIP-7702 is active on the chain (Prague fork) and if the + /// 'address' of the call was pointing to another address at time of execution. + #[prost(bytes = "vec", optional, tag = "34")] + pub address_delegates_to: ::core::option::Option<::prost::alloc::vec::Vec>, #[prost(message, optional, tag = "7")] pub value: ::core::option::Option, #[prost(uint64, tag = "8")] @@ -406,8 +779,27 @@ pub struct Call { pub gas_consumed: u64, #[prost(bytes = "vec", tag = "13")] pub return_data: ::prost::alloc::vec::Vec, + /// Known Issues + /// + /// * Version 3: + /// When call is `CREATE` or `CREATE2`, this field is not populated. A couple of suggestions: + /// + /// 1. You can get the contract's code in the `code_changes` field. + /// 1. In the root `CREATE` call, you can directly use the `TransactionTrace`'s input field. + /// Fixed in `Version 4`, see for information about block versions. #[prost(bytes = "vec", tag = "14")] pub input: ::prost::alloc::vec::Vec, + /// Indicates whether the call executed code. + /// + /// Known Issues + /// + /// * Version 3: + /// This may be incorrectly set to `false` for accounts with code handling native value transfers, + /// as well as for certain precompiles with no input. + /// The value is initially set based on `call.type != CREATE && len(call.input) > 0` + /// and later adjusted if the tracer detects an account without code. + /// + /// Fixed in `Version 4`, see for information about block versions. #[prost(bool, tag = "15")] pub executed_code: bool, #[prost(bool, tag = "16")] @@ -418,6 +810,12 @@ pub struct Call { ::prost::alloc::string::String, ::prost::alloc::string::String, >, + /// Known Issues + /// + /// * Version 3: + /// The data might be not be in order. + /// + /// Fixed in `Version 4`, see for information about block versions. #[prost(message, repeated, tag = "21")] pub storage_changes: ::prost::alloc::vec::Vec, #[prost(message, repeated, tag = "22")] @@ -428,12 +826,21 @@ pub struct Call { pub logs: ::prost::alloc::vec::Vec, #[prost(message, repeated, tag = "26")] pub code_changes: ::prost::alloc::vec::Vec, + /// Known Issues + /// + /// * Version 3: + /// Some gas changes are not correctly tracked: + /// 1. Gas refunded due to data returned to the chain (occurs at the end of a transaction, before buyback). + /// 1. Initial gas allocation (0 -> GasLimit) at the start of a call. + /// 1. Final gas deduction (LeftOver -> 0) at the end of a call (if applicable). + /// Fixed in `Version 4`, see for information about block versions. #[prost(message, repeated, tag = "28")] pub gas_changes: ::prost::alloc::vec::Vec, /// In Ethereum, a call can be either: - /// - Successfull, execution passes without any problem encountered - /// - Failed, execution failed, and remaining gas should be consumed - /// - Reverted, execution failed, but only gas consumed so far is billed, remaining gas is refunded + /// + /// * Successful, execution passes without any problem encountered + /// * Failed, execution failed, and remaining gas should be consumed + /// * Reverted, execution failed, but only gas consumed so far is billed, remaining gas is refunded /// /// When a call is either `failed` or `reverted`, the `status_failed` field /// below is set to `true`. If the status is `reverted`, then both `status_failed` @@ -446,7 +853,7 @@ pub struct Call { /// see above for details about those flags. #[prost(string, tag = "11")] pub failure_reason: ::prost::alloc::string::String, - /// This field represents wheter or not the state changes performed + /// This field represents whether or not the state changes performed /// by this call were correctly recorded by the blockchain. /// /// On Ethereum, a transaction can record state changes even if some @@ -456,28 +863,59 @@ pub struct Call { /// has a status of `SUCCESS`, the chain might have reverted all the state /// changes it performed. /// - /// ```text - /// Trx 1 - /// Call #1 - /// Call #2 - /// Call #3 - /// |--- Failure here - /// Call #4 + /// ```text,text + /// Trx 1 + /// Call #1 + /// Call #2 + /// Call #3 + /// |--- Failure here + /// Call #4 /// ``` /// /// In the transaction above, while Call #2 and Call #3 would have the - /// status `EXECUTED` + /// status `EXECUTED`. + /// + /// If you check all calls and check only `state_reverted` flag, you might be missing + /// some balance changes and nonce changes. This is because when a full transaction fails + /// in ethereum (e.g. `calls.all(x.state_reverted == true)`), there is still the transaction + /// fee that are recorded to the chain. + /// + /// Refer to \[TransactionTrace#status\] field for more details about the handling you must + /// perform. #[prost(bool, tag = "30")] pub state_reverted: bool, + /// Known Issues + /// + /// * Version 3: + /// + /// 1. The block's global ordinal when the call started executing, refer to + /// \[Block\] documentation for further information about ordinals and total ordering. + /// 1. The transaction root call `begin_ordial` is always `0` (also in the GENESIS block), which can cause issues + /// when sorting by this field. To ensure proper execution order, set it as follows: + /// `trx.Calls\[0\].BeginOrdinal = trx.BeginOrdinal`. + /// Fixed in `Version 4`, see for information about block versions. #[prost(uint64, tag = "31")] pub begin_ordinal: u64, + /// Known Issues + /// + /// * Version 3: + /// + /// 1. The block's global ordinal when the call finished executing, refer to + /// \[Block\] documentation for further information about ordinals and total ordering. + /// 1. The root call of the GENESIS block is always `0`. To fix it, you can set it as follows: + /// `rx.Calls\[0\].EndOrdinal = max.Uint64`. + /// Fixed in `Version 4`, see for information about block versions. #[prost(uint64, tag = "32")] pub end_ordinal: u64, + /// Known Issues + /// + /// * Version 4: + /// AccountCreations are NOT SUPPORTED anymore. DO NOT rely on them. + #[deprecated] #[prost(message, repeated, tag = "33")] pub account_creations: ::prost::alloc::vec::Vec, } -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct StorageChange { #[prost(bytes = "vec", tag = "1")] pub address: ::prost::alloc::vec::Vec, @@ -487,30 +925,50 @@ pub struct StorageChange { pub old_value: ::prost::alloc::vec::Vec, #[prost(bytes = "vec", tag = "4")] pub new_value: ::prost::alloc::vec::Vec, + /// The block's global ordinal when the storage change was recorded, refer to \[Block\] + /// documentation for further information about ordinals and total ordering. #[prost(uint64, tag = "5")] pub ordinal: u64, } -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct BalanceChange { + /// Address is the address of the account that has changed balance. #[prost(bytes = "vec", tag = "1")] pub address: ::prost::alloc::vec::Vec, + /// OldValue is the balance of the address before the change. This value + /// can be **nil/null/None** if there was no previous balance for the address. + /// It is safe in those case(s) to consider the balance as being 0. + /// + /// If you consume this from a Substreams, you can safely use: + /// + /// ```text,ignore + /// let old_value = old_value.unwrap_or_default(); + /// ``` #[prost(message, optional, tag = "2")] pub old_value: ::core::option::Option, + /// NewValue is the balance of the address after the change. This value + /// can be **nil/null/None** if there was no previous balance for the address + /// after the change. It is safe in those case(s) to consider the balance as being + /// 0. + /// + /// If you consume this from a Substreams, you can safely use: + /// + /// ```text,ignore + /// let new_value = new_value.unwrap_or_default(); + /// ``` #[prost(message, optional, tag = "3")] pub new_value: ::core::option::Option, + /// Reason is the reason why the balance has changed. This is useful to determine + /// why the balance has changed and what is the context of the change. #[prost(enumeration = "balance_change::Reason", tag = "4")] pub reason: i32, + /// The block's global ordinal when the balance change was recorded, refer to \[Block\] + /// documentation for further information about ordinals and total ordering. #[prost(uint64, tag = "5")] pub ordinal: u64, } /// Nested message and enum types in `BalanceChange`. pub mod balance_change { - /// Obtain all balanche change reasons under deep mind repository: - /// - /// ```shell - /// ack -ho 'BalanceChangeReason\(".*"\)' | grep -Eo '".*"' | sort | uniq - /// ``` #[derive( Clone, Copy, @@ -541,6 +999,14 @@ pub mod balance_change { CallBalanceOverride = 12, /// Used on chain(s) where some Ether burning happens Burn = 15, + Withdrawal = 16, + /// Rewards for Blob processing on BNB chain added in Tycho hard-fork, refers + /// to BNB documentation to check the timestamp at which it was activated. + RewardBlobFee = 17, + /// This reason is used only on Optimism chain. + IncreaseMint = 18, + /// This reason is used only on Optimism chain. + Revert = 19, } impl Reason { /// String value of the enum field names used in the ProtoBuf definition. @@ -549,22 +1015,26 @@ pub mod balance_change { /// (if the ProtoBuf definition does not change) and safe for programmatic use. pub fn as_str_name(&self) -> &'static str { match self { - Reason::Unknown => "REASON_UNKNOWN", - Reason::RewardMineUncle => "REASON_REWARD_MINE_UNCLE", - Reason::RewardMineBlock => "REASON_REWARD_MINE_BLOCK", - Reason::DaoRefundContract => "REASON_DAO_REFUND_CONTRACT", - Reason::DaoAdjustBalance => "REASON_DAO_ADJUST_BALANCE", - Reason::Transfer => "REASON_TRANSFER", - Reason::GenesisBalance => "REASON_GENESIS_BALANCE", - Reason::GasBuy => "REASON_GAS_BUY", - Reason::RewardTransactionFee => "REASON_REWARD_TRANSACTION_FEE", - Reason::RewardFeeReset => "REASON_REWARD_FEE_RESET", - Reason::GasRefund => "REASON_GAS_REFUND", - Reason::TouchAccount => "REASON_TOUCH_ACCOUNT", - Reason::SuicideRefund => "REASON_SUICIDE_REFUND", - Reason::SuicideWithdraw => "REASON_SUICIDE_WITHDRAW", - Reason::CallBalanceOverride => "REASON_CALL_BALANCE_OVERRIDE", - Reason::Burn => "REASON_BURN", + Self::Unknown => "REASON_UNKNOWN", + Self::RewardMineUncle => "REASON_REWARD_MINE_UNCLE", + Self::RewardMineBlock => "REASON_REWARD_MINE_BLOCK", + Self::DaoRefundContract => "REASON_DAO_REFUND_CONTRACT", + Self::DaoAdjustBalance => "REASON_DAO_ADJUST_BALANCE", + Self::Transfer => "REASON_TRANSFER", + Self::GenesisBalance => "REASON_GENESIS_BALANCE", + Self::GasBuy => "REASON_GAS_BUY", + Self::RewardTransactionFee => "REASON_REWARD_TRANSACTION_FEE", + Self::RewardFeeReset => "REASON_REWARD_FEE_RESET", + Self::GasRefund => "REASON_GAS_REFUND", + Self::TouchAccount => "REASON_TOUCH_ACCOUNT", + Self::SuicideRefund => "REASON_SUICIDE_REFUND", + Self::SuicideWithdraw => "REASON_SUICIDE_WITHDRAW", + Self::CallBalanceOverride => "REASON_CALL_BALANCE_OVERRIDE", + Self::Burn => "REASON_BURN", + Self::Withdrawal => "REASON_WITHDRAWAL", + Self::RewardBlobFee => "REASON_REWARD_BLOB_FEE", + Self::IncreaseMint => "REASON_INCREASE_MINT", + Self::Revert => "REASON_REVERT", } } /// Creates an enum from field names used in the ProtoBuf definition. @@ -586,13 +1056,16 @@ pub mod balance_change { "REASON_SUICIDE_WITHDRAW" => Some(Self::SuicideWithdraw), "REASON_CALL_BALANCE_OVERRIDE" => Some(Self::CallBalanceOverride), "REASON_BURN" => Some(Self::Burn), + "REASON_WITHDRAWAL" => Some(Self::Withdrawal), + "REASON_REWARD_BLOB_FEE" => Some(Self::RewardBlobFee), + "REASON_INCREASE_MINT" => Some(Self::IncreaseMint), + "REASON_REVERT" => Some(Self::Revert), _ => None, } } } } -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct NonceChange { #[prost(bytes = "vec", tag = "1")] pub address: ::prost::alloc::vec::Vec, @@ -600,19 +1073,21 @@ pub struct NonceChange { pub old_value: u64, #[prost(uint64, tag = "3")] pub new_value: u64, + /// The block's global ordinal when the nonce change was recorded, refer to \[Block\] + /// documentation for further information about ordinals and total ordering. #[prost(uint64, tag = "4")] pub ordinal: u64, } -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct AccountCreation { #[prost(bytes = "vec", tag = "1")] pub account: ::prost::alloc::vec::Vec, + /// The block's global ordinal when the account creation was recorded, refer to \[Block\] + /// documentation for further information about ordinals and total ordering. #[prost(uint64, tag = "2")] pub ordinal: u64, } -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct CodeChange { #[prost(bytes = "vec", tag = "1")] pub address: ::prost::alloc::vec::Vec, @@ -624,6 +1099,8 @@ pub struct CodeChange { pub new_hash: ::prost::alloc::vec::Vec, #[prost(bytes = "vec", tag = "5")] pub new_code: ::prost::alloc::vec::Vec, + /// The block's global ordinal when the code change was recorded, refer to \[Block\] + /// documentation for further information about ordinals and total ordering. #[prost(uint64, tag = "6")] pub ordinal: u64, } @@ -633,8 +1110,7 @@ pub struct CodeChange { /// /// Hence, we only index some of them, those that are costy like all the calls /// one, log events, return data, etc. -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct GasChange { #[prost(uint64, tag = "1")] pub old_value: u64, @@ -642,16 +1118,13 @@ pub struct GasChange { pub new_value: u64, #[prost(enumeration = "gas_change::Reason", tag = "3")] pub reason: i32, + /// The block's global ordinal when the gas change was recorded, refer to \[Block\] + /// documentation for further information about ordinals and total ordering. #[prost(uint64, tag = "4")] pub ordinal: u64, } /// Nested message and enum types in `GasChange`. pub mod gas_change { - /// Obtain all gas change reasons under deep mind repository: - /// - /// ```shell - /// ack -ho 'GasChangeReason\(".*"\)' | grep -Eo '".*"' | sort | uniq - /// ``` #[derive( Clone, Copy, @@ -666,27 +1139,88 @@ pub mod gas_change { #[repr(i32)] pub enum Reason { Unknown = 0, + /// REASON_CALL is the amount of gas that will be charged for a 'CALL' opcode executed by the EVM Call = 1, + /// REASON_CALL_CODE is the amount of gas that will be charged for a 'CALLCODE' opcode executed by the EVM CallCode = 2, + /// REASON_CALL_DATA_COPY is the amount of gas that will be charged for a 'CALLDATACOPY' opcode executed by the EVM CallDataCopy = 3, + /// REASON_CODE_COPY is the amount of gas that will be charged for a 'CALLDATACOPY' opcode executed by the EVM CodeCopy = 4, + /// REASON_CODE_STORAGE is the amount of gas that will be charged for code storage CodeStorage = 5, + /// REASON_CONTRACT_CREATION is the amount of gas that will be charged for a 'CREATE' opcode executed by the EVM and for the gas + /// burned for a CREATE, today controlled by EIP150 rules ContractCreation = 6, + /// REASON_CONTRACT_CREATION2 is the amount of gas that will be charged for a 'CREATE2' opcode executed by the EVM and for the gas + /// burned for a CREATE2, today controlled by EIP150 rules ContractCreation2 = 7, + /// REASON_DELEGATE_CALL is the amount of gas that will be charged for a 'DELEGATECALL' opcode executed by the EVM DelegateCall = 8, + /// REASON_EVENT_LOG is the amount of gas that will be charged for a 'LOG' opcode executed by the EVM EventLog = 9, + /// REASON_EXT_CODE_COPY is the amount of gas that will be charged for a 'LOG' opcode executed by the EVM ExtCodeCopy = 10, + /// REASON_FAILED_EXECUTION is the burning of the remaining gas when the execution failed without a revert FailedExecution = 11, + /// REASON_INTRINSIC_GAS is the amount of gas that will be charged for the intrinsic cost of the transaction, there is + /// always exactly one of those per transaction IntrinsicGas = 12, + /// GasChangePrecompiledContract is the amount of gas that will be charged for a precompiled contract execution PrecompiledContract = 13, + /// REASON_REFUND_AFTER_EXECUTION is the amount of gas that will be refunded to the caller after the execution of the call, + /// if there is left over at the end of execution RefundAfterExecution = 14, + /// REASON_RETURN is the amount of gas that will be charged for a 'RETURN' opcode executed by the EVM Return = 15, + /// REASON_RETURN_DATA_COPY is the amount of gas that will be charged for a 'RETURNDATACOPY' opcode executed by the EVM ReturnDataCopy = 16, + /// REASON_REVERT is the amount of gas that will be charged for a 'REVERT' opcode executed by the EVM Revert = 17, + /// REASON_SELF_DESTRUCT is the amount of gas that will be charged for a 'SELFDESTRUCT' opcode executed by the EVM SelfDestruct = 18, + /// REASON_STATIC_CALL is the amount of gas that will be charged for a 'STATICALL' opcode executed by the EVM StaticCall = 19, + /// REASON_STATE_COLD_ACCESS is the amount of gas that will be charged for a cold storage access as controlled by EIP2929 rules + /// /// Added in Berlin fork (Geth 1.10+) StateColdAccess = 20, + /// REASON_TX_INITIAL_BALANCE is the initial balance for the call which will be equal to the gasLimit of the call + /// + /// Added as new tracing reason in Geth, available only on some chains + TxInitialBalance = 21, + /// REASON_TX_REFUNDS is the sum of all refunds which happened during the tx execution (e.g. storage slot being cleared) + /// this generates an increase in gas. There is only one such gas change per transaction. + /// + /// Added as new tracing reason in Geth, available only on some chains + TxRefunds = 22, + /// REASON_TX_LEFT_OVER_RETURNED is the amount of gas left over at the end of transaction's execution that will be returned + /// to the chain. This change will always be a negative change as we "drain" left over gas towards 0. If there was no gas + /// left at the end of execution, no such even will be emitted. The returned gas's value in Wei is returned to caller. + /// There is at most one of such gas change per transaction. + /// + /// Added as new tracing reason in Geth, available only on some chains + TxLeftOverReturned = 23, + /// REASON_CALL_INITIAL_BALANCE is the initial balance for the call which will be equal to the gasLimit of the call. There is only + /// one such gas change per call. + /// + /// Added as new tracing reason in Geth, available only on some chains + CallInitialBalance = 24, + /// REASON_CALL_LEFT_OVER_RETURNED is the amount of gas left over that will be returned to the caller, this change will always + /// be a negative change as we "drain" left over gas towards 0. If there was no gas left at the end of execution, no such even + /// will be emitted. + CallLeftOverReturned = 25, + /// REASON_WITNESS_CONTRACT_INIT flags the event of adding to the witness during the contract creation initialization step. + WitnessContractInit = 26, + /// REASON_WITNESS_CONTRACT_CREATION flags the event of adding to the witness during the contract creation finalization step. + WitnessContractCreation = 27, + /// REASON_WITNESS_CODE_CHUNK flags the event of adding one or more contract code chunks to the witness. + WitnessCodeChunk = 28, + /// REASON_WITNESS_CONTRACT_COLLISION_CHECK flags the event of adding to the witness when checking for contract address collision. + WitnessContractCollisionCheck = 29, + /// REASON_TX_DATA_FLOOR is the amount of extra gas the transaction has to pay to reach the minimum gas requirement for the + /// transaction data. This change will always be a negative change. + TxDataFloor = 30, } impl Reason { /// String value of the enum field names used in the ProtoBuf definition. @@ -695,27 +1229,39 @@ pub mod gas_change { /// (if the ProtoBuf definition does not change) and safe for programmatic use. pub fn as_str_name(&self) -> &'static str { match self { - Reason::Unknown => "REASON_UNKNOWN", - Reason::Call => "REASON_CALL", - Reason::CallCode => "REASON_CALL_CODE", - Reason::CallDataCopy => "REASON_CALL_DATA_COPY", - Reason::CodeCopy => "REASON_CODE_COPY", - Reason::CodeStorage => "REASON_CODE_STORAGE", - Reason::ContractCreation => "REASON_CONTRACT_CREATION", - Reason::ContractCreation2 => "REASON_CONTRACT_CREATION2", - Reason::DelegateCall => "REASON_DELEGATE_CALL", - Reason::EventLog => "REASON_EVENT_LOG", - Reason::ExtCodeCopy => "REASON_EXT_CODE_COPY", - Reason::FailedExecution => "REASON_FAILED_EXECUTION", - Reason::IntrinsicGas => "REASON_INTRINSIC_GAS", - Reason::PrecompiledContract => "REASON_PRECOMPILED_CONTRACT", - Reason::RefundAfterExecution => "REASON_REFUND_AFTER_EXECUTION", - Reason::Return => "REASON_RETURN", - Reason::ReturnDataCopy => "REASON_RETURN_DATA_COPY", - Reason::Revert => "REASON_REVERT", - Reason::SelfDestruct => "REASON_SELF_DESTRUCT", - Reason::StaticCall => "REASON_STATIC_CALL", - Reason::StateColdAccess => "REASON_STATE_COLD_ACCESS", + Self::Unknown => "REASON_UNKNOWN", + Self::Call => "REASON_CALL", + Self::CallCode => "REASON_CALL_CODE", + Self::CallDataCopy => "REASON_CALL_DATA_COPY", + Self::CodeCopy => "REASON_CODE_COPY", + Self::CodeStorage => "REASON_CODE_STORAGE", + Self::ContractCreation => "REASON_CONTRACT_CREATION", + Self::ContractCreation2 => "REASON_CONTRACT_CREATION2", + Self::DelegateCall => "REASON_DELEGATE_CALL", + Self::EventLog => "REASON_EVENT_LOG", + Self::ExtCodeCopy => "REASON_EXT_CODE_COPY", + Self::FailedExecution => "REASON_FAILED_EXECUTION", + Self::IntrinsicGas => "REASON_INTRINSIC_GAS", + Self::PrecompiledContract => "REASON_PRECOMPILED_CONTRACT", + Self::RefundAfterExecution => "REASON_REFUND_AFTER_EXECUTION", + Self::Return => "REASON_RETURN", + Self::ReturnDataCopy => "REASON_RETURN_DATA_COPY", + Self::Revert => "REASON_REVERT", + Self::SelfDestruct => "REASON_SELF_DESTRUCT", + Self::StaticCall => "REASON_STATIC_CALL", + Self::StateColdAccess => "REASON_STATE_COLD_ACCESS", + Self::TxInitialBalance => "REASON_TX_INITIAL_BALANCE", + Self::TxRefunds => "REASON_TX_REFUNDS", + Self::TxLeftOverReturned => "REASON_TX_LEFT_OVER_RETURNED", + Self::CallInitialBalance => "REASON_CALL_INITIAL_BALANCE", + Self::CallLeftOverReturned => "REASON_CALL_LEFT_OVER_RETURNED", + Self::WitnessContractInit => "REASON_WITNESS_CONTRACT_INIT", + Self::WitnessContractCreation => "REASON_WITNESS_CONTRACT_CREATION", + Self::WitnessCodeChunk => "REASON_WITNESS_CODE_CHUNK", + Self::WitnessContractCollisionCheck => { + "REASON_WITNESS_CONTRACT_COLLISION_CHECK" + } + Self::TxDataFloor => "REASON_TX_DATA_FLOOR", } } /// Creates an enum from field names used in the ProtoBuf definition. @@ -742,11 +1288,84 @@ pub mod gas_change { "REASON_SELF_DESTRUCT" => Some(Self::SelfDestruct), "REASON_STATIC_CALL" => Some(Self::StaticCall), "REASON_STATE_COLD_ACCESS" => Some(Self::StateColdAccess), + "REASON_TX_INITIAL_BALANCE" => Some(Self::TxInitialBalance), + "REASON_TX_REFUNDS" => Some(Self::TxRefunds), + "REASON_TX_LEFT_OVER_RETURNED" => Some(Self::TxLeftOverReturned), + "REASON_CALL_INITIAL_BALANCE" => Some(Self::CallInitialBalance), + "REASON_CALL_LEFT_OVER_RETURNED" => Some(Self::CallLeftOverReturned), + "REASON_WITNESS_CONTRACT_INIT" => Some(Self::WitnessContractInit), + "REASON_WITNESS_CONTRACT_CREATION" => Some(Self::WitnessContractCreation), + "REASON_WITNESS_CODE_CHUNK" => Some(Self::WitnessCodeChunk), + "REASON_WITNESS_CONTRACT_COLLISION_CHECK" => { + Some(Self::WitnessContractCollisionCheck) + } + "REASON_TX_DATA_FLOOR" => Some(Self::TxDataFloor), _ => None, } } } } +/// HeaderOnlyBlock is used to optimally unpack the \[Block\] structure (note the +/// corresponding message number for the `header` field) while consuming less +/// memory, when only the `header` is desired. +/// +/// WARN: this is a client-side optimization pattern and should be moved in the +/// consuming code. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct HeaderOnlyBlock { + #[prost(message, optional, tag = "5")] + pub header: ::core::option::Option, +} +/// BlockWithRefs is a lightweight block, with traces and transactions +/// purged from the `block` within, and only. It is used in transports +/// to pass block data around. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct BlockWithRefs { + #[prost(string, tag = "1")] + pub id: ::prost::alloc::string::String, + #[prost(message, optional, tag = "2")] + pub block: ::core::option::Option, + #[prost(message, optional, tag = "3")] + pub transaction_trace_refs: ::core::option::Option, + #[prost(bool, tag = "4")] + pub irreversible: bool, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TransactionTraceWithBlockRef { + #[prost(message, optional, tag = "1")] + pub trace: ::core::option::Option, + #[prost(message, optional, tag = "2")] + pub block_ref: ::core::option::Option, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct TransactionRefs { + #[prost(bytes = "vec", repeated, tag = "1")] + pub hashes: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec>, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct BlockRef { + #[prost(bytes = "vec", tag = "1")] + pub hash: ::prost::alloc::vec::Vec, + #[prost(uint64, tag = "2")] + pub number: u64, +} +/// Withdrawal represents a validator withdrawal from the beacon chain to the EVM. +/// Introduced in EIP-4895 (Shanghai hard fork). +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct Withdrawal { + /// Index is the monotonically increasing identifier of the withdrawal + #[prost(uint64, tag = "1")] + pub index: u64, + /// ValidatorIndex is the index of the validator that is withdrawing + #[prost(uint64, tag = "2")] + pub validator_index: u64, + /// Address is the Ethereum address receiving the withdrawn funds + #[prost(bytes = "vec", tag = "3")] + pub address: ::prost::alloc::vec::Vec, + /// Amount is the value of the withdrawal in gwei (1 gwei = 1e9 wei) + #[prost(uint64, tag = "4")] + pub amount: u64, +} #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] #[repr(i32)] pub enum TransactionTraceStatus { @@ -762,10 +1381,10 @@ impl TransactionTraceStatus { /// (if the ProtoBuf definition does not change) and safe for programmatic use. pub fn as_str_name(&self) -> &'static str { match self { - TransactionTraceStatus::Unknown => "UNKNOWN", - TransactionTraceStatus::Succeeded => "SUCCEEDED", - TransactionTraceStatus::Failed => "FAILED", - TransactionTraceStatus::Reverted => "REVERTED", + Self::Unknown => "UNKNOWN", + Self::Succeeded => "SUCCEEDED", + Self::Failed => "FAILED", + Self::Reverted => "REVERTED", } } /// Creates an enum from field names used in the ProtoBuf definition. @@ -798,12 +1417,12 @@ impl CallType { /// (if the ProtoBuf definition does not change) and safe for programmatic use. pub fn as_str_name(&self) -> &'static str { match self { - CallType::Unspecified => "UNSPECIFIED", - CallType::Call => "CALL", - CallType::Callcode => "CALLCODE", - CallType::Delegate => "DELEGATE", - CallType::Static => "STATIC", - CallType::Create => "CREATE", + Self::Unspecified => "UNSPECIFIED", + Self::Call => "CALL", + Self::Callcode => "CALLCODE", + Self::Delegate => "DELEGATE", + Self::Static => "STATIC", + Self::Create => "CREATE", } } /// Creates an enum from field names used in the ProtoBuf definition. diff --git a/chain/ethereum/src/runtime/abi.rs b/chain/ethereum/src/runtime/abi.rs index dcc9564bd7f..4311fed3c48 100644 --- a/chain/ethereum/src/runtime/abi.rs +++ b/chain/ethereum/src/runtime/abi.rs @@ -2,15 +2,18 @@ use super::runtime_adapter::UnresolvedContractCall; use crate::trigger::{ EthereumBlockData, EthereumCallData, EthereumEventData, EthereumTransactionData, }; +use anyhow::anyhow; +use async_trait::async_trait; +use graph::abi; +use graph::prelude::alloy; +use graph::prelude::alloy::consensus::TxReceipt; +use graph::prelude::alloy::network::ReceiptResponse; +use graph::prelude::alloy::rpc::types::{Log, TransactionReceipt}; use graph::{ - prelude::{ - ethabi, - web3::types::{Log, TransactionReceipt, H256}, - BigInt, - }, + prelude::BigInt, runtime::{ - asc_get, asc_new, gas::GasCounter, AscHeap, AscIndexId, AscPtr, AscType, - DeterministicHostError, FromAscObj, IndexForAscTypeId, ToAscObj, + AscHeap, AscIndexId, AscPtr, AscType, DeterministicHostError, FromAscObj, HostExportError, + IndexForAscTypeId, ToAscObj, asc_get, asc_new, asc_new_or_null, gas::GasCounter, }, }; use graph_runtime_derive::AscType; @@ -20,7 +23,7 @@ use graph_runtime_wasm::asc_abi::class::{ }; use semver::Version; -type AscH256 = Uint8Array; +type AscB256 = Uint8Array; type AscH2048 = Uint8Array; pub struct AscLogParamArray(Array>); @@ -37,15 +40,18 @@ impl AscType for AscLogParamArray { } } -impl ToAscObj for Vec { - fn to_asc_obj( +#[async_trait] +impl ToAscObj for &[abi::DynSolParam] { + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, - ) -> Result { - let content: Result, _> = self.iter().map(|x| asc_new(heap, x, gas)).collect(); - let content = content?; - Ok(AscLogParamArray(Array::new(&content, heap, gas)?)) + ) -> Result { + let mut content = Vec::with_capacity(self.len()); + for x in *self { + content.push(asc_new(heap, x, gas).await?); + } + Ok(AscLogParamArray(Array::new(&content, heap, gas).await?)) } } @@ -53,7 +59,7 @@ impl AscIndexId for AscLogParamArray { const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::ArrayEventParam; } -pub struct AscTopicArray(Array>); +pub struct AscTopicArray(Array>); impl AscType for AscTopicArray { fn to_asc_bytes(&self) -> Result, DeterministicHostError> { @@ -68,22 +74,23 @@ impl AscType for AscTopicArray { } } -impl ToAscObj for Vec { - fn to_asc_obj( +#[async_trait] +impl ToAscObj for &[alloy::primitives::B256] { + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, - ) -> Result { - let topics = self - .iter() - .map(|topic| asc_new(heap, topic, gas)) - .collect::, _>>()?; - Ok(AscTopicArray(Array::new(&topics, heap, gas)?)) + ) -> Result { + let mut topics = Vec::with_capacity(self.len()); + for topic in *self { + topics.push(asc_new(heap, topic, gas).await?); + } + Ok(AscTopicArray(Array::new(&topics, heap, gas).await?)) } } impl AscIndexId for AscTopicArray { - const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::ArrayH256; + const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::ArrayB256; } pub struct AscLogArray(Array>); @@ -101,17 +108,19 @@ impl AscType for AscLogArray { } } -impl ToAscObj for Vec { - fn to_asc_obj( +#[async_trait] +impl ToAscObj for &[Log] { + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, - ) -> Result { - let logs = self - .iter() - .map(|log| asc_new(heap, &log, gas)) - .collect::, _>>()?; - Ok(AscLogArray(Array::new(&logs, heap, gas)?)) + ) -> Result { + let mut logs = Vec::with_capacity(self.len()); + for log in *self { + logs.push(asc_new(heap, log, gas).await?); + } + + Ok(AscLogArray(Array::new(&logs, heap, gas).await?)) } } @@ -121,6 +130,7 @@ impl AscIndexId for AscLogArray { #[repr(C)] #[derive(AscType)] +#[allow(non_camel_case_types)] pub struct AscUnresolvedContractCall_0_0_4 { pub contract_name: AscPtr, pub contract_address: AscPtr, @@ -138,13 +148,14 @@ impl FromAscObj for UnresolvedContractCall { asc_call: AscUnresolvedContractCall_0_0_4, heap: &H, gas: &GasCounter, + depth: usize, ) -> Result { Ok(UnresolvedContractCall { - contract_name: asc_get(heap, asc_call.contract_name, gas)?, - contract_address: asc_get(heap, asc_call.contract_address, gas)?, - function_name: asc_get(heap, asc_call.function_name, gas)?, - function_signature: Some(asc_get(heap, asc_call.function_signature, gas)?), - function_args: asc_get(heap, asc_call.function_args, gas)?, + contract_name: asc_get(heap, asc_call.contract_name, gas, depth)?, + contract_address: asc_get(heap, asc_call.contract_address, gas, depth)?, + function_name: asc_get(heap, asc_call.function_name, gas, depth)?, + function_signature: Some(asc_get(heap, asc_call.function_signature, gas, depth)?), + function_args: asc_get(heap, asc_call.function_args, gas, depth)?, }) } } @@ -163,13 +174,14 @@ impl FromAscObj for UnresolvedContractCall { asc_call: AscUnresolvedContractCall, heap: &H, gas: &GasCounter, + depth: usize, ) -> Result { Ok(UnresolvedContractCall { - contract_name: asc_get(heap, asc_call.contract_name, gas)?, - contract_address: asc_get(heap, asc_call.contract_address, gas)?, - function_name: asc_get(heap, asc_call.function_name, gas)?, + contract_name: asc_get(heap, asc_call.contract_name, gas, depth)?, + contract_address: asc_get(heap, asc_call.contract_address, gas, depth)?, + function_name: asc_get(heap, asc_call.function_name, gas, depth)?, function_signature: None, - function_args: asc_get(heap, asc_call.function_args, gas)?, + function_args: asc_get(heap, asc_call.function_args, gas, depth)?, }) } } @@ -177,13 +189,13 @@ impl FromAscObj for UnresolvedContractCall { #[repr(C)] #[derive(AscType)] pub(crate) struct AscEthereumBlock { - pub hash: AscPtr, - pub parent_hash: AscPtr, - pub uncles_hash: AscPtr, + pub hash: AscPtr, + pub parent_hash: AscPtr, + pub uncles_hash: AscPtr, pub author: AscPtr, - pub state_root: AscPtr, - pub transactions_root: AscPtr, - pub receipts_root: AscPtr, + pub state_root: AscPtr, + pub transactions_root: AscPtr, + pub receipts_root: AscPtr, pub number: AscPtr, pub gas_used: AscPtr, pub gas_limit: AscPtr, @@ -199,14 +211,15 @@ impl AscIndexId for AscEthereumBlock { #[repr(C)] #[derive(AscType)] +#[allow(non_camel_case_types)] pub(crate) struct AscEthereumBlock_0_0_6 { - pub hash: AscPtr, - pub parent_hash: AscPtr, - pub uncles_hash: AscPtr, + pub hash: AscPtr, + pub parent_hash: AscPtr, + pub uncles_hash: AscPtr, pub author: AscPtr, - pub state_root: AscPtr, - pub transactions_root: AscPtr, - pub receipts_root: AscPtr, + pub state_root: AscPtr, + pub transactions_root: AscPtr, + pub receipts_root: AscPtr, pub number: AscPtr, pub gas_used: AscPtr, pub gas_limit: AscPtr, @@ -223,8 +236,9 @@ impl AscIndexId for AscEthereumBlock_0_0_6 { #[repr(C)] #[derive(AscType)] +#[allow(non_camel_case_types)] pub(crate) struct AscEthereumTransaction_0_0_1 { - pub hash: AscPtr, + pub hash: AscPtr, pub index: AscPtr, pub from: AscPtr, pub to: AscPtr, @@ -239,8 +253,9 @@ impl AscIndexId for AscEthereumTransaction_0_0_1 { #[repr(C)] #[derive(AscType)] +#[allow(non_camel_case_types)] pub(crate) struct AscEthereumTransaction_0_0_2 { - pub hash: AscPtr, + pub hash: AscPtr, pub index: AscPtr, pub from: AscPtr, pub to: AscPtr, @@ -256,8 +271,9 @@ impl AscIndexId for AscEthereumTransaction_0_0_2 { #[repr(C)] #[derive(AscType)] +#[allow(non_camel_case_types)] pub(crate) struct AscEthereumTransaction_0_0_6 { - pub hash: AscPtr, + pub hash: AscPtr, pub index: AscPtr, pub from: AscPtr, pub to: AscPtr, @@ -306,9 +322,9 @@ pub(crate) struct AscEthereumLog { pub address: AscPtr, pub topics: AscPtr, pub data: AscPtr, - pub block_hash: AscPtr, - pub block_number: AscPtr, - pub transaction_hash: AscPtr, + pub block_hash: AscPtr, + pub block_number: AscPtr, + pub transaction_hash: AscPtr, pub transaction_index: AscPtr, pub log_index: AscPtr, pub transaction_log_index: AscPtr, @@ -323,16 +339,16 @@ impl AscIndexId for AscEthereumLog { #[repr(C)] #[derive(AscType)] pub(crate) struct AscEthereumTransactionReceipt { - pub transaction_hash: AscPtr, + pub transaction_hash: AscPtr, pub transaction_index: AscPtr, - pub block_hash: AscPtr, + pub block_hash: AscPtr, pub block_number: AscPtr, pub cumulative_gas_used: AscPtr, pub gas_used: AscPtr, pub contract_address: AscPtr, pub logs: AscPtr, pub status: AscPtr, - pub root: AscPtr, + pub root: AscPtr, pub logs_bloom: AscPtr, } @@ -344,6 +360,7 @@ impl AscIndexId for AscEthereumTransactionReceipt { /// `receipt` field. #[repr(C)] #[derive(AscType)] +#[allow(non_camel_case_types)] pub(crate) struct AscEthereumEvent_0_0_7 where T: AscType, @@ -390,6 +407,7 @@ impl AscIndexId for AscEthereumCall { #[repr(C)] #[derive(AscType)] +#[allow(non_camel_case_types)] pub(crate) struct AscEthereumCall_0_0_3 where T: AscType, @@ -411,185 +429,177 @@ where const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::EthereumCall; } -impl ToAscObj for EthereumBlockData { - fn to_asc_obj( +#[async_trait] +impl<'a> ToAscObj for EthereumBlockData<'a> { + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, - ) -> Result { + ) -> Result { + let size = asc_new_or_null_u256(heap, self.size(), gas).await?; + Ok(AscEthereumBlock { - hash: asc_new(heap, &self.hash, gas)?, - parent_hash: asc_new(heap, &self.parent_hash, gas)?, - uncles_hash: asc_new(heap, &self.uncles_hash, gas)?, - author: asc_new(heap, &self.author, gas)?, - state_root: asc_new(heap, &self.state_root, gas)?, - transactions_root: asc_new(heap, &self.transactions_root, gas)?, - receipts_root: asc_new(heap, &self.receipts_root, gas)?, - number: asc_new(heap, &BigInt::from(self.number), gas)?, - gas_used: asc_new(heap, &BigInt::from_unsigned_u256(&self.gas_used), gas)?, - gas_limit: asc_new(heap, &BigInt::from_unsigned_u256(&self.gas_limit), gas)?, - timestamp: asc_new(heap, &BigInt::from_unsigned_u256(&self.timestamp), gas)?, - difficulty: asc_new(heap, &BigInt::from_unsigned_u256(&self.difficulty), gas)?, + hash: asc_new(heap, self.hash(), gas).await?, + parent_hash: asc_new(heap, self.parent_hash(), gas).await?, + uncles_hash: asc_new(heap, self.uncles_hash(), gas).await?, + author: asc_new(heap, self.author(), gas).await?, + state_root: asc_new(heap, self.state_root(), gas).await?, + transactions_root: asc_new(heap, self.transactions_root(), gas).await?, + receipts_root: asc_new(heap, self.receipts_root(), gas).await?, + number: asc_new(heap, &BigInt::from(self.number()), gas).await?, + gas_used: asc_new(heap, &BigInt::from(self.gas_used()), gas).await?, + gas_limit: asc_new(heap, &BigInt::from(self.gas_limit()), gas).await?, + timestamp: asc_new(heap, &BigInt::from(self.timestamp()), gas).await?, + difficulty: asc_new(heap, &BigInt::from_unsigned_u256(self.difficulty()), gas).await?, total_difficulty: asc_new( heap, - &BigInt::from_unsigned_u256(&self.total_difficulty), + &BigInt::from_unsigned_u256(self.total_difficulty()), gas, - )?, - size: self - .size - .map(|size| asc_new(heap, &BigInt::from_unsigned_u256(&size), gas)) - .unwrap_or(Ok(AscPtr::null()))?, + ) + .await?, + size, }) } } -impl ToAscObj for EthereumBlockData { - fn to_asc_obj( +#[async_trait] +impl<'a> ToAscObj for EthereumBlockData<'a> { + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, - ) -> Result { + ) -> Result { + let size = asc_new_or_null_u256(heap, self.size(), gas).await?; + let base_fee_per_block = asc_new_or_null_u64(heap, self.base_fee_per_gas(), gas).await?; + Ok(AscEthereumBlock_0_0_6 { - hash: asc_new(heap, &self.hash, gas)?, - parent_hash: asc_new(heap, &self.parent_hash, gas)?, - uncles_hash: asc_new(heap, &self.uncles_hash, gas)?, - author: asc_new(heap, &self.author, gas)?, - state_root: asc_new(heap, &self.state_root, gas)?, - transactions_root: asc_new(heap, &self.transactions_root, gas)?, - receipts_root: asc_new(heap, &self.receipts_root, gas)?, - number: asc_new(heap, &BigInt::from(self.number), gas)?, - gas_used: asc_new(heap, &BigInt::from_unsigned_u256(&self.gas_used), gas)?, - gas_limit: asc_new(heap, &BigInt::from_unsigned_u256(&self.gas_limit), gas)?, - timestamp: asc_new(heap, &BigInt::from_unsigned_u256(&self.timestamp), gas)?, - difficulty: asc_new(heap, &BigInt::from_unsigned_u256(&self.difficulty), gas)?, + hash: asc_new(heap, self.hash(), gas).await?, + parent_hash: asc_new(heap, self.parent_hash(), gas).await?, + uncles_hash: asc_new(heap, self.uncles_hash(), gas).await?, + author: asc_new(heap, self.author(), gas).await?, + state_root: asc_new(heap, self.state_root(), gas).await?, + transactions_root: asc_new(heap, self.transactions_root(), gas).await?, + receipts_root: asc_new(heap, self.receipts_root(), gas).await?, + number: asc_new(heap, &BigInt::from(self.number()), gas).await?, + gas_used: asc_new(heap, &BigInt::from(self.gas_used()), gas).await?, + gas_limit: asc_new(heap, &BigInt::from(self.gas_limit()), gas).await?, + timestamp: asc_new(heap, &BigInt::from(self.timestamp()), gas).await?, + difficulty: asc_new(heap, &BigInt::from_unsigned_u256(self.difficulty()), gas).await?, total_difficulty: asc_new( heap, - &BigInt::from_unsigned_u256(&self.total_difficulty), + &BigInt::from_unsigned_u256(self.total_difficulty()), gas, - )?, - size: self - .size - .map(|size| asc_new(heap, &BigInt::from_unsigned_u256(&size), gas)) - .unwrap_or(Ok(AscPtr::null()))?, - base_fee_per_block: self - .base_fee_per_gas - .map(|base_fee| asc_new(heap, &BigInt::from_unsigned_u256(&base_fee), gas)) - .unwrap_or(Ok(AscPtr::null()))?, + ) + .await?, + size, + base_fee_per_block, }) } } -impl ToAscObj for EthereumTransactionData { - fn to_asc_obj( +#[async_trait] +impl<'a> ToAscObj for EthereumTransactionData<'a> { + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, - ) -> Result { + ) -> Result { Ok(AscEthereumTransaction_0_0_1 { - hash: asc_new(heap, &self.hash, gas)?, - index: asc_new(heap, &BigInt::from(self.index), gas)?, - from: asc_new(heap, &self.from, gas)?, - to: self - .to - .map(|to| asc_new(heap, &to, gas)) - .unwrap_or(Ok(AscPtr::null()))?, - value: asc_new(heap, &BigInt::from_unsigned_u256(&self.value), gas)?, - gas_limit: asc_new(heap, &BigInt::from_unsigned_u256(&self.gas_limit), gas)?, - gas_price: asc_new(heap, &BigInt::from_unsigned_u256(&self.gas_price), gas)?, + hash: asc_new(heap, &self.hash(), gas).await?, + index: asc_new(heap, &BigInt::from(self.index()), gas).await?, + from: asc_new(heap, &self.from(), gas).await?, + to: asc_new_or_null(heap, &self.to(), gas).await?, + value: asc_new(heap, &BigInt::from_unsigned_u256(&self.value()), gas).await?, + gas_limit: asc_new(heap, &BigInt::from(self.gas_limit()), gas).await?, + gas_price: asc_new(heap, &BigInt::from(self.gas_price()), gas).await?, }) } } -impl ToAscObj for EthereumTransactionData { - fn to_asc_obj( +#[async_trait] +impl<'a> ToAscObj for EthereumTransactionData<'a> { + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, - ) -> Result { + ) -> Result { Ok(AscEthereumTransaction_0_0_2 { - hash: asc_new(heap, &self.hash, gas)?, - index: asc_new(heap, &BigInt::from(self.index), gas)?, - from: asc_new(heap, &self.from, gas)?, - to: self - .to - .map(|to| asc_new(heap, &to, gas)) - .unwrap_or(Ok(AscPtr::null()))?, - value: asc_new(heap, &BigInt::from_unsigned_u256(&self.value), gas)?, - gas_limit: asc_new(heap, &BigInt::from_unsigned_u256(&self.gas_limit), gas)?, - gas_price: asc_new(heap, &BigInt::from_unsigned_u256(&self.gas_price), gas)?, - input: asc_new(heap, &*self.input, gas)?, + hash: asc_new(heap, &self.hash(), gas).await?, + index: asc_new(heap, &BigInt::from(self.index()), gas).await?, + from: asc_new(heap, &self.from(), gas).await?, + to: asc_new_or_null(heap, &self.to(), gas).await?, + value: asc_new(heap, &BigInt::from_unsigned_u256(&self.value()), gas).await?, + gas_limit: asc_new(heap, &BigInt::from(self.gas_limit()), gas).await?, + gas_price: asc_new(heap, &BigInt::from(self.gas_price()), gas).await?, + input: asc_new(heap, self.input(), gas).await?, }) } } -impl ToAscObj for EthereumTransactionData { - fn to_asc_obj( +#[async_trait] +impl<'a> ToAscObj for EthereumTransactionData<'a> { + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, - ) -> Result { + ) -> Result { Ok(AscEthereumTransaction_0_0_6 { - hash: asc_new(heap, &self.hash, gas)?, - index: asc_new(heap, &BigInt::from(self.index), gas)?, - from: asc_new(heap, &self.from, gas)?, - to: self - .to - .map(|to| asc_new(heap, &to, gas)) - .unwrap_or(Ok(AscPtr::null()))?, - value: asc_new(heap, &BigInt::from_unsigned_u256(&self.value), gas)?, - gas_limit: asc_new(heap, &BigInt::from_unsigned_u256(&self.gas_limit), gas)?, - gas_price: asc_new(heap, &BigInt::from_unsigned_u256(&self.gas_price), gas)?, - input: asc_new(heap, &*self.input, gas)?, - nonce: asc_new(heap, &BigInt::from_unsigned_u256(&self.nonce), gas)?, + hash: asc_new(heap, &self.hash(), gas).await?, + index: asc_new(heap, &BigInt::from(self.index()), gas).await?, + from: asc_new(heap, &self.from(), gas).await?, + to: asc_new_or_null(heap, &self.to(), gas).await?, + value: asc_new(heap, &BigInt::from_unsigned_u256(&self.value()), gas).await?, + gas_limit: asc_new(heap, &BigInt::from(self.gas_limit()), gas).await?, + gas_price: asc_new(heap, &BigInt::from(self.gas_price()), gas).await?, + input: asc_new(heap, self.input(), gas).await?, + nonce: asc_new(heap, &BigInt::from(self.nonce()), gas).await?, }) } } -impl ToAscObj> for EthereumEventData +#[async_trait] +impl<'a, T, B> ToAscObj> for EthereumEventData<'a> where - T: AscType + AscIndexId, - B: AscType + AscIndexId, - EthereumTransactionData: ToAscObj, - EthereumBlockData: ToAscObj, + T: AscType + AscIndexId + Send, + B: AscType + AscIndexId + Send, + EthereumTransactionData<'a>: ToAscObj, + EthereumBlockData<'a>: ToAscObj, { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, - ) -> Result, DeterministicHostError> { + ) -> Result, HostExportError> { Ok(AscEthereumEvent { - address: asc_new(heap, &self.address, gas)?, - log_index: asc_new(heap, &BigInt::from_unsigned_u256(&self.log_index), gas)?, - transaction_log_index: asc_new( - heap, - &BigInt::from_unsigned_u256(&self.transaction_log_index), - gas, - )?, - log_type: self - .log_type - .clone() - .map(|log_type| asc_new(heap, &log_type, gas)) - .unwrap_or(Ok(AscPtr::null()))?, - block: asc_new::(heap, &self.block, gas)?, - transaction: asc_new::(heap, &self.transaction, gas)?, - params: asc_new(heap, &self.params, gas)?, + address: asc_new(heap, self.address(), gas).await?, + log_index: asc_new(heap, &BigInt::from(self.log_index()), gas).await?, + transaction_log_index: asc_new(heap, &BigInt::from(self.transaction_log_index()), gas) + .await?, + log_type: asc_new_or_null(heap, &self.log_type().as_ref(), gas).await?, + block: asc_new::(heap, &self.block, gas).await?, + transaction: asc_new::(heap, &self.transaction, gas) + .await?, + params: asc_new(heap, &self.params, gas).await?, }) } } -impl ToAscObj> - for (EthereumEventData, Option<&TransactionReceipt>) +#[async_trait] +impl<'a, T, B, Inner> ToAscObj> + for (EthereumEventData<'a>, Option<&TransactionReceipt>) where - T: AscType + AscIndexId, - B: AscType + AscIndexId, - EthereumTransactionData: ToAscObj, - EthereumBlockData: ToAscObj, + T: AscType + AscIndexId + Send, + B: AscType + AscIndexId + Send, + EthereumTransactionData<'a>: ToAscObj, + EthereumBlockData<'a>: ToAscObj, + Inner: Send + Sync, + TransactionReceipt: ToAscObj, { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, - ) -> Result, DeterministicHostError> { + ) -> Result, HostExportError> { let (event_data, optional_receipt) = self; let AscEthereumEvent { address, @@ -599,9 +609,9 @@ where block, transaction, params, - } = event_data.to_asc_obj(heap, gas)?; + } = event_data.to_asc_obj(heap, gas).await?; let receipt = if let Some(receipt_data) = optional_receipt { - asc_new(heap, receipt_data, gas)? + asc_new(heap, receipt_data, gas).await? } else { AscPtr::null() }; @@ -618,166 +628,166 @@ where } } +async fn asc_new_or_null_u256( + heap: &mut H, + value: &Option, + gas: &GasCounter, +) -> Result, HostExportError> { + match value { + Some(value) => asc_new(heap, &BigInt::from_unsigned_u256(value), gas).await, + None => Ok(AscPtr::null()), + } +} + +async fn asc_new_or_null_u64( + heap: &mut H, + value: &Option, + gas: &GasCounter, +) -> Result, HostExportError> { + match value { + Some(value) => asc_new(heap, &BigInt::from(*value), gas).await, + None => Ok(AscPtr::null()), + } +} + +#[async_trait] impl ToAscObj for Log { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, - ) -> Result { + ) -> Result { Ok(AscEthereumLog { - address: asc_new(heap, &self.address, gas)?, - topics: asc_new(heap, &self.topics, gas)?, - data: asc_new(heap, self.data.0.as_slice(), gas)?, - block_hash: self - .block_hash - .map(|block_hash| asc_new(heap, &block_hash, gas)) - .unwrap_or(Ok(AscPtr::null()))?, - block_number: self - .block_number - .map(|block_number| asc_new(heap, &BigInt::from(block_number), gas)) - .unwrap_or(Ok(AscPtr::null()))?, - transaction_hash: self - .transaction_hash - .map(|txn_hash| asc_new(heap, &txn_hash, gas)) - .unwrap_or(Ok(AscPtr::null()))?, - transaction_index: self - .transaction_index - .map(|txn_index| asc_new(heap, &BigInt::from(txn_index), gas)) - .unwrap_or(Ok(AscPtr::null()))?, - log_index: self - .log_index - .map(|log_index| asc_new(heap, &BigInt::from_unsigned_u256(&log_index), gas)) - .unwrap_or(Ok(AscPtr::null()))?, - transaction_log_index: self - .transaction_log_index - .map(|index| asc_new(heap, &BigInt::from_unsigned_u256(&index), gas)) - .unwrap_or(Ok(AscPtr::null()))?, - log_type: self - .log_type - .as_ref() - .map(|log_type| asc_new(heap, &log_type, gas)) - .unwrap_or(Ok(AscPtr::null()))?, - removed: self - .removed - .map(|removed| asc_new(heap, &AscWrapped { inner: removed }, gas)) - .unwrap_or(Ok(AscPtr::null()))?, + address: asc_new(heap, &self.address(), gas).await?, + topics: asc_new(heap, &self.topics(), gas).await?, + data: asc_new(heap, self.data().data.as_ref(), gas).await?, + block_hash: asc_new_or_null(heap, &self.block_hash, gas).await?, + block_number: asc_new_or_null_u64(heap, &self.block_number, gas).await?, + transaction_hash: asc_new_or_null(heap, &self.transaction_hash, gas).await?, + transaction_index: asc_new_or_null_u64(heap, &self.transaction_index, gas).await?, + log_index: asc_new_or_null_u64(heap, &self.log_index, gas).await?, + transaction_log_index: AscPtr::null(), // Non-standard field, not available in alloy + log_type: AscPtr::null(), // Non-standard field, not available in alloy + removed: asc_new( + heap, + &AscWrapped { + inner: self.removed, + }, + gas, + ) + .await?, }) } } -impl ToAscObj for &TransactionReceipt { - fn to_asc_obj( +#[async_trait] +impl ToAscObj + for TransactionReceipt> +{ + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, - ) -> Result { + ) -> Result { + let transaction_index = self + .transaction_index + .ok_or(HostExportError::Unknown(anyhow!( + "Transaction index is missing" + )))?; + let status = match self.inner.status_or_post_state().as_eip658() { + Some(success) => asc_new(heap, &BigInt::from(success as u64), gas).await?, + None => AscPtr::null(), // Pre-EIP-658 (pre-Byzantium) receipt + }; Ok(AscEthereumTransactionReceipt { - transaction_hash: asc_new(heap, &self.transaction_hash, gas)?, - transaction_index: asc_new(heap, &BigInt::from(self.transaction_index), gas)?, - block_hash: self - .block_hash - .map(|block_hash| asc_new(heap, &block_hash, gas)) - .unwrap_or(Ok(AscPtr::null()))?, - block_number: self - .block_number - .map(|block_number| asc_new(heap, &BigInt::from(block_number), gas)) - .unwrap_or(Ok(AscPtr::null()))?, - cumulative_gas_used: asc_new( - heap, - &BigInt::from_unsigned_u256(&self.cumulative_gas_used), - gas, - )?, - gas_used: self - .gas_used - .map(|gas_used| asc_new(heap, &BigInt::from_unsigned_u256(&gas_used), gas)) - .unwrap_or(Ok(AscPtr::null()))?, - contract_address: self - .contract_address - .map(|contract_address| asc_new(heap, &contract_address, gas)) - .unwrap_or(Ok(AscPtr::null()))?, - logs: asc_new(heap, &self.logs, gas)?, - status: self - .status - .map(|status| asc_new(heap, &BigInt::from(status), gas)) - .unwrap_or(Ok(AscPtr::null()))?, - root: self - .root - .map(|root| asc_new(heap, &root, gas)) - .unwrap_or(Ok(AscPtr::null()))?, - logs_bloom: asc_new(heap, self.logs_bloom.as_bytes(), gas)?, + transaction_hash: asc_new(heap, &self.transaction_hash, gas).await?, + transaction_index: asc_new(heap, &BigInt::from(transaction_index), gas).await?, + block_hash: asc_new_or_null(heap, &self.block_hash, gas).await?, + block_number: asc_new_or_null_u64(heap, &self.block_number, gas).await?, + cumulative_gas_used: asc_new(heap, &BigInt::from(self.cumulative_gas_used()), gas) + .await?, + gas_used: asc_new(heap, &BigInt::from(self.gas_used), gas).await?, + contract_address: asc_new_or_null(heap, &self.contract_address, gas).await?, + logs: asc_new(heap, &self.logs(), gas).await?, + status, + root: asc_new_or_null(heap, &self.state_root(), gas).await?, + logs_bloom: asc_new(heap, self.inner.bloom().as_slice(), gas).await?, }) } } -impl ToAscObj for EthereumCallData { - fn to_asc_obj( +#[async_trait] +impl<'a> ToAscObj for EthereumCallData<'a> { + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, - ) -> Result { + ) -> Result { Ok(AscEthereumCall { - address: asc_new(heap, &self.to, gas)?, - block: asc_new(heap, &self.block, gas)?, - transaction: asc_new(heap, &self.transaction, gas)?, - inputs: asc_new(heap, &self.inputs, gas)?, - outputs: asc_new(heap, &self.outputs, gas)?, + address: asc_new(heap, self.to(), gas).await?, + block: asc_new(heap, &self.block, gas).await?, + transaction: asc_new(heap, &self.transaction, gas).await?, + inputs: asc_new(heap, &self.inputs, gas).await?, + outputs: asc_new(heap, &self.outputs, gas).await?, }) } } -impl ToAscObj> - for EthereumCallData +#[async_trait] +impl<'a> ToAscObj> + for EthereumCallData<'a> { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, ) -> Result< AscEthereumCall_0_0_3, - DeterministicHostError, + HostExportError, > { Ok(AscEthereumCall_0_0_3 { - to: asc_new(heap, &self.to, gas)?, - from: asc_new(heap, &self.from, gas)?, - block: asc_new(heap, &self.block, gas)?, - transaction: asc_new(heap, &self.transaction, gas)?, - inputs: asc_new(heap, &self.inputs, gas)?, - outputs: asc_new(heap, &self.outputs, gas)?, + to: asc_new(heap, self.to(), gas).await?, + from: asc_new(heap, self.from(), gas).await?, + block: asc_new(heap, &self.block, gas).await?, + transaction: asc_new(heap, &self.transaction, gas).await?, + inputs: asc_new(heap, &self.inputs, gas).await?, + outputs: asc_new(heap, &self.outputs, gas).await?, }) } } -impl ToAscObj> - for EthereumCallData +#[async_trait] +impl<'a> ToAscObj> + for EthereumCallData<'a> { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, ) -> Result< AscEthereumCall_0_0_3, - DeterministicHostError, + HostExportError, > { Ok(AscEthereumCall_0_0_3 { - to: asc_new(heap, &self.to, gas)?, - from: asc_new(heap, &self.from, gas)?, - block: asc_new(heap, &self.block, gas)?, - transaction: asc_new(heap, &self.transaction, gas)?, - inputs: asc_new(heap, &self.inputs, gas)?, - outputs: asc_new(heap, &self.outputs, gas)?, + to: asc_new(heap, self.to(), gas).await?, + from: asc_new(heap, self.from(), gas).await?, + block: asc_new(heap, &self.block, gas).await?, + transaction: asc_new(heap, &self.transaction, gas).await?, + inputs: asc_new(heap, &self.inputs, gas).await?, + outputs: asc_new(heap, &self.outputs, gas).await?, }) } } -impl ToAscObj for ethabi::LogParam { - fn to_asc_obj( +#[async_trait] +impl ToAscObj for abi::DynSolParam { + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, - ) -> Result { + ) -> Result { Ok(AscLogParam { - name: asc_new(heap, self.name.as_str(), gas)?, - value: asc_new(heap, &self.value, gas)?, + name: asc_new(heap, self.name.as_str(), gas).await?, + value: asc_new(heap, &self.value, gas).await?, }) } } diff --git a/chain/ethereum/src/runtime/runtime_adapter.rs b/chain/ethereum/src/runtime/runtime_adapter.rs index 71e20532ed3..a5d68a761a7 100644 --- a/chain/ethereum/src/runtime/runtime_adapter.rs +++ b/chain/ethereum/src/runtime/runtime_adapter.rs @@ -1,29 +1,47 @@ use std::{sync::Arc, time::Instant}; -use crate::data_source::MappingABI; +use crate::adapter::EthereumRpcError; use crate::{ - capabilities::NodeCapabilities, network::EthereumNetworkAdapters, Chain, DataSource, - EthereumAdapter, EthereumAdapterTrait, EthereumContractCall, EthereumContractCallError, + Chain, ContractCallError, ENV_VARS, EthereumAdapter, EthereumAdapterTrait, + capabilities::NodeCapabilities, network::EthereumNetworkAdapters, }; -use anyhow::{Context, Error}; +use anyhow::{Context, Error, anyhow}; use blockchain::HostFn; +use graph::abi; +use graph::abi::DynSolValueExt; +use graph::blockchain::ChainIdentifier; +use graph::components::subgraph::HostMetrics; +use graph::data::store::ethereum::call; +use graph::data::store::scalar::BigInt; +use graph::data::subgraph::{API_VERSION_0_0_4, API_VERSION_0_0_9}; +use graph::data_source; +use graph::data_source::common::{ContractCall, MappingABI}; use graph::runtime::gas::Gas; use graph::runtime::{AscIndexId, IndexForAscTypeId}; +use graph::slog::debug; use graph::{ blockchain::{self, BlockPtr, HostFnCtx}, cheap_clone::CheapClone, - prelude::{ - ethabi::{self, Address, Token}, - EthereumCallCache, Future01CompatExt, - }, - runtime::{asc_get, asc_new, AscPtr, HostExportError}, - semver::Version, - slog::{info, trace, Logger}, + futures03::FutureExt, + prelude::{EthereumCallCache, alloy::primitives::Address}, + runtime::{AscPtr, HostExportError, asc_get, asc_new}, + slog::Logger, }; -use graph_runtime_wasm::asc_abi::class::{AscEnumArray, EthereumValueKind}; +use graph_runtime_wasm::asc_abi::class::{AscBigInt, AscEnumArray, AscWrapped, EthereumValueKind}; +use itertools::Itertools; use super::abi::{AscUnresolvedContractCall, AscUnresolvedContractCall_0_0_4}; +/// Gas limit for `eth_call`. The value of 50_000_000 is a protocol-wide parameter so this +/// should be changed only for debugging purposes and never on an indexer in the network. This +/// value was chosen because it is the Geth default +/// https://github.com/ethereum/go-ethereum/blob/e4b687cf462870538743b3218906940ae590e7fd/eth/ethconfig/config.go#L91. +/// It is not safe to set something higher because Geth will silently override the gas limit +/// with the default. This means that we do not support indexing against a Geth node with +/// `RPCGasCap` set below 50 million. +// See also f0af4ab0-6b7c-4b68-9141-5b79346a5f61. +const ETH_CALL_GAS: u32 = 50_000_000; + // When making an ethereum call, the maximum ethereum gas is ETH_CALL_GAS which is 50 million. One // unit of Ethereum gas is at least 100ns according to these benchmarks [1], so 1000 of our gas. In // the worst case an Ethereum call could therefore consume 50 billion of our gas. However the @@ -34,50 +52,160 @@ use super::abi::{AscUnresolvedContractCall, AscUnresolvedContractCall_0_0_4}; // [1] - https://www.sciencedirect.com/science/article/abs/pii/S0166531620300900 pub const ETHEREUM_CALL: Gas = Gas::new(5_000_000_000); +// TODO: Determine the appropriate gas cost for `ETH_GET_BALANCE`, initially aligned with `ETHEREUM_CALL`. +pub const ETH_GET_BALANCE: Gas = Gas::new(5_000_000_000); + +// TODO: Determine the appropriate gas cost for `ETH_HAS_CODE`, initially aligned with `ETHEREUM_CALL`. +pub const ETH_HAS_CODE: Gas = Gas::new(5_000_000_000); + pub struct RuntimeAdapter { pub eth_adapters: Arc, pub call_cache: Arc, + pub chain_identifier: Arc, +} + +pub fn eth_call_gas(chain_identifier: &ChainIdentifier) -> Option { + // Check if the current network version is in the eth_call_no_gas list + let should_skip_gas = ENV_VARS + .eth_call_no_gas + .contains(&chain_identifier.net_version); + + if should_skip_gas { + None + } else { + Some(ETH_CALL_GAS) + } } impl blockchain::RuntimeAdapter for RuntimeAdapter { - fn host_fns(&self, ds: &DataSource) -> Result, Error> { - let abis = ds.mapping.abis.clone(); - let call_cache = self.call_cache.cheap_clone(); - // Ethereum calls should prioritise call-only adapters if one is available. - let eth_adapter = self.eth_adapters.call_or_cheapest(Some(&NodeCapabilities { - archive: ds.mapping.requires_archive()?, - traces: false, - }))?; - - let ethereum_call = HostFn { - name: "ethereum.call", - func: Arc::new(move |ctx, wasm_ptr| { - ethereum_call(ð_adapter, call_cache.cheap_clone(), ctx, wasm_ptr, &abis) - .map(|ptr| ptr.wasm_ptr()) - }), + fn host_fns(&self, ds: &data_source::DataSource) -> Result, Error> { + fn create_host_fns( + abis: Arc>>, // Use Arc to ensure `'static` lifetimes. + archive: bool, + call_cache: Arc, + eth_adapters: Arc, + eth_call_gas: Option, + ) -> Vec { + vec![ + HostFn { + name: "ethereum.call", + func: Arc::new({ + let eth_adapters = eth_adapters.clone(); + let call_cache = call_cache.clone(); + let abis = abis.clone(); + move |ctx, wasm_ptr| { + let eth_adapters = eth_adapters.cheap_clone(); + let call_cache = call_cache.cheap_clone(); + let abis = abis.cheap_clone(); + async move { + let eth_adapter = + eth_adapters.call_or_cheapest(Some(&NodeCapabilities { + archive, + traces: false, + }))?; + ethereum_call( + ð_adapter, + call_cache.clone(), + ctx, + wasm_ptr, + &abis, + eth_call_gas, + ) + .await + .map(|ptr| ptr.wasm_ptr()) + } + .boxed() + } + }), + }, + HostFn { + name: "ethereum.getBalance", + func: Arc::new({ + let eth_adapters = eth_adapters.clone(); + move |ctx, wasm_ptr| { + let eth_adapters = eth_adapters.cheap_clone(); + async move { + let eth_adapter = + eth_adapters.unverified_cheapest_with(&NodeCapabilities { + archive, + traces: false, + })?; + eth_get_balance(ð_adapter, ctx, wasm_ptr) + .await + .map(|ptr| ptr.wasm_ptr()) + } + .boxed() + } + }), + }, + HostFn { + name: "ethereum.hasCode", + func: Arc::new({ + move |ctx, wasm_ptr| { + let eth_adapters = eth_adapters.cheap_clone(); + async move { + let eth_adapter = + eth_adapters.unverified_cheapest_with(&NodeCapabilities { + archive, + traces: false, + })?; + eth_has_code(ð_adapter, ctx, wasm_ptr) + .await + .map(|ptr| ptr.wasm_ptr()) + } + .boxed() + } + }), + }, + ] + } + + let host_fns = match ds { + data_source::DataSource::Onchain(onchain_ds) => { + let abis = Arc::new(onchain_ds.mapping.abis.clone()); + let archive = onchain_ds.mapping.requires_archive()?; + let call_cache = self.call_cache.cheap_clone(); + let eth_adapters = self.eth_adapters.cheap_clone(); + let eth_call_gas = eth_call_gas(&self.chain_identifier); + + create_host_fns(abis, archive, call_cache, eth_adapters, eth_call_gas) + } + data_source::DataSource::Subgraph(subgraph_ds) => { + let abis = Arc::new(subgraph_ds.mapping.abis.clone()); + let archive = subgraph_ds.mapping.requires_archive()?; + let call_cache = self.call_cache.cheap_clone(); + let eth_adapters = self.eth_adapters.cheap_clone(); + let eth_call_gas = eth_call_gas(&self.chain_identifier); + + create_host_fns(abis, archive, call_cache, eth_adapters, eth_call_gas) + } + data_source::DataSource::Offchain(_) => vec![], + data_source::DataSource::Amp(_) => vec![], }; - Ok(vec![ethereum_call]) + Ok(host_fns) } } /// function ethereum.call(call: SmartContractCall): Array | null -fn ethereum_call( +async fn ethereum_call( eth_adapter: &EthereumAdapter, call_cache: Arc, ctx: HostFnCtx<'_>, wasm_ptr: u32, abis: &[Arc], + eth_call_gas: Option, ) -> Result, HostExportError> { - ctx.gas.consume_host_fn(ETHEREUM_CALL)?; + ctx.gas + .consume_host_fn_with_metrics(ETHEREUM_CALL, "ethereum_call")?; // For apiVersion >= 0.0.4 the call passed from the mapping includes the // function signature; subgraphs using an apiVersion < 0.0.4 don't pass // the signature along with the call. - let call: UnresolvedContractCall = if ctx.heap.api_version() >= Version::new(0, 0, 4) { - asc_get::<_, AscUnresolvedContractCall_0_0_4, _>(ctx.heap, wasm_ptr.into(), &ctx.gas)? + let call: UnresolvedContractCall = if ctx.heap.api_version() >= &API_VERSION_0_0_4 { + asc_get::<_, AscUnresolvedContractCall_0_0_4, _>(ctx.heap, wasm_ptr.into(), &ctx.gas, 0)? } else { - asc_get::<_, AscUnresolvedContractCall, _>(ctx.heap, wasm_ptr.into(), &ctx.gas)? + asc_get::<_, AscUnresolvedContractCall, _>(ctx.heap, wasm_ptr.into(), &ctx.gas, 0)? }; let result = eth_call( @@ -87,26 +215,101 @@ fn ethereum_call( &ctx.block_ptr, call, abis, - )?; + eth_call_gas, + ctx.metrics.cheap_clone(), + ) + .await?; match result { - Some(tokens) => Ok(asc_new(ctx.heap, tokens.as_slice(), &ctx.gas)?), + Some(tokens) => Ok(asc_new(ctx.heap, tokens.as_slice(), &ctx.gas).await?), None => Ok(AscPtr::null()), } } +async fn eth_get_balance( + eth_adapter: &EthereumAdapter, + ctx: HostFnCtx<'_>, + wasm_ptr: u32, +) -> Result, HostExportError> { + ctx.gas + .consume_host_fn_with_metrics(ETH_GET_BALANCE, "eth_get_balance")?; + + if ctx.heap.api_version() < &API_VERSION_0_0_9 { + return Err(HostExportError::Deterministic(anyhow!( + "ethereum.getBalance call is not supported before API version 0.0.9" + ))); + } + + let logger = &ctx.logger; + let block_ptr = &ctx.block_ptr; + + let address: Address = asc_get(ctx.heap, wasm_ptr.into(), &ctx.gas, 0)?; + + let result = eth_adapter + .get_balance(logger, address, block_ptr.clone()) + .await; + + match result { + Ok(v) => { + let bigint = BigInt::from_unsigned_u256(&v); + Ok(asc_new(ctx.heap, &bigint, &ctx.gas).await?) + } + // Retry on any kind of error + Err(EthereumRpcError::AlloyError(e)) => Err(HostExportError::PossibleReorg(e.into())), + Err(EthereumRpcError::Timeout) => Err(HostExportError::PossibleReorg( + EthereumRpcError::Timeout.into(), + )), + } +} + +async fn eth_has_code( + eth_adapter: &EthereumAdapter, + ctx: HostFnCtx<'_>, + wasm_ptr: u32, +) -> Result>, HostExportError> { + ctx.gas + .consume_host_fn_with_metrics(ETH_HAS_CODE, "eth_has_code")?; + + if ctx.heap.api_version() < &API_VERSION_0_0_9 { + return Err(HostExportError::Deterministic(anyhow!( + "ethereum.hasCode call is not supported before API version 0.0.9" + ))); + } + + let logger = &ctx.logger; + let block_ptr = &ctx.block_ptr; + + let address: Address = asc_get(ctx.heap, wasm_ptr.into(), &ctx.gas, 0)?; + + let result = eth_adapter + .get_code(logger, address, block_ptr.clone()) + .await + .map(|v| !v.0.is_empty()); + + match result { + Ok(v) => Ok(asc_new(ctx.heap, &AscWrapped { inner: v }, &ctx.gas).await?), + // Retry on any kind of error + Err(EthereumRpcError::AlloyError(e)) => Err(HostExportError::PossibleReorg(e.into())), + Err(EthereumRpcError::Timeout) => Err(HostExportError::PossibleReorg( + EthereumRpcError::Timeout.into(), + )), + } +} + /// Returns `Ok(None)` if the call was reverted. -fn eth_call( +async fn eth_call( eth_adapter: &EthereumAdapter, call_cache: Arc, logger: &Logger, block_ptr: &BlockPtr, unresolved_call: UnresolvedContractCall, abis: &[Arc], -) -> Result>, HostExportError> { + eth_call_gas: Option, + metrics: Arc, +) -> Result>, HostExportError> { let start_time = Instant::now(); // Obtain the path to the contract ABI - let contract = abis + let abi = abis .iter() .find(|abi| abi.name == unresolved_call.contract_name) .with_context(|| { @@ -115,97 +318,93 @@ fn eth_call( of the subgraph manifest", unresolved_call.contract_name ) - })? - .contract - .clone(); - - let function = match unresolved_call.function_signature { - // Behavior for apiVersion < 0.0.4: look up function by name; for overloaded - // functions this always picks the same overloaded variant, which is incorrect - // and may lead to encoding/decoding errors - None => contract - .function(unresolved_call.function_name.as_str()) - .with_context(|| { - format!( - "Unknown function \"{}::{}\" called from WASM runtime", - unresolved_call.contract_name, unresolved_call.function_name - ) - })?, - - // Behavior for apiVersion >= 0.0.04: look up function by signature of - // the form `functionName(uint256,string) returns (bytes32,string)`; this - // correctly picks the correct variant of an overloaded function - Some(ref function_signature) => contract - .functions_by_name(unresolved_call.function_name.as_str()) - .with_context(|| { - format!( - "Unknown function \"{}::{}\" called from WASM runtime", - unresolved_call.contract_name, unresolved_call.function_name - ) - })? - .iter() - .find(|f| function_signature == &f.signature()) - .with_context(|| { - format!( - "Unknown function \"{}::{}\" with signature `{}` \ - called from WASM runtime", - unresolved_call.contract_name, - unresolved_call.function_name, - function_signature, - ) - })?, - }; + }) + .map_err(HostExportError::Deterministic)?; - let call = EthereumContractCall { + let function = abi + .function( + &unresolved_call.contract_name, + &unresolved_call.function_name, + unresolved_call.function_signature.as_deref(), + ) + .map_err(HostExportError::Deterministic)?; + + let call = ContractCall { + contract_name: unresolved_call.contract_name.clone(), address: unresolved_call.contract_address, block_ptr: block_ptr.cheap_clone(), function: function.clone(), args: unresolved_call.function_args.clone(), + gas: eth_call_gas, }; // Run Ethereum call in tokio runtime let logger1 = logger.clone(); let call_cache = call_cache.clone(); - let result = match graph::block_on( - eth_adapter.contract_call(&logger1, call, call_cache).compat() - ) { - Ok(tokens) => Ok(Some(tokens)), - Err(EthereumContractCallError::Revert(reason)) => { - info!(logger, "Contract call reverted"; "reason" => reason); - Ok(None) - } + let (result, source) = match eth_adapter.contract_call(&logger1, &call, call_cache).await { + Ok((result, source)) => (Ok(result), source), + Err(e) => (Err(e), call::Source::Rpc), + }; + let result = match result { + Ok(res) => Ok(res), - // Any error reported by the Ethereum node could be due to the block no longer being on - // the main chain. This is very unespecific but we don't want to risk failing a - // subgraph due to a transient error such as a reorg. - Err(EthereumContractCallError::Web3Error(e)) => Err(HostExportError::PossibleReorg(anyhow::anyhow!( + // Any error reported by the Ethereum node could be due to the block no longer being on + // the main chain. This is very unespecific but we don't want to risk failing a + // subgraph due to a transient error such as a reorg. + Err(ContractCallError::AlloyError(e)) => { + Err(HostExportError::PossibleReorg(anyhow::anyhow!( "Ethereum node returned an error when calling function \"{}\" of contract \"{}\": {}", unresolved_call.function_name, unresolved_call.contract_name, e - ))), + ))) + } - // Also retry on timeouts. - Err(EthereumContractCallError::Timeout) => Err(HostExportError::PossibleReorg(anyhow::anyhow!( - "Ethereum node did not respond when calling function \"{}\" of contract \"{}\"", - unresolved_call.function_name, - unresolved_call.contract_name, - ))), + // Also retry on timeouts. + Err(ContractCallError::Timeout) => Err(HostExportError::PossibleReorg(anyhow::anyhow!( + "Ethereum node did not respond when calling function \"{}\" of contract \"{}\"", + unresolved_call.function_name, + unresolved_call.contract_name, + ))), - Err(e) => Err(HostExportError::Unknown(anyhow::anyhow!( - "Failed to call function \"{}\" of contract \"{}\": {}", - unresolved_call.function_name, - unresolved_call.contract_name, - e - ))), - }; + Err(e) => Err(HostExportError::Unknown(anyhow::anyhow!( + "Failed to call function \"{}\" of contract \"{}\": {}", + unresolved_call.function_name, + unresolved_call.contract_name, + e + ))), + }; + + let elapsed = start_time.elapsed(); + + if source.observe() { + metrics.observe_eth_call_execution_time( + elapsed.as_secs_f64(), + &unresolved_call.contract_name, + &unresolved_call.function_name, + ); + } + + let args_as_string = format!("[{}]", values_to_string(&unresolved_call.function_args)); + + let result_as_string = match &result { + Ok(Some(values)) => format!("({})", values_to_string(values)), + Ok(None) => "none".to_owned(), + Err(_err) => "error".to_owned(), + }; - trace!(logger, "Contract call finished"; - "address" => &unresolved_call.contract_address.to_string(), - "contract" => &unresolved_call.contract_name, - "function" => &unresolved_call.function_name, - "function_signature" => &unresolved_call.function_signature, - "time" => format!("{}ms", start_time.elapsed().as_millis())); + debug!( + logger, "Contract call finished"; + "address" => format!("0x{:x}", &unresolved_call.contract_address), + "contract" => &unresolved_call.contract_name, + "signature" => &unresolved_call.function_signature, + "args" => args_as_string, + "time_ms" => format!("{}ms", elapsed.as_millis()), + "result" => result_as_string, + "block_hash" => block_ptr.hash_hex(), + "block_number" => block_ptr.block_number(), + "source" => source.to_string(), + ); result } @@ -216,9 +415,18 @@ pub struct UnresolvedContractCall { pub contract_address: Address, pub function_name: String, pub function_signature: Option, - pub function_args: Vec, + pub function_args: Vec, } impl AscIndexId for AscUnresolvedContractCall { const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::SmartContractCall; } + +#[inline] +fn values_to_string(values: &[abi::DynSolValue]) -> String { + values + .iter() + .map(|x| x.to_string()) + .collect_vec() + .join(", ") +} diff --git a/chain/ethereum/src/tests.rs b/chain/ethereum/src/tests.rs index 9c4a46130e5..a5f0a17bff0 100644 --- a/chain/ethereum/src/tests.rs +++ b/chain/ethereum/src/tests.rs @@ -1,78 +1,115 @@ use std::sync::Arc; use graph::{ - blockchain::{block_stream::BlockWithTriggers, BlockPtr}, + blockchain::{BlockPtr, Trigger, block_stream::BlockWithTriggers}, prelude::{ - web3::types::{Address, Bytes, Log, H160, H256, U64}, EthereumCall, LightEthereumBlock, + alloy::{ + self, + primitives::{Address, B256, Bytes, LogData}, + rpc::types::{Block, Log}, + }, + rand::{self, Rng}, }, - slog::{self, o, Logger}, + slog::{self, Logger, o}, }; use crate::{ chain::BlockFinality, - trigger::{EthereumBlockTriggerType, EthereumTrigger}, + trigger::{EthereumBlockTriggerType, EthereumTrigger, LogRef}, }; +pub trait Random { + fn random() -> Self; +} + +impl Random for B256 { + fn random() -> Self { + let mut rng = rand::rng(); + let mut bytes = [0u8; 32]; + rng.fill(&mut bytes); + Self::from(bytes) + } +} + +impl Random for Address { + fn random() -> Self { + let mut rng = rand::rng(); + let mut bytes = [0u8; 20]; + rng.fill(&mut bytes); + Self::from(bytes) + } +} + +fn create_log(tx_index: u64, log_index: u64) -> Arc { + let log = Log { + inner: alloy::primitives::Log { + address: Address::default(), + data: LogData::new_unchecked(vec![], Bytes::from(vec![])), + }, + block_hash: Some(B256::ZERO), + block_number: Some(0), + block_timestamp: Some(0), + transaction_hash: Some(B256::ZERO), + transaction_index: Some(tx_index), + log_index: Some(log_index), + removed: false, + }; + + Arc::new(log) +} + #[test] fn test_trigger_ordering() { let block1 = EthereumTrigger::Block( - BlockPtr::from((H256::random(), 1u64)), - EthereumBlockTriggerType::Every, + BlockPtr::from((B256::random(), 1u64)), + EthereumBlockTriggerType::End, ); let block2 = EthereumTrigger::Block( - BlockPtr::from((H256::random(), 0u64)), + BlockPtr::from((B256::random(), 0u64)), EthereumBlockTriggerType::WithCallTo(Address::random()), ); - let mut call1 = EthereumCall::default(); - call1.transaction_index = 1; + let call1 = EthereumCall { + transaction_index: 1, + ..Default::default() + }; let call1 = EthereumTrigger::Call(Arc::new(call1)); - let mut call2 = EthereumCall::default(); - call2.transaction_index = 2; - call2.input = Bytes(vec![0]); + let call2 = EthereumCall { + transaction_index: 2, + input: Bytes::from(vec![0]), + ..Default::default() + }; let call2 = EthereumTrigger::Call(Arc::new(call2)); - let mut call3 = EthereumCall::default(); - call3.transaction_index = 3; + let call3 = EthereumCall { + transaction_index: 3, + ..Default::default() + }; let call3 = EthereumTrigger::Call(Arc::new(call3)); // Call with the same tx index as call2 - let mut call4 = EthereumCall::default(); - call4.transaction_index = 2; // different than call2 so they don't get mistaken as the same - call4.input = Bytes(vec![1]); + let call4 = EthereumCall { + transaction_index: 2, + input: Bytes::from(vec![1]), + ..Default::default() + }; let call4 = EthereumTrigger::Call(Arc::new(call4)); - fn create_log(tx_index: u64, log_index: u64) -> Arc { - Arc::new(Log { - address: H160::default(), - topics: vec![], - data: Bytes::default(), - block_hash: Some(H256::zero()), - block_number: Some(U64::zero()), - transaction_hash: Some(H256::zero()), - transaction_index: Some(tx_index.into()), - log_index: Some(log_index.into()), - transaction_log_index: Some(log_index.into()), - log_type: Some("".into()), - removed: Some(false), - }) - } - // Event with transaction_index 1 and log_index 0; // should be the first element after sorting - let log1 = EthereumTrigger::Log(create_log(1, 0), None); + let log1 = EthereumTrigger::Log(LogRef::FullLog(create_log(1, 0), None)); // Event with transaction_index 1 and log_index 1; // should be the second element after sorting - let log2 = EthereumTrigger::Log(create_log(1, 1), None); + let log2 = EthereumTrigger::Log(LogRef::FullLog(create_log(1, 1), None)); // Event with transaction_index 2 and log_index 5; // should come after call1 and before call2 after sorting - let log3 = EthereumTrigger::Log(create_log(2, 5), None); + let log3 = EthereumTrigger::Log(LogRef::FullLog(create_log(2, 5), None)); let triggers = vec![ // Call triggers; these should be in the order 1, 2, 4, 3 after sorting @@ -92,13 +129,9 @@ fn test_trigger_ordering() { let logger = Logger::root(slog::Discard, o!()); - let mut b: LightEthereumBlock = Default::default(); + let b = Block::default(); - // This is necessary because inside of BlockWithTriggers::new - // there's a log for both fields. So just using Default above - // gives None on them. - b.number = Some(Default::default()); - b.hash = Some(Default::default()); + let b = LightEthereumBlock::new(graph::components::ethereum::AnyBlock::from(b)); // Test that `BlockWithTriggers` sorts the triggers. let block_with_triggers = BlockWithTriggers::::new( @@ -107,63 +140,57 @@ fn test_trigger_ordering() { &logger, ); - assert_eq!( - block_with_triggers.trigger_data, - vec![log1, log2, call1, log3, call2, call4, call3, block2, block1] - ); + let expected = vec![log1, log2, call1, log3, call2, call4, call3, block2, block1] + .into_iter() + .map(Trigger::Chain) + .collect::>(); + + assert_eq!(block_with_triggers.trigger_data, expected); } #[test] fn test_trigger_dedup() { let block1 = EthereumTrigger::Block( - BlockPtr::from((H256::random(), 1u64)), - EthereumBlockTriggerType::Every, + BlockPtr::from((B256::random(), 1u64)), + EthereumBlockTriggerType::End, ); let block2 = EthereumTrigger::Block( - BlockPtr::from((H256::random(), 0u64)), + BlockPtr::from((B256::random(), 0u64)), EthereumBlockTriggerType::WithCallTo(Address::random()), ); // duplicate block2 let block3 = block2.clone(); - let mut call1 = EthereumCall::default(); - call1.transaction_index = 1; + let call1 = EthereumCall { + transaction_index: 1, + ..Default::default() + }; let call1 = EthereumTrigger::Call(Arc::new(call1)); - let mut call2 = EthereumCall::default(); - call2.transaction_index = 2; + let call2 = EthereumCall { + transaction_index: 2, + ..Default::default() + }; let call2 = EthereumTrigger::Call(Arc::new(call2)); - let mut call3 = EthereumCall::default(); - call3.transaction_index = 3; + let call3 = EthereumCall { + transaction_index: 3, + ..Default::default() + }; let call3 = EthereumTrigger::Call(Arc::new(call3)); // duplicate call2 - let mut call4 = EthereumCall::default(); - call4.transaction_index = 2; + let call4 = EthereumCall { + transaction_index: 2, + ..Default::default() + }; let call4 = EthereumTrigger::Call(Arc::new(call4)); - fn create_log(tx_index: u64, log_index: u64) -> Arc { - Arc::new(Log { - address: H160::default(), - topics: vec![], - data: Bytes::default(), - block_hash: Some(H256::zero()), - block_number: Some(U64::zero()), - transaction_hash: Some(H256::zero()), - transaction_index: Some(tx_index.into()), - log_index: Some(log_index.into()), - transaction_log_index: Some(log_index.into()), - log_type: Some("".into()), - removed: Some(false), - }) - } - - let log1 = EthereumTrigger::Log(create_log(1, 0), None); - let log2 = EthereumTrigger::Log(create_log(1, 1), None); - let log3 = EthereumTrigger::Log(create_log(2, 5), None); + let log1 = EthereumTrigger::Log(LogRef::FullLog(create_log(1, 0), None)); + let log2 = EthereumTrigger::Log(LogRef::FullLog(create_log(1, 1), None)); + let log3 = EthereumTrigger::Log(LogRef::FullLog(create_log(2, 5), None)); // duplicate logs 2 and 3 let log4 = log2.clone(); let log5 = log3.clone(); @@ -188,13 +215,11 @@ fn test_trigger_dedup() { let logger = Logger::root(slog::Discard, o!()); - let mut b: LightEthereumBlock = Default::default(); + #[allow(unused_variables)] + let b = Block::default(); - // This is necessary because inside of BlockWithTriggers::new - // there's a log for both fields. So just using Default above - // gives None on them. - b.number = Some(Default::default()); - b.hash = Some(Default::default()); + #[allow(unreachable_code)] + let b = LightEthereumBlock::new(graph::components::ethereum::AnyBlock::from(b)); // Test that `BlockWithTriggers` sorts the triggers. let block_with_triggers = BlockWithTriggers::::new( @@ -203,8 +228,10 @@ fn test_trigger_dedup() { &logger, ); - assert_eq!( - block_with_triggers.trigger_data, - vec![log1, log2, call1, log3, call2, call3, block2, block1] - ); + let expected = vec![log1, log2, call1, log3, call2, call3, block2, block1] + .into_iter() + .map(Trigger::Chain) + .collect::>(); + + assert_eq!(block_with_triggers.trigger_data, expected); } diff --git a/chain/ethereum/src/transport.rs b/chain/ethereum/src/transport.rs index b30fd17d84b..5b5414c4a5d 100644 --- a/chain/ethereum/src/transport.rs +++ b/chain/ethereum/src/transport.rs @@ -1,88 +1,305 @@ -use jsonrpc_core::types::Call; -use jsonrpc_core::Value; - -use web3::transports::{http, ipc, ws}; -use web3::RequestId; - +use alloy::transports::{TransportError, TransportErrorKind, TransportFut}; +use graph::components::ethereum::json_patch; +use graph::components::network_provider::ProviderName; +use graph::endpoint::{ConnectionType, EndpointMetrics, RequestLabels}; +use graph::prelude::alloy::rpc::json_rpc::{RequestPacket, ResponsePacket}; +use graph::prelude::alloy::transports::{ipc::IpcConnect, ws::WsConnect}; use graph::prelude::*; use graph::url::Url; -use std::future::Future; +use serde_json::Value; +use std::sync::Arc; +use std::task::{Context, Poll}; +use tower::Service; + +/// Compression method for RPC requests. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum Compression { + #[default] + None, + Gzip, + Brotli, + Deflate, +} + +impl std::fmt::Display for Compression { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Compression::None => write!(f, "none"), + Compression::Gzip => write!(f, "gzip"), + Compression::Brotli => write!(f, "brotli"), + Compression::Deflate => write!(f, "deflate"), + } + } +} -/// Abstraction over the different web3 transports. +/// Abstraction over different transport types for Alloy providers. #[derive(Clone, Debug)] pub enum Transport { - RPC(http::Http), - IPC(ipc::Ipc), - WS(ws::WebSocket), + RPC(alloy::rpc::client::RpcClient), + IPC(IpcConnect), + WS(WsConnect), } impl Transport { /// Creates an IPC transport. + /// + /// Accepts both a raw file path (`/tmp/geth.ipc`) and an `ipc://` URL + /// (`ipc:///tmp/geth.ipc`). When a URL is provided the file path is + /// extracted so that the underlying IPC connector receives a plain path. #[cfg(unix)] pub async fn new_ipc(ipc: &str) -> Self { - ipc::Ipc::new(ipc) - .await - .map(Transport::IPC) - .expect("Failed to connect to Ethereum IPC") + let path = Url::parse(ipc) + .ok() + .map(|u| u.path().to_string()) + .unwrap_or_else(|| ipc.to_string()); + let transport = IpcConnect::new(path); + + Transport::IPC(transport) + } + + #[cfg(not(unix))] + pub async fn new_ipc(_ipc: &str) -> Self { + panic!("IPC connections are not supported on non-Unix platforms") } /// Creates a WebSocket transport. pub async fn new_ws(ws: &str) -> Self { - ws::WebSocket::new(ws) - .await - .map(Transport::WS) - .expect("Failed to connect to Ethereum WS") + let transport = WsConnect::new(ws.to_string()); + + Transport::WS(transport) } /// Creates a JSON-RPC over HTTP transport. /// - /// Note: JSON-RPC over HTTP doesn't always support subscribing to new - /// blocks (one such example is Infura's HTTP endpoint). - pub fn new_rpc(rpc: Url, headers: ::http::HeaderMap) -> Self { - // Unwrap: This only fails if something is wrong with the system's TLS config. - let client = reqwest::Client::builder() - .default_headers(headers) - .build() - .unwrap(); - Transport::RPC(http::Http::with_client(client, rpc)) + /// Set `no_eip2718` to true for chains that don't return the `type` field + /// in transaction receipts (pre-EIP-2718 chains). Use provider feature `no_eip2718`. + pub fn new_rpc( + rpc: Url, + headers: graph::http::HeaderMap, + metrics: Arc, + provider: impl AsRef, + no_eip2718: bool, + compression: Compression, + ) -> Self { + let mut client_builder = reqwest::Client::builder().default_headers(headers); + + match compression { + Compression::None => {} + Compression::Gzip => { + client_builder = client_builder.gzip(true); + } + Compression::Brotli => { + client_builder = client_builder.brotli(true); + } + Compression::Deflate => { + client_builder = client_builder.deflate(true); + } + } + + let client = client_builder.build().expect("Failed to build HTTP client"); + + let patching_transport = PatchingHttp::new(client, rpc, no_eip2718); + let metrics_transport = + MetricsHttp::new(patching_transport, metrics, provider.as_ref().into()); + let rpc_client = alloy::rpc::client::RpcClient::new(metrics_transport, false); + + Transport::RPC(rpc_client) } } -impl web3::Transport for Transport { - type Out = Box> + Send + Unpin>; +/// Custom HTTP transport wrapper that collects metrics +#[derive(Clone)] +pub struct MetricsHttp { + inner: PatchingHttp, + metrics: Arc, + provider: ProviderName, +} - fn prepare(&self, method: &str, params: Vec) -> (RequestId, Call) { - match self { - Transport::RPC(http) => http.prepare(method, params), - Transport::IPC(ipc) => ipc.prepare(method, params), - Transport::WS(ws) => ws.prepare(method, params), +impl MetricsHttp { + pub fn new(inner: PatchingHttp, metrics: Arc, provider: ProviderName) -> Self { + Self { + inner, + metrics, + provider, } } +} - fn send(&self, id: RequestId, request: Call) -> Self::Out { - match self { - Transport::RPC(http) => Box::new(http.send(id, request)), - Transport::IPC(ipc) => Box::new(ipc.send(id, request)), - Transport::WS(ws) => Box::new(ws.send(id, request)), - } +// Implement tower::Service trait for MetricsHttp to intercept RPC calls +impl Service for MetricsHttp { + type Response = ResponsePacket; + type Error = TransportError; + type Future = TransportFut<'static>; + + fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { + self.inner.poll_ready(cx) + } + + fn call(&mut self, request: RequestPacket) -> Self::Future { + let metrics = self.metrics.clone(); + let provider = self.provider.clone(); + let mut inner = self.inner.clone(); + + Box::pin(async move { + // Extract method name from request + let method = match &request { + RequestPacket::Single(req) => req.method().to_string(), + RequestPacket::Batch(reqs) => reqs + .first() + .map(|r| r.method().to_string()) + .unwrap_or_else(|| "batch".to_string()), + }; + + let labels = RequestLabels { + provider, + req_type: method.into(), + conn_type: ConnectionType::Rpc, + }; + + // Call inner transport and track metrics + let result = inner.call(request).await; + + match &result { + Ok(_) => metrics.success(&labels), + Err(_) => metrics.failure(&labels), + } + + result + }) } } -impl web3::BatchTransport for Transport { - type Batch = Box< - dyn Future>, web3::error::Error>> - + Send - + Unpin, - >; - - fn send_batch(&self, requests: T) -> Self::Batch - where - T: IntoIterator, - { - match self { - Transport::RPC(http) => Box::new(http.send_batch(requests)), - Transport::IPC(ipc) => Box::new(ipc.send_batch(requests)), - Transport::WS(ws) => Box::new(ws.send_batch(requests)), +/// HTTP transport that patches receipts for chains that don't support EIP-2718 (typed transactions). +/// When `no_eip2718` is set, adds missing `type` field to receipts. +#[derive(Clone)] +pub struct PatchingHttp { + client: reqwest::Client, + url: Url, + no_eip2718: bool, +} + +impl PatchingHttp { + pub fn new(client: reqwest::Client, url: Url, no_eip2718: bool) -> Self { + Self { + client, + url, + no_eip2718, + } + } + + fn is_receipt_method(method: &str) -> bool { + method == "eth_getTransactionReceipt" || method == "eth_getBlockReceipts" + } + + fn patch_rpc_response(response: &mut Value) -> bool { + response + .get_mut("result") + .map(json_patch::patch_receipts) + .unwrap_or(false) + } + + fn patch_response(body: &[u8]) -> Option> { + let mut json: Value = serde_json::from_slice(body).ok()?; + + let patched = match &mut json { + Value::Object(_) => Self::patch_rpc_response(&mut json), + Value::Array(batch) => { + let mut patched = false; + for r in batch { + patched |= Self::patch_rpc_response(r); + } + patched + } + _ => false, + }; + + if patched { + serde_json::to_vec(&json).ok() + } else { + None } } } + +impl Service for PatchingHttp { + type Response = ResponsePacket; + type Error = TransportError; + type Future = TransportFut<'static>; + + fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn call(&mut self, request: RequestPacket) -> Self::Future { + let client = self.client.clone(); + let url = self.url.clone(); + let no_eip2718 = self.no_eip2718; + + let should_patch = if no_eip2718 { + match &request { + RequestPacket::Single(req) => Self::is_receipt_method(req.method()), + RequestPacket::Batch(reqs) => { + reqs.iter().any(|r| Self::is_receipt_method(r.method())) + } + } + } else { + false + }; + + Box::pin(async move { + let resp = client + .post(url) + .json(&request) + .headers(request.headers()) + .send() + .await + .map_err(TransportErrorKind::custom)?; + + let status = resp.status(); + let body = resp.bytes().await.map_err(TransportErrorKind::custom)?; + + if !status.is_success() { + return Err(TransportErrorKind::http_error( + status.as_u16(), + String::from_utf8_lossy(&body).into_owned(), + )); + } + + if should_patch && let Some(patched) = Self::patch_response(&body) { + return serde_json::from_slice(&patched).map_err(|err| { + TransportError::deser_err(err, String::from_utf8_lossy(&patched)) + }); + } + serde_json::from_slice(&body) + .map_err(|err| TransportError::deser_err(err, String::from_utf8_lossy(&body))) + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn patch_response_single() { + let body = br#"{"jsonrpc":"2.0","id":1,"result":{"status":"0x1"}}"#; + let patched = PatchingHttp::patch_response(body).unwrap(); + let json: Value = serde_json::from_slice(&patched).unwrap(); + assert_eq!(json["result"]["type"], "0x0"); + } + + #[test] + fn patch_response_returns_none_when_type_exists() { + let body = br#"{"jsonrpc":"2.0","id":1,"result":{"status":"0x1","type":"0x2"}}"#; + assert!(PatchingHttp::patch_response(body).is_none()); + } + + #[test] + fn patch_response_batch() { + let body = br#"[{"jsonrpc":"2.0","id":1,"result":{"status":"0x1"}},{"jsonrpc":"2.0","id":2,"result":{"status":"0x1"}}]"#; + let patched = PatchingHttp::patch_response(body).unwrap(); + let json: Value = serde_json::from_slice(&patched).unwrap(); + assert_eq!(json[0]["result"]["type"], "0x0"); + assert_eq!(json[1]["result"]["type"], "0x0"); + } +} diff --git a/chain/ethereum/src/trigger.rs b/chain/ethereum/src/trigger.rs index 9b609668b1f..b5d51d9a379 100644 --- a/chain/ethereum/src/trigger.rs +++ b/chain/ethereum/src/trigger.rs @@ -1,31 +1,28 @@ +use async_trait::async_trait; +use graph::abi; +use graph::blockchain::MappingTriggerTrait; use graph::blockchain::TriggerData; +use graph::components::ethereum::AnyTransaction; +use graph::components::ethereum::AnyTransactionReceiptBare as AlloyTransactionReceipt; use graph::data::subgraph::API_VERSION_0_0_2; use graph::data::subgraph::API_VERSION_0_0_6; use graph::data::subgraph::API_VERSION_0_0_7; -use graph::prelude::ethabi::ethereum_types::H160; -use graph::prelude::ethabi::ethereum_types::H256; -use graph::prelude::ethabi::ethereum_types::U128; -use graph::prelude::ethabi::ethereum_types::U256; -use graph::prelude::ethabi::ethereum_types::U64; -use graph::prelude::ethabi::Address; -use graph::prelude::ethabi::Bytes; -use graph::prelude::ethabi::LogParam; -use graph::prelude::web3::types::Block; -use graph::prelude::web3::types::Log; -use graph::prelude::web3::types::Transaction; -use graph::prelude::web3::types::TransactionReceipt; +use graph::data_source::common::DeclaredCall; use graph::prelude::BlockNumber; use graph::prelude::BlockPtr; +use graph::prelude::LightEthereumBlock; +use graph::prelude::alloy::consensus::Transaction as TransactionTrait; +use graph::prelude::alloy::network::TransactionResponse; +use graph::prelude::alloy::primitives::{Address, B256, U256}; +use graph::prelude::alloy::rpc::types::Log; use graph::prelude::{CheapClone, EthereumCall}; -use graph::runtime::asc_new; -use graph::runtime::gas::GasCounter; use graph::runtime::AscHeap; use graph::runtime::AscPtr; -use graph::runtime::DeterministicHostError; +use graph::runtime::HostExportError; +use graph::runtime::asc_new; +use graph::runtime::gas::GasCounter; use graph::semver::Version; use graph_runtime_wasm::module::ToAscPtr; -use std::convert::TryFrom; -use std::ops::Deref; use std::{cmp::Ordering, sync::Arc}; use crate::runtime::abi::AscEthereumBlock; @@ -38,44 +35,59 @@ use crate::runtime::abi::AscEthereumTransaction_0_0_1; use crate::runtime::abi::AscEthereumTransaction_0_0_2; use crate::runtime::abi::AscEthereumTransaction_0_0_6; -// ETHDEP: This should be defined in only one place. -type LightEthereumBlock = Block; +static U256_DEFAULT: U256 = U256::ZERO; pub enum MappingTrigger { Log { block: Arc, - transaction: Arc, + transaction: Arc, log: Arc, - params: Vec, - receipt: Option>, + params: Vec, + receipt: Option>, + calls: Vec, }, Call { block: Arc, - transaction: Arc, + transaction: Arc, call: Arc, - inputs: Vec, - outputs: Vec, + inputs: Vec, + outputs: Vec, }, Block { block: Arc, }, } +impl MappingTriggerTrait for MappingTrigger { + fn error_context(&self) -> String { + let transaction_id = match self { + MappingTrigger::Log { log, .. } => log.transaction_hash, + MappingTrigger::Call { call, .. } => call.transaction_hash, + MappingTrigger::Block { .. } => None, + }; + + match transaction_id { + Some(tx_hash) => format!("transaction {:x}", tx_hash), + None => String::new(), + } + } +} + // Logging the block is too verbose, so this strips the block from the trigger for Debug. impl std::fmt::Debug for MappingTrigger { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { #[derive(Debug)] enum MappingTriggerWithoutBlock { Log { - _transaction: Arc, + _transaction: Arc, _log: Arc, - _params: Vec, + _params: Vec, }, Call { - _transaction: Arc, + _transaction: Arc, _call: Arc, - _inputs: Vec, - _outputs: Vec, + _inputs: Vec, + _outputs: Vec, }, Block, } @@ -87,6 +99,7 @@ impl std::fmt::Debug for MappingTrigger { log, params, receipt: _, + calls: _, } => MappingTriggerWithoutBlock::Log { _transaction: transaction.cheap_clone(), _log: log.cheap_clone(), @@ -111,12 +124,13 @@ impl std::fmt::Debug for MappingTrigger { } } +#[async_trait] impl ToAscPtr for MappingTrigger { - fn to_asc_ptr( + async fn to_asc_ptr( self, heap: &mut H, gas: &GasCounter, - ) -> Result, DeterministicHostError> { + ) -> Result, HostExportError> { Ok(match self { MappingTrigger::Log { block, @@ -124,18 +138,16 @@ impl ToAscPtr for MappingTrigger { log, params, receipt, + calls: _, } => { let api_version = heap.api_version(); - let ethereum_event_data = EthereumEventData { - block: EthereumBlockData::from(block.as_ref()), - transaction: EthereumTransactionData::from(transaction.deref()), - address: log.address, - log_index: log.log_index.unwrap_or(U256::zero()), - transaction_log_index: log.log_index.unwrap_or(U256::zero()), - log_type: log.log_type.clone(), - params, - }; - if api_version >= API_VERSION_0_0_7 { + let ethereum_event_data = EthereumEventData::new( + block.as_ref(), + transaction.as_ref(), + log.as_ref(), + ¶ms, + ); + if api_version >= &API_VERSION_0_0_7 { asc_new::< AscEthereumEvent_0_0_7< AscEthereumTransaction_0_0_6, @@ -143,28 +155,31 @@ impl ToAscPtr for MappingTrigger { >, _, _, - >(heap, &(ethereum_event_data, receipt.as_deref()), gas)? + >(heap, &(ethereum_event_data, receipt.as_deref()), gas) + .await? .erase() - } else if api_version >= API_VERSION_0_0_6 { + } else if api_version >= &API_VERSION_0_0_6 { asc_new::< AscEthereumEvent, _, _, - >(heap, ðereum_event_data, gas)? + >(heap, ðereum_event_data, gas) + .await? .erase() - } else if api_version >= API_VERSION_0_0_2 { + } else if api_version >= &API_VERSION_0_0_2 { asc_new::< AscEthereumEvent, _, _, - >(heap, ðereum_event_data, gas)? + >(heap, ðereum_event_data, gas) + .await? .erase() } else { asc_new::< AscEthereumEvent, _, _, - >(heap, ðereum_event_data, gas)? + >(heap, ðereum_event_data, gas).await? .erase() } } @@ -175,49 +190,115 @@ impl ToAscPtr for MappingTrigger { inputs, outputs, } => { - let call = EthereumCallData { - to: call.to, - from: call.from, - block: EthereumBlockData::from(block.as_ref()), - transaction: EthereumTransactionData::from(transaction.deref()), - inputs, - outputs, - }; - if heap.api_version() >= Version::new(0, 0, 6) { + let call = EthereumCallData::new(&block, &transaction, &call, &inputs, &outputs); + if heap.api_version() >= &Version::new(0, 0, 6) { asc_new::< AscEthereumCall_0_0_3, _, _, - >(heap, &call, gas)? + >(heap, &call, gas) + .await? .erase() - } else if heap.api_version() >= Version::new(0, 0, 3) { + } else if heap.api_version() >= &Version::new(0, 0, 3) { asc_new::< AscEthereumCall_0_0_3, _, _, - >(heap, &call, gas)? + >(heap, &call, gas) + .await? .erase() } else { - asc_new::(heap, &call, gas)?.erase() + asc_new::(heap, &call, gas) + .await? + .erase() } } MappingTrigger::Block { block } => { let block = EthereumBlockData::from(block.as_ref()); - if heap.api_version() >= Version::new(0, 0, 6) { - asc_new::(heap, &block, gas)?.erase() + if heap.api_version() >= &Version::new(0, 0, 6) { + asc_new::(heap, &block, gas) + .await? + .erase() } else { - asc_new::(heap, &block, gas)?.erase() + asc_new::(heap, &block, gas) + .await? + .erase() } } }) } } +#[derive(Clone, Debug)] +pub struct LogPosition { + pub index: usize, + pub receipt: Arc, + pub requires_transaction_receipt: bool, +} + +#[derive(Clone, Debug)] +pub enum LogRef { + FullLog(Arc, Option>), + LogPosition(LogPosition), +} + +impl LogRef { + pub fn log(&self) -> &Log { + match self { + LogRef::FullLog(log, _) => log.as_ref(), + LogRef::LogPosition(pos) => pos.receipt.logs().get(pos.index).unwrap(), + } + } + + /// Returns the transaction receipt if it's available and required. + /// + /// For `FullLog` variants, returns the receipt if present. + /// For `LogPosition` variants, only returns the receipt if the + /// `requires_transaction_receipt` flag is true, otherwise returns None + /// even though the receipt is stored internally. + pub fn receipt(&self) -> Option<&Arc> { + match self { + LogRef::FullLog(_, receipt) => receipt.as_ref(), + LogRef::LogPosition(pos) => { + if pos.requires_transaction_receipt { + Some(&pos.receipt) + } else { + None + } + } + } + } + + pub fn log_index(&self) -> Option { + self.log().log_index + } + + pub fn transaction_index(&self) -> Option { + self.log().transaction_index + } + + fn transaction_hash(&self) -> Option { + self.log().transaction_hash + } + + pub fn block_hash(&self) -> Option { + self.log().block_hash + } + + pub fn block_number(&self) -> Option { + self.log().block_number + } + + pub fn address(&self) -> &Address { + &self.log().inner.address + } +} + #[derive(Clone, Debug)] pub enum EthereumTrigger { Block(BlockPtr, EthereumBlockTriggerType), Call(Arc), - Log(Arc, Option>), + Log(LogRef), } impl PartialEq for EthereumTrigger { @@ -229,12 +310,9 @@ impl PartialEq for EthereumTrigger { (Self::Call(a), Self::Call(b)) => a == b, - (Self::Log(a, a_receipt), Self::Log(b, b_receipt)) => { - a.transaction_hash == b.transaction_hash - && a.log_index == b.log_index - && a_receipt == b_receipt + (Self::Log(a), Self::Log(b)) => { + a.transaction_hash() == b.transaction_hash() && a.log_index() == b.log_index() } - _ => false, } } @@ -244,7 +322,8 @@ impl Eq for EthereumTrigger {} #[derive(Clone, Debug, PartialEq, Eq)] pub enum EthereumBlockTriggerType { - Every, + Start, + End, WithCallTo(Address), } @@ -253,17 +332,31 @@ impl EthereumTrigger { match self { EthereumTrigger::Block(block_ptr, _) => block_ptr.number, EthereumTrigger::Call(call) => call.block_number, - EthereumTrigger::Log(log, _) => { - i32::try_from(log.block_number.unwrap().as_u64()).unwrap() + EthereumTrigger::Log(log_ref) => { + i32::try_from(log_ref.block_number().unwrap()).unwrap() } } } - pub fn block_hash(&self) -> H256 { + pub fn block_hash(&self) -> B256 { match self { - EthereumTrigger::Block(block_ptr, _) => block_ptr.hash_as_h256(), + EthereumTrigger::Block(block_ptr, _) => block_ptr.hash.as_b256(), EthereumTrigger::Call(call) => call.block_hash, - EthereumTrigger::Log(log, _) => log.block_hash.unwrap(), + EthereumTrigger::Log(log_ref) => log_ref.block_hash().unwrap(), + } + } + + /// `None` means the trigger matches any address. + pub fn address(&self) -> Option<&Address> { + match self { + EthereumTrigger::Block(_, EthereumBlockTriggerType::WithCallTo(address)) => { + Some(address) + } + EthereumTrigger::Call(call) => Some(&call.to), + EthereumTrigger::Log(log_ref) => Some(log_ref.address()), + // Unfiltered block triggers match any data source address. + EthereumTrigger::Block(_, EthereumBlockTriggerType::End) => None, + EthereumTrigger::Block(_, EthereumBlockTriggerType::Start) => None, } } } @@ -271,10 +364,14 @@ impl EthereumTrigger { impl Ord for EthereumTrigger { fn cmp(&self, other: &Self) -> Ordering { match (self, other) { + // Block triggers with `EthereumBlockTriggerType::Start` always come + (Self::Block(_, EthereumBlockTriggerType::Start), _) => Ordering::Less, + (_, Self::Block(_, EthereumBlockTriggerType::Start)) => Ordering::Greater, + // Keep the order when comparing two block triggers (Self::Block(..), Self::Block(..)) => Ordering::Equal, - // Block triggers always come last + // Block triggers with `EthereumBlockTriggerType::End` always come last (Self::Block(..), _) => Ordering::Greater, (_, Self::Block(..)) => Ordering::Less, @@ -282,28 +379,26 @@ impl Ord for EthereumTrigger { (Self::Call(a), Self::Call(b)) => a.transaction_index.cmp(&b.transaction_index), // Events are ordered by their log index - (Self::Log(a, _), Self::Log(b, _)) => a.log_index.cmp(&b.log_index), + (Self::Log(a), Self::Log(b)) => a.log_index().cmp(&b.log_index()), // Calls vs. events are logged by their tx index; // if they are from the same transaction, events come first - (Self::Call(a), Self::Log(b, _)) - if a.transaction_index == b.transaction_index.unwrap().as_u64() => + (Self::Call(a), Self::Log(b)) + if a.transaction_index == b.transaction_index().unwrap() => { Ordering::Greater } - (Self::Log(a, _), Self::Call(b)) - if a.transaction_index.unwrap().as_u64() == b.transaction_index => + (Self::Log(a), Self::Call(b)) + if a.transaction_index().unwrap() == b.transaction_index => { Ordering::Less } - (Self::Call(a), Self::Log(b, _)) => a - .transaction_index - .cmp(&b.transaction_index.unwrap().as_u64()), - (Self::Log(a, _), Self::Call(b)) => a - .transaction_index - .unwrap() - .as_u64() - .cmp(&b.transaction_index), + (Self::Call(a), Self::Log(b)) => { + a.transaction_index.cmp(&b.transaction_index().unwrap()) + } + (Self::Log(a), Self::Call(b)) => { + a.transaction_index().unwrap().cmp(&b.transaction_index) + } } } } @@ -317,7 +412,7 @@ impl PartialOrd for EthereumTrigger { impl TriggerData for EthereumTrigger { fn error_context(&self) -> std::string::String { let transaction_id = match self { - EthereumTrigger::Log(log, _) => log.transaction_hash, + EthereumTrigger::Log(log) => log.transaction_hash(), EthereumTrigger::Call(call) => call.transaction_hash, EthereumTrigger::Block(..) => None, }; @@ -332,102 +427,225 @@ impl TriggerData for EthereumTrigger { None => String::new(), } } + + fn address_match(&self) -> Option<&[u8]> { + self.address().map(|address| address.as_slice()) + } } /// Ethereum block data. -#[derive(Clone, Debug, Default)] -pub struct EthereumBlockData { - pub hash: H256, - pub parent_hash: H256, - pub uncles_hash: H256, - pub author: H160, - pub state_root: H256, - pub transactions_root: H256, - pub receipts_root: H256, - pub number: U64, - pub gas_used: U256, - pub gas_limit: U256, - pub timestamp: U256, - pub difficulty: U256, - pub total_difficulty: U256, - pub size: Option, - pub base_fee_per_gas: Option, +#[derive(Clone, Debug)] +pub struct EthereumBlockData<'a> { + block: &'a LightEthereumBlock, } -impl<'a, T> From<&'a Block> for EthereumBlockData { - fn from(block: &'a Block) -> EthereumBlockData { - EthereumBlockData { - hash: block.hash.unwrap(), - parent_hash: block.parent_hash, - uncles_hash: block.uncles_hash, - author: block.author, - state_root: block.state_root, - transactions_root: block.transactions_root, - receipts_root: block.receipts_root, - number: block.number.unwrap(), - gas_used: block.gas_used, - gas_limit: block.gas_limit, - timestamp: block.timestamp, - difficulty: block.difficulty, - total_difficulty: block.total_difficulty.unwrap_or_default(), - size: block.size, - base_fee_per_gas: block.base_fee_per_gas, - } +impl<'a> From<&'a LightEthereumBlock> for EthereumBlockData<'a> { + fn from(block: &'a LightEthereumBlock) -> EthereumBlockData<'a> { + EthereumBlockData { block } + } +} + +impl<'a> EthereumBlockData<'a> { + pub fn hash(&self) -> &B256 { + &self.block.inner().header.hash + } + + pub fn parent_hash(&self) -> &B256 { + &self.block.inner().header.parent_hash + } + + pub fn uncles_hash(&self) -> &B256 { + &self.block.inner().header.ommers_hash + } + + pub fn author(&self) -> &Address { + &self.block.inner().header.beneficiary + } + + pub fn state_root(&self) -> &B256 { + &self.block.inner().header.state_root + } + + pub fn transactions_root(&self) -> &B256 { + &self.block.inner().header.transactions_root + } + + pub fn receipts_root(&self) -> &B256 { + &self.block.inner().header.receipts_root + } + + pub fn number(&self) -> u64 { + self.block.number_u64() + } + + pub fn gas_used(&self) -> u64 { + self.block.inner().header.gas_used + } + + pub fn gas_limit(&self) -> u64 { + self.block.inner().header.gas_limit + } + + pub fn timestamp(&self) -> u64 { + self.block.inner().header.timestamp + } + + pub fn difficulty(&self) -> &U256 { + &self.block.inner().header.difficulty + } + + pub fn total_difficulty(&self) -> &U256 { + self.block + .inner() + .header + .total_difficulty + .as_ref() + .unwrap_or(&U256_DEFAULT) + } + + pub fn size(&self) -> &Option { + &self.block.inner().header.size + } + + pub fn base_fee_per_gas(&self) -> &Option { + &self.block.inner().header.base_fee_per_gas } } /// Ethereum transaction data. #[derive(Clone, Debug)] -pub struct EthereumTransactionData { - pub hash: H256, - pub index: U128, - pub from: H160, - pub to: Option, - pub value: U256, - pub gas_limit: U256, - pub gas_price: U256, - pub input: Bytes, - pub nonce: U256, +pub struct EthereumTransactionData<'a> { + tx: &'a AnyTransaction, + base_fee_per_gas: Option, } -impl From<&'_ Transaction> for EthereumTransactionData { - fn from(tx: &Transaction) -> EthereumTransactionData { - // unwrap: this is always `Some` for txns that have been mined - // (see https://github.com/tomusdrw/rust-web3/pull/407) - let from = tx.from.unwrap(); +impl<'a> EthereumTransactionData<'a> { + // We don't implement `From` because it causes confusion with the `from` + // accessor method + fn new(tx: &'a AnyTransaction, base_fee_per_gas: Option) -> EthereumTransactionData<'a> { EthereumTransactionData { - hash: tx.hash, - index: tx.transaction_index.unwrap().as_u64().into(), - from, - to: tx.to, - value: tx.value, - gas_limit: tx.gas, - gas_price: tx.gas_price.unwrap_or(U256::zero()), // EIP-1559 made this optional. - input: tx.input.0.clone(), - nonce: tx.nonce, + tx, + base_fee_per_gas, } } + + pub fn hash(&self) -> B256 { + self.tx.tx_hash() + } + + pub fn index(&self) -> u64 { + self.tx.transaction_index.unwrap() + } + + pub fn from(&self) -> Address { + self.tx.from() + } + + pub fn to(&self) -> Option
{ + self.tx.to() + } + + pub fn value(&self) -> U256 { + self.tx.value() + } + + pub fn gas_limit(&self) -> u64 { + self.tx.gas_limit() + } + + pub fn gas_price(&self) -> u128 { + self.tx.effective_gas_price(self.base_fee_per_gas) + } + + pub fn input(&self) -> &[u8] { + self.tx.input() + } + + pub fn nonce(&self) -> u64 { + self.tx.nonce() + } } /// An Ethereum event logged from a specific contract address and block. #[derive(Debug, Clone)] -pub struct EthereumEventData { - pub address: Address, - pub log_index: U256, - pub transaction_log_index: U256, - pub log_type: Option, - pub block: EthereumBlockData, - pub transaction: EthereumTransactionData, - pub params: Vec, +pub struct EthereumEventData<'a> { + pub block: EthereumBlockData<'a>, + pub transaction: EthereumTransactionData<'a>, + pub params: &'a [abi::DynSolParam], + log: &'a Log, +} + +impl<'a> EthereumEventData<'a> { + pub fn new( + block: &'a LightEthereumBlock, + tx: &'a AnyTransaction, + log: &'a Log, + params: &'a [abi::DynSolParam], + ) -> Self { + EthereumEventData { + block: EthereumBlockData::from(block), + transaction: EthereumTransactionData::new(tx, block.base_fee_per_gas()), + log, + params, + } + } + + pub fn address(&self) -> &Address { + &self.log.inner.address + } + + pub fn log_index(&self) -> u64 { + self.log.log_index.unwrap_or(0) + } + + pub fn transaction_log_index(&self) -> u64 { + // We purposely use the `log_index` here. Geth does not support + // `transaction_log_index`, and subgraphs that use it only care that + // it identifies the log, the specific value is not important. Still + // this will change the output of subgraphs that use this field. + // + // This was initially changed in commit b95c6953 + self.log.log_index.unwrap_or(0) + } + + pub fn log_type(&self) -> Option { + // This field was present in old rust-web3 Block, but alloy doesn't have it. + None + } } /// An Ethereum call executed within a transaction within a block to a contract address. #[derive(Debug, Clone)] -pub struct EthereumCallData { - pub from: Address, - pub to: Address, - pub block: EthereumBlockData, - pub transaction: EthereumTransactionData, - pub inputs: Vec, - pub outputs: Vec, +pub struct EthereumCallData<'a> { + pub block: EthereumBlockData<'a>, + pub transaction: EthereumTransactionData<'a>, + pub inputs: &'a [abi::DynSolParam], + pub outputs: &'a [abi::DynSolParam], + call: &'a EthereumCall, +} + +impl<'a> EthereumCallData<'a> { + fn new( + block: &'a LightEthereumBlock, + transaction: &'a AnyTransaction, + call: &'a EthereumCall, + inputs: &'a [abi::DynSolParam], + outputs: &'a [abi::DynSolParam], + ) -> EthereumCallData<'a> { + EthereumCallData { + block: EthereumBlockData::from(block), + transaction: EthereumTransactionData::new(transaction, block.base_fee_per_gas()), + inputs, + outputs, + call, + } + } + + pub fn from(&self) -> &Address { + &self.call.from + } + + pub fn to(&self) -> &Address { + &self.call.to + } } diff --git a/chain/ethereum/tests/README.md b/chain/ethereum/tests/README.md new file mode 100644 index 00000000000..e0444bc179f --- /dev/null +++ b/chain/ethereum/tests/README.md @@ -0,0 +1,5 @@ +Put integration tests for this crate into +`store/test-store/tests/chain/ethereum`. This avoids cyclic dev-dependencies +which make rust-analyzer nearly unusable. Once [this +issue](https://github.com/rust-lang/rust-analyzer/issues/14167) has been +fixed, we can move tests back here diff --git a/chain/ethereum/tests/manifest.rs b/chain/ethereum/tests/manifest.rs deleted file mode 100644 index 5d1c7bb3a84..00000000000 --- a/chain/ethereum/tests/manifest.rs +++ /dev/null @@ -1,793 +0,0 @@ -use std::collections::HashMap; -use std::sync::Arc; -use std::time::Duration; - -use graph::data::subgraph::schema::SubgraphError; -use graph::data::subgraph::{SPEC_VERSION_0_0_4, SPEC_VERSION_0_0_7}; -use graph::data_source::DataSourceTemplate; -use graph::prelude::{ - anyhow, async_trait, serde_yaml, tokio, DeploymentHash, Entity, Link, Logger, SubgraphManifest, - SubgraphManifestValidationError, UnvalidatedSubgraphManifest, -}; -use graph::{ - blockchain::NodeCapabilities as _, - components::{ - link_resolver::{JsonValueStream, LinkResolver as LinkResolverTrait}, - store::EntityType, - }, - data::subgraph::SubgraphFeature, -}; - -use graph_chain_ethereum::{Chain, NodeCapabilities}; -use semver::Version; -use test_store::LOGGER; - -const GQL_SCHEMA: &str = "type Thing @entity { id: ID! }"; -const GQL_SCHEMA_FULLTEXT: &str = include_str!("full-text.graphql"); -const MAPPING_WITH_IPFS_FUNC_WASM: &[u8] = include_bytes!("ipfs-on-ethereum-contracts.wasm"); -const ABI: &str = "[{\"type\":\"function\", \"inputs\": [{\"name\": \"i\",\"type\": \"uint256\"}],\"name\":\"get\",\"outputs\": [{\"type\": \"address\",\"name\": \"o\"}]}]"; -const FILE: &str = "{}"; -const FILE_CID: &str = "bafkreigkhuldxkyfkoaye4rgcqcwr45667vkygd45plwq6hawy7j4rbdky"; - -#[derive(Default, Debug, Clone)] -struct TextResolver { - texts: HashMap>, -} - -impl TextResolver { - fn add(&mut self, link: &str, text: &impl AsRef<[u8]>) { - self.texts.insert(link.to_owned(), text.as_ref().to_vec()); - } -} - -#[async_trait] -impl LinkResolverTrait for TextResolver { - fn with_timeout(&self, _timeout: Duration) -> Box { - Box::new(self.clone()) - } - - fn with_retries(&self) -> Box { - Box::new(self.clone()) - } - - async fn cat(&self, _logger: &Logger, link: &Link) -> Result, anyhow::Error> { - self.texts - .get(&link.link) - .ok_or(anyhow!("No text for {}", &link.link)) - .map(Clone::clone) - } - - async fn get_block(&self, _logger: &Logger, _link: &Link) -> Result, anyhow::Error> { - unimplemented!() - } - - async fn json_stream( - &self, - _logger: &Logger, - _link: &Link, - ) -> Result { - unimplemented!() - } -} - -async fn resolve_manifest( - text: &str, - max_spec_version: Version, -) -> SubgraphManifest { - let mut resolver = TextResolver::default(); - let id = DeploymentHash::new("Qmmanifest").unwrap(); - - resolver.add(id.as_str(), &text); - resolver.add("/ipfs/Qmschema", &GQL_SCHEMA); - resolver.add("/ipfs/Qmabi", &ABI); - resolver.add("/ipfs/Qmmapping", &MAPPING_WITH_IPFS_FUNC_WASM); - resolver.add(FILE_CID, &FILE); - - let resolver: Arc = Arc::new(resolver); - - let raw = serde_yaml::from_str(text).unwrap(); - SubgraphManifest::resolve_from_raw(id, raw, &resolver, &LOGGER, max_spec_version) - .await - .expect("Parsing simple manifest works") -} - -async fn resolve_unvalidated(text: &str) -> UnvalidatedSubgraphManifest { - let mut resolver = TextResolver::default(); - let id = DeploymentHash::new("Qmmanifest").unwrap(); - - resolver.add(id.as_str(), &text); - resolver.add("/ipfs/Qmschema", &GQL_SCHEMA); - - let resolver: Arc = Arc::new(resolver); - - let raw = serde_yaml::from_str(text).unwrap(); - UnvalidatedSubgraphManifest::resolve(id, raw, &resolver, &LOGGER, SPEC_VERSION_0_0_4.clone()) - .await - .expect("Parsing simple manifest works") -} - -// Some of these manifest tests should be made chain-independent, but for -// now we just run them for the ethereum `Chain` - -#[tokio::test] -async fn simple_manifest() { - const YAML: &str = " -dataSources: [] -schema: - file: - /: /ipfs/Qmschema -specVersion: 0.0.2 -"; - - let manifest = resolve_manifest(YAML, SPEC_VERSION_0_0_4).await; - - assert_eq!("Qmmanifest", manifest.id.as_str()); - assert!(manifest.graft.is_none()); -} - -#[tokio::test] -async fn ipfs_manifest() { - let yaml = " -schema: - file: - /: /ipfs/Qmschema -dataSources: [] -templates: - - name: IpfsSource - kind: file/ipfs - mapping: - apiVersion: 0.0.6 - language: wasm/assemblyscript - entities: - - TestEntity - file: - /: /ipfs/Qmmapping - handler: handleFile -specVersion: 0.0.7 -"; - - let manifest = resolve_manifest(yaml, SPEC_VERSION_0_0_7).await; - - assert_eq!("Qmmanifest", manifest.id.as_str()); - assert_eq!(manifest.data_sources.len(), 0); - let data_source = match &manifest.templates[0] { - DataSourceTemplate::Offchain(ds) => ds, - DataSourceTemplate::Onchain(_) => unreachable!(), - }; - assert_eq!(data_source.kind, "file/ipfs"); -} - -#[tokio::test] -async fn graft_manifest() { - const YAML: &str = " -dataSources: [] -schema: - file: - /: /ipfs/Qmschema -graft: - base: Qmbase - block: 12345 -specVersion: 0.0.2 -"; - - let manifest = resolve_manifest(YAML, SPEC_VERSION_0_0_4).await; - - assert_eq!("Qmmanifest", manifest.id.as_str()); - let graft = manifest.graft.expect("The manifest has a graft base"); - assert_eq!("Qmbase", graft.base.as_str()); - assert_eq!(12345, graft.block); -} - -#[test] -fn graft_failed_subgraph() { - const YAML: &str = " -dataSources: [] -schema: - file: - /: /ipfs/Qmschema -graft: - base: Qmbase - block: 0 -specVersion: 0.0.2 -"; - - test_store::run_test_sequentially(|store| async move { - let subgraph_store = store.subgraph_store(); - - let unvalidated = resolve_unvalidated(YAML).await; - let subgraph = DeploymentHash::new("Qmbase").unwrap(); - - // Creates base subgraph at block 0 (genesis). - let deployment = test_store::create_test_subgraph(&subgraph, GQL_SCHEMA).await; - - // Adds an example entity. - let mut thing = Entity::new(); - thing.set("id", "datthing"); - test_store::insert_entities(&deployment, vec![(EntityType::from("Thing"), thing)]) - .await - .unwrap(); - - let error = SubgraphError { - subgraph_id: deployment.hash.clone(), - message: "deterministic error".to_string(), - block_ptr: Some(test_store::BLOCKS[1].clone()), - handler: None, - deterministic: true, - }; - - // Fails the base subgraph at block 1 (and advances the pointer). - test_store::transact_errors( - &store, - &deployment, - test_store::BLOCKS[1].clone(), - vec![error], - ) - .await - .unwrap(); - - // Make sure there are no GraftBaseInvalid errors. - // - // This is allowed because: - // - base: failed at block 1 - // - graft: starts at block 0 - // - // Meaning that the graft will fail just like it's parent - // but it started at a valid previous block. - assert!( - !unvalidated - .validate(subgraph_store.clone(), true) - .await - .expect_err("Validation must fail") - .into_iter() - .any(|e| matches!(&e, SubgraphManifestValidationError::GraftBaseInvalid(_))), - "There shouldn't be a GraftBaseInvalid error" - ); - - // Resolve the graft normally. - let manifest = resolve_manifest(YAML, SPEC_VERSION_0_0_4).await; - - assert_eq!("Qmmanifest", manifest.id.as_str()); - let graft = manifest.graft.expect("The manifest has a graft base"); - assert_eq!("Qmbase", graft.base.as_str()); - assert_eq!(0, graft.block); - }) -} - -#[test] -fn graft_invalid_manifest() { - const YAML: &str = " -dataSources: [] -schema: - file: - /: /ipfs/Qmschema -graft: - base: Qmbase - block: 1 -specVersion: 0.0.2 -"; - - test_store::run_test_sequentially(|store| async move { - let subgraph_store = store.subgraph_store(); - - let unvalidated = resolve_unvalidated(YAML).await; - let subgraph = DeploymentHash::new("Qmbase").unwrap(); - - // - // Validation against subgraph that hasn't synced anything fails - // - let deployment = test_store::create_test_subgraph(&subgraph, GQL_SCHEMA).await; - // This check is awkward since the test manifest has other problems - // that the validation complains about as setting up a valid manifest - // would be a bit more work; we just want to make sure that - // graft-related checks work - let msg = unvalidated - .validate(subgraph_store.clone(), true) - .await - .expect_err("Validation must fail") - .into_iter() - .find(|e| matches!(e, SubgraphManifestValidationError::GraftBaseInvalid(_))) - .expect("There must be a GraftBaseInvalid error") - .to_string(); - assert_eq!( - "the graft base is invalid: failed to graft onto `Qmbase` since \ - it has not processed any blocks", - msg - ); - - let mut thing = Entity::new(); - thing.set("id", "datthing"); - test_store::insert_entities(&deployment, vec![(EntityType::from("Thing"), thing)]) - .await - .unwrap(); - - // Validation against subgraph that has not reached the graft point fails - let unvalidated = resolve_unvalidated(YAML).await; - let msg = unvalidated - .validate(subgraph_store.clone(), true) - .await - .expect_err("Validation must fail") - .into_iter() - .find(|e| matches!(e, SubgraphManifestValidationError::GraftBaseInvalid(_))) - .expect("There must be a GraftBaseInvalid error") - .to_string(); - assert_eq!( - "the graft base is invalid: failed to graft onto `Qmbase` \ - at block 1 since it has only processed block 0", - msg - ); - - let error = SubgraphError { - subgraph_id: deployment.hash.clone(), - message: "deterministic error".to_string(), - block_ptr: Some(test_store::BLOCKS[1].clone()), - handler: None, - deterministic: true, - }; - - test_store::transact_errors( - &store, - &deployment, - test_store::BLOCKS[1].clone(), - vec![error], - ) - .await - .unwrap(); - - // This check is bit awkward, but we just want to be sure there is a - // GraftBaseInvalid error. - // - // The validation error happens because: - // - base: failed at block 1 - // - graft: starts at block 1 - // - // Since we start grafts at N + 1, we can't allow a graft to be created - // at the failed block. They (developers) should choose a previous valid - // block. - let unvalidated = resolve_unvalidated(YAML).await; - let msg = unvalidated - .validate(subgraph_store, true) - .await - .expect_err("Validation must fail") - .into_iter() - .find(|e| matches!(e, SubgraphManifestValidationError::GraftBaseInvalid(_))) - .expect("There must be a GraftBaseInvalid error") - .to_string(); - assert_eq!( - "the graft base is invalid: failed to graft onto `Qmbase` \ - at block 1 since it's not healthy. You can graft it starting at block 0 backwards", - msg - ); - }) -} - -#[tokio::test] -async fn parse_call_handlers() { - const YAML: &str = " -dataSources: - - kind: ethereum/contract - name: Factory - network: mainnet - source: - abi: Factory - startBlock: 9562480 - mapping: - kind: ethereum/events - apiVersion: 0.0.4 - language: wasm/assemblyscript - entities: - - TestEntity - file: - /: /ipfs/Qmmapping - abis: - - name: Factory - file: - /: /ipfs/Qmabi - callHandlers: - - function: get(address) - handler: handleget -schema: - file: - /: /ipfs/Qmschema -specVersion: 0.0.2 -"; - - let manifest = resolve_manifest(YAML, SPEC_VERSION_0_0_4).await; - let onchain_data_sources = manifest - .data_sources - .iter() - .filter_map(|ds| ds.as_onchain().cloned()) - .collect::>(); - let required_capabilities = NodeCapabilities::from_data_sources(&onchain_data_sources); - - assert_eq!("Qmmanifest", manifest.id.as_str()); - assert_eq!(true, required_capabilities.traces); -} - -#[test] -fn undeclared_grafting_feature_causes_feature_validation_error() { - const YAML: &str = " -specVersion: 0.0.4 -dataSources: [] -schema: - file: - /: /ipfs/Qmschema -graft: - base: Qmbase - block: 1 -"; - test_store::run_test_sequentially(|store| async move { - let store = store.subgraph_store(); - let unvalidated = resolve_unvalidated(YAML).await; - let error_msg = unvalidated - .validate(store.clone(), true) - .await - .expect_err("Validation must fail") - .into_iter() - .find(|e| { - matches!( - e, - SubgraphManifestValidationError::FeatureValidationError(_) - ) - }) - .expect("There must be a FeatureValidation error") - .to_string(); - assert_eq!( - "The feature `grafting` is used by the subgraph but it is not declared in the manifest.", - error_msg - ) - }) -} - -#[test] -fn declared_grafting_feature_causes_no_feature_validation_errors() { - const YAML: &str = " -specVersion: 0.0.4 -features: - - grafting -dataSources: [] -schema: - file: - /: /ipfs/Qmschema -graft: - base: Qmbase - block: 1 -"; - test_store::run_test_sequentially(|store| async move { - let store = store.subgraph_store(); - let unvalidated = resolve_unvalidated(YAML).await; - assert!(unvalidated - .validate(store.clone(), true) - .await - .expect_err("Validation must fail") - .into_iter() - .find(|e| { - matches!( - e, - SubgraphManifestValidationError::FeatureValidationError(_) - ) - }) - .is_none()); - let manifest = resolve_manifest(YAML, SPEC_VERSION_0_0_4).await; - assert!(manifest.features.contains(&SubgraphFeature::Grafting)) - }) -} - -#[test] -fn declared_non_fatal_errors_feature_causes_no_feature_validation_errors() { - const YAML: &str = " -specVersion: 0.0.4 -features: - - nonFatalErrors -dataSources: [] -schema: - file: - /: /ipfs/Qmschema -"; - test_store::run_test_sequentially(|store| async move { - let store = store.subgraph_store(); - let unvalidated = resolve_unvalidated(YAML).await; - assert!(unvalidated - .validate(store.clone(), true) - .await - .expect_err("Validation must fail") - .into_iter() - .find(|e| { - matches!( - e, - SubgraphManifestValidationError::FeatureValidationError(_) - ) - }) - .is_none()); - - let manifest = resolve_manifest(YAML, SPEC_VERSION_0_0_4).await; - assert!(manifest.features.contains(&SubgraphFeature::NonFatalErrors)) - }); -} - -#[test] -fn declared_full_text_search_feature_causes_no_feature_validation_errors() { - const YAML: &str = " -specVersion: 0.0.4 -features: - - fullTextSearch -dataSources: [] -schema: - file: - /: /ipfs/Qmschema -"; - - test_store::run_test_sequentially(|store| async move { - let store = store.subgraph_store(); - let unvalidated: UnvalidatedSubgraphManifest = { - let mut resolver = TextResolver::default(); - let id = DeploymentHash::new("Qmmanifest").unwrap(); - resolver.add(id.as_str(), &YAML); - resolver.add("/ipfs/Qmabi", &ABI); - resolver.add("/ipfs/Qmschema", &GQL_SCHEMA_FULLTEXT); - - let resolver: Arc = Arc::new(resolver); - - let raw = serde_yaml::from_str(YAML).unwrap(); - UnvalidatedSubgraphManifest::resolve( - id, - raw, - &resolver, - &LOGGER, - SPEC_VERSION_0_0_4.clone(), - ) - .await - .expect("Parsing simple manifest works") - }; - - assert!(unvalidated - .validate(store.clone(), true) - .await - .expect_err("Validation must fail") - .into_iter() - .find(|e| { - matches!( - e, - SubgraphManifestValidationError::FeatureValidationError(_) - ) - }) - .is_none()); - - let manifest = resolve_manifest(YAML, SPEC_VERSION_0_0_4).await; - assert!(manifest.features.contains(&SubgraphFeature::FullTextSearch)) - }); -} - -#[test] -fn undeclared_full_text_search_feature_causes_no_feature_validation_errors() { - const YAML: &str = " -specVersion: 0.0.4 - -dataSources: [] -schema: - file: - /: /ipfs/Qmschema -"; - - test_store::run_test_sequentially(|store| async move { - let store = store.subgraph_store(); - let unvalidated: UnvalidatedSubgraphManifest = { - let mut resolver = TextResolver::default(); - let id = DeploymentHash::new("Qmmanifest").unwrap(); - resolver.add(id.as_str(), &YAML); - resolver.add("/ipfs/Qmabi", &ABI); - resolver.add("/ipfs/Qmschema", &GQL_SCHEMA_FULLTEXT); - - let resolver: Arc = Arc::new(resolver); - - let raw = serde_yaml::from_str(YAML).unwrap(); - UnvalidatedSubgraphManifest::resolve( - id, - raw, - &resolver, - &LOGGER, - SPEC_VERSION_0_0_4.clone(), - ) - .await - .expect("Parsing simple manifest works") - }; - - let error_msg = unvalidated - .validate(store.clone(), true) - .await - .expect_err("Validation must fail") - .into_iter() - .find(|e| { - matches!( - e, - SubgraphManifestValidationError::FeatureValidationError(_) - ) - }) - .expect("There must be a FeatureValidationError") - .to_string(); - - assert_eq!( - "The feature `fullTextSearch` is used by the subgraph but it is not declared in the manifest.", - error_msg - ); - }); -} - -#[test] -fn undeclared_ipfs_on_ethereum_contracts_feature_causes_feature_validation_error() { - const YAML: &str = " -specVersion: 0.0.4 -schema: - file: - /: /ipfs/Qmschema -dataSources: - - kind: ethereum/contract - name: Factory - network: mainnet - source: - abi: Factory - startBlock: 9562480 - mapping: - kind: ethereum/events - apiVersion: 0.0.4 - language: wasm/assemblyscript - entities: - - TestEntity - file: - /: /ipfs/Qmmapping - abis: - - name: Factory - file: - /: /ipfs/Qmabi - callHandlers: - - function: get(address) - handler: handleget -"; - - test_store::run_test_sequentially(|store| async move { - let store = store.subgraph_store(); - let unvalidated: UnvalidatedSubgraphManifest = { - let mut resolver = TextResolver::default(); - let id = DeploymentHash::new("Qmmanifest").unwrap(); - resolver.add(id.as_str(), &YAML); - resolver.add("/ipfs/Qmabi", &ABI); - resolver.add("/ipfs/Qmschema", &GQL_SCHEMA); - resolver.add("/ipfs/Qmmapping", &MAPPING_WITH_IPFS_FUNC_WASM); - - let resolver: Arc = Arc::new(resolver); - - let raw = serde_yaml::from_str(YAML).unwrap(); - UnvalidatedSubgraphManifest::resolve( - id, - raw, - &resolver, - &LOGGER, - SPEC_VERSION_0_0_4.clone(), - ) - .await - .expect("Parsing simple manifest works") - }; - - let error_msg = unvalidated - .validate(store.clone(), true) - .await - .expect_err("Validation must fail") - .into_iter() - .find(|e| { - matches!( - e, - SubgraphManifestValidationError::FeatureValidationError(_) - ) - }) - .expect("There must be a FeatureValidationError") - .to_string(); - - assert_eq!( - "The feature `ipfsOnEthereumContracts` is used by the subgraph but it is not declared in the manifest.", - error_msg - ); - }); -} - -#[test] -fn declared_ipfs_on_ethereum_contracts_feature_causes_no_errors() { - const YAML: &str = " -specVersion: 0.0.4 -schema: - file: - /: /ipfs/Qmschema -features: - - ipfsOnEthereumContracts -dataSources: - - kind: ethereum/contract - name: Factory - network: mainnet - source: - abi: Factory - startBlock: 9562480 - mapping: - kind: ethereum/events - apiVersion: 0.0.4 - language: wasm/assemblyscript - entities: - - TestEntity - file: - /: /ipfs/Qmmapping - abis: - - name: Factory - file: - /: /ipfs/Qmabi - callHandlers: - - function: get(address) - handler: handleget -"; - - test_store::run_test_sequentially(|store| async move { - let store = store.subgraph_store(); - let unvalidated: UnvalidatedSubgraphManifest = { - let mut resolver = TextResolver::default(); - let id = DeploymentHash::new("Qmmanifest").unwrap(); - resolver.add(id.as_str(), &YAML); - resolver.add("/ipfs/Qmabi", &ABI); - resolver.add("/ipfs/Qmschema", &GQL_SCHEMA); - resolver.add("/ipfs/Qmmapping", &MAPPING_WITH_IPFS_FUNC_WASM); - - let resolver: Arc = Arc::new(resolver); - - let raw = serde_yaml::from_str(YAML).unwrap(); - UnvalidatedSubgraphManifest::resolve( - id, - raw, - &resolver, - &LOGGER, - SPEC_VERSION_0_0_4.clone(), - ) - .await - .expect("Parsing simple manifest works") - }; - - assert!(unvalidated - .validate(store.clone(), true) - .await - .expect_err("Validation must fail") - .into_iter() - .find(|e| { - matches!( - e, - SubgraphManifestValidationError::FeatureValidationError(_) - ) - }) - .is_none()); - }); -} - -#[test] -fn can_detect_features_in_subgraphs_with_spec_version_lesser_than_0_0_4() { - const YAML: &str = " -specVersion: 0.0.2 -features: - - nonFatalErrors -dataSources: [] -schema: - file: - /: /ipfs/Qmschema -"; - test_store::run_test_sequentially(|store| async move { - let store = store.subgraph_store(); - let unvalidated = resolve_unvalidated(YAML).await; - assert!(unvalidated - .validate(store.clone(), true) - .await - .expect_err("Validation must fail") - .into_iter() - .find(|e| { - matches!( - e, - SubgraphManifestValidationError::FeatureValidationError(_) - ) - }) - .is_none()); - - let manifest = resolve_manifest(YAML, SPEC_VERSION_0_0_4).await; - assert!(manifest.features.contains(&SubgraphFeature::NonFatalErrors)) - }); -} diff --git a/chain/near/Cargo.toml b/chain/near/Cargo.toml index d41b901159e..d21f3755d0e 100644 --- a/chain/near/Cargo.toml +++ b/chain/near/Cargo.toml @@ -4,17 +4,19 @@ version.workspace = true edition.workspace = true [build-dependencies] -tonic-build = { workspace = true } +tonic-prost-build = { workspace = true } [dependencies] -base64 = "0.20" +async-trait = { workspace = true } graph = { path = "../../graph" } prost = { workspace = true } prost-types = { workspace = true } -serde = "1.0" +serde = { workspace = true } +anyhow = "1" graph-runtime-wasm = { path = "../../runtime/wasm" } graph-runtime-derive = { path = "../../runtime/derive" } [dev-dependencies] -diesel = { version = "1.4.7", features = ["postgres", "serde_json", "numeric", "r2d2"] } +diesel = { workspace = true } +tokio = { workspace = true } diff --git a/chain/near/build.rs b/chain/near/build.rs index 73c33efb26f..feafe4718f9 100644 --- a/chain/near/build.rs +++ b/chain/near/build.rs @@ -1,7 +1,8 @@ fn main() { println!("cargo:rerun-if-changed=proto"); - tonic_build::configure() + tonic_prost_build::configure() .out_dir("src/protobuf") - .compile(&["proto/codec.proto"], &["proto"]) + .extern_path(".sf.near.codec.v1", "crate::codec::pbcodec") + .compile_protos(&["proto/near.proto"], &["proto"]) .expect("Failed to compile Firehose NEAR proto(s)"); } diff --git a/chain/near/proto/codec.proto b/chain/near/proto/near.proto similarity index 100% rename from chain/near/proto/codec.proto rename to chain/near/proto/near.proto diff --git a/chain/near/src/adapter.rs b/chain/near/src/adapter.rs index 89c95b20c28..31c7dca7c97 100644 --- a/chain/near/src/adapter.rs +++ b/chain/near/src/adapter.rs @@ -1,9 +1,10 @@ use std::collections::HashSet; use crate::data_source::PartialAccounts; -use crate::{data_source::DataSource, Chain}; +use crate::{Chain, data_source::DataSource}; use graph::blockchain as bc; use graph::firehose::{BasicReceiptFilter, PrefixSuffixPair}; +use graph::itertools::Itertools; use graph::prelude::*; use prost::Message; use prost_types::Any; @@ -17,6 +18,31 @@ pub struct TriggerFilter { pub(crate) receipt_filter: NearReceiptFilter, } +impl TriggerFilter { + pub fn to_module_params(&self) -> String { + let matches = self.receipt_filter.accounts.iter().join(","); + let partial_matches = self + .receipt_filter + .partial_accounts + .iter() + .map(|(starts_with, ends_with)| match (starts_with, ends_with) { + (None, None) => unreachable!(), + (None, Some(e)) => format!(",{}", e), + (Some(s), None) => format!("{},", s), + (Some(s), Some(e)) => format!("{},{}", s, e), + }) + .join("\n"); + + format!( + "{},{}\n{}\n{}", + self.receipt_filter.accounts.len(), + self.receipt_filter.partial_accounts.len(), + matches, + partial_matches + ) + } +} + impl bc::TriggerFilter for TriggerFilter { fn extend<'a>(&mut self, data_sources: impl Iterator + Clone) { let TriggerFilter { @@ -225,7 +251,7 @@ mod test { use std::collections::HashSet; use super::NearBlockFilter; - use crate::adapter::{TriggerFilter, BASIC_RECEIPT_FILTER_TYPE_URL}; + use crate::adapter::{BASIC_RECEIPT_FILTER_TYPE_URL, TriggerFilter}; use graph::{ blockchain::TriggerFilter as _, firehose::{BasicReceiptFilter, PrefixSuffixPair}, @@ -244,6 +270,7 @@ mod test { partial_accounts: HashSet::new(), }, }; + assert_eq!(filter.to_module_params(), "0,0\n\n"); assert_eq!(filter.to_firehose_filter(), vec![]); } @@ -312,7 +339,7 @@ mod test { let firehose_filter = decode_filter(filter); assert_eq!(firehose_filter.accounts, vec![String::from("acc1"),],); - let expected_pairs = vec![ + let expected_pairs = [ PrefixSuffixPair { prefix: "acc3".to_string(), suffix: "acc4".to_string(), @@ -329,8 +356,7 @@ mod test { let pairs = firehose_filter.prefix_and_suffix_pairs; assert_eq!(pairs.len(), 3); - assert_eq!( - true, + assert!( expected_pairs.iter().all(|x| pairs.contains(x)), "{:?}", pairs diff --git a/chain/near/src/chain.rs b/chain/near/src/chain.rs index 59e838a9322..5698301b5e8 100644 --- a/chain/near/src/chain.rs +++ b/chain/near/src/chain.rs @@ -1,36 +1,46 @@ +use async_trait::async_trait; use graph::blockchain::client::ChainClient; -use graph::blockchain::BlockchainKind; +use graph::blockchain::firehose_block_ingestor::FirehoseBlockIngestor; +use graph::blockchain::{ + BlockIngestor, BlockchainKind, NoopDecoderHook, NoopRuntimeAdapter, TriggerFilterWrapper, +}; use graph::cheap_clone::CheapClone; +use graph::components::network_provider::ChainName; +use graph::components::store::{ChainHeadStore, DeploymentCursorTracker, SourceableStore}; use graph::data::subgraph::UnifiedMappingApiVersion; use graph::firehose::{FirehoseEndpoint, FirehoseEndpoints}; -use graph::prelude::{MetricsRegistry, TryFutureExt}; +use graph::futures03::TryFutureExt; +use graph::prelude::MetricsRegistry; use graph::{ anyhow::Result, blockchain::{ + BlockHash, BlockPtr, Blockchain, EmptyNodeCapabilities, IngestorError, + RuntimeAdapter as RuntimeAdapterTrait, block_stream::{ BlockStreamEvent, BlockWithTriggers, FirehoseError, FirehoseMapper as FirehoseMapperTrait, TriggersAdapter as TriggersAdapterTrait, }, firehose_block_stream::FirehoseBlockStream, - BlockHash, BlockPtr, Blockchain, EmptyNodeCapabilities, IngestorError, - RuntimeAdapter as RuntimeAdapterTrait, }, components::store::DeploymentLocator, firehose::{self as firehose, ForkStep}, - prelude::{async_trait, o, BlockNumber, ChainStore, Error, Logger, LoggerFactory}, + prelude::{BlockNumber, Error, Logger, LoggerFactory, o}, }; use prost::Message; +use std::collections::BTreeSet; use std::sync::Arc; use crate::adapter::TriggerFilter; +use crate::codec::Block; use crate::data_source::{DataSourceTemplate, UnresolvedDataSourceTemplate}; -use crate::runtime::RuntimeAdapter; use crate::trigger::{self, NearTrigger}; use crate::{ codec, data_source::{DataSource, UnresolvedDataSource}, }; -use graph::blockchain::block_stream::{BlockStream, BlockStreamBuilder, FirehoseCursor}; +use graph::blockchain::block_stream::{ + BlockStream, BlockStreamBuilder, BlockStreamError, BlockStreamMapper, FirehoseCursor, +}; pub struct NearStreamBuilder {} @@ -54,23 +64,19 @@ impl BlockStreamBuilder for NearStreamBuilder { ) .unwrap_or_else(|_| panic!("no adapter for network {}", chain.name)); - let firehose_endpoint = chain.chain_client().firehose_endpoint()?; - let logger = chain .logger_factory .subgraph_logger(&deployment) .new(o!("component" => "FirehoseBlockStream")); - let firehose_mapper = Arc::new(FirehoseMapper {}); + let firehose_mapper = Arc::new(FirehoseMapper { adapter, filter }); Ok(Box::new(FirehoseBlockStream::new( deployment.hash, - firehose_endpoint, + chain.chain_client(), subgraph_current_block, block_cursor, firehose_mapper, - adapter, - filter, start_blocks, logger, chain.metrics_registry.clone(), @@ -79,11 +85,12 @@ impl BlockStreamBuilder for NearStreamBuilder { async fn build_polling( &self, - _chain: Arc, + _chain: &Chain, _deployment: DeploymentLocator, _start_blocks: Vec, + _source_subgraph_stores: Vec>, _subgraph_current_block: Option, - _filter: Arc<::TriggerFilter>, + _filter: Arc>, _unified_api_version: UnifiedMappingApiVersion, ) -> Result>> { todo!() @@ -92,10 +99,10 @@ impl BlockStreamBuilder for NearStreamBuilder { pub struct Chain { logger_factory: LoggerFactory, - name: String, + name: ChainName, client: Arc>, - chain_store: Arc, - metrics_registry: Arc, + chain_head_store: Arc, + metrics_registry: Arc, block_stream_builder: Arc>, } @@ -108,19 +115,18 @@ impl std::fmt::Debug for Chain { impl Chain { pub fn new( logger_factory: LoggerFactory, - name: String, - chain_store: Arc, + name: ChainName, + chain_head_store: Arc, firehose_endpoints: FirehoseEndpoints, - metrics_registry: Arc, - block_stream_builder: Arc>, + metrics_registry: Arc, ) -> Self { Chain { logger_factory, name, + chain_head_store, client: Arc::new(ChainClient::new_firehose(firehose_endpoints)), - chain_store, metrics_registry, - block_stream_builder, + block_stream_builder: Arc::new(NearStreamBuilder {}), } } } @@ -148,6 +154,8 @@ impl Blockchain for Chain { type NodeCapabilities = EmptyNodeCapabilities; + type DecoderHook = NoopDecoderHook; + fn triggers_adapter( &self, _loc: &DeploymentLocator, @@ -158,23 +166,23 @@ impl Blockchain for Chain { Ok(Arc::new(adapter)) } - async fn new_firehose_block_stream( + async fn new_block_stream( &self, deployment: DeploymentLocator, - block_cursor: FirehoseCursor, + store: impl DeploymentCursorTracker, start_blocks: Vec, - subgraph_current_block: Option, - filter: Arc, + _source_subgraph_stores: Vec>, + filter: Arc>, unified_api_version: UnifiedMappingApiVersion, ) -> Result>, Error> { self.block_stream_builder .build_firehose( self, deployment, - block_cursor, + store.firehose_cursor(), start_blocks, - subgraph_current_block, - filter, + store.block_ptr(), + filter.chain_filter.clone(), unified_api_version, ) .await @@ -189,22 +197,13 @@ impl Blockchain for Chain { _logger: &Logger, _cursor: FirehoseCursor, ) -> Result { - unimplemented!("This chain does not support Dynamic Data Sources. is_refetch_block_required always returns false, this shouldn't be called.") + unimplemented!( + "This chain does not support Dynamic Data Sources. is_refetch_block_required always returns false, this shouldn't be called." + ) } - async fn new_polling_block_stream( - &self, - _deployment: DeploymentLocator, - _start_blocks: Vec, - _subgraph_current_block: Option, - _filter: Arc, - _unified_api_version: UnifiedMappingApiVersion, - ) -> Result>, Error> { - panic!("NEAR does not support polling block stream") - } - - fn chain_store(&self) -> Arc { - self.chain_store.clone() + async fn chain_head_ptr(&self) -> Result, Error> { + self.chain_head_store.cheap_clone().chain_head_ptr().await } async fn block_pointer_from_number( @@ -212,7 +211,7 @@ impl Blockchain for Chain { logger: &Logger, number: BlockNumber, ) -> Result { - let firehose_endpoint = self.client.firehose_endpoint()?; + let firehose_endpoint = self.client.firehose_endpoint().await?; firehose_endpoint .block_ptr_for_number::(logger, number) @@ -220,13 +219,26 @@ impl Blockchain for Chain { .await } - fn runtime_adapter(&self) -> Arc> { - Arc::new(RuntimeAdapter {}) + async fn runtime( + &self, + ) -> anyhow::Result<(Arc>, Self::DecoderHook)> { + Ok((Arc::new(NoopRuntimeAdapter::default()), NoopDecoderHook)) } fn chain_client(&self) -> Arc> { self.client.clone() } + + async fn block_ingestor(&self) -> anyhow::Result> { + let ingestor = FirehoseBlockIngestor::::new( + self.chain_head_store.cheap_clone(), + self.chain_client(), + self.logger_factory + .component_logger("NearFirehoseBlockIngestor", None), + self.name.clone(), + ); + Ok(Box::new(ingestor)) + } } pub struct TriggersAdapter {} @@ -238,10 +250,22 @@ impl TriggersAdapterTrait for TriggersAdapter { _from: BlockNumber, _to: BlockNumber, _filter: &TriggerFilter, - ) -> Result>, Error> { + ) -> Result<(Vec>, BlockNumber), Error> { panic!("Should never be called since not used by FirehoseBlockStream") } + async fn load_block_ptrs_by_numbers( + &self, + _logger: Logger, + _block_numbers: BTreeSet, + ) -> Result> { + unimplemented!() + } + + async fn chain_head_ptr(&self) -> Result, Error> { + unimplemented!() + } + async fn triggers_in_block( &self, logger: &Logger, @@ -312,6 +336,7 @@ impl TriggersAdapterTrait for TriggersAdapter { &self, _ptr: BlockPtr, _offset: BlockNumber, + _root: Option, ) -> Result, Error> { panic!("Should never be called since FirehoseBlockStream cannot resolve it") } @@ -327,18 +352,54 @@ impl TriggersAdapterTrait for TriggersAdapter { } } -pub struct FirehoseMapper {} +pub struct FirehoseMapper { + adapter: Arc>, + filter: Arc, +} + +#[async_trait] +impl BlockStreamMapper for FirehoseMapper { + fn decode_block( + &self, + output: Option<&[u8]>, + ) -> Result, BlockStreamError> { + let block = match output { + Some(block) => codec::Block::decode(block)?, + None => { + return Err(anyhow::anyhow!( + "near mapper is expected to always have a block" + )) + .map_err(BlockStreamError::from); + } + }; + + Ok(Some(block)) + } + + async fn block_with_triggers( + &self, + logger: &Logger, + block: codec::Block, + ) -> Result, BlockStreamError> { + self.adapter + .triggers_in_block(logger, block, self.filter.as_ref()) + .await + .map_err(BlockStreamError::from) + } +} #[async_trait] impl FirehoseMapperTrait for FirehoseMapper { + fn trigger_filter(&self) -> &TriggerFilter { + self.filter.as_ref() + } + async fn to_block_stream_event( &self, logger: &Logger, response: &firehose::Response, - adapter: &Arc>, - filter: &TriggerFilter, ) -> Result, FirehoseError> { - let step = ForkStep::from_i32(response.step).unwrap_or_else(|| { + let step = ForkStep::try_from(response.step).unwrap_or_else(|_| { panic!( "unknown step i32 value {}, maybe you forgot update & re-regenerate the protobuf definitions?", response.step @@ -357,12 +418,13 @@ impl FirehoseMapperTrait for FirehoseMapper { // // Check about adding basic information about the block in the bstream::BlockResponseV2 or maybe // define a slimmed down stuct that would decode only a few fields and ignore all the rest. - let block = codec::Block::decode(any_block.value.as_ref())?; + // unwrap: Input cannot be None so output will be error or block. + let block = self.decode_block(Some(any_block.value.as_ref()))?.unwrap(); use ForkStep::*; match step { StepNew => Ok(BlockStreamEvent::ProcessBlock( - adapter.triggers_in_block(logger, block, filter).await?, + self.block_with_triggers(logger, block).await?, FirehoseCursor::from(response.cursor.clone()), )), @@ -379,7 +441,9 @@ impl FirehoseMapperTrait for FirehoseMapper { } StepFinal => { - panic!("irreversible step is not handled and should not be requested in the Firehose request") + panic!( + "irreversible step is not handled and should not be requested in the Firehose request" + ) } StepUnset => { @@ -417,22 +481,23 @@ mod test { use std::{collections::HashSet, sync::Arc, vec}; use graph::{ - blockchain::{block_stream::BlockWithTriggers, DataSource as _, TriggersAdapter as _}, - prelude::{tokio, Link}, + blockchain::{DataSource as _, TriggersAdapter as _, block_stream::BlockWithTriggers}, + data::subgraph::LATEST_VERSION, + prelude::Link, semver::Version, - slog::{self, o, Logger}, + slog::{self, Logger, o}, }; use crate::{ + Chain, adapter::{NearReceiptFilter, TriggerFilter}, codec::{ - self, execution_outcome, receipt, Block, BlockHeader, DataReceiver, ExecutionOutcome, - ExecutionOutcomeWithId, IndexerExecutionOutcomeWithReceipt, IndexerShard, - ReceiptAction, SuccessValueExecutionStatus, + self, Block, BlockHeader, DataReceiver, ExecutionOutcome, ExecutionOutcomeWithId, + IndexerExecutionOutcomeWithReceipt, IndexerShard, ReceiptAction, + SuccessValueExecutionStatus, execution_outcome, receipt, }, - data_source::{DataSource, Mapping, PartialAccounts, ReceiptHandler, NEAR_KIND}, + data_source::{DataSource, Mapping, NEAR_KIND, PartialAccounts, ReceiptHandler}, trigger::{NearTrigger, ReceiptWithOutcome}, - Chain, }; use super::TriggersAdapter; @@ -440,7 +505,7 @@ mod test { #[test] fn validate_empty() { let ds = new_data_source(None, None); - let errs = ds.validate(); + let errs = ds.validate(LATEST_VERSION); assert_eq!(errs.len(), 1, "{:?}", ds); assert_eq!(errs[0].to_string(), "subgraph source address is required"); } @@ -448,7 +513,7 @@ mod test { #[test] fn validate_empty_account_none_partial() { let ds = new_data_source(None, Some(PartialAccounts::default())); - let errs = ds.validate(); + let errs = ds.validate(LATEST_VERSION); assert_eq!(errs.len(), 1, "{:?}", ds); assert_eq!(errs[0].to_string(), "subgraph source address is required"); } @@ -462,7 +527,7 @@ mod test { suffixes: vec!["x.near".to_string()], }), ); - let errs = ds.validate(); + let errs = ds.validate(LATEST_VERSION); assert_eq!(errs.len(), 0, "{:?}", ds); } @@ -476,18 +541,17 @@ mod test { }), ); let errs: Vec = ds - .validate() + .validate(LATEST_VERSION) .into_iter() .map(|err| err.to_string()) .collect(); assert_eq!(errs.len(), 2, "{:?}", ds); - let expected_errors = vec![ + let expected_errors = [ "partial account prefixes can't have empty values".to_string(), "partial account suffixes can't have empty values".to_string(), ]; - assert_eq!( - true, + assert!( expected_errors.iter().all(|err| errs.contains(err)), "{:?}", errs @@ -497,7 +561,7 @@ mod test { #[test] fn validate_empty_partials() { let ds = new_data_source(Some("x.near".to_string()), None); - let errs = ds.validate(); + let errs = ds.validate(LATEST_VERSION); assert_eq!(errs.len(), 0, "{:?}", ds); } @@ -573,8 +637,7 @@ mod test { case.name, receipt.partial_accounts, ); - assert_eq!( - true, + assert!( case.expected .iter() .all(|x| receipt.partial_accounts.contains(x)), @@ -742,7 +805,7 @@ mod test { } } - #[tokio::test] + #[graph::test] async fn test_trigger_filter_empty() { let account1: String = "account1".into(); @@ -760,7 +823,7 @@ mod test { assert_eq!(block_with_triggers.trigger_count(), 0); } - #[tokio::test] + #[graph::test] async fn test_trigger_filter_every_block() { let account1: String = "account1".into(); @@ -786,7 +849,7 @@ mod test { assert_eq!(height, vec![1]); } - #[tokio::test] + #[graph::test] async fn test_trigger_filter_every_receipt() { let account1: String = "account1".into(); @@ -818,14 +881,14 @@ mod test { .trigger_data .clone() .into_iter() - .filter_map(|x| match x { - crate::trigger::NearTrigger::Block(b) => b.header.clone().map(|x| x.height), + .filter_map(|x| match x.as_chain() { + Some(crate::trigger::NearTrigger::Block(b)) => b.header.clone().map(|x| x.height), _ => None, }) .collect() } - fn new_success_block(height: u64, receiver_id: &String) -> codec::Block { + fn new_success_block(height: u64, receiver_id: &str) -> codec::Block { codec::Block { header: Some(BlockHeader { height, @@ -837,12 +900,12 @@ mod test { receipt: Some(crate::codec::Receipt { receipt: Some(receipt::Receipt::Action(ReceiptAction { output_data_receivers: vec![DataReceiver { - receiver_id: receiver_id.clone(), + receiver_id: receiver_id.to_string(), ..Default::default() }], ..Default::default() })), - receiver_id: receiver_id.clone(), + receiver_id: receiver_id.to_string(), ..Default::default() }), execution_outcome: Some(ExecutionOutcomeWithId { @@ -873,6 +936,7 @@ mod test { source: crate::data_source::Source { account, start_block: 10, + end_block: None, accounts: partial_accounts, }, mapping: Mapping { @@ -891,7 +955,7 @@ mod test { } } - fn new_receipt_with_outcome(receiver_id: &String, block: Arc) -> ReceiptWithOutcome { + fn new_receipt_with_outcome(receiver_id: &str, block: Arc) -> ReceiptWithOutcome { ReceiptWithOutcome { outcome: ExecutionOutcomeWithId { outcome: Some(ExecutionOutcome { @@ -906,12 +970,12 @@ mod test { receipt: codec::Receipt { receipt: Some(receipt::Receipt::Action(ReceiptAction { output_data_receivers: vec![DataReceiver { - receiver_id: receiver_id.clone(), + receiver_id: receiver_id.to_string(), ..Default::default() }], ..Default::default() })), - receiver_id: receiver_id.clone(), + receiver_id: receiver_id.to_string(), ..Default::default() }, block, diff --git a/chain/near/src/codec.rs b/chain/near/src/codec.rs index 854e9dc1341..569e8cf5243 100644 --- a/chain/near/src/codec.rs +++ b/chain/near/src/codec.rs @@ -3,18 +3,17 @@ pub mod pbcodec; use graph::{ - blockchain::Block as BlockchainBlock, - blockchain::BlockPtr, - prelude::{hex, web3::types::H256, BlockNumber}, + blockchain::{Block as BlockchainBlock, BlockPtr, BlockTime}, + prelude::{BlockNumber, alloy::primitives::B256, hex}, }; use std::convert::TryFrom; use std::fmt::LowerHex; pub use pbcodec::*; -impl From<&CryptoHash> for H256 { +impl From<&CryptoHash> for B256 { fn from(input: &CryptoHash) -> Self { - H256::from_slice(&input.bytes) + B256::from_slice(&input.bytes) } } @@ -27,7 +26,7 @@ impl LowerHex for &CryptoHash { impl BlockHeader { pub fn parent_ptr(&self) -> Option { match (self.prev_hash.as_ref(), self.prev_height) { - (Some(hash), number) => Some(BlockPtr::from((H256::from(hash), number))), + (Some(hash), number) => Some(BlockPtr::from((B256::from(hash), number))), _ => None, } } @@ -35,7 +34,7 @@ impl BlockHeader { impl<'a> From<&'a BlockHeader> for BlockPtr { fn from(b: &'a BlockHeader) -> BlockPtr { - BlockPtr::from((H256::from(b.hash.as_ref().unwrap()), b.height)) + BlockPtr::from((B256::from(b.hash.as_ref().unwrap()), b.height)) } } @@ -71,6 +70,10 @@ impl BlockchainBlock for Block { fn parent_ptr(&self) -> Option { self.parent_ptr() } + + fn timestamp(&self) -> BlockTime { + block_time_from_header(self.header()) + } } impl HeaderOnlyBlock { @@ -97,6 +100,10 @@ impl BlockchainBlock for HeaderOnlyBlock { fn parent_ptr(&self) -> Option { self.header().parent_ptr() } + + fn timestamp(&self) -> BlockTime { + block_time_from_header(self.header()) + } } impl execution_outcome::Status { @@ -108,3 +115,25 @@ impl execution_outcome::Status { } } } + +fn block_time_from_header(header: &BlockHeader) -> BlockTime { + // The timstamp is in ns since the epoch + let ts = i64::try_from(header.timestamp_nanosec).unwrap(); + let secs = ts / 1_000_000_000; + let ns: u32 = (ts % 1_000_000_000) as u32; + BlockTime::since_epoch(secs, ns) +} + +#[test] +fn timestamp_conversion() { + // 2020-07-21T21:50:10Z in ns + let ts = 1_595_368_210_762_782_796; + let header = BlockHeader { + timestamp_nanosec: ts, + ..Default::default() + }; + assert_eq!( + 1595368210, + block_time_from_header(&header).as_secs_since_epoch() + ); +} diff --git a/chain/near/src/data_source.rs b/chain/near/src/data_source.rs index 0742b05e8c8..01484905a58 100644 --- a/chain/near/src/data_source.rs +++ b/chain/near/src/data_source.rs @@ -1,22 +1,26 @@ +use async_trait::async_trait; +use graph::anyhow::Context; use graph::blockchain::{Block, TriggerWithHandler}; +use graph::components::link_resolver::LinkResolverContext; use graph::components::store::StoredDynamicDataSource; -use graph::data::subgraph::DataSourceContext; +use graph::components::subgraph::InstanceDSTemplateInfo; +use graph::data::subgraph::{DataSourceContext, DeploymentHash}; use graph::prelude::SubgraphManifestValidationError; use graph::{ - anyhow::{anyhow, Error}, + anyhow::{Error, anyhow}, blockchain::{self, Blockchain}, - prelude::{ - async_trait, info, BlockNumber, CheapClone, DataSourceTemplateInfo, Deserialize, Link, - LinkResolver, Logger, - }, + prelude::{BlockNumber, CheapClone, Deserialize, Link, LinkResolver, Logger}, semver, }; +use std::collections::HashSet; use std::sync::Arc; use crate::chain::Chain; use crate::trigger::{NearTrigger, ReceiptWithOutcome}; pub const NEAR_KIND: &str = "near"; +const BLOCK_HANDLER_KIND: &str = "block"; +const RECEIPT_HANDLER_KIND: &str = "receipt"; /// Runtime representation of a data source. #[derive(Clone, Debug)] @@ -31,7 +35,10 @@ pub struct DataSource { } impl blockchain::DataSource for DataSource { - fn from_template_info(_template_info: DataSourceTemplateInfo) -> Result { + fn from_template_info( + _info: InstanceDSTemplateInfo, + _template: &graph::data_source::DataSourceTemplate, + ) -> Result { Err(anyhow!("Near subgraphs do not support templates")) // How this might be implemented if/when Near gets support for templates: @@ -74,6 +81,24 @@ impl blockchain::DataSource for DataSource { self.source.start_block } + fn handler_kinds(&self) -> HashSet<&str> { + let mut kinds = HashSet::new(); + + if self.handler_for_block().is_some() { + kinds.insert(BLOCK_HANDLER_KIND); + } + + if self.handler_for_receipt().is_some() { + kinds.insert(RECEIPT_HANDLER_KIND); + } + + kinds + } + + fn end_block(&self) -> Option { + self.source.end_block + } + fn match_and_decode( &self, trigger: &::TriggerData, @@ -141,6 +166,7 @@ impl blockchain::DataSource for DataSource { trigger.cheap_clone(), handler.clone(), block.ptr(), + block.timestamp(), ))) } @@ -202,7 +228,7 @@ impl blockchain::DataSource for DataSource { todo!() } - fn validate(&self) -> Vec { + fn validate(&self, _: &semver::Version) -> Vec { let mut errors = Vec::new(); if self.kind != NEAR_KIND { @@ -306,9 +332,11 @@ pub struct UnresolvedDataSource { impl blockchain::UnresolvedDataSource for UnresolvedDataSource { async fn resolve( self, + deployment_hash: &DeploymentHash, resolver: &Arc, logger: &Logger, _manifest_idx: u32, + _spec_version: &semver::Version, ) -> Result { let UnresolvedDataSource { kind, @@ -319,9 +347,12 @@ impl blockchain::UnresolvedDataSource for UnresolvedDataSource { context, } = self; - info!(logger, "Resolve data source"; "name" => &name, "source_account" => format_args!("{:?}", source.account), "source_start_block" => source.start_block); - - let mapping = mapping.resolve(resolver, logger).await?; + let mapping = mapping.resolve(deployment_hash, resolver, logger).await.with_context(|| { + format!( + "failed to resolve data source {} with source_account {:?} and source_start_block {}", + name, source.account, source.start_block + ) + })?; DataSource::from_manifest(kind, network, name, source, mapping, context) } @@ -342,9 +373,11 @@ pub type DataSourceTemplate = BaseDataSourceTemplate; impl blockchain::UnresolvedDataSourceTemplate for UnresolvedDataSourceTemplate { async fn resolve( self, + deployment_hash: &DeploymentHash, resolver: &Arc, logger: &Logger, _manifest_idx: u32, + _spec_version: &semver::Version, ) -> Result { let UnresolvedDataSourceTemplate { kind, @@ -353,13 +386,16 @@ impl blockchain::UnresolvedDataSourceTemplate for UnresolvedDataSourceTem mapping, } = self; - info!(logger, "Resolve data source template"; "name" => &name); + let mapping = mapping + .resolve(deployment_hash, resolver, logger) + .await + .with_context(|| format!("failed to resolve data source template {}", name))?; Ok(DataSourceTemplate { kind, network, name, - mapping: mapping.resolve(resolver, logger).await?, + mapping, }) } } @@ -380,6 +416,10 @@ impl blockchain::DataSourceTemplate for DataSourceTemplate { fn manifest_idx(&self) -> u32 { unreachable!("near does not support dynamic data sources") } + + fn kind(&self) -> &str { + &self.kind + } } #[derive(Clone, Debug, Default, Hash, Eq, PartialEq, Deserialize)] @@ -398,6 +438,7 @@ pub struct UnresolvedMapping { impl UnresolvedMapping { pub async fn resolve( self, + deployment_hash: &DeploymentHash, resolver: &Arc, logger: &Logger, ) -> Result { @@ -412,8 +453,10 @@ impl UnresolvedMapping { let api_version = semver::Version::parse(&api_version)?; - info!(logger, "Resolve mapping"; "link" => &link.link); - let module_bytes = resolver.cat(logger, &link).await?; + let module_bytes = resolver + .cat(&LinkResolverContext::new(deployment_hash, logger), &link) + .await + .with_context(|| format!("failed to resolve mapping {}", link.link))?; Ok(Mapping { api_version, @@ -463,10 +506,12 @@ impl PartialAccounts { } #[derive(Clone, Debug, Hash, Eq, PartialEq, Deserialize)] +#[serde(rename_all = "camelCase")] pub(crate) struct Source { // A data source that does not have an account or accounts can only have block handlers. pub(crate) account: Option, - #[serde(rename = "startBlock", default)] + #[serde(default)] pub(crate) start_block: BlockNumber, + pub(crate) end_block: Option, pub(crate) accounts: Option, } diff --git a/chain/near/src/runtime/abi.rs b/chain/near/src/runtime/abi.rs index f9142fa16c7..b74d8f5f7ff 100644 --- a/chain/near/src/runtime/abi.rs +++ b/chain/near/src/runtime/abi.rs @@ -1,189 +1,204 @@ use crate::codec; use crate::trigger::ReceiptWithOutcome; +use async_trait::async_trait; use graph::anyhow::anyhow; use graph::runtime::gas::GasCounter; -use graph::runtime::{asc_new, AscHeap, AscPtr, DeterministicHostError, ToAscObj}; +use graph::runtime::{AscHeap, AscPtr, DeterministicHostError, HostExportError, ToAscObj, asc_new}; use graph_runtime_wasm::asc_abi::class::{Array, AscEnum, EnumPayload, Uint8Array}; pub(crate) use super::generated::*; +#[async_trait] impl ToAscObj for codec::Block { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, - ) -> Result { + ) -> Result { Ok(AscBlock { - author: asc_new(heap, &self.author, gas)?, - header: asc_new(heap, self.header(), gas)?, - chunks: asc_new(heap, &self.chunk_headers, gas)?, + author: asc_new(heap, &self.author, gas).await?, + header: asc_new(heap, self.header(), gas).await?, + chunks: asc_new(heap, &self.chunk_headers, gas).await?, }) } } +#[async_trait] impl ToAscObj for codec::BlockHeader { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, - ) -> Result { - let chunk_mask = Array::new(self.chunk_mask.as_ref(), heap, gas)?; + ) -> Result { + let chunk_mask = Array::new(self.chunk_mask.as_ref(), heap, gas).await?; Ok(AscBlockHeader { height: self.height, prev_height: self.prev_height, - epoch_id: asc_new(heap, self.epoch_id.as_ref().unwrap(), gas)?, - next_epoch_id: asc_new(heap, self.next_epoch_id.as_ref().unwrap(), gas)?, - hash: asc_new(heap, self.hash.as_ref().unwrap(), gas)?, - prev_hash: asc_new(heap, self.prev_hash.as_ref().unwrap(), gas)?, - prev_state_root: asc_new(heap, self.prev_state_root.as_ref().unwrap(), gas)?, - chunk_receipts_root: asc_new(heap, self.chunk_receipts_root.as_ref().unwrap(), gas)?, - chunk_headers_root: asc_new(heap, self.chunk_headers_root.as_ref().unwrap(), gas)?, - chunk_tx_root: asc_new(heap, self.chunk_tx_root.as_ref().unwrap(), gas)?, - outcome_root: asc_new(heap, self.outcome_root.as_ref().unwrap(), gas)?, + epoch_id: asc_new(heap, self.epoch_id.as_ref().unwrap(), gas).await?, + next_epoch_id: asc_new(heap, self.next_epoch_id.as_ref().unwrap(), gas).await?, + hash: asc_new(heap, self.hash.as_ref().unwrap(), gas).await?, + prev_hash: asc_new(heap, self.prev_hash.as_ref().unwrap(), gas).await?, + prev_state_root: asc_new(heap, self.prev_state_root.as_ref().unwrap(), gas).await?, + chunk_receipts_root: asc_new(heap, self.chunk_receipts_root.as_ref().unwrap(), gas) + .await?, + chunk_headers_root: asc_new(heap, self.chunk_headers_root.as_ref().unwrap(), gas) + .await?, + chunk_tx_root: asc_new(heap, self.chunk_tx_root.as_ref().unwrap(), gas).await?, + outcome_root: asc_new(heap, self.outcome_root.as_ref().unwrap(), gas).await?, chunks_included: self.chunks_included, - challenges_root: asc_new(heap, self.challenges_root.as_ref().unwrap(), gas)?, + challenges_root: asc_new(heap, self.challenges_root.as_ref().unwrap(), gas).await?, timestamp_nanosec: self.timestamp_nanosec, - random_value: asc_new(heap, self.random_value.as_ref().unwrap(), gas)?, - validator_proposals: asc_new(heap, &self.validator_proposals, gas)?, - chunk_mask: AscPtr::alloc_obj(chunk_mask, heap, gas)?, - gas_price: asc_new(heap, self.gas_price.as_ref().unwrap(), gas)?, + random_value: asc_new(heap, self.random_value.as_ref().unwrap(), gas).await?, + validator_proposals: asc_new(heap, &self.validator_proposals, gas).await?, + chunk_mask: AscPtr::alloc_obj(chunk_mask, heap, gas).await?, + gas_price: asc_new(heap, self.gas_price.as_ref().unwrap(), gas).await?, block_ordinal: self.block_ordinal, - total_supply: asc_new(heap, self.total_supply.as_ref().unwrap(), gas)?, - challenges_result: asc_new(heap, &self.challenges_result, gas)?, - last_final_block: asc_new(heap, self.last_final_block.as_ref().unwrap(), gas)?, - last_ds_final_block: asc_new(heap, self.last_ds_final_block.as_ref().unwrap(), gas)?, - next_bp_hash: asc_new(heap, self.next_bp_hash.as_ref().unwrap(), gas)?, - block_merkle_root: asc_new(heap, self.block_merkle_root.as_ref().unwrap(), gas)?, - epoch_sync_data_hash: asc_new(heap, self.epoch_sync_data_hash.as_slice(), gas)?, - approvals: asc_new(heap, &self.approvals, gas)?, - signature: asc_new(heap, &self.signature.as_ref().unwrap(), gas)?, + total_supply: asc_new(heap, self.total_supply.as_ref().unwrap(), gas).await?, + challenges_result: asc_new(heap, &self.challenges_result, gas).await?, + last_final_block: asc_new(heap, self.last_final_block.as_ref().unwrap(), gas).await?, + last_ds_final_block: asc_new(heap, self.last_ds_final_block.as_ref().unwrap(), gas) + .await?, + next_bp_hash: asc_new(heap, self.next_bp_hash.as_ref().unwrap(), gas).await?, + block_merkle_root: asc_new(heap, self.block_merkle_root.as_ref().unwrap(), gas).await?, + epoch_sync_data_hash: asc_new(heap, self.epoch_sync_data_hash.as_slice(), gas).await?, + approvals: asc_new(heap, &self.approvals, gas).await?, + signature: asc_new(heap, &self.signature.as_ref().unwrap(), gas).await?, latest_protocol_version: self.latest_protocol_version, }) } } +#[async_trait] impl ToAscObj for codec::ChunkHeader { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, - ) -> Result { + ) -> Result { Ok(AscChunkHeader { - chunk_hash: asc_new(heap, self.chunk_hash.as_slice(), gas)?, - signature: asc_new(heap, &self.signature.as_ref().unwrap(), gas)?, - prev_block_hash: asc_new(heap, self.prev_block_hash.as_slice(), gas)?, - prev_state_root: asc_new(heap, self.prev_state_root.as_slice(), gas)?, - encoded_merkle_root: asc_new(heap, self.encoded_merkle_root.as_slice(), gas)?, + chunk_hash: asc_new(heap, self.chunk_hash.as_slice(), gas).await?, + signature: asc_new(heap, &self.signature.as_ref().unwrap(), gas).await?, + prev_block_hash: asc_new(heap, self.prev_block_hash.as_slice(), gas).await?, + prev_state_root: asc_new(heap, self.prev_state_root.as_slice(), gas).await?, + encoded_merkle_root: asc_new(heap, self.encoded_merkle_root.as_slice(), gas).await?, encoded_length: self.encoded_length, height_created: self.height_created, height_included: self.height_included, shard_id: self.shard_id, gas_used: self.gas_used, gas_limit: self.gas_limit, - balance_burnt: asc_new(heap, self.balance_burnt.as_ref().unwrap(), gas)?, - outgoing_receipts_root: asc_new(heap, self.outgoing_receipts_root.as_slice(), gas)?, - tx_root: asc_new(heap, self.tx_root.as_slice(), gas)?, - validator_proposals: asc_new(heap, &self.validator_proposals, gas)?, + balance_burnt: asc_new(heap, self.balance_burnt.as_ref().unwrap(), gas).await?, + outgoing_receipts_root: asc_new(heap, self.outgoing_receipts_root.as_slice(), gas) + .await?, + tx_root: asc_new(heap, self.tx_root.as_slice(), gas).await?, + validator_proposals: asc_new(heap, &self.validator_proposals, gas).await?, _padding: 0, }) } } +#[async_trait] impl ToAscObj for Vec { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, - ) -> Result { - let content: Result, _> = self.iter().map(|x| asc_new(heap, x, gas)).collect(); - let content = content?; - Ok(AscChunkHeaderArray(Array::new(&content, heap, gas)?)) + ) -> Result { + let mut content = Vec::new(); + for x in self { + content.push(asc_new(heap, x, gas).await?); + } + Ok(AscChunkHeaderArray(Array::new(&content, heap, gas).await?)) } } +#[async_trait] impl ToAscObj for ReceiptWithOutcome { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, - ) -> Result { + ) -> Result { Ok(AscReceiptWithOutcome { - outcome: asc_new(heap, &self.outcome, gas)?, - receipt: asc_new(heap, &self.receipt, gas)?, - block: asc_new(heap, self.block.as_ref(), gas)?, + outcome: asc_new(heap, &self.outcome, gas).await?, + receipt: asc_new(heap, &self.receipt, gas).await?, + block: asc_new(heap, self.block.as_ref(), gas).await?, }) } } +#[async_trait] impl ToAscObj for codec::Receipt { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, - ) -> Result { + ) -> Result { let action = match self.receipt.as_ref().unwrap() { codec::receipt::Receipt::Action(action) => action, codec::receipt::Receipt::Data(_) => { - return Err(DeterministicHostError::from(anyhow!( - "Data receipt are now allowed" - ))); + return Err( + DeterministicHostError::from(anyhow!("Data receipt are now allowed")).into(), + ); } }; Ok(AscActionReceipt { - id: asc_new(heap, &self.receipt_id.as_ref().unwrap(), gas)?, - predecessor_id: asc_new(heap, &self.predecessor_id, gas)?, - receiver_id: asc_new(heap, &self.receiver_id, gas)?, - signer_id: asc_new(heap, &action.signer_id, gas)?, - signer_public_key: asc_new(heap, action.signer_public_key.as_ref().unwrap(), gas)?, - gas_price: asc_new(heap, action.gas_price.as_ref().unwrap(), gas)?, - output_data_receivers: asc_new(heap, &action.output_data_receivers, gas)?, - input_data_ids: asc_new(heap, &action.input_data_ids, gas)?, - actions: asc_new(heap, &action.actions, gas)?, + id: asc_new(heap, &self.receipt_id.as_ref().unwrap(), gas).await?, + predecessor_id: asc_new(heap, &self.predecessor_id, gas).await?, + receiver_id: asc_new(heap, &self.receiver_id, gas).await?, + signer_id: asc_new(heap, &action.signer_id, gas).await?, + signer_public_key: asc_new(heap, action.signer_public_key.as_ref().unwrap(), gas) + .await?, + gas_price: asc_new(heap, action.gas_price.as_ref().unwrap(), gas).await?, + output_data_receivers: asc_new(heap, &action.output_data_receivers, gas).await?, + input_data_ids: asc_new(heap, &action.input_data_ids, gas).await?, + actions: asc_new(heap, &action.actions, gas).await?, }) } } +#[async_trait] impl ToAscObj for codec::Action { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, - ) -> Result { + ) -> Result { let (kind, payload) = match self.action.as_ref().unwrap() { codec::action::Action::CreateAccount(action) => ( AscActionKind::CreateAccount, - asc_new(heap, action, gas)?.to_payload(), + asc_new(heap, action, gas).await?.to_payload(), ), codec::action::Action::DeployContract(action) => ( AscActionKind::DeployContract, - asc_new(heap, action, gas)?.to_payload(), + asc_new(heap, action, gas).await?.to_payload(), ), codec::action::Action::FunctionCall(action) => ( AscActionKind::FunctionCall, - asc_new(heap, action, gas)?.to_payload(), + asc_new(heap, action, gas).await?.to_payload(), ), codec::action::Action::Transfer(action) => ( AscActionKind::Transfer, - asc_new(heap, action, gas)?.to_payload(), + asc_new(heap, action, gas).await?.to_payload(), ), codec::action::Action::Stake(action) => ( AscActionKind::Stake, - asc_new(heap, action, gas)?.to_payload(), + asc_new(heap, action, gas).await?.to_payload(), ), codec::action::Action::AddKey(action) => ( AscActionKind::AddKey, - asc_new(heap, action, gas)?.to_payload(), + asc_new(heap, action, gas).await?.to_payload(), ), codec::action::Action::DeleteKey(action) => ( AscActionKind::DeleteKey, - asc_new(heap, action, gas)?.to_payload(), + asc_new(heap, action, gas).await?.to_payload(), ), codec::action::Action::DeleteAccount(action) => ( AscActionKind::DeleteAccount, - asc_new(heap, action, gas)?.to_payload(), + asc_new(heap, action, gas).await?.to_payload(), ), }; @@ -195,122 +210,133 @@ impl ToAscObj for codec::Action { } } +#[async_trait] impl ToAscObj for Vec { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, - ) -> Result { - let content: Result, _> = self.iter().map(|x| asc_new(heap, x, gas)).collect(); - let content = content?; - Ok(AscActionEnumArray(Array::new(&content, heap, gas)?)) + ) -> Result { + let mut content = Vec::new(); + for x in self { + content.push(asc_new(heap, x, gas).await?); + } + Ok(AscActionEnumArray(Array::new(&content, heap, gas).await?)) } } +#[async_trait] impl ToAscObj for codec::CreateAccountAction { - fn to_asc_obj( + async fn to_asc_obj( &self, _heap: &mut H, _gas: &GasCounter, - ) -> Result { + ) -> Result { Ok(AscCreateAccountAction {}) } } +#[async_trait] impl ToAscObj for codec::DeployContractAction { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, - ) -> Result { + ) -> Result { Ok(AscDeployContractAction { - code: asc_new(heap, self.code.as_slice(), gas)?, + code: asc_new(heap, self.code.as_slice(), gas).await?, }) } } +#[async_trait] impl ToAscObj for codec::FunctionCallAction { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, - ) -> Result { + ) -> Result { Ok(AscFunctionCallAction { - method_name: asc_new(heap, &self.method_name, gas)?, - args: asc_new(heap, self.args.as_slice(), gas)?, + method_name: asc_new(heap, &self.method_name, gas).await?, + args: asc_new(heap, self.args.as_slice(), gas).await?, gas: self.gas, - deposit: asc_new(heap, self.deposit.as_ref().unwrap(), gas)?, + deposit: asc_new(heap, self.deposit.as_ref().unwrap(), gas).await?, _padding: 0, }) } } +#[async_trait] impl ToAscObj for codec::TransferAction { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, - ) -> Result { + ) -> Result { Ok(AscTransferAction { - deposit: asc_new(heap, self.deposit.as_ref().unwrap(), gas)?, + deposit: asc_new(heap, self.deposit.as_ref().unwrap(), gas).await?, }) } } +#[async_trait] impl ToAscObj for codec::StakeAction { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, - ) -> Result { + ) -> Result { Ok(AscStakeAction { - stake: asc_new(heap, self.stake.as_ref().unwrap(), gas)?, - public_key: asc_new(heap, self.public_key.as_ref().unwrap(), gas)?, + stake: asc_new(heap, self.stake.as_ref().unwrap(), gas).await?, + public_key: asc_new(heap, self.public_key.as_ref().unwrap(), gas).await?, }) } } +#[async_trait] impl ToAscObj for codec::AddKeyAction { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, - ) -> Result { + ) -> Result { Ok(AscAddKeyAction { - public_key: asc_new(heap, self.public_key.as_ref().unwrap(), gas)?, - access_key: asc_new(heap, self.access_key.as_ref().unwrap(), gas)?, + public_key: asc_new(heap, self.public_key.as_ref().unwrap(), gas).await?, + access_key: asc_new(heap, self.access_key.as_ref().unwrap(), gas).await?, }) } } +#[async_trait] impl ToAscObj for codec::AccessKey { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, - ) -> Result { + ) -> Result { Ok(AscAccessKey { nonce: self.nonce, - permission: asc_new(heap, self.permission.as_ref().unwrap(), gas)?, + permission: asc_new(heap, self.permission.as_ref().unwrap(), gas).await?, _padding: 0, }) } } +#[async_trait] impl ToAscObj for codec::AccessKeyPermission { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, - ) -> Result { + ) -> Result { let (kind, payload) = match self.permission.as_ref().unwrap() { codec::access_key_permission::Permission::FunctionCall(permission) => ( AscAccessKeyPermissionKind::FunctionCall, - asc_new(heap, permission, gas)?.to_payload(), + asc_new(heap, permission, gas).await?.to_payload(), ), codec::access_key_permission::Permission::FullAccess(permission) => ( AscAccessKeyPermissionKind::FullAccess, - asc_new(heap, permission, gas)?.to_payload(), + asc_new(heap, permission, gas).await?.to_payload(), ), }; @@ -322,133 +348,147 @@ impl ToAscObj for codec::AccessKeyPermission { } } +#[async_trait] impl ToAscObj for codec::FunctionCallPermission { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, - ) -> Result { + ) -> Result { Ok(AscFunctionCallPermission { // The `allowance` field is one of the few fields that can actually be None for real allowance: match self.allowance.as_ref() { - Some(allowance) => asc_new(heap, allowance, gas)?, + Some(allowance) => asc_new(heap, allowance, gas).await?, None => AscPtr::null(), }, - receiver_id: asc_new(heap, &self.receiver_id, gas)?, - method_names: asc_new(heap, &self.method_names, gas)?, + receiver_id: asc_new(heap, &self.receiver_id, gas).await?, + method_names: asc_new(heap, &self.method_names, gas).await?, }) } } +#[async_trait] impl ToAscObj for codec::FullAccessPermission { - fn to_asc_obj( + async fn to_asc_obj( &self, _heap: &mut H, _gas: &GasCounter, - ) -> Result { + ) -> Result { Ok(AscFullAccessPermission {}) } } +#[async_trait] impl ToAscObj for codec::DeleteKeyAction { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, - ) -> Result { + ) -> Result { Ok(AscDeleteKeyAction { - public_key: asc_new(heap, self.public_key.as_ref().unwrap(), gas)?, + public_key: asc_new(heap, self.public_key.as_ref().unwrap(), gas).await?, }) } } +#[async_trait] impl ToAscObj for codec::DeleteAccountAction { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, - ) -> Result { + ) -> Result { Ok(AscDeleteAccountAction { - beneficiary_id: asc_new(heap, &self.beneficiary_id, gas)?, + beneficiary_id: asc_new(heap, &self.beneficiary_id, gas).await?, }) } } +#[async_trait] impl ToAscObj for codec::DataReceiver { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, - ) -> Result { + ) -> Result { Ok(AscDataReceiver { - data_id: asc_new(heap, self.data_id.as_ref().unwrap(), gas)?, - receiver_id: asc_new(heap, &self.receiver_id, gas)?, + data_id: asc_new(heap, self.data_id.as_ref().unwrap(), gas).await?, + receiver_id: asc_new(heap, &self.receiver_id, gas).await?, }) } } +#[async_trait] impl ToAscObj for Vec { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, - ) -> Result { - let content: Result, _> = self.iter().map(|x| asc_new(heap, x, gas)).collect(); - let content = content?; - Ok(AscDataReceiverArray(Array::new(&content, heap, gas)?)) + ) -> Result { + let mut content = Vec::new(); + for x in self { + content.push(asc_new(heap, x, gas).await?); + } + Ok(AscDataReceiverArray(Array::new(&content, heap, gas).await?)) } } +#[async_trait] impl ToAscObj for codec::ExecutionOutcomeWithId { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, - ) -> Result { + ) -> Result { let outcome = self.outcome.as_ref().unwrap(); Ok(AscExecutionOutcome { - proof: asc_new(heap, &self.proof.as_ref().unwrap().path, gas)?, - block_hash: asc_new(heap, self.block_hash.as_ref().unwrap(), gas)?, - id: asc_new(heap, self.id.as_ref().unwrap(), gas)?, - logs: asc_new(heap, &outcome.logs, gas)?, - receipt_ids: asc_new(heap, &outcome.receipt_ids, gas)?, + proof: asc_new(heap, &self.proof.as_ref().unwrap().path, gas).await?, + block_hash: asc_new(heap, self.block_hash.as_ref().unwrap(), gas).await?, + id: asc_new(heap, self.id.as_ref().unwrap(), gas).await?, + logs: asc_new(heap, &outcome.logs, gas).await?, + receipt_ids: asc_new(heap, &outcome.receipt_ids, gas).await?, gas_burnt: outcome.gas_burnt, - tokens_burnt: asc_new(heap, outcome.tokens_burnt.as_ref().unwrap(), gas)?, - executor_id: asc_new(heap, &outcome.executor_id, gas)?, - status: asc_new(heap, outcome.status.as_ref().unwrap(), gas)?, + tokens_burnt: asc_new(heap, outcome.tokens_burnt.as_ref().unwrap(), gas).await?, + executor_id: asc_new(heap, &outcome.executor_id, gas).await?, + status: asc_new(heap, outcome.status.as_ref().unwrap(), gas).await?, }) } } +#[async_trait] impl ToAscObj for codec::execution_outcome::Status { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, - ) -> Result { + ) -> Result { let (kind, payload) = match self { codec::execution_outcome::Status::SuccessValue(value) => { let bytes = &value.value; ( AscSuccessStatusKind::Value, - asc_new(heap, bytes.as_slice(), gas)?.to_payload(), + asc_new(heap, bytes.as_slice(), gas).await?.to_payload(), ) } codec::execution_outcome::Status::SuccessReceiptId(receipt_id) => ( AscSuccessStatusKind::ReceiptId, - asc_new(heap, receipt_id.id.as_ref().unwrap(), gas)?.to_payload(), + asc_new(heap, receipt_id.id.as_ref().unwrap(), gas) + .await? + .to_payload(), ), codec::execution_outcome::Status::Failure(_) => { return Err(DeterministicHostError::from(anyhow!( "Failure execution status are not allowed" - ))); + )) + .into()); } codec::execution_outcome::Status::Unknown(_) => { return Err(DeterministicHostError::from(anyhow!( "Unknown execution status are not allowed" - ))); + )) + .into()); } }; @@ -460,14 +500,15 @@ impl ToAscObj for codec::execution_outcome::Status { } } +#[async_trait] impl ToAscObj for codec::MerklePathItem { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, - ) -> Result { + ) -> Result { Ok(AscMerklePathItem { - hash: asc_new(heap, self.hash.as_ref().unwrap(), gas)?, + hash: asc_new(heap, self.hash.as_ref().unwrap(), gas).await?, direction: match self.direction { 0 => AscDirection::Left, 1 => AscDirection::Right, @@ -475,31 +516,38 @@ impl ToAscObj for codec::MerklePathItem { return Err(DeterministicHostError::from(anyhow!( "Invalid direction value {}", x - ))) + )) + .into()); } }, }) } } +#[async_trait] impl ToAscObj for Vec { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, - ) -> Result { - let content: Result, _> = self.iter().map(|x| asc_new(heap, x, gas)).collect(); - let content = content?; - Ok(AscMerklePathItemArray(Array::new(&content, heap, gas)?)) + ) -> Result { + let mut content = Vec::new(); + for x in self { + content.push(asc_new(heap, x, gas).await?); + } + Ok(AscMerklePathItemArray( + Array::new(&content, heap, gas).await?, + )) } } +#[async_trait] impl ToAscObj for codec::Signature { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, - ) -> Result { + ) -> Result { Ok(AscSignature { kind: match self.r#type { 0 => 0, @@ -508,32 +556,37 @@ impl ToAscObj for codec::Signature { return Err(DeterministicHostError::from(anyhow!( "Invalid signature type {}", value, - ))) + )) + .into()); } }, - bytes: asc_new(heap, self.bytes.as_slice(), gas)?, + bytes: asc_new(heap, self.bytes.as_slice(), gas).await?, }) } } +#[async_trait] impl ToAscObj for Vec { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, - ) -> Result { - let content: Result, _> = self.iter().map(|x| asc_new(heap, x, gas)).collect(); - let content = content?; - Ok(AscSignatureArray(Array::new(&content, heap, gas)?)) + ) -> Result { + let mut content = Vec::new(); + for x in self { + content.push(asc_new(heap, x, gas).await?); + } + Ok(AscSignatureArray(Array::new(&content, heap, gas).await?)) } } +#[async_trait] impl ToAscObj for codec::PublicKey { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, - ) -> Result { + ) -> Result { Ok(AscPublicKey { kind: match self.r#type { 0 => 0, @@ -542,96 +595,114 @@ impl ToAscObj for codec::PublicKey { return Err(DeterministicHostError::from(anyhow!( "Invalid public key type {}", value, - ))) + )) + .into()); } }, - bytes: asc_new(heap, self.bytes.as_slice(), gas)?, + bytes: asc_new(heap, self.bytes.as_slice(), gas).await?, }) } } +#[async_trait] impl ToAscObj for codec::ValidatorStake { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, - ) -> Result { + ) -> Result { Ok(AscValidatorStake { - account_id: asc_new(heap, &self.account_id, gas)?, - public_key: asc_new(heap, self.public_key.as_ref().unwrap(), gas)?, - stake: asc_new(heap, self.stake.as_ref().unwrap(), gas)?, + account_id: asc_new(heap, &self.account_id, gas).await?, + public_key: asc_new(heap, self.public_key.as_ref().unwrap(), gas).await?, + stake: asc_new(heap, self.stake.as_ref().unwrap(), gas).await?, }) } } +#[async_trait] impl ToAscObj for Vec { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, - ) -> Result { - let content: Result, _> = self.iter().map(|x| asc_new(heap, x, gas)).collect(); - let content = content?; - Ok(AscValidatorStakeArray(Array::new(&content, heap, gas)?)) + ) -> Result { + let mut content = Vec::new(); + for x in self { + content.push(asc_new(heap, x, gas).await?); + } + Ok(AscValidatorStakeArray( + Array::new(&content, heap, gas).await?, + )) } } +#[async_trait] impl ToAscObj for codec::SlashedValidator { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, - ) -> Result { + ) -> Result { Ok(AscSlashedValidator { - account_id: asc_new(heap, &self.account_id, gas)?, + account_id: asc_new(heap, &self.account_id, gas).await?, is_double_sign: self.is_double_sign, }) } } +#[async_trait] impl ToAscObj for Vec { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, - ) -> Result { - let content: Result, _> = self.iter().map(|x| asc_new(heap, x, gas)).collect(); - let content = content?; - Ok(AscSlashedValidatorArray(Array::new(&content, heap, gas)?)) + ) -> Result { + let mut content = Vec::new(); + for x in self { + content.push(asc_new(heap, x, gas).await?); + } + Ok(AscSlashedValidatorArray( + Array::new(&content, heap, gas).await?, + )) } } +#[async_trait] impl ToAscObj for codec::CryptoHash { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, - ) -> Result { - self.bytes.to_asc_obj(heap, gas) + ) -> Result { + self.bytes.to_asc_obj(heap, gas).await } } +#[async_trait] impl ToAscObj for Vec { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, - ) -> Result { - let content: Result, _> = self.iter().map(|x| asc_new(heap, x, gas)).collect(); - let content = content?; - Ok(AscCryptoHashArray(Array::new(&content, heap, gas)?)) + ) -> Result { + let mut content = Vec::new(); + for x in self { + content.push(asc_new(heap, x, gas).await?); + } + Ok(AscCryptoHashArray(Array::new(&content, heap, gas).await?)) } } +#[async_trait] impl ToAscObj for codec::BigInt { - fn to_asc_obj( + async fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, - ) -> Result { + ) -> Result { // Bytes are reversed to align with BigInt bytes endianess let reversed: Vec = self.bytes.iter().rev().copied().collect(); - reversed.to_asc_obj(heap, gas) + reversed.to_asc_obj(heap, gas).await } } diff --git a/chain/near/src/runtime/generated.rs b/chain/near/src/runtime/generated.rs index 153eb8b5ab5..d8fe2937f43 100644 --- a/chain/near/src/runtime/generated.rs +++ b/chain/near/src/runtime/generated.rs @@ -227,20 +227,15 @@ impl AscIndexId for AscSignature { } #[repr(u32)] -#[derive(AscType, Copy, Clone)] +#[derive(AscType, Copy, Clone, Default)] pub(crate) enum AscAccessKeyPermissionKind { + #[default] FunctionCall, FullAccess, } impl AscValue for AscAccessKeyPermissionKind {} -impl Default for AscAccessKeyPermissionKind { - fn default() -> Self { - Self::FunctionCall - } -} - #[repr(C)] #[derive(AscType)] pub(crate) struct AscFunctionCallPermission { @@ -293,8 +288,9 @@ impl AscIndexId for AscDataReceiver { } #[repr(u32)] -#[derive(AscType, Copy, Clone)] +#[derive(AscType, Copy, Clone, Default)] pub(crate) enum AscActionKind { + #[default] CreateAccount, DeployContract, FunctionCall, @@ -307,12 +303,6 @@ pub(crate) enum AscActionKind { impl AscValue for AscActionKind {} -impl Default for AscActionKind { - fn default() -> Self { - Self::CreateAccount - } -} - #[repr(C)] #[derive(AscType)] pub(crate) struct AscCreateAccountAction {} @@ -424,20 +414,15 @@ impl AscIndexId for AscActionReceipt { } #[repr(u32)] -#[derive(AscType, Copy, Clone)] +#[derive(AscType, Copy, Clone, Default)] pub(crate) enum AscSuccessStatusKind { + #[default] Value, ReceiptId, } impl AscValue for AscSuccessStatusKind {} -impl Default for AscSuccessStatusKind { - fn default() -> Self { - Self::Value - } -} - pub struct AscSuccessStatusEnum(pub(crate) AscEnum); impl AscType for AscSuccessStatusEnum { @@ -458,20 +443,15 @@ impl AscIndexId for AscSuccessStatusEnum { } #[repr(u32)] -#[derive(AscType, Copy, Clone)] +#[derive(AscType, Copy, Clone, Default)] pub(crate) enum AscDirection { + #[default] Left, Right, } impl AscValue for AscDirection {} -impl Default for AscDirection { - fn default() -> Self { - Self::Left - } -} - #[repr(C)] #[derive(AscType)] pub(crate) struct AscMerklePathItem { diff --git a/chain/near/src/runtime/mod.rs b/chain/near/src/runtime/mod.rs index f44391caffd..31e18de7dd8 100644 --- a/chain/near/src/runtime/mod.rs +++ b/chain/near/src/runtime/mod.rs @@ -1,6 +1,3 @@ -pub use runtime_adapter::RuntimeAdapter; - pub mod abi; -pub mod runtime_adapter; mod generated; diff --git a/chain/near/src/runtime/runtime_adapter.rs b/chain/near/src/runtime/runtime_adapter.rs deleted file mode 100644 index c5fa9e15059..00000000000 --- a/chain/near/src/runtime/runtime_adapter.rs +++ /dev/null @@ -1,11 +0,0 @@ -use crate::{data_source::DataSource, Chain}; -use blockchain::HostFn; -use graph::{anyhow::Error, blockchain}; - -pub struct RuntimeAdapter {} - -impl blockchain::RuntimeAdapter for RuntimeAdapter { - fn host_fns(&self, _ds: &DataSource) -> Result, Error> { - Ok(vec![]) - } -} diff --git a/chain/near/src/trigger.rs b/chain/near/src/trigger.rs index 6fc31e8aefe..bf9794d2b20 100644 --- a/chain/near/src/trigger.rs +++ b/chain/near/src/trigger.rs @@ -1,10 +1,13 @@ +use async_trait::async_trait; use graph::blockchain::Block; +use graph::blockchain::MappingTriggerTrait; use graph::blockchain::TriggerData; -use graph::cheap_clone::CheapClone; -use graph::prelude::hex; -use graph::prelude::web3::types::H256; +use graph::derive::CheapClone; use graph::prelude::BlockNumber; -use graph::runtime::{asc_new, gas::GasCounter, AscHeap, AscPtr, DeterministicHostError}; +use graph::prelude::alloy::primitives::B256; +use graph::prelude::hex; +use graph::runtime::HostExportError; +use graph::runtime::{AscHeap, AscPtr, asc_new, gas::GasCounter}; use graph_runtime_wasm::module::ToAscPtr; use std::{cmp::Ordering, sync::Arc}; @@ -13,6 +16,7 @@ use crate::codec; // Logging the block is too verbose, so this strips the block from the trigger for Debug. impl std::fmt::Debug for NearTrigger { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + #[allow(unused)] #[derive(Debug)] pub enum MappingTriggerWithoutBlock<'a> { Block, @@ -35,34 +39,26 @@ impl std::fmt::Debug for NearTrigger { } } +#[async_trait] impl ToAscPtr for NearTrigger { - fn to_asc_ptr( + async fn to_asc_ptr( self, heap: &mut H, gas: &GasCounter, - ) -> Result, DeterministicHostError> { + ) -> Result, HostExportError> { Ok(match self { - NearTrigger::Block(block) => asc_new(heap, block.as_ref(), gas)?.erase(), - NearTrigger::Receipt(receipt) => asc_new(heap, receipt.as_ref(), gas)?.erase(), + NearTrigger::Block(block) => asc_new(heap, block.as_ref(), gas).await?.erase(), + NearTrigger::Receipt(receipt) => asc_new(heap, receipt.as_ref(), gas).await?.erase(), }) } } -#[derive(Clone)] +#[derive(Clone, CheapClone)] pub enum NearTrigger { Block(Arc), Receipt(Arc), } -impl CheapClone for NearTrigger { - fn cheap_clone(&self) -> NearTrigger { - match self { - NearTrigger::Block(block) => NearTrigger::Block(block.cheap_clone()), - NearTrigger::Receipt(receipt) => NearTrigger::Receipt(receipt.cheap_clone()), - } - } -} - impl PartialEq for NearTrigger { fn eq(&self, other: &Self) -> bool { match (self, other) { @@ -84,10 +80,26 @@ impl NearTrigger { } } - pub fn block_hash(&self) -> H256 { + pub fn block_hash(&self) -> B256 { + match self { + NearTrigger::Block(block) => block.ptr().hash.as_b256(), + NearTrigger::Receipt(receipt) => receipt.block.ptr().hash.as_b256(), + } + } + + fn error_context(&self) -> std::string::String { match self { - NearTrigger::Block(block) => block.ptr().hash_as_h256(), - NearTrigger::Receipt(receipt) => receipt.block.ptr().hash_as_h256(), + NearTrigger::Block(..) => { + format!("Block #{} ({})", self.block_number(), self.block_hash()) + } + NearTrigger::Receipt(receipt) => { + format!( + "receipt id {}, block #{} ({})", + hex::encode(&receipt.receipt.receipt_id.as_ref().unwrap().bytes), + self.block_number(), + self.block_hash() + ) + } } } } @@ -116,20 +128,18 @@ impl PartialOrd for NearTrigger { } impl TriggerData for NearTrigger { - fn error_context(&self) -> std::string::String { - match self { - NearTrigger::Block(..) => { - format!("Block #{} ({})", self.block_number(), self.block_hash()) - } - NearTrigger::Receipt(receipt) => { - format!( - "receipt id {}, block #{} ({})", - hex::encode(&receipt.receipt.receipt_id.as_ref().unwrap().bytes), - self.block_number(), - self.block_hash() - ) - } - } + fn error_context(&self) -> String { + self.error_context() + } + + fn address_match(&self) -> Option<&[u8]> { + None + } +} + +impl MappingTriggerTrait for NearTrigger { + fn error_context(&self) -> String { + self.error_context() } } @@ -142,29 +152,30 @@ pub struct ReceiptWithOutcome { #[cfg(test)] mod tests { - use std::convert::TryFrom; - use super::*; use graph::{ anyhow::anyhow, + components::metrics::gas::GasMetrics, data::subgraph::API_VERSION_0_0_5, - prelude::{hex, BigInt}, - runtime::gas::GasCounter, + prelude::{BigInt, hex}, + runtime::{DeterministicHostError, HostExportError, gas::GasCounter}, util::mem::init_slice, }; - #[test] - fn block_trigger_to_asc_ptr() { + #[graph::test] + async fn block_trigger_to_asc_ptr() { let mut heap = BytesHeap::new(API_VERSION_0_0_5); let trigger = NearTrigger::Block(Arc::new(block())); - let result = trigger.to_asc_ptr(&mut heap, &GasCounter::default()); + let result = trigger + .to_asc_ptr(&mut heap, &GasCounter::new(GasMetrics::mock())) + .await; assert!(result.is_ok()); } - #[test] - fn receipt_trigger_to_asc_ptr() { + #[graph::test] + async fn receipt_trigger_to_asc_ptr() { let mut heap = BytesHeap::new(API_VERSION_0_0_5); let trigger = NearTrigger::Receipt(Arc::new(ReceiptWithOutcome { block: Arc::new(block()), @@ -172,7 +183,9 @@ mod tests { receipt: receipt().unwrap(), })); - let result = trigger.to_asc_ptr(&mut heap, &GasCounter::default()); + let result = trigger + .to_asc_ptr(&mut heap, &GasCounter::new(GasMetrics::mock())) + .await; assert!(result.is_ok()); } @@ -392,8 +405,7 @@ mod tests { } fn big_int(input: u64) -> Option { - let value = - BigInt::try_from(input).unwrap_or_else(|_| panic!("Invalid BigInt value {}", input)); + let value = BigInt::from(input); let bytes = value.to_signed_bytes_le(); Some(codec::BigInt { bytes }) @@ -435,8 +447,9 @@ mod tests { } } + #[async_trait] impl AscHeap for BytesHeap { - fn raw_new( + async fn raw_new( &mut self, bytes: &[u8], _gas: &GasCounter, @@ -488,14 +501,14 @@ mod tests { Ok(init_slice(src, buffer)) } - fn api_version(&self) -> graph::semver::Version { - self.api_version.clone() + fn api_version(&self) -> &graph::semver::Version { + &self.api_version } - fn asc_type_id( + async fn asc_type_id( &mut self, type_id_index: graph::runtime::IndexForAscTypeId, - ) -> Result { + ) -> Result { // Not totally clear what is the purpose of this method, why not a default implementation here? Ok(type_id_index as u32) } diff --git a/chain/substreams/Cargo.toml b/chain/substreams/Cargo.toml deleted file mode 100644 index ad557e27c4f..00000000000 --- a/chain/substreams/Cargo.toml +++ /dev/null @@ -1,32 +0,0 @@ -[package] -name = "graph-chain-substreams" -version.workspace = true -edition.workspace = true - -[build-dependencies] -tonic-build = { workspace = true } - -[dependencies] -async-stream = "0.3" -envconfig = "0.10.0" -futures = "0.1.21" -http = "0.2.4" -jsonrpc-core = "18.0.0" -graph = { path = "../../graph" } -graph-runtime-wasm = { path = "../../runtime/wasm" } -lazy_static = "1.2.0" -serde = "1.0" -prost = { workspace = true } -prost-types = { workspace = true } -dirs-next = "2.0" -anyhow = "1.0" -tiny-keccak = "1.5.0" -hex = "0.4.3" -semver = "1.0.16" -base64 = "0.20.0" - -itertools = "0.10.5" - -[dev-dependencies] -graph-core = { path = "../../core" } -tokio = { version = "1", features = ["full"] } diff --git a/chain/substreams/build.rs b/chain/substreams/build.rs deleted file mode 100644 index 8cccc11fe3a..00000000000 --- a/chain/substreams/build.rs +++ /dev/null @@ -1,8 +0,0 @@ -fn main() { - println!("cargo:rerun-if-changed=proto"); - tonic_build::configure() - .protoc_arg("--experimental_allow_proto3_optional") - .out_dir("src/protobuf") - .compile(&["proto/codec.proto"], &["proto"]) - .expect("Failed to compile Substreams entity proto(s)"); -} diff --git a/chain/substreams/examples/README.md b/chain/substreams/examples/README.md deleted file mode 100644 index afd1882b337..00000000000 --- a/chain/substreams/examples/README.md +++ /dev/null @@ -1,13 +0,0 @@ -## Substreams example - -1. Set environmental variables -```bash -$> export SUBSTREAMS_API_TOKEN=your_sf_token -$> export SUBSTREAMS_ENDPOINT=your_sf_endpoint # you can also not define this one and use the default specified endpoint -$> export SUBSTREAMS_PACKAGE=path_to_your_spkg -``` - -2. Run `substreams` example -```bash -cargo run -p graph-chain-substreams --example substreams [module_name] # for graph entities run `graph_out` -``` diff --git a/chain/substreams/examples/substreams.rs b/chain/substreams/examples/substreams.rs deleted file mode 100644 index a6f74692f52..00000000000 --- a/chain/substreams/examples/substreams.rs +++ /dev/null @@ -1,95 +0,0 @@ -use anyhow::{format_err, Context, Error}; -use graph::blockchain::block_stream::BlockStreamEvent; -use graph::blockchain::substreams_block_stream::SubstreamsBlockStream; -use graph::firehose::SubgraphLimit; -use graph::prelude::{info, tokio, DeploymentHash, Registry}; -use graph::tokio_stream::StreamExt; -use graph::{env::env_var, firehose::FirehoseEndpoint, log::logger, substreams}; -use graph_chain_substreams::mapper::Mapper; -use graph_core::MetricsRegistry; -use prost::Message; -use std::env; -use std::sync::Arc; - -#[tokio::main] -async fn main() -> Result<(), Error> { - let module_name = env::args().nth(1).unwrap(); - - let token_env = env_var("SUBSTREAMS_API_TOKEN", "".to_string()); - let mut token: Option = None; - if !token_env.is_empty() { - token = Some(token_env); - } - - let endpoint = env_var( - "SUBSTREAMS_ENDPOINT", - "https://api.streamingfast.io".to_string(), - ); - - let package_file = env_var("SUBSTREAMS_PACKAGE", "".to_string()); - if package_file.is_empty() { - panic!("Environment variable SUBSTREAMS_PACKAGE must be set"); - } - - let package = read_package(&package_file)?; - - let logger = logger(true); - // Set up Prometheus registry - let prometheus_registry = Arc::new(Registry::new()); - let metrics_registry = Arc::new(MetricsRegistry::new( - logger.clone(), - prometheus_registry.clone(), - )); - - let firehose = Arc::new(FirehoseEndpoint::new( - "substreams", - &endpoint, - token, - false, - false, - SubgraphLimit::Unlimited, - )); - - let mut stream: SubstreamsBlockStream = - SubstreamsBlockStream::new( - DeploymentHash::new("substreams".to_string()).unwrap(), - firehose.clone(), - None, - None, - Arc::new(Mapper {}), - package.modules.clone(), - module_name.to_string(), - vec![12369621], - vec![], - logger.clone(), - metrics_registry, - ); - - loop { - match stream.next().await { - None => { - break; - } - Some(event) => match event { - Err(_) => {} - Ok(block_stream_event) => match block_stream_event { - BlockStreamEvent::Revert(_, _) => {} - BlockStreamEvent::ProcessBlock(block_with_trigger, _) => { - for change in block_with_trigger.block.changes.entity_changes { - for field in change.fields { - info!(&logger, "field: {:?}", field); - } - } - } - }, - }, - } - } - - Ok(()) -} - -fn read_package(file: &str) -> Result { - let content = std::fs::read(file).context(format_err!("read package {}", file))?; - substreams::Package::decode(content.as_ref()).context("decode command") -} diff --git a/chain/substreams/proto/codec.proto b/chain/substreams/proto/codec.proto deleted file mode 100644 index a24dcb97310..00000000000 --- a/chain/substreams/proto/codec.proto +++ /dev/null @@ -1,46 +0,0 @@ -syntax = "proto3"; - -package substreams.entity.v1; - -message EntityChanges { - repeated EntityChange entity_changes = 5; -} - -message EntityChange { - string entity = 1; - string id = 2; - uint64 ordinal = 3; - enum Operation { - UNSET = 0; // Protobuf default should not be used, this is used so that the consume can ensure that the value was actually specified - CREATE = 1; - UPDATE = 2; - DELETE = 3; - } - Operation operation = 4; - repeated Field fields = 5; -} - -message Value { - oneof typed { - int32 int32 = 1; - string bigdecimal = 2; - string bigint = 3; - string string = 4; - bytes bytes = 5; - bool bool = 6; - - //reserved 7 to 9; // For future types - - Array array = 10; - } -} - -message Array { - repeated Value value = 1; -} - -message Field { - string name = 1; - optional Value new_value = 3; - optional Value old_value = 5; -} diff --git a/chain/substreams/src/block_stream.rs b/chain/substreams/src/block_stream.rs deleted file mode 100644 index 441d28dc1d1..00000000000 --- a/chain/substreams/src/block_stream.rs +++ /dev/null @@ -1,78 +0,0 @@ -use anyhow::Result; -use std::sync::Arc; - -use graph::{ - blockchain::{ - block_stream::{ - BlockStream, BlockStreamBuilder as BlockStreamBuilderTrait, FirehoseCursor, - }, - substreams_block_stream::SubstreamsBlockStream, - Blockchain, - }, - components::store::DeploymentLocator, - data::subgraph::UnifiedMappingApiVersion, - prelude::{async_trait, BlockNumber, BlockPtr}, - slog::o, -}; - -use crate::{mapper::Mapper, Chain, TriggerFilter}; - -pub struct BlockStreamBuilder {} - -impl BlockStreamBuilder { - pub fn new() -> Self { - Self {} - } -} - -#[async_trait] -/// Substreams doesn't actually use Firehose, the configuration for firehose and the grpc substream -/// is very similar, so we can re-use the configuration and the builder for it. -/// This is probably something to improve but for now it works. -impl BlockStreamBuilderTrait for BlockStreamBuilder { - async fn build_firehose( - &self, - chain: &Chain, - deployment: DeploymentLocator, - block_cursor: FirehoseCursor, - _start_blocks: Vec, - subgraph_current_block: Option, - filter: Arc, - _unified_api_version: UnifiedMappingApiVersion, - ) -> Result>> { - let firehose_endpoint = chain.chain_client().firehose_endpoint()?; - - let mapper = Arc::new(Mapper {}); - - let logger = chain - .logger_factory - .subgraph_logger(&deployment) - .new(o!("component" => "SubstreamsBlockStream")); - - Ok(Box::new(SubstreamsBlockStream::new( - deployment.hash, - firehose_endpoint, - subgraph_current_block, - block_cursor.as_ref().clone(), - mapper, - filter.modules.clone(), - filter.module_name.clone(), - filter.start_block.map(|x| vec![x]).unwrap_or_default(), - vec![], - logger, - chain.metrics_registry.clone(), - ))) - } - - async fn build_polling( - &self, - _chain: Arc, - _deployment: DeploymentLocator, - _start_blocks: Vec, - _subgraph_current_block: Option, - _filter: Arc, - _unified_api_version: UnifiedMappingApiVersion, - ) -> Result>> { - unimplemented!("polling block stream is not support for substreams") - } -} diff --git a/chain/substreams/src/chain.rs b/chain/substreams/src/chain.rs deleted file mode 100644 index b42eecc6118..00000000000 --- a/chain/substreams/src/chain.rs +++ /dev/null @@ -1,185 +0,0 @@ -use crate::{data_source::*, EntityChanges, TriggerData, TriggerFilter, TriggersAdapter}; -use anyhow::Error; -use graph::blockchain::client::ChainClient; -use graph::blockchain::EmptyNodeCapabilities; -use graph::firehose::FirehoseEndpoints; -use graph::prelude::{BlockHash, LoggerFactory, MetricsRegistry}; -use graph::{ - blockchain::{ - self, - block_stream::{BlockStream, BlockStreamBuilder, FirehoseCursor}, - BlockPtr, Blockchain, BlockchainKind, IngestorError, RuntimeAdapter as RuntimeAdapterTrait, - }, - components::store::DeploymentLocator, - data::subgraph::UnifiedMappingApiVersion, - prelude::{async_trait, BlockNumber, ChainStore}, - slog::Logger, -}; -use std::sync::Arc; - -#[derive(Default, Debug, Clone)] -pub struct Block { - pub hash: BlockHash, - pub number: BlockNumber, - pub changes: EntityChanges, -} - -impl blockchain::Block for Block { - fn ptr(&self) -> BlockPtr { - BlockPtr { - hash: self.hash.clone(), - number: self.number, - } - } - - fn parent_ptr(&self) -> Option { - None - } -} - -pub struct Chain { - chain_store: Arc, - block_stream_builder: Arc>, - - pub(crate) logger_factory: LoggerFactory, - pub(crate) client: Arc>, - pub(crate) metrics_registry: Arc, -} - -impl Chain { - pub fn new( - logger_factory: LoggerFactory, - firehose_endpoints: FirehoseEndpoints, - metrics_registry: Arc, - chain_store: Arc, - block_stream_builder: Arc>, - ) -> Self { - Self { - logger_factory, - client: Arc::new(ChainClient::new_firehose(firehose_endpoints)), - metrics_registry, - chain_store, - block_stream_builder, - } - } -} - -impl std::fmt::Debug for Chain { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "chain: substreams") - } -} - -#[async_trait] -impl Blockchain for Chain { - const KIND: BlockchainKind = BlockchainKind::Substreams; - - type Client = (); - type Block = Block; - type DataSource = DataSource; - type UnresolvedDataSource = UnresolvedDataSource; - - type DataSourceTemplate = NoopDataSourceTemplate; - type UnresolvedDataSourceTemplate = NoopDataSourceTemplate; - - /// Trigger data as parsed from the triggers adapter. - type TriggerData = TriggerData; - - /// Decoded trigger ready to be processed by the mapping. - /// New implementations should have this be the same as `TriggerData`. - type MappingTrigger = TriggerData; - - /// Trigger filter used as input to the triggers adapter. - type TriggerFilter = TriggerFilter; - - type NodeCapabilities = EmptyNodeCapabilities; - - fn triggers_adapter( - &self, - _log: &DeploymentLocator, - _capabilities: &Self::NodeCapabilities, - _unified_api_version: UnifiedMappingApiVersion, - ) -> Result>, Error> { - Ok(Arc::new(TriggersAdapter {})) - } - - async fn new_firehose_block_stream( - &self, - deployment: DeploymentLocator, - block_cursor: FirehoseCursor, - start_blocks: Vec, - subgraph_current_block: Option, - filter: Arc, - unified_api_version: UnifiedMappingApiVersion, - ) -> Result>, Error> { - self.block_stream_builder - .build_firehose( - self, - deployment, - block_cursor, - start_blocks, - subgraph_current_block, - filter, - unified_api_version, - ) - .await - } - - fn is_refetch_block_required(&self) -> bool { - false - } - async fn refetch_firehose_block( - &self, - _logger: &Logger, - _cursor: FirehoseCursor, - ) -> Result { - unimplemented!("This chain does not support Dynamic Data Sources. is_refetch_block_required always returns false, this shouldn't be called.") - } - - async fn new_polling_block_stream( - &self, - _deployment: DeploymentLocator, - _start_blocks: Vec, - _subgraph_current_block: Option, - _filter: Arc, - _unified_api_version: UnifiedMappingApiVersion, - ) -> Result>, Error> { - unimplemented!("this should never be called for substreams") - } - - fn chain_store(&self) -> Arc { - self.chain_store.clone() - } - - async fn block_pointer_from_number( - &self, - _logger: &Logger, - number: BlockNumber, - ) -> Result { - // This is the same thing TriggersAdapter does, not sure if it's going to work but - // we also don't yet have a good way of getting this value until we sort out the - // chain store. - // TODO(filipe): Fix this once the chain_store is correctly setup for substreams. - Ok(BlockPtr { - hash: BlockHash::from(vec![0xff; 32]), - number, - }) - } - fn runtime_adapter(&self) -> Arc> { - Arc::new(RuntimeAdapter {}) - } - - fn chain_client(&self) -> Arc> { - self.client.clone() - } -} - -pub struct RuntimeAdapter {} -impl RuntimeAdapterTrait for RuntimeAdapter { - fn host_fns( - &self, - _ds: &::DataSource, - ) -> Result, Error> { - todo!() - } -} diff --git a/chain/substreams/src/codec.rs b/chain/substreams/src/codec.rs deleted file mode 100644 index 31781baa201..00000000000 --- a/chain/substreams/src/codec.rs +++ /dev/null @@ -1,5 +0,0 @@ -#[rustfmt::skip] -#[path = "protobuf/substreams.entity.v1.rs"] -mod pbsubstreamsentity; - -pub use pbsubstreamsentity::*; diff --git a/chain/substreams/src/data_source.rs b/chain/substreams/src/data_source.rs deleted file mode 100644 index 9e3389189ef..00000000000 --- a/chain/substreams/src/data_source.rs +++ /dev/null @@ -1,405 +0,0 @@ -use std::sync::Arc; - -use anyhow::{anyhow, Error}; -use graph::{ - blockchain, - cheap_clone::CheapClone, - components::link_resolver::LinkResolver, - prelude::{async_trait, BlockNumber, DataSourceTemplateInfo, Link}, - slog::Logger, -}; - -use prost::Message; -use serde::Deserialize; - -use crate::{chain::Chain, Block, TriggerData}; - -pub const SUBSTREAMS_KIND: &str = "substreams"; - -const DYNAMIC_DATA_SOURCE_ERROR: &str = "Substreams do not support dynamic data sources"; -const TEMPLATE_ERROR: &str = "Substreams do not support templates"; - -const ALLOWED_MAPPING_KIND: [&str; 1] = ["substreams/graph-entities"]; - -#[derive(Clone, Debug, PartialEq)] -/// Represents the DataSource portion of the manifest once it has been parsed -/// and the substream spkg has been downloaded + parsed. -pub struct DataSource { - pub kind: String, - pub network: Option, - pub name: String, - pub(crate) source: Source, - pub mapping: Mapping, - pub context: Arc>, - pub initial_block: Option, -} - -impl blockchain::DataSource for DataSource { - fn from_template_info(_template_info: DataSourceTemplateInfo) -> Result { - Err(anyhow!("Substreams does not support templates")) - } - - fn address(&self) -> Option<&[u8]> { - None - } - - fn start_block(&self) -> BlockNumber { - self.initial_block.unwrap_or(0) - } - - fn name(&self) -> &str { - &self.name - } - - fn kind(&self) -> &str { - &self.kind - } - - fn network(&self) -> Option<&str> { - self.network.as_deref() - } - - fn context(&self) -> Arc> { - self.context.cheap_clone() - } - - fn creation_block(&self) -> Option { - None - } - - fn api_version(&self) -> semver::Version { - self.mapping.api_version.clone() - } - - // runtime is not needed for substreams, it will cause the host creation to be skipped. - fn runtime(&self) -> Option>> { - None - } - - // match_and_decode only seems to be used on the default trigger processor which substreams - // bypasses so it should be fine to leave it unimplemented. - fn match_and_decode( - &self, - _trigger: &TriggerData, - _block: &Arc, - _logger: &Logger, - ) -> Result>, Error> { - unimplemented!() - } - - fn is_duplicate_of(&self, _other: &Self) -> bool { - todo!() - } - - fn as_stored_dynamic_data_source(&self) -> graph::components::store::StoredDynamicDataSource { - unimplemented!("{}", DYNAMIC_DATA_SOURCE_ERROR) - } - - fn validate(&self) -> Vec { - let mut errs = vec![]; - - if &self.kind != SUBSTREAMS_KIND { - errs.push(anyhow!( - "data source has invalid `kind`, expected {} but found {}", - SUBSTREAMS_KIND, - self.kind - )) - } - - if self.name.is_empty() { - errs.push(anyhow!("name cannot be empty")); - } - - if !ALLOWED_MAPPING_KIND.contains(&self.mapping.kind.as_str()) { - errs.push(anyhow!( - "mapping kind has to be one of {:?}, found {}", - ALLOWED_MAPPING_KIND, - self.mapping.kind - )) - } - - errs - } - - fn from_stored_dynamic_data_source( - _template: &::DataSourceTemplate, - _stored: graph::components::store::StoredDynamicDataSource, - ) -> Result { - Err(anyhow!(DYNAMIC_DATA_SOURCE_ERROR)) - } -} - -#[derive(Clone, Debug, Default, PartialEq)] -/// Module name comes from the manifest, package is the parsed spkg file. -pub struct Source { - pub module_name: String, - pub package: graph::substreams::Package, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct Mapping { - pub api_version: semver::Version, - pub kind: String, -} - -#[derive(Clone, Debug, Deserialize, PartialEq, Eq)] -/// Raw representation of the data source for deserialization purposes. -pub struct UnresolvedDataSource { - pub kind: String, - pub network: Option, - pub name: String, - pub(crate) source: UnresolvedSource, - pub mapping: UnresolvedMapping, -} - -#[derive(Clone, Debug, Default, Hash, Eq, PartialEq, Deserialize)] -#[serde(rename_all = "camelCase")] -/// Text api_version, before parsing and validation. -pub struct UnresolvedMapping { - pub api_version: String, - pub kind: String, -} - -#[async_trait] -impl blockchain::UnresolvedDataSource for UnresolvedDataSource { - async fn resolve( - self, - resolver: &Arc, - logger: &Logger, - _manifest_idx: u32, - ) -> Result { - let content = resolver.cat(logger, &self.source.package.file).await?; - - let package = graph::substreams::Package::decode(content.as_ref())?; - - let initial_block: Option = match package.modules { - Some(ref modules) => modules.modules.iter().map(|x| x.initial_block).min(), - None => None, - }; - - let initial_block: Option = initial_block - .map_or(Ok(None), |x: u64| TryInto::::try_into(x).map(Some)) - .map_err(anyhow::Error::from)?; - - Ok(DataSource { - kind: SUBSTREAMS_KIND.into(), - network: self.network, - name: self.name, - source: Source { - module_name: self.source.package.module_name, - package, - }, - mapping: Mapping { - api_version: semver::Version::parse(&self.mapping.api_version)?, - kind: self.mapping.kind, - }, - context: Arc::new(None), - initial_block, - }) - } -} - -#[derive(Clone, Debug, Default, Hash, Eq, PartialEq, Deserialize)] -#[serde(rename_all = "camelCase")] -/// Source is a part of the manifest and this is needed for parsing. -pub struct UnresolvedSource { - package: UnresolvedPackage, -} - -#[derive(Clone, Debug, Default, Hash, Eq, PartialEq, Deserialize)] -#[serde(rename_all = "camelCase")] -/// The unresolved Package section of the manifest. -pub struct UnresolvedPackage { - pub module_name: String, - pub file: Link, -} - -#[derive(Debug, Clone, Default, Deserialize)] -/// This is necessary for the Blockchain trait associated types, substreams do not support -/// data source templates so this is a noop and is not expected to be called. -pub struct NoopDataSourceTemplate {} - -impl blockchain::DataSourceTemplate for NoopDataSourceTemplate { - fn name(&self) -> &str { - unimplemented!("{}", TEMPLATE_ERROR); - } - - fn api_version(&self) -> semver::Version { - unimplemented!("{}", TEMPLATE_ERROR); - } - - fn runtime(&self) -> Option>> { - unimplemented!("{}", TEMPLATE_ERROR); - } - - fn manifest_idx(&self) -> u32 { - todo!() - } -} - -#[async_trait] -impl blockchain::UnresolvedDataSourceTemplate for NoopDataSourceTemplate { - async fn resolve( - self, - _resolver: &Arc, - _logger: &Logger, - _manifest_idx: u32, - ) -> Result { - unimplemented!("{}", TEMPLATE_ERROR) - } -} - -#[cfg(test)] -mod test { - use std::{str::FromStr, sync::Arc}; - - use anyhow::Error; - use graph::{ - blockchain::{DataSource as _, UnresolvedDataSource as _}, - components::link_resolver::LinkResolver, - prelude::{async_trait, serde_yaml, JsonValueStream, Link}, - slog::{o, Discard, Logger}, - }; - - use crate::{DataSource, Mapping, UnresolvedDataSource, UnresolvedMapping, SUBSTREAMS_KIND}; - - const EMPTY_PACKAGE: graph::substreams::Package = graph::substreams::Package { - proto_files: vec![], - version: 0, - modules: None, - module_meta: vec![], - package_meta: vec![], - }; - - #[test] - fn parse_data_source() { - let ds: UnresolvedDataSource = serde_yaml::from_str(TEMPLATE_DATA_SOURCE).unwrap(); - let expected = UnresolvedDataSource { - kind: SUBSTREAMS_KIND.into(), - network: Some("mainnet".into()), - name: "Uniswap".into(), - source: crate::UnresolvedSource { - package: crate::UnresolvedPackage { - module_name: "output".into(), - file: Link { - link: "/ipfs/QmbHnhUFZa6qqqRyubUYhXntox1TCBxqryaBM1iNGqVJzT".into(), - }, - }, - }, - mapping: UnresolvedMapping { - api_version: "0.0.7".into(), - kind: "substreams/graph-entities".into(), - }, - }; - assert_eq!(ds, expected); - } - - #[tokio::test] - async fn data_source_conversion() { - let ds: UnresolvedDataSource = serde_yaml::from_str(TEMPLATE_DATA_SOURCE).unwrap(); - let link_resolver: Arc = Arc::new(NoopLinkResolver {}); - let logger = Logger::root(Discard, o!()); - let ds: DataSource = ds.resolve(&link_resolver, &logger, 0).await.unwrap(); - let expected = DataSource { - kind: SUBSTREAMS_KIND.into(), - network: Some("mainnet".into()), - name: "Uniswap".into(), - source: crate::Source { - module_name: "output".into(), - package: EMPTY_PACKAGE, - }, - mapping: Mapping { - api_version: semver::Version::from_str("0.0.7").unwrap(), - kind: "substreams/graph-entities".into(), - }, - context: Arc::new(None), - initial_block: None, - }; - assert_eq!(ds, expected); - } - - #[test] - fn data_source_validation() { - let mut ds = gen_data_source(); - assert_eq!(true, ds.validate().is_empty()); - - ds.network = None; - assert_eq!(true, ds.validate().is_empty()); - - ds.kind = "asdasd".into(); - ds.name = "".into(); - ds.mapping.kind = "asdasd".into(); - let errs: Vec = ds.validate().into_iter().map(|e| e.to_string()).collect(); - assert_eq!( - errs, - vec![ - "data source has invalid `kind`, expected substreams but found asdasd", - "name cannot be empty", - "mapping kind has to be one of [\"substreams/graph-entities\"], found asdasd" - ] - ); - } - - fn gen_data_source() -> DataSource { - DataSource { - kind: SUBSTREAMS_KIND.into(), - network: Some("mainnet".into()), - name: "Uniswap".into(), - source: crate::Source { - module_name: "".to_string(), - package: EMPTY_PACKAGE, - }, - mapping: Mapping { - api_version: semver::Version::from_str("0.0.7").unwrap(), - kind: "substreams/graph-entities".into(), - }, - context: Arc::new(None), - initial_block: None, - } - } - - const TEMPLATE_DATA_SOURCE: &str = r#" - kind: substreams - name: Uniswap - network: mainnet - source: - package: - moduleName: output - file: - /: /ipfs/QmbHnhUFZa6qqqRyubUYhXntox1TCBxqryaBM1iNGqVJzT - # This IPFs path would be generated from a local path at deploy time - mapping: - kind: substreams/graph-entities - apiVersion: 0.0.7 - "#; - - #[derive(Debug)] - struct NoopLinkResolver {} - - #[async_trait] - impl LinkResolver for NoopLinkResolver { - fn with_timeout(&self, _timeout: std::time::Duration) -> Box { - unimplemented!() - } - - fn with_retries(&self) -> Box { - unimplemented!() - } - - async fn cat(&self, _logger: &Logger, _link: &Link) -> Result, Error> { - Ok(vec![]) - } - - async fn get_block(&self, _logger: &Logger, _link: &Link) -> Result, Error> { - unimplemented!() - } - - async fn json_stream( - &self, - _logger: &Logger, - _link: &Link, - ) -> Result { - unimplemented!() - } - } -} diff --git a/chain/substreams/src/lib.rs b/chain/substreams/src/lib.rs deleted file mode 100644 index 60215c453cc..00000000000 --- a/chain/substreams/src/lib.rs +++ /dev/null @@ -1,15 +0,0 @@ -mod block_stream; -mod chain; -mod codec; -mod data_source; -mod trigger; - -pub mod mapper; - -pub use block_stream::BlockStreamBuilder; -pub use chain::*; -pub use codec::EntityChanges; -pub use data_source::*; -pub use trigger::*; - -pub use codec::Field; diff --git a/chain/substreams/src/mapper.rs b/chain/substreams/src/mapper.rs deleted file mode 100644 index e9d5ba06862..00000000000 --- a/chain/substreams/src/mapper.rs +++ /dev/null @@ -1,105 +0,0 @@ -use crate::{Block, Chain, EntityChanges, TriggerData}; -use graph::blockchain::block_stream::SubstreamsError::{ - MultipleModuleOutputError, UnexpectedStoreDeltaOutput, -}; -use graph::blockchain::block_stream::{ - BlockStreamEvent, BlockWithTriggers, FirehoseCursor, SubstreamsError, SubstreamsMapper, -}; -use graph::prelude::{async_trait, BlockHash, BlockNumber, BlockPtr, Logger}; -use graph::substreams::module_output::Data; -use graph::substreams::{BlockScopedData, Clock, ForkStep}; -use prost::Message; - -pub struct Mapper {} - -#[async_trait] -impl SubstreamsMapper for Mapper { - async fn to_block_stream_event( - &self, - logger: &Logger, - block_scoped_data: &BlockScopedData, - ) -> Result>, SubstreamsError> { - let BlockScopedData { - outputs, - clock, - step, - cursor: _, - } = block_scoped_data; - - let step = ForkStep::from_i32(*step).unwrap_or_else(|| { - panic!( - "unknown step i32 value {}, maybe you forgot update & re-regenerate the protobuf definitions?", - step - ) - }); - - if outputs.is_empty() { - return Ok(None); - } - - if outputs.len() > 1 { - return Err(MultipleModuleOutputError); - } - - //todo: handle step - let module_output = &block_scoped_data.outputs[0]; - let cursor = &block_scoped_data.cursor; - - let clock = match clock { - Some(clock) => clock, - None => return Err(SubstreamsError::MissingClockError), - }; - - let Clock { - id: hash, - number, - timestamp: _, - } = clock; - - let hash: BlockHash = hash.as_str().try_into()?; - let number: BlockNumber = *number as BlockNumber; - - match module_output.data.as_ref() { - Some(Data::MapOutput(msg)) => { - let changes: EntityChanges = Message::decode(msg.value.as_slice()) - .map_err(SubstreamsError::DecodingError)?; - - use ForkStep::*; - match step { - StepIrreversible | StepNew => Ok(Some(BlockStreamEvent::ProcessBlock( - // Even though the trigger processor for substreams doesn't care about TriggerData - // there are a bunch of places in the runner that check if trigger data - // empty and skip processing if so. This will prolly breakdown - // close to head so we will need to improve things. - - // TODO(filipe): Fix once either trigger data can be empty - // or we move the changes into trigger data. - BlockWithTriggers::new( - Block { - hash, - number, - changes, - }, - vec![TriggerData {}], - logger, - ), - FirehoseCursor::from(cursor.clone()), - ))), - StepUndo => { - let parent_ptr = BlockPtr { hash, number }; - - Ok(Some(BlockStreamEvent::Revert( - parent_ptr, - FirehoseCursor::from(cursor.clone()), - ))) - } - StepUnknown => { - panic!("unknown step should not happen in the Firehose response") - } - } - } - Some(Data::DebugStoreDeltas(_)) => Err(UnexpectedStoreDeltaOutput), - _ => Err(SubstreamsError::ModuleOutputNotPresentOrUnexpected), - } - } -} diff --git a/chain/substreams/src/protobuf/substreams.entity.v1.rs b/chain/substreams/src/protobuf/substreams.entity.v1.rs deleted file mode 100644 index 47368e25fba..00000000000 --- a/chain/substreams/src/protobuf/substreams.entity.v1.rs +++ /dev/null @@ -1,109 +0,0 @@ -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct EntityChanges { - #[prost(message, repeated, tag = "5")] - pub entity_changes: ::prost::alloc::vec::Vec, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct EntityChange { - #[prost(string, tag = "1")] - pub entity: ::prost::alloc::string::String, - #[prost(string, tag = "2")] - pub id: ::prost::alloc::string::String, - #[prost(uint64, tag = "3")] - pub ordinal: u64, - #[prost(enumeration = "entity_change::Operation", tag = "4")] - pub operation: i32, - #[prost(message, repeated, tag = "5")] - pub fields: ::prost::alloc::vec::Vec, -} -/// Nested message and enum types in `EntityChange`. -pub mod entity_change { - #[derive( - Clone, - Copy, - Debug, - PartialEq, - Eq, - Hash, - PartialOrd, - Ord, - ::prost::Enumeration - )] - #[repr(i32)] - pub enum Operation { - /// Protobuf default should not be used, this is used so that the consume can ensure that the value was actually specified - Unset = 0, - Create = 1, - Update = 2, - Delete = 3, - } - impl Operation { - /// String value of the enum field names used in the ProtoBuf definition. - /// - /// The values are not transformed in any way and thus are considered stable - /// (if the ProtoBuf definition does not change) and safe for programmatic use. - pub fn as_str_name(&self) -> &'static str { - match self { - Operation::Unset => "UNSET", - Operation::Create => "CREATE", - Operation::Update => "UPDATE", - Operation::Delete => "DELETE", - } - } - /// Creates an enum from field names used in the ProtoBuf definition. - pub fn from_str_name(value: &str) -> ::core::option::Option { - match value { - "UNSET" => Some(Self::Unset), - "CREATE" => Some(Self::Create), - "UPDATE" => Some(Self::Update), - "DELETE" => Some(Self::Delete), - _ => None, - } - } - } -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Value { - #[prost(oneof = "value::Typed", tags = "1, 2, 3, 4, 5, 6, 10")] - pub typed: ::core::option::Option, -} -/// Nested message and enum types in `Value`. -pub mod value { - #[allow(clippy::derive_partial_eq_without_eq)] - #[derive(Clone, PartialEq, ::prost::Oneof)] - pub enum Typed { - #[prost(int32, tag = "1")] - Int32(i32), - #[prost(string, tag = "2")] - Bigdecimal(::prost::alloc::string::String), - #[prost(string, tag = "3")] - Bigint(::prost::alloc::string::String), - #[prost(string, tag = "4")] - String(::prost::alloc::string::String), - #[prost(bytes, tag = "5")] - Bytes(::prost::alloc::vec::Vec), - #[prost(bool, tag = "6")] - Bool(bool), - #[prost(message, tag = "10")] - Array(super::Array), - } -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Array { - #[prost(message, repeated, tag = "1")] - pub value: ::prost::alloc::vec::Vec, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Field { - #[prost(string, tag = "1")] - pub name: ::prost::alloc::string::String, - #[prost(message, optional, tag = "3")] - pub new_value: ::core::option::Option, - #[prost(message, optional, tag = "5")] - pub old_value: ::core::option::Option, -} diff --git a/chain/substreams/src/trigger.rs b/chain/substreams/src/trigger.rs deleted file mode 100644 index f4a7729c78c..00000000000 --- a/chain/substreams/src/trigger.rs +++ /dev/null @@ -1,411 +0,0 @@ -use std::{collections::HashMap, str::FromStr, sync::Arc}; - -use anyhow::Error; -use graph::{ - blockchain::{self, block_stream::BlockWithTriggers, BlockPtr, EmptyNodeCapabilities}, - components::{ - store::{DeploymentLocator, EntityKey, EntityType, SubgraphFork}, - subgraph::{MappingError, ProofOfIndexingEvent, SharedProofOfIndexing}, - }, - data::store::scalar::Bytes, - data_source::{self, CausalityRegion}, - prelude::{ - anyhow, async_trait, BigDecimal, BigInt, BlockHash, BlockNumber, BlockState, Entity, - RuntimeHostBuilder, Value, - }, - slog::Logger, - substreams::Modules, -}; -use graph_runtime_wasm::module::ToAscPtr; -use lazy_static::__Deref; - -use crate::codec; -use crate::{codec::entity_change::Operation, Block, Chain, NoopDataSourceTemplate}; - -#[derive(Eq, PartialEq, PartialOrd, Ord, Debug)] -pub struct TriggerData {} - -impl blockchain::TriggerData for TriggerData { - // TODO(filipe): Can this be improved with some data from the block? - fn error_context(&self) -> String { - "Failed to process substreams block".to_string() - } -} - -impl ToAscPtr for TriggerData { - // substreams doesn't rely on wasm on the graph-node so this is not needed. - fn to_asc_ptr( - self, - _heap: &mut H, - _gas: &graph::runtime::gas::GasCounter, - ) -> Result, graph::runtime::DeterministicHostError> { - unimplemented!() - } -} - -#[derive(Debug, Clone, Default)] -pub struct TriggerFilter { - pub(crate) modules: Option, - pub(crate) module_name: String, - pub(crate) start_block: Option, - pub(crate) data_sources_len: u8, -} - -// TriggerFilter should bypass all triggers and just rely on block since all the data received -// should already have been processed. -impl blockchain::TriggerFilter for TriggerFilter { - fn extend_with_template(&mut self, _data_source: impl Iterator) { - } - - /// this function is not safe to call multiple times, only one DataSource is supported for - /// - fn extend<'a>( - &mut self, - mut data_sources: impl Iterator + Clone, - ) { - let Self { - modules, - module_name, - start_block, - data_sources_len, - } = self; - - if *data_sources_len >= 1 { - return; - } - - if let Some(ds) = data_sources.next() { - *data_sources_len = 1; - *modules = ds.source.package.modules.clone(); - *module_name = ds.source.module_name.clone(); - *start_block = ds.initial_block; - } - } - - fn node_capabilities(&self) -> EmptyNodeCapabilities { - EmptyNodeCapabilities::default() - } - - fn to_firehose_filter(self) -> Vec { - unimplemented!("this should never be called for this type") - } -} - -pub struct TriggersAdapter {} - -#[async_trait] -impl blockchain::TriggersAdapter for TriggersAdapter { - async fn ancestor_block( - &self, - _ptr: BlockPtr, - _offset: BlockNumber, - ) -> Result, Error> { - unimplemented!() - } - - async fn scan_triggers( - &self, - _from: BlockNumber, - _to: BlockNumber, - _filter: &TriggerFilter, - ) -> Result>, Error> { - unimplemented!() - } - - async fn triggers_in_block( - &self, - _logger: &Logger, - _block: Block, - _filter: &TriggerFilter, - ) -> Result, Error> { - unimplemented!() - } - - async fn is_on_main_chain(&self, _ptr: BlockPtr) -> Result { - unimplemented!() - } - - async fn parent_ptr(&self, block: &BlockPtr) -> Result, Error> { - // This seems to work for a lot of the firehose chains. - Ok(Some(BlockPtr { - hash: BlockHash::from(vec![0xff; 32]), - number: block.number.saturating_sub(1), - })) - } -} - -fn write_poi_event( - proof_of_indexing: &SharedProofOfIndexing, - poi_event: &ProofOfIndexingEvent, - causality_region: &str, - logger: &Logger, -) { - if let Some(proof_of_indexing) = proof_of_indexing { - let mut proof_of_indexing = proof_of_indexing.deref().borrow_mut(); - proof_of_indexing.write(logger, causality_region, poi_event); - } -} - -pub struct TriggerProcessor { - pub locator: DeploymentLocator, -} - -impl TriggerProcessor { - pub fn new(locator: DeploymentLocator) -> Self { - Self { locator } - } -} - -#[async_trait] -impl graph::prelude::TriggerProcessor for TriggerProcessor -where - T: RuntimeHostBuilder, -{ - async fn process_trigger( - &self, - logger: &Logger, - _hosts: &[Arc], - block: &Arc, - _trigger: &data_source::TriggerData, - mut state: BlockState, - proof_of_indexing: &SharedProofOfIndexing, - causality_region: &str, - _debug_fork: &Option>, - _subgraph_metrics: &Arc, - ) -> Result, MappingError> { - for entity_change in block.changes.entity_changes.iter() { - match entity_change.operation() { - Operation::Unset => { - // Potentially an issue with the server side or - // we are running an outdated version. In either case we should abort. - return Err(MappingError::Unknown(anyhow!("Detected UNSET entity operation, either a server error or there's a new type of operation and we're running an outdated protobuf"))); - } - Operation::Create | Operation::Update => { - let entity_type: &str = &entity_change.entity; - let entity_id: String = entity_change.id.clone(); - let key = EntityKey { - entity_type: EntityType::new(entity_type.to_string()), - entity_id: entity_id.clone().into(), - causality_region: CausalityRegion::ONCHAIN, // Substreams don't currently support offchain data - }; - let mut data: HashMap = HashMap::from_iter(vec![]); - - for field in entity_change.fields.iter() { - let new_value: &codec::value::Typed = match &field.new_value { - Some(codec::Value { - typed: Some(new_value), - }) => new_value, - _ => continue, - }; - - let value: Value = decode_value(new_value)?; - *data.entry(field.name.clone()).or_insert(Value::Null) = value; - } - - write_poi_event( - proof_of_indexing, - &ProofOfIndexingEvent::SetEntity { - entity_type, - id: &entity_id, - data: &data, - }, - causality_region, - logger, - ); - - state.entity_cache.set(key, Entity::from(data))?; - } - Operation::Delete => { - let entity_type: &str = &entity_change.entity; - let entity_id: String = entity_change.id.clone(); - let key = EntityKey { - entity_type: EntityType::new(entity_type.to_string()), - entity_id: entity_id.clone().into(), - causality_region: CausalityRegion::ONCHAIN, // Substreams don't currently support offchain data - }; - - state.entity_cache.remove(key); - - write_poi_event( - proof_of_indexing, - &ProofOfIndexingEvent::RemoveEntity { - entity_type, - id: &entity_id, - }, - causality_region, - logger, - ) - } - } - } - - Ok(state) - } -} - -fn decode_value(value: &crate::codec::value::Typed) -> Result { - use codec::value::Typed; - - match value { - Typed::Int32(new_value) => Ok(Value::Int(*new_value)), - - Typed::Bigdecimal(new_value) => BigDecimal::from_str(new_value) - .map(Value::BigDecimal) - .map_err(|err| MappingError::Unknown(anyhow::Error::from(err))), - - Typed::Bigint(new_value) => BigInt::from_str(new_value) - .map(Value::BigInt) - .map_err(|err| MappingError::Unknown(anyhow::Error::from(err))), - - Typed::String(new_value) => { - let mut string = new_value.clone(); - - // Strip null characters since they are not accepted by Postgres. - if string.contains('\u{0000}') { - string = string.replace('\u{0000}', ""); - } - Ok(Value::String(string)) - } - - Typed::Bytes(new_value) => base64::decode(new_value) - .map(|bs| Value::Bytes(Bytes::from(bs.as_ref()))) - .map_err(|err| MappingError::Unknown(anyhow::Error::from(err))), - - Typed::Bool(new_value) => Ok(Value::Bool(*new_value)), - - Typed::Array(arr) => arr - .value - .iter() - .filter_map(|item| item.typed.as_ref().map(decode_value)) - .collect::, MappingError>>() - .map(Value::List), - } -} - -#[cfg(test)] -mod test { - use std::{ops::Add, str::FromStr}; - - use crate::codec::value::Typed; - use crate::codec::{Array, Value}; - use crate::trigger::decode_value; - use graph::{ - data::store::scalar::Bytes, - prelude::{BigDecimal, BigInt, Value as GraphValue}, - }; - - #[test] - fn validate_substreams_field_types() { - struct Case { - name: String, - value: Value, - expected_value: GraphValue, - } - - let cases = vec![ - Case { - name: "string value".to_string(), - value: Value { - typed: Some(Typed::String( - "d4325ee72c39999e778a9908f5fb0803f78e30c441a5f2ce5c65eee0e0eba59d" - .to_string(), - )), - }, - expected_value: GraphValue::String( - "d4325ee72c39999e778a9908f5fb0803f78e30c441a5f2ce5c65eee0e0eba59d".to_string(), - ), - }, - Case { - name: "bytes value".to_string(), - value: Value { - typed: Some(Typed::Bytes( - base64::encode( - hex::decode( - "445247fe150195bd866516594e087e1728294aa831613f4d48b8ec618908519f", - ) - .unwrap(), - ) - .into_bytes(), - )), - }, - expected_value: GraphValue::Bytes( - Bytes::from_str( - "0x445247fe150195bd866516594e087e1728294aa831613f4d48b8ec618908519f", - ) - .unwrap(), - ), - }, - Case { - name: "int value for block".to_string(), - value: Value { - typed: Some(Typed::Int32(12369760)), - }, - expected_value: GraphValue::Int(12369760), - }, - Case { - name: "negative int value".to_string(), - value: Value { - typed: Some(Typed::Int32(-12369760)), - }, - expected_value: GraphValue::Int(-12369760), - }, - Case { - name: "big int".to_string(), - value: Value { - typed: Some(Typed::Bigint("123".to_string())), - }, - expected_value: GraphValue::BigInt(BigInt::from(123u64)), - }, - Case { - name: "big int > u64".to_string(), - value: Value { - typed: Some(Typed::Bigint( - BigInt::from(u64::MAX).add(BigInt::from(1)).to_string(), - )), - }, - expected_value: GraphValue::BigInt(BigInt::from(u64::MAX).add(BigInt::from(1))), - }, - Case { - name: "big decimal value".to_string(), - value: Value { - typed: Some(Typed::Bigdecimal("3133363633312e35".to_string())), - }, - expected_value: GraphValue::BigDecimal(BigDecimal::new( - BigInt::from(3133363633312u64), - 35, - )), - }, - Case { - name: "bool value".to_string(), - value: Value { - typed: Some(Typed::Bool(true)), - }, - expected_value: GraphValue::Bool(true), - }, - Case { - name: "string array".to_string(), - value: Value { - typed: Some(Typed::Array(Array { - value: vec![ - Value { - typed: Some(Typed::String("1".to_string())), - }, - Value { - typed: Some(Typed::String("2".to_string())), - }, - Value { - typed: Some(Typed::String("3".to_string())), - }, - ], - })), - }, - expected_value: GraphValue::List(vec!["1".into(), "2".into(), "3".into()]), - }, - ]; - - for case in cases.into_iter() { - let value: GraphValue = decode_value(&case.value.typed.unwrap()).unwrap(); - assert_eq!(case.expected_value, value, "failed case: {}", case.name) - } - } -} diff --git a/core/Cargo.toml b/core/Cargo.toml index 8700190cab5..a4b38367213 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -4,40 +4,36 @@ version.workspace = true edition.workspace = true [dependencies] -async-trait = "0.1.50" -atomic_refcell = "0.1.9" -async-stream = "0.3" +async-trait = { workspace = true} bytes = "1.0" -futures01 = { package = "futures", version = "0.1.31" } -futures = { version = "0.3.4", features = ["compat"] } graph = { path = "../graph" } -# This dependency is temporary. The multiblockchain refactoring is not -# finished as long as this dependency exists -graph-chain-arweave = { path = "../chain/arweave" } graph-chain-ethereum = { path = "../chain/ethereum" } graph-chain-near = { path = "../chain/near" } -graph-chain-cosmos = { path = "../chain/cosmos" } -graph-chain-substreams = { path = "../chain/substreams" } -lazy_static = "1.2.0" -lru_time_cache = "0.11" -semver = "1.0.16" -serde = "1.0" -serde_json = "1.0" -serde_yaml = "0.8" +graph-runtime-wasm = { path = "../runtime/wasm" } +serde_yaml = { workspace = true } +tokio = { workspace = true } +tokio-retry = { workspace = true } # Switch to crates.io once tower 0.5 is released tower = { git = "https://github.com/tower-rs/tower.git", features = ["full"] } -graph-runtime-wasm = { path = "../runtime/wasm" } -cid = "0.10.1" +thiserror = { workspace = true } anyhow = "1.0" +# Dependencies related to Amp subgraphs +alloy.workspace = true +arrow.workspace = true +chrono.workspace = true +futures.workspace = true +indoc.workspace = true +itertools.workspace = true +parking_lot.workspace = true +prometheus.workspace = true +slog.workspace = true +strum.workspace = true +tokio-util.workspace = true + [dev-dependencies] tower-test = { git = "https://github.com/tower-rs/tower.git" } -graph-mock = { path = "../mock" } -test-store = { path = "../store/test-store" } -hex = "0.4.3" -graphql-parser = "0.4.0" -pretty_assertions = "1.3.0" -anyhow = "1.0" -ipfs-api-backend-hyper = "0.6" -ipfs-api = { version = "0.17.0", features = ["with-hyper-rustls"], default-features = false } -uuid = { version = "1.3.0", features = ["v4"] } +wiremock = "0.6.5" + +[lints] +workspace = true diff --git a/core/graphman/Cargo.toml b/core/graphman/Cargo.toml new file mode 100644 index 00000000000..858cc1b9012 --- /dev/null +++ b/core/graphman/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "graphman" +version.workspace = true +edition.workspace = true + +[dependencies] +anyhow = { workspace = true } +diesel = { workspace = true } +diesel-async = { workspace = true } +graph = { workspace = true } +graph-store-postgres = { workspace = true } +graphman-store = { workspace = true } +itertools = { workspace = true } +thiserror = { workspace = true } +tokio = { workspace = true } diff --git a/core/graphman/src/commands/deployment/info.rs b/core/graphman/src/commands/deployment/info.rs new file mode 100644 index 00000000000..155a821d715 --- /dev/null +++ b/core/graphman/src/commands/deployment/info.rs @@ -0,0 +1,82 @@ +use std::collections::HashMap; +use std::sync::Arc; + +use anyhow::anyhow; +use graph::blockchain::BlockPtr; +use graph::components::store::BlockNumber; +use graph::components::store::DeploymentId; +use graph::components::store::StatusStore; +use graph::data::subgraph::schema::SubgraphHealth; +use graph_store_postgres::ConnectionPool; +use graph_store_postgres::Store; +use itertools::Itertools; + +use crate::GraphmanError; +use crate::deployment::Deployment; +use crate::deployment::DeploymentSelector; +use crate::deployment::DeploymentVersionSelector; + +#[derive(Clone, Debug)] +pub struct DeploymentStatus { + pub is_paused: Option, + pub is_synced: bool, + pub health: SubgraphHealth, + pub earliest_block_number: BlockNumber, + pub latest_block: Option, + pub chain_head_block: Option, +} + +pub async fn load_deployments( + primary_pool: ConnectionPool, + deployment: &DeploymentSelector, + version: &DeploymentVersionSelector, +) -> Result, GraphmanError> { + let mut primary_conn = primary_pool.get().await?; + + crate::deployment::load_deployments(&mut primary_conn, deployment, version).await +} + +pub async fn load_deployment_statuses( + store: Arc, + deployments: &[Deployment], +) -> Result, GraphmanError> { + use graph::data::subgraph::status::Filter; + + let deployment_ids = deployments + .iter() + .map(|deployment| DeploymentId::new(deployment.id)) + .collect_vec(); + + let deployment_statuses = store + .status(Filter::DeploymentIds(deployment_ids)) + .await? + .into_iter() + .map(|status| { + let id = status.id.0; + + let chain = status + .chains + .first() + .ok_or_else(|| { + GraphmanError::Store(anyhow!( + "deployment status has no chains on deployment '{id}'" + )) + })? + .to_owned(); + + Ok(( + id, + DeploymentStatus { + is_paused: status.paused, + is_synced: status.synced, + health: status.health, + earliest_block_number: chain.earliest_block_number.to_owned(), + latest_block: chain.latest_block.map(|x| x.to_ptr()), + chain_head_block: chain.chain_head_block.map(|x| x.to_ptr()), + }, + )) + }) + .collect::>()?; + + Ok(deployment_statuses) +} diff --git a/core/graphman/src/commands/deployment/mod.rs b/core/graphman/src/commands/deployment/mod.rs new file mode 100644 index 00000000000..4cac2277bbe --- /dev/null +++ b/core/graphman/src/commands/deployment/mod.rs @@ -0,0 +1,5 @@ +pub mod info; +pub mod pause; +pub mod reassign; +pub mod resume; +pub mod unassign; diff --git a/core/graphman/src/commands/deployment/pause.rs b/core/graphman/src/commands/deployment/pause.rs new file mode 100644 index 00000000000..e2405b5f68b --- /dev/null +++ b/core/graphman/src/commands/deployment/pause.rs @@ -0,0 +1,91 @@ +use std::sync::Arc; + +use anyhow::anyhow; +use graph::components::store::DeploymentLocator; +use graph::components::store::StoreEvent; +use graph_store_postgres::ConnectionPool; +use graph_store_postgres::NotificationSender; +use graph_store_postgres::command_support::catalog; +use graph_store_postgres::command_support::catalog::Site; +use thiserror::Error; + +use crate::GraphmanError; +use crate::deployment::DeploymentSelector; +use crate::deployment::DeploymentVersionSelector; + +pub struct ActiveDeployment { + locator: DeploymentLocator, + site: Site, +} + +#[derive(Debug, Error)] +pub enum PauseDeploymentError { + #[error("deployment '{0}' is already paused")] + AlreadyPaused(String), + + #[error(transparent)] + Common(#[from] GraphmanError), +} + +impl ActiveDeployment { + pub fn locator(&self) -> &DeploymentLocator { + &self.locator + } +} + +pub async fn load_active_deployment( + primary_pool: ConnectionPool, + deployment: &DeploymentSelector, +) -> Result { + let mut primary_conn = primary_pool + .get_permitted() + .await + .map_err(GraphmanError::from)?; + + let locator = crate::deployment::load_deployment_locator( + &mut primary_conn, + deployment, + &DeploymentVersionSelector::All, + ) + .await?; + + let mut catalog_conn = catalog::Connection::new(primary_conn); + + let site = catalog_conn + .locate_site(locator.clone()) + .await + .map_err(GraphmanError::from)? + .ok_or_else(|| { + GraphmanError::Store(anyhow!("deployment site not found for '{locator}'")) + })?; + + let (_, is_paused) = catalog_conn + .assignment_status(&site) + .await + .map_err(GraphmanError::from)? + .ok_or_else(|| { + GraphmanError::Store(anyhow!("assignment status not found for '{locator}'")) + })?; + + if is_paused { + return Err(PauseDeploymentError::AlreadyPaused(locator.to_string())); + } + + Ok(ActiveDeployment { locator, site }) +} + +pub async fn pause_active_deployment( + primary_pool: ConnectionPool, + notification_sender: Arc, + active_deployment: ActiveDeployment, +) -> Result<(), GraphmanError> { + let primary_conn = primary_pool.get_permitted().await?; + let mut catalog_conn = catalog::Connection::new(primary_conn); + + let changes = catalog_conn.pause_subgraph(&active_deployment.site).await?; + catalog_conn + .send_store_event(¬ification_sender, &StoreEvent::new(changes)) + .await?; + + Ok(()) +} diff --git a/core/graphman/src/commands/deployment/reassign.rs b/core/graphman/src/commands/deployment/reassign.rs new file mode 100644 index 00000000000..f4279946917 --- /dev/null +++ b/core/graphman/src/commands/deployment/reassign.rs @@ -0,0 +1,145 @@ +use std::sync::Arc; + +use anyhow::anyhow; +use graph::components::store::DeploymentLocator; +use graph::components::store::StoreEvent; +use graph::prelude::AssignmentChange; +use graph::prelude::NodeId; +use graph_store_postgres::ConnectionPool; +use graph_store_postgres::NotificationSender; +use graph_store_postgres::command_support::catalog; +use graph_store_postgres::command_support::catalog::Site; +use thiserror::Error; + +use crate::GraphmanError; +use crate::deployment::DeploymentSelector; +use crate::deployment::DeploymentVersionSelector; + +pub struct Deployment { + locator: DeploymentLocator, + site: Site, +} + +impl Deployment { + pub fn locator(&self) -> &DeploymentLocator { + &self.locator + } + + pub async fn assigned_node( + &self, + primary_pool: ConnectionPool, + ) -> Result, GraphmanError> { + let primary_conn = primary_pool + .get_permitted() + .await + .map_err(GraphmanError::from)?; + let mut catalog_conn = catalog::Connection::new(primary_conn); + let node = catalog_conn + .assigned_node(&self.site) + .await + .map_err(GraphmanError::from)?; + Ok(node) + } +} + +#[derive(Debug, Error)] +pub enum ReassignDeploymentError { + #[error("deployment '{0}' is already assigned to '{1}'")] + AlreadyAssigned(String, String), + + #[error(transparent)] + Common(#[from] GraphmanError), +} + +#[derive(Clone, Debug)] +pub enum ReassignResult { + Ok, + CompletedWithWarnings(Vec), +} + +pub async fn load_deployment( + primary_pool: ConnectionPool, + deployment: &DeploymentSelector, +) -> Result { + let mut primary_conn = primary_pool + .get_permitted() + .await + .map_err(GraphmanError::from)?; + + let locator = crate::deployment::load_deployment_locator( + &mut primary_conn, + deployment, + &DeploymentVersionSelector::All, + ) + .await?; + + let mut catalog_conn = catalog::Connection::new(primary_conn); + + let site = catalog_conn + .locate_site(locator.clone()) + .await + .map_err(GraphmanError::from)? + .ok_or_else(|| { + GraphmanError::Store(anyhow!("deployment site not found for '{locator}'")) + })?; + + Ok(Deployment { locator, site }) +} + +pub async fn reassign_deployment( + primary_pool: ConnectionPool, + notification_sender: Arc, + deployment: &Deployment, + node: &NodeId, + curr_node: Option, +) -> Result { + let primary_conn = primary_pool + .get_permitted() + .await + .map_err(GraphmanError::from)?; + let mut catalog_conn = catalog::Connection::new(primary_conn); + let changes: Vec = match &curr_node { + Some(curr) => { + if curr == node { + vec![] + } else { + catalog_conn + .reassign_subgraph(&deployment.site, node) + .await + .map_err(GraphmanError::from)? + } + } + None => catalog_conn + .assign_subgraph(&deployment.site, node) + .await + .map_err(GraphmanError::from)?, + }; + + if changes.is_empty() { + return Err(ReassignDeploymentError::AlreadyAssigned( + deployment.locator.to_string(), + node.to_string(), + )); + } + + catalog_conn + .send_store_event(¬ification_sender, &StoreEvent::new(changes)) + .await + .map_err(GraphmanError::from)?; + + let mirror = catalog::Mirror::primary_only(primary_pool); + let count = mirror + .assignments(node) + .await + .map_err(GraphmanError::from)? + .len(); + if count == 1 { + let warning_msg = format!( + "This is the only deployment assigned to '{}'. Please make sure that the node ID is spelled correctly.", + node.as_str() + ); + Ok(ReassignResult::CompletedWithWarnings(vec![warning_msg])) + } else { + Ok(ReassignResult::Ok) + } +} diff --git a/core/graphman/src/commands/deployment/resume.rs b/core/graphman/src/commands/deployment/resume.rs new file mode 100644 index 00000000000..4fa0162f83e --- /dev/null +++ b/core/graphman/src/commands/deployment/resume.rs @@ -0,0 +1,93 @@ +use std::sync::Arc; + +use anyhow::anyhow; +use graph::components::store::DeploymentLocator; +use graph::prelude::StoreEvent; +use graph_store_postgres::ConnectionPool; +use graph_store_postgres::NotificationSender; +use graph_store_postgres::command_support::catalog; +use graph_store_postgres::command_support::catalog::Site; +use thiserror::Error; + +use crate::GraphmanError; +use crate::deployment::DeploymentSelector; +use crate::deployment::DeploymentVersionSelector; + +pub struct PausedDeployment { + locator: DeploymentLocator, + site: Site, +} + +#[derive(Debug, Error)] +pub enum ResumeDeploymentError { + #[error("deployment '{0}' is not paused")] + NotPaused(String), + + #[error(transparent)] + Common(#[from] GraphmanError), +} + +impl PausedDeployment { + pub fn locator(&self) -> &DeploymentLocator { + &self.locator + } +} + +pub async fn load_paused_deployment( + primary_pool: ConnectionPool, + deployment: &DeploymentSelector, +) -> Result { + let mut primary_conn = primary_pool + .get_permitted() + .await + .map_err(GraphmanError::from)?; + + let locator = crate::deployment::load_deployment_locator( + &mut primary_conn, + deployment, + &DeploymentVersionSelector::All, + ) + .await?; + + let mut catalog_conn = catalog::Connection::new(primary_conn); + + let site = catalog_conn + .locate_site(locator.clone()) + .await + .map_err(GraphmanError::from)? + .ok_or_else(|| { + GraphmanError::Store(anyhow!("deployment site not found for '{locator}'")) + })?; + + let (_, is_paused) = catalog_conn + .assignment_status(&site) + .await + .map_err(GraphmanError::from)? + .ok_or_else(|| { + GraphmanError::Store(anyhow!("assignment status not found for '{locator}'")) + })?; + + if !is_paused { + return Err(ResumeDeploymentError::NotPaused(locator.to_string())); + } + + Ok(PausedDeployment { locator, site }) +} + +pub async fn resume_paused_deployment( + primary_pool: ConnectionPool, + notification_sender: Arc, + paused_deployment: PausedDeployment, +) -> Result<(), GraphmanError> { + let primary_conn = primary_pool.get_permitted().await?; + let mut catalog_conn = catalog::Connection::new(primary_conn); + + let changes = catalog_conn + .resume_subgraph(&paused_deployment.site) + .await?; + catalog_conn + .send_store_event(¬ification_sender, &StoreEvent::new(changes)) + .await?; + + Ok(()) +} diff --git a/core/graphman/src/commands/deployment/unassign.rs b/core/graphman/src/commands/deployment/unassign.rs new file mode 100644 index 00000000000..cef72e947f1 --- /dev/null +++ b/core/graphman/src/commands/deployment/unassign.rs @@ -0,0 +1,88 @@ +use std::sync::Arc; + +use anyhow::anyhow; +use graph::components::store::DeploymentLocator; +use graph::components::store::StoreEvent; +use graph_store_postgres::ConnectionPool; +use graph_store_postgres::NotificationSender; +use graph_store_postgres::command_support::catalog; +use graph_store_postgres::command_support::catalog::Site; +use thiserror::Error; + +use crate::GraphmanError; +use crate::deployment::DeploymentSelector; +use crate::deployment::DeploymentVersionSelector; + +pub struct AssignedDeployment { + locator: DeploymentLocator, + site: Site, +} + +impl AssignedDeployment { + pub fn locator(&self) -> &DeploymentLocator { + &self.locator + } +} + +#[derive(Debug, Error)] +pub enum UnassignDeploymentError { + #[error("deployment '{0}' is already unassigned")] + AlreadyUnassigned(String), + + #[error(transparent)] + Common(#[from] GraphmanError), +} + +pub async fn load_assigned_deployment( + primary_pool: ConnectionPool, + deployment: &DeploymentSelector, +) -> Result { + let mut primary_conn = primary_pool + .get_permitted() + .await + .map_err(GraphmanError::from)?; + + let locator = crate::deployment::load_deployment_locator( + &mut primary_conn, + deployment, + &DeploymentVersionSelector::All, + ) + .await?; + + let mut catalog_conn = catalog::Connection::new(primary_conn); + + let site = catalog_conn + .locate_site(locator.clone()) + .await + .map_err(GraphmanError::from)? + .ok_or_else(|| { + GraphmanError::Store(anyhow!("deployment site not found for '{locator}'")) + })?; + + match catalog_conn + .assigned_node(&site) + .await + .map_err(GraphmanError::from)? + { + Some(_) => Ok(AssignedDeployment { locator, site }), + None => Err(UnassignDeploymentError::AlreadyUnassigned( + locator.to_string(), + )), + } +} + +pub async fn unassign_deployment( + primary_pool: ConnectionPool, + notification_sender: Arc, + deployment: AssignedDeployment, +) -> Result<(), GraphmanError> { + let primary_conn = primary_pool.get_permitted().await?; + let mut catalog_conn = catalog::Connection::new(primary_conn); + + let changes = catalog_conn.unassign_subgraph(&deployment.site).await?; + catalog_conn + .send_store_event(¬ification_sender, &StoreEvent::new(changes)) + .await?; + + Ok(()) +} diff --git a/core/graphman/src/commands/mod.rs b/core/graphman/src/commands/mod.rs new file mode 100644 index 00000000000..98629027b58 --- /dev/null +++ b/core/graphman/src/commands/mod.rs @@ -0,0 +1 @@ +pub mod deployment; diff --git a/core/graphman/src/deployment.rs b/core/graphman/src/deployment.rs new file mode 100644 index 00000000000..6538017edaa --- /dev/null +++ b/core/graphman/src/deployment.rs @@ -0,0 +1,157 @@ +use anyhow::anyhow; +use diesel::BoolExpressionMethods; +use diesel::ExpressionMethods; +use diesel::JoinOnDsl; +use diesel::NullableExpressionMethods; +use diesel::PgTextExpressionMethods; +use diesel::QueryDsl; +use diesel::Queryable; +use diesel::dsl::sql; +use diesel::sql_types::Text; +use diesel_async::RunQueryDsl; +use graph::components::store::DeploymentId; +use graph::components::store::DeploymentLocator; +use graph::data::subgraph::DeploymentHash; +use graph_store_postgres::AsyncPgConnection; +use graph_store_postgres::command_support::catalog; +use itertools::Itertools; + +use crate::GraphmanError; + +#[derive(Clone, Debug, Queryable)] +pub struct Deployment { + pub id: i32, + pub hash: String, + pub namespace: String, + pub name: String, + pub node_id: Option, + pub shard: String, + pub chain: String, + pub version_status: String, + pub is_active: bool, +} + +#[derive(Clone, Debug)] +pub enum DeploymentSelector { + Name(String), + Subgraph { hash: String, shard: Option }, + Schema(String), + All, +} + +#[derive(Clone, Debug)] +pub enum DeploymentVersionSelector { + Current, + Pending, + Used, + All, +} + +impl Deployment { + pub fn locator(&self) -> DeploymentLocator { + DeploymentLocator::new( + DeploymentId::new(self.id), + DeploymentHash::new(self.hash.clone()).unwrap(), + ) + } +} + +pub(crate) async fn load_deployments( + primary_conn: &mut AsyncPgConnection, + deployment: &DeploymentSelector, + version: &DeploymentVersionSelector, +) -> Result, GraphmanError> { + use catalog::deployment_schemas as ds; + use catalog::subgraph as sg; + use catalog::subgraph_deployment_assignment as sgda; + use catalog::subgraph_version as sgv; + + let mut query = ds::table + .inner_join(sgv::table.on(sgv::deployment.eq(ds::subgraph))) + .inner_join(sg::table.on(sgv::subgraph.eq(sg::id))) + .left_outer_join(sgda::table.on(sgda::id.eq(ds::id))) + .select(( + ds::id, + sgv::deployment, + ds::name, + sg::name, + sgda::node_id.nullable(), + ds::shard, + ds::network, + sql::( + "( + case + when subgraphs.subgraph.pending_version = subgraphs.subgraph_version.id + then 'pending' + when subgraphs.subgraph.current_version = subgraphs.subgraph_version.id + then 'current' + else + 'unused' + end + ) status", + ), + ds::active, + )) + .into_boxed(); + + match deployment { + DeploymentSelector::Name(name) => { + let pattern = format!("%{}%", name.replace("%", "")); + query = query.filter(sg::name.ilike(pattern)); + } + DeploymentSelector::Subgraph { hash, shard } => { + query = query.filter(ds::subgraph.eq(hash)); + + if let Some(shard) = shard { + query = query.filter(ds::shard.eq(shard)); + } + } + DeploymentSelector::Schema(name) => { + query = query.filter(ds::name.eq(name)); + } + DeploymentSelector::All => { + // No query changes required. + } + }; + + let current_version_filter = sg::current_version.eq(sgv::id.nullable()); + let pending_version_filter = sg::pending_version.eq(sgv::id.nullable()); + + match version { + DeploymentVersionSelector::Current => { + query = query.filter(current_version_filter); + } + DeploymentVersionSelector::Pending => { + query = query.filter(pending_version_filter); + } + DeploymentVersionSelector::Used => { + query = query.filter(current_version_filter.or(pending_version_filter)); + } + DeploymentVersionSelector::All => { + // No query changes required. + } + } + + query.load(primary_conn).await.map_err(Into::into) +} + +pub(crate) async fn load_deployment_locator( + primary_conn: &mut AsyncPgConnection, + deployment: &DeploymentSelector, + version: &DeploymentVersionSelector, +) -> Result { + let deployment_locator = load_deployments(primary_conn, deployment, version) + .await? + .into_iter() + .map(|deployment| deployment.locator()) + .unique() + .exactly_one() + .map_err(|err| { + let count = err.into_iter().count(); + GraphmanError::Store(anyhow!( + "expected exactly one deployment for '{deployment:?}', found {count}" + )) + })?; + + Ok(deployment_locator) +} diff --git a/core/graphman/src/error.rs b/core/graphman/src/error.rs new file mode 100644 index 00000000000..731b2574f0e --- /dev/null +++ b/core/graphman/src/error.rs @@ -0,0 +1,19 @@ +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum GraphmanError { + #[error("store error: {0:#}")] + Store(#[source] anyhow::Error), +} + +impl From for GraphmanError { + fn from(err: graph::components::store::StoreError) -> Self { + Self::Store(err.into()) + } +} + +impl From for GraphmanError { + fn from(err: diesel::result::Error) -> Self { + Self::Store(err.into()) + } +} diff --git a/core/graphman/src/execution_tracker.rs b/core/graphman/src/execution_tracker.rs new file mode 100644 index 00000000000..806d78defed --- /dev/null +++ b/core/graphman/src/execution_tracker.rs @@ -0,0 +1,86 @@ +use std::sync::Arc; +use std::time::Duration; + +use anyhow::Result; +use graphman_store::ExecutionId; +use graphman_store::GraphmanStore; +use tokio::sync::Notify; + +/// The execution status is updated at this interval. +const DEFAULT_HEARTBEAT_INTERVAL: Duration = Duration::from_secs(20); + +/// Used with long-running command executions to maintain their status as active. +pub struct GraphmanExecutionTracker { + id: ExecutionId, + heartbeat_stopper: Arc, + store: Arc, +} + +impl GraphmanExecutionTracker +where + S: GraphmanStore + Send + Sync + 'static, +{ + /// Creates a new execution tracker that spawns a separate background task that keeps + /// the execution active by periodically updating its status. + pub fn new(store: Arc, id: ExecutionId) -> Self { + let heartbeat_stopper = Arc::new(Notify::new()); + + let tracker = Self { + id, + store, + heartbeat_stopper, + }; + + tracker.spawn_heartbeat(); + tracker + } + + fn spawn_heartbeat(&self) { + let id = self.id; + let heartbeat_stopper = self.heartbeat_stopper.clone(); + let store = self.store.clone(); + + graph::spawn(async move { + store.mark_execution_as_running(id).await.unwrap(); + + let stop_heartbeat = heartbeat_stopper.notified(); + tokio::pin!(stop_heartbeat); + + loop { + tokio::select! { + biased; + + _ = &mut stop_heartbeat => { + break; + }, + + _ = tokio::time::sleep(DEFAULT_HEARTBEAT_INTERVAL) => { + store.mark_execution_as_running(id).await.unwrap(); + }, + } + } + }); + } + + /// Completes the execution with an error. + pub async fn track_failure(self, error_message: String) -> Result<()> { + self.heartbeat_stopper.notify_one(); + + self.store + .mark_execution_as_failed(self.id, error_message) + .await + } + + /// Completes the execution with a success. + pub async fn track_success(self) -> Result<()> { + self.heartbeat_stopper.notify_one(); + + self.store.mark_execution_as_succeeded(self.id).await + } +} + +impl Drop for GraphmanExecutionTracker { + fn drop(&mut self) { + self.heartbeat_stopper.notify_one(); + } +} diff --git a/core/graphman/src/lib.rs b/core/graphman/src/lib.rs new file mode 100644 index 00000000000..71f8e77a848 --- /dev/null +++ b/core/graphman/src/lib.rs @@ -0,0 +1,15 @@ +//! This crate contains graphman commands that can be executed via +//! the GraphQL API as well as via the CLI. +//! +//! Each command is broken into small execution steps to allow different interfaces to perform +//! some additional interface-specific operations between steps. An example of this is printing +//! intermediate information to the user in the CLI, or prompting for additional input. + +mod error; + +pub mod commands; +pub mod deployment; +pub mod execution_tracker; + +pub use self::error::GraphmanError; +pub use self::execution_tracker::GraphmanExecutionTracker; diff --git a/core/graphman_store/Cargo.toml b/core/graphman_store/Cargo.toml new file mode 100644 index 00000000000..fee9daff663 --- /dev/null +++ b/core/graphman_store/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "graphman-store" +version.workspace = true +edition.workspace = true + +[dependencies] +anyhow = { workspace = true } +async-trait = { workspace = true } +chrono = { workspace = true } +diesel = { workspace = true } +strum = { workspace = true } diff --git a/core/graphman_store/src/lib.rs b/core/graphman_store/src/lib.rs new file mode 100644 index 00000000000..8b1965df7cd --- /dev/null +++ b/core/graphman_store/src/lib.rs @@ -0,0 +1,129 @@ +//! This crate allows graphman commands to store data in a persistent storage. +//! +//! Note: The trait is extracted as a separate crate to avoid cyclic dependencies between graphman +//! commands and store implementations. + +use anyhow::Result; +use async_trait::async_trait; +use chrono::DateTime; +use chrono::Utc; +use diesel::AsExpression; +use diesel::FromSqlRow; +use diesel::Queryable; +use diesel::deserialize::FromSql; +use diesel::pg::Pg; +use diesel::pg::PgValue; +use diesel::serialize::Output; +use diesel::serialize::ToSql; +use diesel::sql_types::BigSerial; +use diesel::sql_types::Varchar; +use strum::Display; +use strum::EnumString; +use strum::IntoStaticStr; + +/// Describes all the capabilities that graphman commands need from a persistent storage. +/// +/// The primary use case for this is background execution of commands. +#[async_trait] +pub trait GraphmanStore { + /// Creates a new pending execution of the specified type. + /// The implementation is expected to manage execution IDs and return unique IDs on each call. + /// + /// Creating a new execution does not mean that a command is actually running or will run. + async fn new_execution(&self, kind: CommandKind) -> Result; + + /// Returns all stored execution data. + async fn load_execution(&self, id: ExecutionId) -> Result; + + /// When an execution begins to make progress, this method is used to update its status. + /// + /// For long-running commands, it is expected that this method will be called at some interval + /// to show that the execution is still making progress. + /// + /// The implementation is expected to not allow updating the status of completed executions. + async fn mark_execution_as_running(&self, id: ExecutionId) -> Result<()>; + + /// This is a finalizing operation and is expected to be called only once, + /// when an execution fails. + /// + /// The implementation is not expected to prevent overriding the final state of an execution. + async fn mark_execution_as_failed(&self, id: ExecutionId, error_message: String) -> Result<()>; + + /// This is a finalizing operation and is expected to be called only once, + /// when an execution succeeds. + /// + /// The implementation is not expected to prevent overriding the final state of an execution. + async fn mark_execution_as_succeeded(&self, id: ExecutionId) -> Result<()>; +} + +/// Data stored about a command execution. +#[derive(Clone, Debug, Queryable)] +pub struct Execution { + pub id: ExecutionId, + pub kind: CommandKind, + pub status: ExecutionStatus, + pub error_message: Option, + pub created_at: DateTime, + pub updated_at: Option>, + pub completed_at: Option>, +} + +/// A unique ID of a command execution. +#[derive(Clone, Copy, Debug, AsExpression, FromSqlRow)] +#[diesel(sql_type = BigSerial)] +pub struct ExecutionId(pub i64); + +/// Types of commands that can store data about their execution. +#[derive(Clone, Copy, Debug, AsExpression, FromSqlRow, Display, IntoStaticStr, EnumString)] +#[diesel(sql_type = Varchar)] +#[strum(serialize_all = "snake_case")] +pub enum CommandKind { + RestartDeployment, +} + +/// All possible states of a command execution. +#[derive(Clone, Copy, Debug, AsExpression, FromSqlRow, Display, IntoStaticStr, EnumString)] +#[diesel(sql_type = Varchar)] +#[strum(serialize_all = "snake_case")] +pub enum ExecutionStatus { + Initializing, + Running, + Failed, + Succeeded, +} + +impl FromSql for ExecutionId { + fn from_sql(bytes: PgValue) -> diesel::deserialize::Result { + Ok(ExecutionId(i64::from_sql(bytes)?)) + } +} + +impl ToSql for ExecutionId { + fn to_sql<'b>(&'b self, out: &mut Output<'b, '_, Pg>) -> diesel::serialize::Result { + >::to_sql(&self.0, &mut out.reborrow()) + } +} + +impl FromSql for CommandKind { + fn from_sql(bytes: PgValue) -> diesel::deserialize::Result { + Ok(std::str::from_utf8(bytes.as_bytes())?.parse()?) + } +} + +impl ToSql for CommandKind { + fn to_sql<'b>(&'b self, out: &mut Output<'b, '_, Pg>) -> diesel::serialize::Result { + >::to_sql(self.into(), &mut out.reborrow()) + } +} + +impl FromSql for ExecutionStatus { + fn from_sql(bytes: PgValue) -> diesel::deserialize::Result { + Ok(std::str::from_utf8(bytes.as_bytes())?.parse()?) + } +} + +impl ToSql for ExecutionStatus { + fn to_sql<'b>(&'b self, out: &mut Output<'b, '_, Pg>) -> diesel::serialize::Result { + >::to_sql(self.into(), &mut out.reborrow()) + } +} diff --git a/core/src/amp_subgraph/manager.rs b/core/src/amp_subgraph/manager.rs new file mode 100644 index 00000000000..61ec421ca17 --- /dev/null +++ b/core/src/amp_subgraph/manager.rs @@ -0,0 +1,171 @@ +use std::sync::Arc; + +use alloy::primitives::BlockNumber; +use anyhow::Context; +use async_trait::async_trait; +use graph::{ + amp, + components::{ + link_resolver::{LinkResolver, LinkResolverContext}, + metrics::MetricsRegistry, + store::{DeploymentLocator, SubgraphStore}, + subgraph::SubgraphInstanceManager, + }, + env::EnvVars, + log::factory::LoggerFactory, + prelude::CheapClone, +}; +use slog::{debug, error}; +use tokio_util::sync::CancellationToken; + +use super::{Metrics, Monitor, runner}; + +/// Manages Amp subgraph runner futures. +/// +/// Creates and schedules Amp subgraph runner futures for execution on demand. +/// Also handles stopping previously started Amp subgraph runners. +pub struct Manager { + logger_factory: LoggerFactory, + metrics_registry: Arc, + env_vars: Arc, + monitor: Monitor, + subgraph_store: Arc, + link_resolver: Arc, + amp_client: Arc, +} + +impl Manager +where + SS: SubgraphStore, + NC: amp::Client, +{ + /// Creates a new Amp subgraph manager. + pub fn new( + logger_factory: &LoggerFactory, + metrics_registry: Arc, + env_vars: Arc, + cancel_token: &CancellationToken, + subgraph_store: Arc, + link_resolver: Arc, + amp_client: Arc, + ) -> Self { + let logger = logger_factory.component_logger("AmpSubgraphManager", None); + let logger_factory = logger_factory.with_parent(logger); + + let monitor = Monitor::new(&logger_factory, cancel_token); + + Self { + logger_factory, + metrics_registry, + env_vars, + monitor, + subgraph_store, + link_resolver, + amp_client, + } + } +} + +#[async_trait] +impl SubgraphInstanceManager for Manager +where + SS: SubgraphStore, + NC: amp::Client + Send + Sync + 'static, +{ + async fn start_subgraph( + self: Arc, + deployment: DeploymentLocator, + stop_block: Option, + ) { + let manager = self.cheap_clone(); + + self.monitor.start( + deployment.cheap_clone(), + Box::new(move |cancel_token| { + Box::pin(async move { + let logger = manager.logger_factory.subgraph_logger(&deployment); + + let store = manager + .subgraph_store + .cheap_clone() + .writable(logger.cheap_clone(), deployment.id, Vec::new().into()) + .await + .context("failed to create writable store")?; + + let metrics = Metrics::new( + &logger, + manager.metrics_registry.cheap_clone(), + store.cheap_clone(), + deployment.hash.cheap_clone(), + ); + + let link_resolver = manager + .link_resolver + .for_manifest(&deployment.hash.to_string()) + .context("failed to create link resolver")?; + + let manifest_bytes = link_resolver + .cat( + &LinkResolverContext::new(&deployment.hash, &logger), + &deployment.hash.to_ipfs_link(), + ) + .await + .context("failed to load subgraph manifest")?; + + let raw_manifest = serde_yaml::from_slice(&manifest_bytes) + .context("failed to parse subgraph manifest")?; + + let mut manifest = amp::Manifest::resolve::( + &logger, + manager.link_resolver.cheap_clone(), + manager.amp_client.cheap_clone(), + manager.env_vars.max_spec_version.cheap_clone(), + deployment.hash.cheap_clone(), + raw_manifest, + ) + .await?; + + if let Some(stop_block) = stop_block { + for data_source in manifest.data_sources.iter_mut() { + data_source.source.end_block = stop_block as BlockNumber; + } + } + + store + .start_subgraph_deployment(&logger) + .await + .context("failed to start subgraph deployment")?; + + let runner_context = runner::Context::new( + &logger, + &manager.env_vars.amp, + manager.amp_client.cheap_clone(), + store, + deployment.hash.cheap_clone(), + manifest, + metrics, + ); + + let runner_result = runner::new_runner(runner_context, cancel_token).await; + + match manager.subgraph_store.stop_subgraph(&deployment).await { + Ok(()) => { + debug!(logger, "Subgraph writer stopped"); + } + Err(e) => { + error!(logger, "Failed to stop subgraph writer"; + "e" => ?e + ); + } + } + + runner_result + }) + }), + ); + } + + async fn stop_subgraph(&self, deployment: DeploymentLocator) { + self.monitor.stop(deployment); + } +} diff --git a/core/src/amp_subgraph/metrics.rs b/core/src/amp_subgraph/metrics.rs new file mode 100644 index 00000000000..7088cf9a48f --- /dev/null +++ b/core/src/amp_subgraph/metrics.rs @@ -0,0 +1,267 @@ +use std::{sync::Arc, time::Duration}; + +use alloy::primitives::BlockNumber; +use graph::{ + cheap_clone::CheapClone, + components::{ + metrics::{MetricsRegistry, stopwatch::StopwatchMetrics}, + store::WritableStore, + }, + prelude::DeploymentHash, +}; +use indoc::indoc; +use prometheus::{IntCounter, IntGauge}; +use slog::Logger; + +/// Contains metrics specific to a deployment. +pub(super) struct Metrics { + pub(super) deployment_status: DeploymentStatus, + pub(super) deployment_head: DeploymentHead, + pub(super) deployment_target: DeploymentTarget, + pub(super) deployment_synced: DeploymentSynced, + pub(super) indexing_duration: IndexingDuration, + pub(super) blocks_processed: BlocksProcessed, + pub(super) stopwatch: StopwatchMetrics, +} + +impl Metrics { + /// Creates new deployment specific metrics. + pub(super) fn new( + logger: &Logger, + metrics_registry: Arc, + store: Arc, + deployment: DeploymentHash, + ) -> Self { + let stopwatch = StopwatchMetrics::new( + logger.cheap_clone(), + deployment.cheap_clone(), + "process", + metrics_registry.cheap_clone(), + store.shard().to_string(), + ); + + let const_labels = [ + ("deployment", deployment.to_string()), + ("shard", store.shard().to_string()), + ]; + + Self { + deployment_status: DeploymentStatus::new(&metrics_registry, const_labels.clone()), + deployment_head: DeploymentHead::new(&metrics_registry, const_labels.clone()), + deployment_target: DeploymentTarget::new(&metrics_registry, const_labels.clone()), + deployment_synced: DeploymentSynced::new(&metrics_registry, const_labels.clone()), + indexing_duration: IndexingDuration::new(&metrics_registry, const_labels.clone()), + blocks_processed: BlocksProcessed::new(&metrics_registry, const_labels.clone()), + stopwatch, + } + } +} + +/// Reports the current indexing status of a deployment. +pub(super) struct DeploymentStatus(IntGauge); + +impl DeploymentStatus { + const STATUS_STARTING: i64 = 1; + const STATUS_RUNNING: i64 = 2; + const STATUS_STOPPED: i64 = 3; + const STATUS_FAILED: i64 = 4; + + fn new( + metrics_registry: &MetricsRegistry, + const_labels: impl IntoIterator, + ) -> Self { + let int_gauge = metrics_registry + .new_int_gauge( + "deployment_status", + indoc!( + " + Indicates the current indexing status of a deployment. + Possible values: + 1 - graph-node is preparing to start indexing; + 2 - deployment is being indexed; + 3 - indexing is stopped by request; + 4 - indexing failed; + " + ), + const_labels, + ) + .expect("failed to register `deployment_status` gauge"); + + Self(int_gauge) + } + + /// Records that the graph-node is preparing to start indexing. + pub fn starting(&self) { + self.0.set(Self::STATUS_STARTING); + } + + /// Records that the deployment is being indexed. + pub fn running(&self) { + self.0.set(Self::STATUS_RUNNING); + } + + /// Records that the indexing stopped by request. + pub fn stopped(&self) { + self.0.set(Self::STATUS_STOPPED); + } + + /// Records that the indexing failed. + pub fn failed(&self) { + self.0.set(Self::STATUS_FAILED); + } +} + +/// Tracks the most recent block number processed by a deployment. +pub(super) struct DeploymentHead(IntGauge); + +impl DeploymentHead { + fn new( + metrics_registry: &MetricsRegistry, + const_labels: impl IntoIterator, + ) -> Self { + let int_gauge = metrics_registry + .new_int_gauge( + "deployment_head", + "Tracks the most recent block number processed by a deployment", + const_labels + .into_iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + // TODO: Pass the network from the Amp manifest + .chain([("network".to_string(), "".to_string())]), + ) + .expect("failed to register `deployment_head` gauge"); + + Self(int_gauge) + } + + /// Updates the most recent block number processed by this deployment. + pub(super) fn update(&self, new_most_recent_block_number: BlockNumber) { + self.0.set( + i64::try_from(new_most_recent_block_number) + .expect("new most recent block number does not fit into `i64`"), + ); + } +} + +/// Tracks the maximum block number currently available for indexing within a deployment. +pub(super) struct DeploymentTarget(IntGauge); + +impl DeploymentTarget { + fn new( + metrics_registry: &MetricsRegistry, + const_labels: impl IntoIterator, + ) -> Self { + let int_gauge = metrics_registry + .new_int_gauge( + "deployment_target", + "Tracks the maximum block number currently available for indexing within a deployment", + const_labels, + ) + .expect("failed to register `amp_deployment_target` gauge"); + + Self(int_gauge) + } + + /// Updates the target block number of this deployment. + pub(super) fn update(&self, new_target_block_number: BlockNumber) { + self.0.set( + i64::try_from(new_target_block_number) + .expect("new target block number does not fit into `i64`"), + ); + } +} + +/// Indicates whether a deployment has reached the chain head or the end block since it was deployed. +pub(super) struct DeploymentSynced(IntGauge); + +impl DeploymentSynced { + const NOT_SYNCED: i64 = 0; + const SYNCED: i64 = 1; + + pub fn new( + metrics_registry: &MetricsRegistry, + const_labels: impl IntoIterator, + ) -> Self { + let int_gauge = metrics_registry + .new_int_gauge( + "deployment_synced", + indoc!( + " + Indicates whether a deployment has reached the chain head or the end block since it was deployed. + Possible values: + 0 - deployment is not synced; + 1 - deployment is synced; + " + ), + const_labels, + ) + .expect("failed to register `deployment_synced` gauge"); + + Self(int_gauge) + } + + /// Records the current sync status of this deployment. + pub fn record(&self, synced: bool) { + self.0.set(if synced { + Self::SYNCED + } else { + Self::NOT_SYNCED + }); + } +} + +/// Tracks the total duration in seconds of deployment indexing. +#[derive(Clone)] +pub(super) struct IndexingDuration(IntCounter); + +impl IndexingDuration { + fn new( + metrics_registry: &MetricsRegistry, + const_labels: impl IntoIterator, + ) -> Self { + let int_counter = metrics_registry + .new_int_counter( + "deployment_indexing_duration_seconds", + "Tracks the total duration in seconds of deployment indexing", + const_labels, + ) + .expect("failed to register `amp_deployment_indexing_duration_seconds` counter"); + + Self(int_counter) + } + + /// Records a new indexing duration of this deployment. + pub(super) fn record(&self, duration: Duration) { + self.0.inc_by(duration.as_secs()) + } +} + +/// Tracks the total number of blocks processed by a deployment. +pub(super) struct BlocksProcessed(IntCounter); + +impl BlocksProcessed { + fn new( + metrics_registry: &MetricsRegistry, + const_labels: impl IntoIterator, + ) -> Self { + let int_counter = metrics_registry + .new_int_counter( + "deployment_blocks_processed_count", + "Tracks the total number of blocks processed by a deployment", + const_labels, + ) + .expect("failed to register `deployment_blocks_processed_count` counter"); + + Self(int_counter) + } + + /// Records a new processed block. + pub(super) fn record_one(&self) { + self.record(1); + } + + /// Records the new processed blocks. + pub(super) fn record(&self, number_of_blocks_processed: usize) { + self.0.inc_by(number_of_blocks_processed as u64); + } +} diff --git a/core/src/amp_subgraph/mod.rs b/core/src/amp_subgraph/mod.rs new file mode 100644 index 00000000000..3d3846742aa --- /dev/null +++ b/core/src/amp_subgraph/mod.rs @@ -0,0 +1,8 @@ +mod manager; +mod metrics; +mod monitor; +mod runner; + +use self::{metrics::Metrics, monitor::Monitor}; + +pub use self::manager::Manager; diff --git a/core/src/amp_subgraph/monitor.rs b/core/src/amp_subgraph/monitor.rs new file mode 100644 index 00000000000..314f464c0d1 --- /dev/null +++ b/core/src/amp_subgraph/monitor.rs @@ -0,0 +1,563 @@ +//! This module is responsible for executing subgraph runner futures. +//! +//! # Terminology used in this module +//! +//! `active subgraph` - A subgraph that was started and is still kept in memory in the list of started subgraphs. +//! `running subgraph` - A subgraph that has an instance that is making progress or stopping. +//! `subgraph instance` - A background task that executes the subgraph runner future. + +use std::{ + collections::{HashMap, hash_map::Entry}, + fmt, + sync::{ + Arc, + atomic::{AtomicU32, Ordering::SeqCst}, + }, + time::Duration, +}; + +use anyhow::Result; +use futures::future::BoxFuture; +use graph::{ + cheap_clone::CheapClone, components::store::DeploymentLocator, log::factory::LoggerFactory, +}; +use slog::{Logger, debug, error, info, warn}; +use tokio::{ + sync::mpsc::{self, error::SendError}, + task::JoinHandle, + time::timeout, +}; +use tokio_util::sync::CancellationToken; + +/// Represents the maximum amount of time a subgraph instance is allowed to run +/// after it receives a cancel signal. +/// +/// If a subgraph instance does not complete its execution in this amount of time +/// it is considered unresponsive and is aborted. +const SUBGRAPH_INSTANCE_GRACE_PERIOD: Duration = { + if cfg!(test) { + Duration::from_millis(300) + } else if cfg!(debug_assertions) { + Duration::from_secs(30) + } else { + Duration::from_secs(300) + } +}; + +/// Represents the subgraph runner future. +/// +/// This is the future that performs the subgraph indexing. +/// It is expected to return only on deterministic failures or when indexing is completed. +/// All retry functionality must be handled internally by this future. +pub(super) type BoxRunner = + Box BoxFuture<'static, Result<()>> + Send + 'static>; + +/// Manages the lifecycle of subgraph runners. +/// +/// Ensures that there is at most one subgraph instance running +/// for any subgraph deployment at any point in time. +/// Handles starting, stopping and restarting subgraphs. +pub(super) struct Monitor { + logger_factory: Arc, + + /// Every subgraph instance is assigned a cancel token derived from this token. + /// + /// This means that the `Monitor` can send cancel signals to all subgraph instances at once, + /// and to each subgraph instance individually. + cancel_token: CancellationToken, + + /// The channel that is used to send subgraph commands. + /// + /// Every subgraph start and stop request results in a command that is sent to the + /// background task that manages the subgraph instances. + command_tx: mpsc::UnboundedSender, + + /// When a subgraph starts it is assigned a sequential ID. + /// The ID is then kept in memory in the list of active subgraphs. + /// + /// When the subgraph completes execution it should be removed from the + /// list of active subgraphs, so that it can be restarted. + /// + /// This ID is required to be able to check if the active subgraph + /// is the same subgraph instance that was stopped. + /// + /// If the IDs do not match, it means that the subgraph was force restarted, + /// ignoring the state of the previous subgraph instance, or that the subgraph + /// was restarted after the previous subgraph instance completed its execution + /// but before the remove request was processed. + subgraph_instance_id: Arc, +} + +impl Monitor { + /// Creates a new subgraph monitor. + /// + /// Spawns a background task that manages the subgraph start and stop requests. + /// + /// A new cancel token is derived from the `cancel_token` and only the derived token is used by the + /// subgraph monitor and its background task. + pub(super) fn new(logger_factory: &LoggerFactory, cancel_token: &CancellationToken) -> Self { + let logger = logger_factory.component_logger("AmpSubgraphMonitor", None); + let logger_factory = Arc::new(logger_factory.with_parent(logger)); + + // A derived token makes sure it is not possible to accidentally cancel the parent token + let cancel_token = cancel_token.child_token(); + + // It is safe to use an unbounded channel here, because it's pretty much unrealistic that the + // command processor will fall behind so much that the channel buffer will take up all the memory. + // The command processor is non-blocking and delegates long-running processes to detached tasks. + let (command_tx, command_rx) = mpsc::unbounded_channel::(); + + tokio::spawn(Self::command_processor( + logger_factory.cheap_clone(), + cancel_token.cheap_clone(), + command_tx.clone(), + command_rx, + )); + + Self { + logger_factory, + cancel_token, + command_tx, + subgraph_instance_id: Arc::new(AtomicU32::new(0)), + } + } + + /// Starts a subgraph. + /// + /// Sends a subgraph start request to this subgraph monitor that + /// eventually starts the subgraph. + /// + /// # Behaviour + /// + /// - If the subgraph is not active, it starts when the request is processed + /// - If the subgraph is active, it stops, and then restarts + /// - Ensures that there is only one subgraph instance for this subgraph deployment + /// - Multiple consecutive calls in a short time period force restart the subgraph, + /// aborting the active subgraph instance + pub(super) fn start(&self, deployment: DeploymentLocator, runner: BoxRunner) { + let logger = self + .logger_factory + .subgraph_logger(&deployment) + .new(slog::o!("method" => "start")); + + info!(logger, "Starting subgraph"); + log_send_error( + &logger, + self.command_tx.send(Command::Start { + id: self.subgraph_instance_id.fetch_add(1, SeqCst), + deployment, + runner, + }), + ); + } + + /// Stops the subgraph. + /// + /// Sends a subgraph stop request to this subgraph monitor that + /// eventually stops the subgraph. + /// + /// # Behaviour + /// + /// - If the subgraph is not active does nothing + /// - If the subgraph is active, sends a cancel signal that gracefully stops the subgraph + /// - If the subgraph fails to stop after an extended period of time it aborts + pub(super) fn stop(&self, deployment: DeploymentLocator) { + let logger = self + .logger_factory + .subgraph_logger(&deployment) + .new(slog::o!("method" => "stop")); + + info!(logger, "Stopping subgraph"); + log_send_error(&logger, self.command_tx.send(Command::Stop { deployment })); + } + + /// Processes commands sent through the command channel. + /// + /// Tracks active subgraphs and keeps a list of pending start commands. + /// Pending start commands are start commands that execute after the related subgraph stops. + async fn command_processor( + logger_factory: Arc, + cancel_token: CancellationToken, + command_tx: mpsc::UnboundedSender, + mut command_rx: mpsc::UnboundedReceiver, + ) { + let logger = logger_factory.component_logger("CommandProcessor", None); + let mut subgraph_instances: HashMap = HashMap::new(); + let mut pending_start_commands: HashMap = HashMap::new(); + + loop { + tokio::select! { + Some(command) = command_rx.recv() => { + match &command { + Command::Start { .. } => { + Self::process_start_command( + &logger_factory, + &cancel_token, + &mut subgraph_instances, + &mut pending_start_commands, + &command_tx, + command + ); + }, + Command::Stop { .. } => { + Self::process_stop_command( + &logger_factory, + &mut subgraph_instances, + &mut pending_start_commands, + command + ); + }, + Command::Clear { .. } => { + Self::process_clear_command( + &logger_factory, + &mut subgraph_instances, + &mut pending_start_commands, + &command_tx, + command + ); + }, + } + }, + _ = cancel_token.cancelled() => { + debug!(logger, "Stopping command processor"); + + // All active subgraphs will shutdown gracefully + // because their cancel tokens are derived from this cancelled token. + return; + } + } + } + } + + /// Starts a subgraph. + /// + /// # Behaviour + /// + /// - If the subgraph is not active, it starts right away + /// - If the subgraph is active, a cancel signal is sent to the active subgraph instance + /// and this start request is stored in the list of pending start commands + /// - If the subgraph is active and there is already a pending start command, + /// the active subgraph instance aborts, and the subgraph force restarts right away + /// - If the subgraph is active, but its instance is not actually running, + /// the subgraph starts right away + fn process_start_command( + logger_factory: &LoggerFactory, + cancel_token: &CancellationToken, + subgraph_instances: &mut HashMap, + pending_start_commands: &mut HashMap, + command_tx: &mpsc::UnboundedSender, + command: Command, + ) { + let Command::Start { + id, + deployment, + runner, + } = command + else { + unreachable!(); + }; + + let logger = logger_factory.subgraph_logger(&deployment); + let command_logger = logger.new(slog::o!("command" => "start")); + + let cancel_token = cancel_token.child_token(); + let pending_start_command = pending_start_commands.remove(&deployment); + + match subgraph_instances.entry(deployment.cheap_clone()) { + Entry::Vacant(entry) => { + debug!(command_logger, "Subgraph is not active, starting"); + + let subgraph_instance = Self::start_subgraph( + logger, + cancel_token, + id, + deployment, + runner, + command_tx.clone(), + ); + + entry.insert(subgraph_instance); + } + Entry::Occupied(mut entry) => { + let subgraph_instance = entry.get_mut(); + subgraph_instance.cancel_token.cancel(); + + if pending_start_command.is_some() { + debug!(command_logger, "Subgraph is active, force restarting"); + + subgraph_instance.handle.abort(); + + *subgraph_instance = Self::start_subgraph( + logger, + cancel_token, + id, + deployment, + runner, + command_tx.clone(), + ); + + return; + } + + if subgraph_instance.handle.is_finished() { + debug!(command_logger, "Subgraph is not running, starting"); + + *subgraph_instance = Self::start_subgraph( + logger, + cancel_token, + id, + deployment, + runner, + command_tx.clone(), + ); + + return; + } + + debug!(command_logger, "Gracefully restarting subgraph"); + + pending_start_commands.insert( + deployment.cheap_clone(), + Command::Start { + id, + deployment, + runner, + }, + ); + } + } + } + + /// Stops a subgraph. + /// + /// # Behaviour + /// + /// - If the subgraph is not active, does nothing + /// - If the subgraph is active, sends a cancel signal to the active subgraph instance + fn process_stop_command( + logger_factory: &LoggerFactory, + subgraph_instances: &mut HashMap, + pending_start_commands: &mut HashMap, + command: Command, + ) { + let Command::Stop { deployment } = command else { + unreachable!(); + }; + + let logger = logger_factory + .subgraph_logger(&deployment) + .new(slog::o!("command" => "stop")); + + if let Some(subgraph_instance) = subgraph_instances.get(&deployment) { + debug!(logger, "Sending cancel signal"); + subgraph_instance.cancel_token.cancel(); + } else { + debug!(logger, "Subgraph is not active"); + } + + pending_start_commands.remove(&deployment); + } + + /// Removes a subgraph from the list of active subgraphs allowing the subgraph to be restarted. + fn process_clear_command( + logger_factory: &LoggerFactory, + subgraph_instances: &mut HashMap, + pending_start_commands: &mut HashMap, + command_tx: &mpsc::UnboundedSender, + command: Command, + ) { + let Command::Clear { id, deployment } = command else { + unreachable!(); + }; + + let logger = logger_factory + .subgraph_logger(&deployment) + .new(slog::o!("command" => "clear")); + + match subgraph_instances.get(&deployment) { + Some(subgraph_instance) if subgraph_instance.id == id => { + debug!(logger, "Removing active subgraph"); + subgraph_instances.remove(&deployment); + } + Some(_subgraph_instance) => { + debug!(logger, "Active subgraph does not need to be removed"); + return; + } + None => { + debug!(logger, "Subgraph is not active"); + } + } + + if let Some(pending_start_command) = pending_start_commands.remove(&deployment) { + debug!(logger, "Resending a pending start command"); + log_send_error(&logger, command_tx.send(pending_start_command)); + } + } + + /// Spawns a background task that executes the subgraph runner future. + /// + /// An additional background task is spawned to handle the graceful shutdown of the subgraph runner, + /// and to ensure correct behaviour even if the subgraph runner panics. + fn start_subgraph( + logger: Logger, + cancel_token: CancellationToken, + id: u32, + deployment: DeploymentLocator, + runner: BoxRunner, + command_tx: mpsc::UnboundedSender, + ) -> SubgraphInstance { + let mut runner_handle = tokio::spawn({ + let logger = logger.new(slog::o!("process" => "subgraph_runner")); + let cancel_token = cancel_token.cheap_clone(); + + async move { + info!(logger, "Subgraph started"); + + match runner(cancel_token).await { + Ok(()) => { + info!(logger, "Subgraph stopped"); + } + Err(e) => { + error!(logger, "Subgraph failed"; + "error" => ?e + ); + } + } + } + }); + + let supervisor_handle = tokio::spawn({ + let logger = logger.new(slog::o!("process" => "subgraph_supervisor")); + let cancel_token = cancel_token.cheap_clone(); + + fn handle_runner_result(logger: &Logger, result: Result<(), tokio::task::JoinError>) { + match result { + Ok(()) => { + debug!(logger, "Subgraph completed execution"); + } + Err(e) if e.is_panic() => { + error!(logger, "Subgraph panicked"; + "error" => ?e + ); + } + Err(e) => { + error!(logger, "Subgraph failed"; + "error" => ?e + ); + } + } + } + + async move { + debug!(logger, "Subgraph supervisor started"); + + tokio::select! { + _ = cancel_token.cancelled() => { + debug!(logger, "Received cancel signal, waiting for subgraph to stop"); + + match timeout(SUBGRAPH_INSTANCE_GRACE_PERIOD, &mut runner_handle).await { + Ok(result) => { + handle_runner_result(&logger, result); + }, + Err(_) => { + warn!(logger, "Subgraph did not stop after grace period, aborting"); + + runner_handle.abort(); + let _ = runner_handle.await; + + warn!(logger, "Subgraph aborted"); + } + } + }, + result = &mut runner_handle => { + handle_runner_result(&logger, result); + cancel_token.cancel(); + } + } + + debug!(logger, "Sending clear command"); + log_send_error(&logger, command_tx.send(Command::Clear { id, deployment })); + } + }); + + SubgraphInstance { + id, + handle: supervisor_handle, + cancel_token, + } + } +} + +impl Drop for Monitor { + fn drop(&mut self) { + // Send cancel signals to all active subgraphs so that they don't remain without an associated monitor + self.cancel_token.cancel(); + } +} + +/// Represents a background task that executes the subgraph runner future. +struct SubgraphInstance { + id: u32, + handle: JoinHandle<()>, + cancel_token: CancellationToken, +} + +/// Every command used by the subgraph monitor. +enum Command { + /// A request to start executing the subgraph runner future. + Start { + id: u32, + deployment: DeploymentLocator, + runner: BoxRunner, + }, + + /// A request to stop executing the subgraph runner future. + Stop { deployment: DeploymentLocator }, + + /// A request to remove the subgraph from the list of active subgraphs. + Clear { + id: u32, + deployment: DeploymentLocator, + }, +} + +impl fmt::Debug for Command { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Start { + id, + deployment, + runner: _, + } => f + .debug_struct("Start") + .field("id", id) + .field("deployment", deployment) + .finish_non_exhaustive(), + Self::Stop { deployment } => f + .debug_struct("Stop") + .field("deployment", deployment) + .finish(), + Self::Clear { id, deployment } => f + .debug_struct("Clear") + .field("id", id) + .field("deployment", deployment) + .finish(), + } + } +} + +fn log_send_error(logger: &Logger, result: Result<(), SendError>) { + match result { + Ok(()) => { + // No need to log anything + } + + // This should only happen if the parent cancel token of the subgraph monitor was cancelled + Err(e) => { + error!(logger, "Failed to send command"; + "command" => ?e.0, + "error" => ?e + ); + } + } +} diff --git a/core/src/amp_subgraph/runner/compat.rs b/core/src/amp_subgraph/runner/compat.rs new file mode 100644 index 00000000000..f7152965cf3 --- /dev/null +++ b/core/src/amp_subgraph/runner/compat.rs @@ -0,0 +1,52 @@ +//! This is a temporary compatibility module until the graph-node is fully migrated to `alloy`. + +use alloy::primitives::{BlockHash, BlockNumber}; +use chrono::{DateTime, Utc}; + +mod legacy { + pub(super) use graph::{ + blockchain::{BlockHash, BlockPtr, BlockTime}, + components::store::BlockNumber, + data::store::scalar::Timestamp, + }; +} + +pub(super) trait Compat { + fn compat(&self) -> T; +} + +impl Compat for BlockNumber { + fn compat(&self) -> legacy::BlockNumber { + (*self).try_into().unwrap() + } +} + +impl Compat for legacy::BlockNumber { + fn compat(&self) -> BlockNumber { + (*self).try_into().unwrap() + } +} + +impl Compat for BlockHash { + fn compat(&self) -> legacy::BlockHash { + legacy::BlockHash(self.0.into()) + } +} + +impl Compat for legacy::BlockHash { + fn compat(&self) -> BlockHash { + BlockHash::from_slice(&self.0) + } +} + +impl Compat for DateTime { + fn compat(&self) -> legacy::BlockTime { + legacy::Timestamp(*self).into() + } +} + +impl Compat for (BlockNumber, BlockHash) { + fn compat(&self) -> legacy::BlockPtr { + legacy::BlockPtr::new(self.1.compat(), self.0.compat()) + } +} diff --git a/core/src/amp_subgraph/runner/context.rs b/core/src/amp_subgraph/runner/context.rs new file mode 100644 index 00000000000..d8b5b7b638e --- /dev/null +++ b/core/src/amp_subgraph/runner/context.rs @@ -0,0 +1,97 @@ +use std::sync::Arc; + +use alloy::primitives::{BlockHash, BlockNumber}; +use graph::{ + amp::{Codec, Manifest, log::Logger as _}, + cheap_clone::CheapClone, + components::store::WritableStore, + data::subgraph::DeploymentHash, + env::AmpEnv, + util::backoff::ExponentialBackoff, +}; +use slog::Logger; + +use super::Compat; +use crate::amp_subgraph::Metrics; + +pub(in super::super) struct Context { + pub(super) logger: Logger, + pub(super) client: Arc, + pub(super) store: Arc, + pub(super) buffer_size: usize, + pub(super) block_range: usize, + pub(super) backoff: ExponentialBackoff, + pub(super) deployment: DeploymentHash, + pub(super) manifest: Manifest, + pub(super) metrics: Metrics, + pub(super) codec: Codec, +} + +impl Context { + pub(in super::super) fn new( + logger: &Logger, + env: &AmpEnv, + client: Arc, + store: Arc, + deployment: DeploymentHash, + manifest: Manifest, + metrics: Metrics, + ) -> Self { + let logger = logger.component("AmpSubgraphRunner"); + let backoff = ExponentialBackoff::new(env.query_retry_min_delay, env.query_retry_max_delay); + let codec = Codec::new(manifest.schema.cheap_clone()); + + Self { + logger, + client, + store, + buffer_size: env.buffer_size, + block_range: env.block_range, + backoff, + deployment, + manifest, + metrics, + codec, + } + } + + pub(super) fn indexing_completed(&self) -> bool { + let Some(last_synced_block) = self.latest_synced_block() else { + return false; + }; + + self.manifest + .data_sources + .iter() + .all(|data_source| last_synced_block >= data_source.source.end_block) + } + + pub(super) fn latest_synced_block(&self) -> Option { + self.latest_synced_block_ptr() + .map(|(block_number, _)| block_number) + } + + pub(super) fn latest_synced_block_ptr(&self) -> Option<(BlockNumber, BlockHash)> { + self.store + .block_ptr() + .map(|block_ptr| (block_ptr.number.compat(), block_ptr.hash.compat())) + } + + pub(super) fn start_block(&self) -> BlockNumber { + self.manifest + .data_sources + .iter() + .map(|data_source| data_source.source.start_block) + .min() + .unwrap() + } + + pub(super) fn end_block(&self) -> BlockNumber { + self.manifest + .data_sources + .iter() + .map(|data_source| data_source.source.end_block) + .max() + .unwrap() + } +} diff --git a/core/src/amp_subgraph/runner/data_processing.rs b/core/src/amp_subgraph/runner/data_processing.rs new file mode 100644 index 00000000000..d5bf991d505 --- /dev/null +++ b/core/src/amp_subgraph/runner/data_processing.rs @@ -0,0 +1,267 @@ +use std::sync::Arc; + +use alloy::primitives::{BlockHash, BlockNumber}; +use anyhow::anyhow; +use arrow::array::RecordBatch; +use chrono::{DateTime, Utc}; +use graph::{ + amp::{ + codec::{DecodeOutput, DecodedEntity, Decoder, utils::auto_block_timestamp_decoder}, + stream_aggregator::{RecordBatchGroup, RecordBatchGroups, StreamRecordBatch}, + }, + blockchain::block_stream::FirehoseCursor, + cheap_clone::CheapClone, + components::store::{EntityCache, EntityLfuCache, ModificationsAndCache, SeqGenerator}, +}; +use slog::{debug, trace}; + +use super::{Compat, Context, Error, data_stream::TablePtr}; + +pub(super) async fn process_record_batch_groups( + cx: &mut Context, + mut entity_lfu_cache: EntityLfuCache, + record_batch_groups: RecordBatchGroups, + stream_table_ptr: Arc<[TablePtr]>, + latest_block: BlockNumber, +) -> Result { + if record_batch_groups.is_empty() { + debug!(cx.logger, "Received no record batch groups"); + return Ok(entity_lfu_cache); + } + + let from_block = record_batch_groups + .first_key_value() + .map(|((block, _), _)| *block) + .unwrap(); + + let to_block = record_batch_groups + .last_key_value() + .map(|((block, _), _)| *block) + .unwrap(); + + debug!(cx.logger, "Processing record batch groups"; + "from_block" => from_block, + "to_block" => to_block + ); + + for ((block_number, block_hash), record_batch_group) in record_batch_groups { + trace!(cx.logger, "Processing record batch group"; + "block" => block_number, + "record_batches_count" => record_batch_group.record_batches.len() + ); + + entity_lfu_cache = process_record_batch_group( + cx, + entity_lfu_cache, + block_number, + block_hash, + record_batch_group, + &stream_table_ptr, + latest_block, + ) + .await + .map_err(|e| { + e.context(format!( + "failed to process record batch group at block '{block_number}'" + )) + })?; + + cx.metrics.deployment_head.update(block_number); + cx.metrics.blocks_processed.record_one(); + + trace!(cx.logger, "Completed processing record batch group"; + "block" => block_number + ); + } + + debug!(cx.logger, "Completed processing record batch groups"; + "from_block" => from_block, + "to_block" => to_block + ); + + Ok(entity_lfu_cache) +} + +async fn process_record_batch_group( + cx: &mut Context, + entity_lfu_cache: EntityLfuCache, + block_number: BlockNumber, + block_hash: BlockHash, + record_batch_group: RecordBatchGroup, + stream_table_ptr: &[TablePtr], + latest_block: BlockNumber, +) -> Result { + let _section = cx + .metrics + .stopwatch + .start_section("process_record_batch_group"); + + let RecordBatchGroup { record_batches } = record_batch_group; + + if record_batches.is_empty() { + debug!(cx.logger, "Record batch group is empty"); + return Ok(entity_lfu_cache); + } + + let mut entity_cache = EntityCache::with_current( + cx.store.cheap_clone(), + entity_lfu_cache, + SeqGenerator::new(block_number.compat()), + ); + + let block_timestamp = if cx.manifest.schema.has_aggregations() { + decode_block_timestamp(&record_batches) + .map_err(|e| e.context("failed to decode block timestamp"))? + } else { + // TODO: Block timestamp is only required for subgraph aggregations. + // Make it optional at the store level. + DateTime::::MIN_UTC + }; + + for record_batch in record_batches { + let StreamRecordBatch { + stream_index, + record_batch, + } = record_batch; + + process_record_batch( + cx, + &mut entity_cache, + record_batch, + stream_table_ptr[stream_index], + ) + .await + .map_err(|e| { + e.context(format!( + "failed to process record batch for stream '{stream_index}'" + )) + })?; + } + + let section = cx.metrics.stopwatch.start_section("as_modifications"); + let ModificationsAndCache { + modifications, + entity_lfu_cache, + evict_stats: _, + } = entity_cache + .as_modifications(block_number.compat(), &cx.metrics.stopwatch) + .await + .map_err(Error::from) + .map_err(|e| e.context("failed to extract entity modifications from the state"))?; + section.end(); + + let _section = cx.metrics.stopwatch.start_section("transact_block"); + let is_close_to_chain_head = latest_block.saturating_sub(block_number) <= 100; + + cx.store + .transact_block_operations( + (block_number, block_hash).compat(), + block_timestamp.compat(), + FirehoseCursor::None, + modifications, + &cx.metrics.stopwatch, + Vec::new(), + Vec::new(), + Vec::new(), + false, + is_close_to_chain_head, + ) + .await + .map_err(Error::from) + .map_err(|e| e.context("failed to transact block operations"))?; + + if is_close_to_chain_head { + cx.metrics.deployment_synced.record(true); + } + + Ok(entity_lfu_cache) +} + +async fn process_record_batch( + cx: &mut Context, + entity_cache: &mut EntityCache, + record_batch: RecordBatch, + (i, j): TablePtr, +) -> Result<(), Error> { + let _section = cx.metrics.stopwatch.start_section("process_record_batch"); + + let table = &cx.manifest.data_sources[i].transformer.tables[j]; + let entity_name = &table.name; + + let DecodeOutput { + entity_type, + id_type, + decoded_entities, + } = cx + .codec + .decode(record_batch, entity_name.as_str()) + .map_err(|e| { + Error::Deterministic( + e.context(format!("failed to decode entities of type '{entity_name}'")), + ) + })?; + + for decoded_entity in decoded_entities { + let DecodedEntity { + key, + mut entity_data, + } = decoded_entity; + + let key = match key { + Some(key) => key, + None => { + let entity_id = entity_cache.seq_gen.id(id_type).map_err(|e| { + Error::Deterministic(e.context(format!( + "failed to generate a new id for an entity of type '{entity_name}'" + ))) + })?; + + entity_data.push(("id".into(), entity_id.clone().into())); + entity_type.key(entity_id) + } + }; + + let entity_id = key.entity_id.clone(); + let entity = cx.manifest.schema.make_entity(entity_data).map_err(|e| { + Error::Deterministic(anyhow!(e).context(format!( + "failed to create a new entity of type '{entity_name}' with id '{entity_id}'" + ))) + })?; + + entity_cache.set(key, entity, None).await.map_err(|e| { + Error::Deterministic(e.context(format!( + "failed to store a new entity of type '{entity_name}' with id '{entity_id}'" + ))) + })?; + } + + Ok(()) +} + +/// Decodes the block timestamp from the first matching column in `record_batches`. +/// +/// Iterates through the provided record batches and returns the timestamp from +/// the first batch that contains a valid block timestamp column. +/// +/// # Preconditions +/// +/// All entries in `record_batches` must belong to the same record batch group. +fn decode_block_timestamp(record_batches: &[StreamRecordBatch]) -> Result, Error> { + let mut last_error: Option = None; + + for record_batch in record_batches { + match auto_block_timestamp_decoder(&record_batch.record_batch) { + Ok((_, decoder)) => { + return decoder + .decode(0) + .map_err(Error::Deterministic)? + .ok_or_else(|| Error::Deterministic(anyhow!("block timestamp is empty"))); + } + Err(e) => { + last_error = Some(Error::Deterministic(e)); + } + } + } + + Err(last_error.unwrap()) +} diff --git a/core/src/amp_subgraph/runner/data_stream.rs b/core/src/amp_subgraph/runner/data_stream.rs new file mode 100644 index 00000000000..693ae9384a7 --- /dev/null +++ b/core/src/amp_subgraph/runner/data_stream.rs @@ -0,0 +1,232 @@ +use std::{collections::HashMap, ops::RangeInclusive, sync::Arc}; + +use alloy::primitives::BlockNumber; +use anyhow::anyhow; +use futures::{ + StreamExt, TryStreamExt, + stream::{self, BoxStream}, +}; +use graph::{ + amp::{ + Client, + client::ResponseBatch, + error::IsDeterministic, + manifest::DataSource, + stream_aggregator::{RecordBatchGroups, StreamAggregator}, + }, + cheap_clone::CheapClone, + prelude::StopwatchMetrics, +}; +use slog::{debug, warn}; + +use super::{Context, Error}; + +pub(super) type TablePtr = (usize, usize); + +pub(super) fn new_data_stream( + cx: &Context, + latest_block: BlockNumber, +) -> BoxStream<'static, Result<(RecordBatchGroups, Arc<[TablePtr]>), Error>> +where + AC: Client + Send + Sync + 'static, +{ + let logger = cx.logger.new(slog::o!("process" => "new_data_stream")); + let client = cx.client.cheap_clone(); + let manifest = cx.manifest.clone(); + let buffer_size = cx.buffer_size; + let block_range = cx.block_range; + let stopwatch = cx.metrics.stopwatch.cheap_clone(); + + debug!(logger, "Creating data stream"; + "from_block" => cx.latest_synced_block().unwrap_or(BlockNumber::MIN), + "to_block" => latest_block, + "start_block" => cx.start_block(), + "block_range" => block_range, + ); + + // State: (latest_queried_block, end_block, is_first) + let initial_state = (cx.latest_synced_block(), BlockNumber::MIN, true); + + stream::unfold( + initial_state, + move |(latest_queried_block, mut end_block, is_first)| { + let block_ranges = next_block_ranges( + &manifest.data_sources, + block_range, + latest_queried_block, + latest_block, + ); + + if block_ranges.is_empty() { + if is_first { + warn!(logger, "There are no unprocessed block ranges"); + } + return futures::future::ready(None); + } + + let start_block = block_ranges.values().map(|r| *r.start()).min().unwrap(); + end_block = end_block.max(block_ranges.values().map(|r| *r.end()).max().unwrap()); + + let (query_streams, table_ptrs) = + build_query_streams(&*client, &logger, &manifest.data_sources, &block_ranges); + + let data_stream = build_data_stream( + &logger, + query_streams, + table_ptrs, + buffer_size, + &stopwatch, + start_block, + ); + + debug!(logger, "Created a new data stream"; + "latest_queried_block" => latest_queried_block, + "start_block" => start_block, + "end_block" => end_block, + ); + futures::future::ready(Some((data_stream, (Some(end_block), end_block, false)))) + }, + ) + .flatten() + .boxed() +} + +fn build_query_streams( + client: &AC, + logger: &slog::Logger, + data_sources: &[DataSource], + block_ranges: &HashMap>, +) -> ( + Vec<(String, BoxStream<'static, Result>)>, + Arc<[TablePtr]>, +) { + let total_queries: usize = data_sources + .iter() + .map(|ds| ds.transformer.tables.len()) + .sum(); + + let mut query_streams = Vec::with_capacity(total_queries); + let mut table_ptrs = Vec::with_capacity(total_queries); + + for (i, data_source) in data_sources.iter().enumerate() { + let Some(block_range) = block_ranges.get(&i) else { + continue; + }; + + for (j, table) in data_source.transformer.tables.iter().enumerate() { + let query = table.query.build_with_block_range(block_range); + let stream = client.query(logger, query, None); + let stream_name = format!("{}.{}", data_source.name, table.name); + + query_streams.push((stream_name, stream)); + table_ptrs.push((i, j)); + } + } + + (query_streams, table_ptrs.into()) +} + +fn build_data_stream( + logger: &slog::Logger, + query_streams: Vec<(String, BoxStream<'static, Result>)>, + table_ptrs: Arc<[TablePtr]>, + buffer_size: usize, + stopwatch: &StopwatchMetrics, + start_block: BlockNumber, +) -> BoxStream<'static, Result<(RecordBatchGroups, Arc<[TablePtr]>), Error>> +where + E: std::error::Error + IsDeterministic + Send + Sync + 'static, +{ + let mut start_block_checked = false; + let mut load_first_record_batch_group_section = + Some(stopwatch.start_section("load_first_record_batch_group")); + + StreamAggregator::new(logger, query_streams, buffer_size) + .map_ok(move |response| (response, table_ptrs.cheap_clone())) + .map_err(Error::from) + .map(move |result| { + if load_first_record_batch_group_section.is_some() { + let _section = load_first_record_batch_group_section.take(); + } + + match result { + Ok(response) => { + if !start_block_checked { + if let Some(((first_block, _), _)) = response.0.first_key_value() + && *first_block < start_block + { + return Err(Error::NonDeterministic(anyhow!("chain reorg"))); + } + + start_block_checked = true; + } + + Ok(response) + } + Err(e) => Err(e), + } + }) + .boxed() +} + +fn next_block_ranges( + data_sources: &[DataSource], + block_range: usize, + latest_queried_block: Option, + latest_block: BlockNumber, +) -> HashMap> { + let block_ranges = data_sources + .iter() + .enumerate() + .filter_map(|(i, data_source)| { + next_block_range(block_range, data_source, latest_queried_block, latest_block) + .map(|block_range| (i, block_range)) + }) + .collect::>(); + + let Some(min_block_range) = block_ranges + .iter() + .min_by_key(|(_, block_range)| *block_range.start()) + .map(|(_, min_block_range)| min_block_range.clone()) + else { + return HashMap::new(); + }; + + block_ranges + .into_iter() + .filter(|(_, block_range)| block_range.start() <= min_block_range.end()) + .collect() +} + +fn next_block_range( + block_range: usize, + data_source: &DataSource, + latest_queried_block: Option, + latest_block: BlockNumber, +) -> Option> { + let start_block = match latest_queried_block { + Some(latest_queried_block) => { + if latest_queried_block >= data_source.source.end_block { + return None; + } + + latest_queried_block + 1 + } + None => data_source.source.start_block, + }; + + let end_block = [ + start_block.saturating_add(block_range as BlockNumber), + data_source.source.end_block, + latest_block, + ] + .into_iter() + .min() + .unwrap(); + + if start_block > end_block { + return None; + } + + Some(start_block..=end_block) +} diff --git a/core/src/amp_subgraph/runner/error.rs b/core/src/amp_subgraph/runner/error.rs new file mode 100644 index 00000000000..8c7077e1c68 --- /dev/null +++ b/core/src/amp_subgraph/runner/error.rs @@ -0,0 +1,43 @@ +use graph::amp::error::IsDeterministic; +use thiserror::Error; + +#[derive(Debug, Error)] +pub(super) enum Error { + #[error("runner failed with a non-deterministic error: {0:#}")] + NonDeterministic(#[source] anyhow::Error), + + #[error("runner failed with a deterministic error: {0:#}")] + Deterministic(#[source] anyhow::Error), +} + +impl Error { + pub(super) fn context(self, context: C) -> Self + where + C: std::fmt::Display + Send + Sync + 'static, + { + match self { + Self::NonDeterministic(e) => Self::NonDeterministic(e.context(context)), + Self::Deterministic(e) => Self::Deterministic(e.context(context)), + } + } + + pub(super) fn is_deterministic(&self) -> bool { + match self { + Self::Deterministic(_) => true, + Self::NonDeterministic(_) => false, + } + } +} + +impl From for Error +where + T: std::error::Error + IsDeterministic + Send + Sync + 'static, +{ + fn from(e: T) -> Self { + if e.is_deterministic() { + Self::Deterministic(e.into()) + } else { + Self::NonDeterministic(e.into()) + } + } +} diff --git a/core/src/amp_subgraph/runner/latest_blocks.rs b/core/src/amp_subgraph/runner/latest_blocks.rs new file mode 100644 index 00000000000..17bdc23dd29 --- /dev/null +++ b/core/src/amp_subgraph/runner/latest_blocks.rs @@ -0,0 +1,180 @@ +use alloy::primitives::BlockNumber; +use anyhow::anyhow; +use arrow::array::RecordBatch; +use futures::{StreamExt, TryFutureExt, future::try_join_all, stream::BoxStream}; +use graph::amp::{ + Client, + client::ResponseBatch, + codec::{Decoder, utils::block_number_decoder}, + error::IsDeterministic, + manifest::DataSource, +}; +use itertools::Itertools; +use slog::debug; + +use super::{Context, Error}; + +pub(super) type TablePtr = (usize, usize); + +pub(super) struct LatestBlocks(Vec<(TablePtr, BlockNumber)>); + +impl LatestBlocks { + pub(super) async fn load(cx: &Context) -> Result + where + AC: Client, + { + debug!(cx.logger, "Loading latest blocks"); + let _section = cx.metrics.stopwatch.start_section("load_latest_blocks"); + + let latest_block_futs = cx + .manifest + .data_sources + .iter() + .enumerate() + .flat_map(|(i, data_source)| { + data_source + .source + .tables + .iter() + .enumerate() + .map(move |(j, table)| ((i, j), &data_source.source.dataset, table)) + }) + .unique_by(|(_, dataset, table)| (dataset.to_string(), table.to_string())) + .map(|(table_ptr, dataset, table)| { + latest_block(cx, dataset, table) + .map_ok(move |latest_block| (table_ptr, latest_block)) + .map_err(move |e| { + e.context(format!( + "failed to load latest block for '{dataset}.{table}'" + )) + }) + }); + + try_join_all(latest_block_futs).await.map(Self) + } + + pub(super) fn filter_completed(self, cx: &Context) -> Self + where + AC: Client, + { + let latest_synced_block = cx.latest_synced_block(); + + Self( + self.0 + .into_iter() + .filter(|((i, _), _)| { + !indexing_completed(&cx.manifest.data_sources[*i], &latest_synced_block) + }) + .collect(), + ) + } + + pub(super) fn min(&self) -> BlockNumber { + self.0 + .iter() + .min_by_key(|(_, latest_block)| *latest_block) + .map(|(_, latest_block)| *latest_block) + .unwrap() + } + + pub(super) async fn changed(self, cx: &Context) -> Result<(), Error> + where + AC: Client, + { + debug!(cx.logger, "Waiting for new blocks"); + let _section = cx.metrics.stopwatch.start_section("latest_blocks_changed"); + + let min_latest_block = self.min(); + let latest_synced_block = cx.latest_synced_block(); + + let latest_block_changed_futs = self + .0 + .into_iter() + .filter(|(_, latest_block)| *latest_block == min_latest_block) + .filter(|((i, _), _)| { + !indexing_completed(&cx.manifest.data_sources[*i], &latest_synced_block) + }) + .map(|((i, j), latest_block)| { + let source = &cx.manifest.data_sources[i].source; + let dataset = &source.dataset; + let table = &source.tables[j]; + + latest_block_changed(cx, dataset, table, latest_block).map_err(move |e| { + e.context(format!( + "failed to check if the latest block changed in '{dataset}.{table}'" + )) + }) + }); + + let _response = try_join_all(latest_block_changed_futs).await?; + + Ok(()) + } + + pub(super) fn iter(&self) -> impl Iterator { + self.0.iter() + } +} + +fn indexing_completed(data_source: &DataSource, latest_synced_block: &Option) -> bool { + latest_synced_block + .as_ref() + .is_some_and(|latest_synced_block| *latest_synced_block >= data_source.source.end_block) +} + +async fn latest_block( + cx: &Context, + dataset: &str, + table: &str, +) -> Result +where + AC: Client, +{ + let query = format!("SELECT MAX(_block_num) FROM {dataset}.{table}"); + let stream = cx.client.query(&cx.logger, query, None); + let record_batch = read_once(stream).await?; + + let latest_block = block_number_decoder(&record_batch, 0) + .map_err(Error::Deterministic)? + .decode(0) + .map_err(Error::Deterministic)? + .ok_or_else(|| Error::NonDeterministic(anyhow!("table is empty")))?; + + Ok(latest_block) +} + +async fn latest_block_changed( + cx: &Context, + dataset: &str, + table: &str, + latest_block: BlockNumber, +) -> Result<(), Error> +where + AC: Client, +{ + let query = format!( + "SELECT _block_num FROM {dataset}.{table} WHERE _block_num > {latest_block} SETTINGS stream = true" + ); + let stream = cx.client.query(&cx.logger, query, None); + let _record_batch = read_once(stream).await?; + + Ok(()) +} + +async fn read_once( + mut stream: BoxStream<'static, Result>, +) -> Result +where + E: std::error::Error + IsDeterministic + Send + Sync + 'static, +{ + let response = stream + .next() + .await + .ok_or_else(|| Error::NonDeterministic(anyhow!("stream is empty")))? + .map_err(Error::from)?; + + match response { + ResponseBatch::Batch { data } => Ok(data), + _ => Err(Error::NonDeterministic(anyhow!("response is empty"))), + } +} diff --git a/core/src/amp_subgraph/runner/mod.rs b/core/src/amp_subgraph/runner/mod.rs new file mode 100644 index 00000000000..4e0e85294b6 --- /dev/null +++ b/core/src/amp_subgraph/runner/mod.rs @@ -0,0 +1,190 @@ +mod compat; +mod context; +mod data_processing; +mod data_stream; +mod error; +mod latest_blocks; +mod reorg_handler; + +use std::time::{Duration, Instant}; + +use anyhow::Result; +use futures::StreamExt; +use graph::{ + amp::Client, cheap_clone::CheapClone, components::store::EntityLfuCache, + data::subgraph::schema::SubgraphError, util::lfu_cache::LfuCache, +}; +use slog::{debug, error, warn}; +use tokio_util::sync::CancellationToken; + +use self::{ + compat::Compat, data_processing::process_record_batch_groups, data_stream::new_data_stream, + error::Error, latest_blocks::LatestBlocks, reorg_handler::check_and_handle_reorg, +}; + +pub(super) use self::context::Context; + +pub(super) async fn new_runner( + mut cx: Context, + cancel_token: CancellationToken, +) -> Result<()> +where + AC: Client + Send + Sync + 'static, +{ + let indexing_duration_handle = tokio::spawn({ + let mut instant = Instant::now(); + let indexing_duration = cx.metrics.indexing_duration.clone(); + + async move { + loop { + tokio::time::sleep(Duration::from_secs(1)).await; + + let prev_instant = std::mem::replace(&mut instant, Instant::now()); + indexing_duration.record(prev_instant.elapsed()); + } + } + }); + + let result = cancel_token + .run_until_cancelled(run_indexing_with_retries(&mut cx)) + .await; + + indexing_duration_handle.abort(); + + match result { + Some(result) => result?, + None => { + debug!(cx.logger, "Processed cancel signal"); + } + } + + cx.metrics.deployment_status.stopped(); + + debug!(cx.logger, "Waiting for the store to finish processing"); + cx.store.flush().await?; + Ok(()) +} + +async fn run_indexing(cx: &mut Context) -> Result<(), Error> +where + AC: Client + Send + Sync + 'static, +{ + cx.metrics.deployment_status.starting(); + + if let Some(latest_synced_block) = cx.latest_synced_block() { + cx.metrics.deployment_head.update(latest_synced_block); + } + + cx.metrics + .deployment_synced + .record(cx.store.is_deployment_synced()); + + loop { + cx.metrics.deployment_status.running(); + + debug!(cx.logger, "Running indexing"; + "latest_synced_block_ptr" => ?cx.latest_synced_block_ptr() + ); + + let mut latest_blocks = LatestBlocks::load(cx).await?; + check_and_handle_reorg(cx, &latest_blocks).await?; + + if cx.indexing_completed() { + cx.metrics.deployment_synced.record(true); + + debug!(cx.logger, "Indexing completed"); + return Ok(()); + } + + latest_blocks = latest_blocks.filter_completed(cx); + let latest_block = latest_blocks.min(); + + cx.metrics + .deployment_target + .update(latest_block.min(cx.end_block())); + + let mut deployment_is_failed = cx.store.health().await?.is_failed(); + let mut entity_lfu_cache: EntityLfuCache = LfuCache::new(); + let mut stream = new_data_stream(cx, latest_block); + + while let Some(result) = stream.next().await { + let (record_batch_groups, stream_table_ptr) = result?; + + entity_lfu_cache = process_record_batch_groups( + cx, + entity_lfu_cache, + record_batch_groups, + stream_table_ptr, + latest_block, + ) + .await?; + + if deployment_is_failed && let Some(block_ptr) = cx.store.block_ptr() { + cx.store.unfail_non_deterministic_error(&block_ptr).await?; + deployment_is_failed = false; + } + } + + // Check if the Amp Flight server has data covering through every data + // source's endBlock. This handles the case where endBlock has no entity + // data — the persisted block pointer never advances to endBlock, but the + // server's latest block confirms all queries have been served. + if latest_block >= cx.end_block() { + cx.metrics.deployment_synced.record(true); + + debug!(cx.logger, "Indexing completed; endBlock reached via server latest block"; + "latest_block" => latest_block, + "end_block" => cx.end_block() + ); + return Ok(()); + } + + debug!(cx.logger, "Completed indexing iteration"; + "latest_synced_block_ptr" => ?cx.latest_synced_block_ptr() + ); + + latest_blocks.changed(cx).await?; + cx.backoff.reset(); + } +} + +async fn run_indexing_with_retries(cx: &mut Context) -> Result<()> +where + AC: Client + Send + Sync + 'static, +{ + loop { + match run_indexing(cx).await { + Ok(()) => return Ok(()), + Err(e) => { + cx.metrics.deployment_status.failed(); + + let deterministic = e.is_deterministic(); + + cx.store + .fail_subgraph(SubgraphError { + subgraph_id: cx.deployment.cheap_clone(), + message: format!("{e:#}"), + block_ptr: None, // TODO: Find a way to propagate the block ptr here + handler: None, + deterministic, + }) + .await?; + + if deterministic { + error!(cx.logger, "Subgraph failed with a deterministic error"; + "e" => ?e + ); + return Err(e.into()); + } + + warn!(cx.logger, "Subgraph failed with a non-deterministic error"; + "e" => ?e, + "retry_delay_seconds" => cx.backoff.delay().as_secs() + ); + + cx.backoff.sleep_async().await; + debug!(cx.logger, "Restarting indexing"); + } + } + } +} diff --git a/core/src/amp_subgraph/runner/reorg_handler.rs b/core/src/amp_subgraph/runner/reorg_handler.rs new file mode 100644 index 00000000000..8f59169304b --- /dev/null +++ b/core/src/amp_subgraph/runner/reorg_handler.rs @@ -0,0 +1,163 @@ +use alloy::primitives::{BlockHash, BlockNumber}; +use anyhow::anyhow; +use futures::{StreamExt, TryFutureExt, future::try_join_all}; +use graph::{ + amp::{ + Client, + client::{LatestBlockBeforeReorg, RequestMetadata, ResponseBatch, ResumeStreamingQuery}, + }, + blockchain::block_stream::FirehoseCursor, +}; +use itertools::Itertools; +use slog::debug; + +use super::{Compat, Context, Error, LatestBlocks}; + +pub(super) async fn check_and_handle_reorg( + cx: &Context, + latest_blocks: &LatestBlocks, +) -> Result<(), Error> +where + AC: Client, +{ + let logger = cx + .logger + .new(slog::o!("process" => "check_and_handle_reorg")); + + let Some((latest_synced_block_number, latest_synced_block_hash)) = cx.latest_synced_block_ptr() + else { + debug!(logger, "There are no synced blocks; Skipping reorg check"); + return Ok(()); + }; + + debug!(logger, "Running reorg check"); + + let Some(latest_block_before_reorg) = detect_deepest_reorg( + cx, + latest_blocks, + latest_synced_block_number, + latest_synced_block_hash, + ) + .await? + else { + debug!(logger, "Successfully checked for reorg: No reorg detected"; + "latest_synced_block" => latest_synced_block_number + ); + return Ok(()); + }; + + let _section = cx.metrics.stopwatch.start_section("handle_reorg"); + + debug!(logger, "Handling reorg"; + "latest_synced_block" => latest_synced_block_number, + "latest_block_before_reorg" => ?latest_block_before_reorg.block_number + ); + + let (block_number, block_hash) = match ( + latest_block_before_reorg.block_number, + latest_block_before_reorg.block_hash, + ) { + (Some(block_number), Some(block_hash)) => (block_number, block_hash), + (_, _) => { + // TODO: Handle reorgs to the genesis block + return Err(Error::Deterministic(anyhow!( + "invalid reorg: rewind to the genesis block not supported" + ))); + } + }; + + if block_number > latest_synced_block_number { + return Err(Error::Deterministic(anyhow!( + "invalid reorg: latest block before reorg cannot be higher than the invalidated block" + ))); + } else if block_number == latest_synced_block_number && block_hash == latest_synced_block_hash { + return Err(Error::Deterministic(anyhow!( + "invalid reorg: latest block before reorg cannot be equal to the invalidated block" + ))); + } + + cx.store + .revert_block_operations((block_number, block_hash).compat(), FirehoseCursor::None) + .await + .map_err(Error::from)?; + + Ok(()) +} + +async fn detect_deepest_reorg( + cx: &Context, + latest_blocks: &LatestBlocks, + latest_synced_block_number: BlockNumber, + latest_synced_block_hash: BlockHash, +) -> Result, Error> +where + AC: Client, +{ + let detect_reorg_futs = latest_blocks + .iter() + .filter(|(_, latest_block)| *latest_block >= latest_synced_block_number) + .map(|((i, j), _)| { + let data_source = &cx.manifest.data_sources[*i]; + let network = &data_source.network; + let dataset = &data_source.source.dataset; + let table = &data_source.source.tables[*j]; + + detect_reorg( + cx, + network, + dataset, + table, + latest_synced_block_number, + latest_synced_block_hash, + ) + .map_err(move |e| e.context(format!("failed to detect reorg in '{dataset}.{table}'"))) + }); + + let deepest_reorg = try_join_all(detect_reorg_futs) + .await? + .into_iter() + .flatten() + .min_by_key(|latest_block_before_reorg| latest_block_before_reorg.block_number); + + Ok(deepest_reorg) +} + +async fn detect_reorg( + cx: &Context, + network: &str, + dataset: &str, + table: &str, + latest_synced_block_number: BlockNumber, + latest_synced_block_hash: BlockHash, +) -> Result, Error> +where + AC: Client, +{ + let query = format!("SELECT _block_num FROM {dataset}.{table} SETTINGS stream = true"); + let mut stream = cx.client.query( + &cx.logger, + query, + Some(RequestMetadata { + resume_streaming_query: Some(vec![ResumeStreamingQuery { + network: network.to_string(), + block_number: latest_synced_block_number, + block_hash: latest_synced_block_hash, + }]), + }), + ); + + let response = stream + .next() + .await + .ok_or_else(|| Error::NonDeterministic(anyhow!("stream is empty")))? + .map_err(Error::from)?; + + match response { + ResponseBatch::Batch { .. } => Ok(None), + ResponseBatch::Reorg(reorg) => reorg + .into_iter() + .exactly_one() + .map_err(|_e| Error::Deterministic(anyhow!("multi-chain datasets are not supported"))) + .map(Some), + } +} diff --git a/core/src/lib.rs b/core/src/lib.rs index 972a45e508f..118ce3ebaae 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -1,12 +1,6 @@ -pub mod polling_monitor; - -mod link_resolver; -mod metrics; -mod subgraph; +mod subgraph_manifest; -pub use crate::link_resolver::LinkResolver; -pub use crate::metrics::MetricsRegistry; -pub use crate::subgraph::{ - SubgraphAssignmentProvider, SubgraphInstanceManager, SubgraphRegistrar, SubgraphRunner, - SubgraphTriggerProcessor, -}; +pub mod amp_subgraph; +pub mod polling_monitor; +pub mod subgraph; +pub mod subgraph_provider; diff --git a/core/src/link_resolver.rs b/core/src/link_resolver.rs deleted file mode 100644 index cebc39d5d6a..00000000000 --- a/core/src/link_resolver.rs +++ /dev/null @@ -1,406 +0,0 @@ -use std::sync::{Arc, Mutex}; -use std::time::Duration; - -use anyhow::anyhow; -use async_trait::async_trait; -use bytes::BytesMut; -use futures01::{stream::poll_fn, try_ready}; -use futures03::stream::FuturesUnordered; -use graph::env::EnvVars; -use graph::util::futures::RetryConfigNoTimeout; -use lru_time_cache::LruCache; -use serde_json::Value; - -use graph::{ - ipfs_client::{IpfsClient, StatApi}, - prelude::{LinkResolver as LinkResolverTrait, *}, -}; - -fn retry_policy( - always_retry: bool, - op: &'static str, - logger: &Logger, -) -> RetryConfigNoTimeout { - // Even if retries were not requested, networking errors are still retried until we either get - // a valid HTTP response or a timeout. - if always_retry { - retry(op, logger).no_limit() - } else { - retry(op, logger) - .no_limit() - .when(|res: &Result<_, reqwest::Error>| match res { - Ok(_) => false, - Err(e) => !(e.is_status() || e.is_timeout()), - }) - } - .no_timeout() // The timeout should be set in the internal future. -} - -/// The IPFS APIs don't have a quick "do you have the file" function. Instead, we -/// just rely on whether an API times out. That makes sense for IPFS, but not for -/// our application. We want to be able to quickly select from a potential list -/// of clients where hopefully one already has the file, and just get the file -/// from that. -/// -/// The strategy here then is to use a stat API as a proxy for "do you have the -/// file". Whichever client has or gets the file first wins. This API is a good -/// choice, because it doesn't involve us actually starting to download the file -/// from each client, which would be wasteful of bandwidth and memory in the -/// case multiple clients respond in a timely manner. In addition, we may make -/// good use of the stat returned. -async fn select_fastest_client_with_stat( - clients: Arc>>, - logger: Logger, - api: StatApi, - path: String, - timeout: Duration, - do_retry: bool, -) -> Result<(u64, Arc), Error> { - let mut err: Option = None; - - let mut stats: FuturesUnordered<_> = clients - .iter() - .enumerate() - .map(|(i, c)| { - let c = c.cheap_clone(); - let path = path.clone(); - retry_policy(do_retry, "IPFS stat", &logger).run(move || { - let path = path.clone(); - let c = c.cheap_clone(); - async move { - c.stat_size(api, path, timeout) - .map_ok(move |s| (s, i)) - .await - } - }) - }) - .collect(); - - while let Some(result) = stats.next().await { - match result { - Ok((stat, index)) => { - return Ok((stat, clients[index].cheap_clone())); - } - Err(e) => err = Some(e.into()), - } - } - - Err(err.unwrap_or_else(|| { - anyhow!( - "No IPFS clients were supplied to handle the call to object.stat. File: {}", - path - ) - })) -} - -// Returns an error if the stat is bigger than `max_file_bytes` -fn restrict_file_size(path: &str, size: u64, max_file_bytes: usize) -> Result<(), Error> { - if size > max_file_bytes as u64 { - return Err(anyhow!( - "IPFS file {} is too large. It can be at most {} bytes but is {} bytes", - path, - max_file_bytes, - size - )); - } - Ok(()) -} - -#[derive(Clone)] -pub struct LinkResolver { - clients: Arc>>, - cache: Arc>>>, - timeout: Duration, - retry: bool, - env_vars: Arc, -} - -impl LinkResolver { - pub fn new(clients: Vec, env_vars: Arc) -> Self { - Self { - clients: Arc::new(clients.into_iter().map(Arc::new).collect()), - cache: Arc::new(Mutex::new(LruCache::with_capacity( - env_vars.mappings.max_ipfs_cache_size as usize, - ))), - timeout: env_vars.mappings.ipfs_timeout, - retry: false, - env_vars, - } - } -} - -impl Debug for LinkResolver { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("LinkResolver") - .field("timeout", &self.timeout) - .field("retry", &self.retry) - .field("env_vars", &self.env_vars) - .finish() - } -} - -impl CheapClone for LinkResolver { - fn cheap_clone(&self) -> Self { - self.clone() - } -} - -#[async_trait] -impl LinkResolverTrait for LinkResolver { - fn with_timeout(&self, timeout: Duration) -> Box { - let mut s = self.cheap_clone(); - s.timeout = timeout; - Box::new(s) - } - - fn with_retries(&self) -> Box { - let mut s = self.cheap_clone(); - s.retry = true; - Box::new(s) - } - - /// Supports links of the form `/ipfs/ipfs_hash` or just `ipfs_hash`. - async fn cat(&self, logger: &Logger, link: &Link) -> Result, Error> { - // Discard the `/ipfs/` prefix (if present) to get the hash. - let path = link.link.trim_start_matches("/ipfs/").to_owned(); - - if let Some(data) = self.cache.lock().unwrap().get(&path) { - trace!(logger, "IPFS cache hit"; "hash" => &path); - return Ok(data.clone()); - } - trace!(logger, "IPFS cache miss"; "hash" => &path); - - let (size, client) = select_fastest_client_with_stat( - self.clients.cheap_clone(), - logger.cheap_clone(), - StatApi::Files, - path.clone(), - self.timeout, - self.retry, - ) - .await?; - - let max_cache_file_size = self.env_vars.mappings.max_ipfs_cache_file_size; - let max_file_size = self.env_vars.mappings.max_ipfs_file_bytes; - restrict_file_size(&path, size, max_file_size)?; - - let req_path = path.clone(); - let timeout = self.timeout; - let data = retry_policy(self.retry, "ipfs.cat", logger) - .run(move || { - let path = req_path.clone(); - let client = client.clone(); - async move { Ok(client.cat_all(&path, timeout).await?.to_vec()) } - }) - .await?; - - // The size reported by `files/stat` is not guaranteed to be exact, so check the limit again. - restrict_file_size(&path, data.len() as u64, max_file_size)?; - - // Only cache files if they are not too large - if data.len() <= max_cache_file_size { - let mut cache = self.cache.lock().unwrap(); - if !cache.contains_key(&path) { - cache.insert(path.clone(), data.clone()); - } - } else { - debug!(logger, "File too large for cache"; - "path" => path, - "size" => data.len() - ); - } - - Ok(data) - } - - async fn get_block(&self, logger: &Logger, link: &Link) -> Result, Error> { - trace!(logger, "IPFS block get"; "hash" => &link.link); - let (size, client) = select_fastest_client_with_stat( - self.clients.cheap_clone(), - logger.cheap_clone(), - StatApi::Block, - link.link.clone(), - self.timeout, - self.retry, - ) - .await?; - - let max_file_size = self.env_vars.mappings.max_ipfs_file_bytes; - restrict_file_size(&link.link, size, max_file_size)?; - - let link = link.link.clone(); - let data = retry_policy(self.retry, "ipfs.getBlock", logger) - .run(move || { - let link = link.clone(); - let client = client.clone(); - async move { - let data = client.get_block(link.clone()).await?.to_vec(); - Result::, reqwest::Error>::Ok(data) - } - }) - .await?; - - Ok(data) - } - - async fn json_stream(&self, logger: &Logger, link: &Link) -> Result { - // Discard the `/ipfs/` prefix (if present) to get the hash. - let path = link.link.trim_start_matches("/ipfs/"); - - let (size, client) = select_fastest_client_with_stat( - self.clients.cheap_clone(), - logger.cheap_clone(), - StatApi::Files, - path.to_string(), - self.timeout, - self.retry, - ) - .await?; - - let max_file_size = self.env_vars.mappings.max_ipfs_map_file_size; - restrict_file_size(path, size, max_file_size)?; - - let mut stream = client.cat(path, None).await?.fuse().boxed().compat(); - - let mut buf = BytesMut::with_capacity(1024); - - // Count the number of lines we've already successfully deserialized. - // We need that to adjust the line number in error messages from serde_json - // to translate from line numbers in the snippet we are deserializing - // to the line number in the overall file - let mut count = 0; - - let stream: JsonValueStream = Box::pin( - poll_fn(move || -> Poll, Error> { - loop { - if let Some(offset) = buf.iter().position(|b| *b == b'\n') { - let line_bytes = buf.split_to(offset + 1); - count += 1; - if line_bytes.len() > 1 { - let line = std::str::from_utf8(&line_bytes)?; - let res = match serde_json::from_str::(line) { - Ok(v) => Ok(Async::Ready(Some(JsonStreamValue { - value: v, - line: count, - }))), - Err(e) => { - // Adjust the line number in the serde error. This - // is fun because we can only get at the full error - // message, and not the error message without line number - let msg = e.to_string(); - let msg = msg.split(" at line ").next().unwrap(); - Err(anyhow!( - "{} at line {} column {}: '{}'", - msg, - e.line() + count - 1, - e.column(), - line - )) - } - }; - return res; - } - } else { - // We only get here if there is no complete line in buf, and - // it is therefore ok to immediately pass an Async::NotReady - // from stream through. - // If we get a None from poll, but still have something in buf, - // that means the input was not terminated with a newline. We - // add that so that the last line gets picked up in the next - // run through the loop. - match try_ready!(stream.poll().map_err(|e| anyhow::anyhow!("{}", e))) { - Some(b) => buf.extend_from_slice(&b), - None if !buf.is_empty() => buf.extend_from_slice(&[b'\n']), - None => return Ok(Async::Ready(None)), - } - } - } - }) - .compat(), - ); - - Ok(stream) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use graph::env::EnvVars; - use serde_json::json; - - #[tokio::test] - async fn max_file_size() { - let mut env_vars = EnvVars::default(); - env_vars.mappings.max_ipfs_file_bytes = 200; - - let file: &[u8] = &[0u8; 201]; - let client = IpfsClient::localhost(); - let resolver = super::LinkResolver::new(vec![client.clone()], Arc::new(env_vars)); - - let logger = Logger::root(slog::Discard, o!()); - - let link = client.add(file.into()).await.unwrap().hash; - let err = LinkResolver::cat(&resolver, &logger, &Link { link: link.clone() }) - .await - .unwrap_err(); - assert_eq!( - err.to_string(), - format!( - "IPFS file {} is too large. It can be at most 200 bytes but is 212 bytes", - link - ) - ); - } - - async fn json_round_trip(text: &'static str, env_vars: EnvVars) -> Result, Error> { - let client = IpfsClient::localhost(); - let resolver = super::LinkResolver::new(vec![client.clone()], Arc::new(env_vars)); - - let logger = Logger::root(slog::Discard, o!()); - let link = client.add(text.as_bytes().into()).await.unwrap().hash; - - let stream = LinkResolver::json_stream(&resolver, &logger, &Link { link }).await?; - stream.map_ok(|sv| sv.value).try_collect().await - } - - #[tokio::test] - async fn read_json_stream() { - let values = json_round_trip("\"with newline\"\n", EnvVars::default()).await; - assert_eq!(vec![json!("with newline")], values.unwrap()); - - let values = json_round_trip("\"without newline\"", EnvVars::default()).await; - assert_eq!(vec![json!("without newline")], values.unwrap()); - - let values = json_round_trip("\"two\" \n \"things\"", EnvVars::default()).await; - assert_eq!(vec![json!("two"), json!("things")], values.unwrap()); - - let values = json_round_trip( - "\"one\"\n \"two\" \n [\"bad\" \n \"split\"]", - EnvVars::default(), - ) - .await; - assert_eq!( - "EOF while parsing a list at line 4 column 0: ' [\"bad\" \n'", - values.unwrap_err().to_string() - ); - } - - #[tokio::test] - async fn ipfs_map_file_size() { - let file = "\"small test string that trips the size restriction\""; - let mut env_vars = EnvVars::default(); - env_vars.mappings.max_ipfs_map_file_size = file.len() - 1; - - let err = json_round_trip(file, env_vars).await.unwrap_err(); - - assert!(err.to_string().contains(" is too large")); - - env_vars = EnvVars::default(); - let values = json_round_trip(file, env_vars).await; - assert_eq!( - vec!["small test string that trips the size restriction"], - values.unwrap() - ); - } -} diff --git a/core/src/metrics/mod.rs b/core/src/metrics/mod.rs deleted file mode 100644 index 047d6b24132..00000000000 --- a/core/src/metrics/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -mod registry; - -pub use registry::MetricsRegistry; diff --git a/core/src/metrics/registry.rs b/core/src/metrics/registry.rs deleted file mode 100644 index 067cf4d9faf..00000000000 --- a/core/src/metrics/registry.rs +++ /dev/null @@ -1,331 +0,0 @@ -use std::collections::HashMap; -use std::sync::{Arc, RwLock}; - -use graph::components::metrics::{counter_with_labels, gauge_with_labels}; -use graph::prelude::{MetricsRegistry as MetricsRegistryTrait, *}; - -#[derive(Clone)] -pub struct MetricsRegistry { - logger: Logger, - registry: Arc, - register_errors: Box, - unregister_errors: Box, - registered_metrics: Box, - - /// Global metrics are lazily initialized and identified by - /// the `Desc.id` that hashes the name and const label values - global_counters: Arc>>, - global_counter_vecs: Arc>>, - global_gauges: Arc>>, - global_gauge_vecs: Arc>>, - global_histogram_vecs: Arc>>, -} - -impl MetricsRegistry { - pub fn new(logger: Logger, registry: Arc) -> Self { - // Generate internal metrics - let register_errors = Self::gen_register_errors_counter(registry.clone()); - let unregister_errors = Self::gen_unregister_errors_counter(registry.clone()); - let registered_metrics = Self::gen_registered_metrics_gauge(registry.clone()); - - MetricsRegistry { - logger: logger.new(o!("component" => String::from("MetricsRegistry"))), - registry, - register_errors, - unregister_errors, - registered_metrics, - global_counters: Arc::new(RwLock::new(HashMap::new())), - global_counter_vecs: Arc::new(RwLock::new(HashMap::new())), - global_gauges: Arc::new(RwLock::new(HashMap::new())), - global_gauge_vecs: Arc::new(RwLock::new(HashMap::new())), - global_histogram_vecs: Arc::new(RwLock::new(HashMap::new())), - } - } - - fn gen_register_errors_counter(registry: Arc) -> Box { - let opts = Opts::new( - String::from("metrics_register_errors"), - String::from("Counts Prometheus metrics register errors"), - ); - let counter = Box::new( - Counter::with_opts(opts).expect("failed to create `metrics_register_errors` counter"), - ); - registry - .register(counter.clone()) - .expect("failed to register `metrics_register_errors` counter"); - counter - } - - fn gen_unregister_errors_counter(registry: Arc) -> Box { - let opts = Opts::new( - String::from("metrics_unregister_errors"), - String::from("Counts Prometheus metrics unregister errors"), - ); - let counter = Box::new( - Counter::with_opts(opts).expect("failed to create `metrics_unregister_errors` counter"), - ); - registry - .register(counter.clone()) - .expect("failed to register `metrics_unregister_errors` counter"); - counter - } - - fn gen_registered_metrics_gauge(registry: Arc) -> Box { - let opts = Opts::new( - String::from("registered_metrics"), - String::from("Tracks the number of registered metrics on the node"), - ); - let gauge = - Box::new(Gauge::with_opts(opts).expect("failed to create `registered_metrics` gauge")); - registry - .register(gauge.clone()) - .expect("failed to register `registered_metrics` gauge"); - gauge - } - - fn global_counter_vec_internal( - &self, - name: &str, - help: &str, - deployment: Option<&str>, - variable_labels: &[&str], - ) -> Result { - let opts = Opts::new(name, help); - let opts = match deployment { - None => opts, - Some(deployment) => opts.const_label("deployment", deployment), - }; - let counters = CounterVec::new(opts, variable_labels)?; - let id = counters.desc().first().unwrap().id; - let maybe_counter = self.global_counter_vecs.read().unwrap().get(&id).cloned(); - if let Some(counters) = maybe_counter { - Ok(counters) - } else { - self.register(name, Box::new(counters.clone())); - self.global_counter_vecs - .write() - .unwrap() - .insert(id, counters.clone()); - Ok(counters) - } - } -} - -impl MetricsRegistryTrait for MetricsRegistry { - fn register(&self, name: &str, c: Box) { - let err = match self.registry.register(c).err() { - None => { - self.registered_metrics.inc(); - return; - } - Some(err) => { - self.register_errors.inc(); - err - } - }; - match err { - PrometheusError::AlreadyReg => { - error!( - self.logger, - "registering metric [{}] failed because it was already registered", name, - ); - } - PrometheusError::InconsistentCardinality { expect, got } => { - error!( - self.logger, - "registering metric [{}] failed due to inconsistent caridinality, expected = {} got = {}", - name, - expect, - got, - ); - } - PrometheusError::Msg(msg) => { - error!( - self.logger, - "registering metric [{}] failed because: {}", name, msg, - ); - } - PrometheusError::Io(err) => { - error!( - self.logger, - "registering metric [{}] failed due to io error: {}", name, err, - ); - } - PrometheusError::Protobuf(err) => { - error!( - self.logger, - "registering metric [{}] failed due to protobuf error: {}", name, err - ); - } - }; - } - - fn global_counter( - &self, - name: &str, - help: &str, - const_labels: HashMap, - ) -> Result { - let counter = counter_with_labels(name, help, const_labels)?; - let id = counter.desc().first().unwrap().id; - let maybe_counter = self.global_counters.read().unwrap().get(&id).cloned(); - if let Some(counter) = maybe_counter { - Ok(counter) - } else { - self.register(name, Box::new(counter.clone())); - self.global_counters - .write() - .unwrap() - .insert(id, counter.clone()); - Ok(counter) - } - } - - fn global_counter_vec( - &self, - name: &str, - help: &str, - variable_labels: &[&str], - ) -> Result { - self.global_counter_vec_internal(name, help, None, variable_labels) - } - - fn global_deployment_counter_vec( - &self, - name: &str, - help: &str, - subgraph: &str, - variable_labels: &[&str], - ) -> Result { - self.global_counter_vec_internal(name, help, Some(subgraph), variable_labels) - } - - fn global_gauge( - &self, - name: &str, - help: &str, - const_labels: HashMap, - ) -> Result { - let gauge = gauge_with_labels(name, help, const_labels)?; - let id = gauge.desc().first().unwrap().id; - let maybe_gauge = self.global_gauges.read().unwrap().get(&id).cloned(); - if let Some(gauge) = maybe_gauge { - Ok(gauge) - } else { - self.register(name, Box::new(gauge.clone())); - self.global_gauges - .write() - .unwrap() - .insert(id, gauge.clone()); - Ok(gauge) - } - } - - fn global_gauge_vec( - &self, - name: &str, - help: &str, - variable_labels: &[&str], - ) -> Result { - let opts = Opts::new(name, help); - let gauges = GaugeVec::new(opts, variable_labels)?; - let id = gauges.desc().first().unwrap().id; - let maybe_gauge = self.global_gauge_vecs.read().unwrap().get(&id).cloned(); - if let Some(gauges) = maybe_gauge { - Ok(gauges) - } else { - self.register(name, Box::new(gauges.clone())); - self.global_gauge_vecs - .write() - .unwrap() - .insert(id, gauges.clone()); - Ok(gauges) - } - } - - fn global_histogram_vec( - &self, - name: &str, - help: &str, - variable_labels: &[&str], - ) -> Result { - let opts = HistogramOpts::new(name, help); - let histograms = HistogramVec::new(opts, variable_labels)?; - let id = histograms.desc().first().unwrap().id; - let maybe_histogram = self.global_histogram_vecs.read().unwrap().get(&id).cloned(); - if let Some(histograms) = maybe_histogram { - Ok(histograms) - } else { - self.register(name, Box::new(histograms.clone())); - self.global_histogram_vecs - .write() - .unwrap() - .insert(id, histograms.clone()); - Ok(histograms) - } - } - - fn unregister(&self, metric: Box) { - match self.registry.unregister(metric) { - Ok(_) => { - self.registered_metrics.dec(); - } - Err(e) => { - self.unregister_errors.inc(); - error!(self.logger, "Unregistering metric failed = {:?}", e,); - } - }; - } -} - -#[test] -fn global_counters_are_shared() { - use graph::log; - - let logger = log::logger(false); - let prom_reg = Arc::new(Registry::new()); - let registry = MetricsRegistry::new(logger, prom_reg); - - fn check_counters( - registry: &MetricsRegistry, - name: &str, - const_labels: HashMap, - ) { - let c1 = registry - .global_counter(name, "help me", const_labels.clone()) - .expect("first test counter"); - let c2 = registry - .global_counter(name, "help me", const_labels) - .expect("second test counter"); - let desc1 = c1.desc(); - let desc2 = c2.desc(); - let d1 = desc1.first().unwrap(); - let d2 = desc2.first().unwrap(); - - // Registering the same metric with the same name and - // const labels twice works and returns the same metric (logically) - assert_eq!(d1.id, d2.id, "counters: {}", name); - - // They share the reported values - c1.inc_by(7.0); - c2.inc_by(2.0); - assert_eq!(9.0, c1.get(), "counters: {}", name); - assert_eq!(9.0, c2.get(), "counters: {}", name); - } - - check_counters(®istry, "nolabels", HashMap::new()); - - let const_labels = { - let mut map = HashMap::new(); - map.insert("pool".to_owned(), "main".to_owned()); - map - }; - check_counters(®istry, "pool", const_labels); - - let const_labels = { - let mut map = HashMap::new(); - map.insert("pool".to_owned(), "replica0".to_owned()); - map - }; - check_counters(®istry, "pool", const_labels); -} diff --git a/core/src/polling_monitor/arweave_service.rs b/core/src/polling_monitor/arweave_service.rs new file mode 100644 index 00000000000..cc42f7af5ea --- /dev/null +++ b/core/src/polling_monitor/arweave_service.rs @@ -0,0 +1,50 @@ +use anyhow::Error; +use bytes::Bytes; +use graph::futures03::future::BoxFuture; +use graph::{ + components::link_resolver::{ArweaveResolver, FileSizeLimit}, + data_source::offchain::Base64, + derive::CheapClone, + prelude::CheapClone, +}; +use std::{sync::Arc, time::Duration}; +use tower::{ServiceBuilder, ServiceExt, buffer::Buffer}; + +pub type ArweaveService = Buffer, Error>>>; + +pub fn arweave_service( + client: Arc, + rate_limit: u16, + max_file_size: FileSizeLimit, +) -> ArweaveService { + let arweave = ArweaveServiceInner { + client, + max_file_size, + }; + + let svc = ServiceBuilder::new() + .rate_limit(rate_limit.into(), Duration::from_secs(1)) + .service_fn(move |req| arweave.cheap_clone().call_inner(req)) + .boxed(); + + // The `Buffer` makes it so the rate limit is shared among clones. + // Make it unbounded to avoid any risk of starvation. + Buffer::new(svc, u32::MAX as usize) +} + +#[derive(Clone, CheapClone)] +struct ArweaveServiceInner { + client: Arc, + max_file_size: FileSizeLimit, +} + +impl ArweaveServiceInner { + async fn call_inner(self, req: Base64) -> Result, Error> { + self.client + .get_with_limit(&req, &self.max_file_size) + .await + .map(Bytes::from) + .map(Some) + .map_err(Error::from) + } +} diff --git a/core/src/polling_monitor/ipfs_service.rs b/core/src/polling_monitor/ipfs_service.rs index 284d25063db..6847bb82acd 100644 --- a/core/src/polling_monitor/ipfs_service.rs +++ b/core/src/polling_monitor/ipfs_service.rs @@ -1,97 +1,80 @@ -use anyhow::{anyhow, Error}; -use bytes::Bytes; -use futures::future::BoxFuture; -use graph::{ - ipfs_client::{CidFile, IpfsClient, StatApi}, - prelude::CheapClone, -}; +use std::sync::Arc; use std::time::Duration; -use tower::{buffer::Buffer, ServiceBuilder, ServiceExt}; -const CLOUDFLARE_TIMEOUT: u16 = 524; -const GATEWAY_TIMEOUT: u16 = 504; +use anyhow::Error; +use anyhow::anyhow; +use bytes::Bytes; +use graph::futures03::future::BoxFuture; +use graph::ipfs::{ContentPath, IpfsClient, IpfsContext, RetryPolicy}; +use graph::{derive::CheapClone, prelude::CheapClone}; +use tower::{ServiceBuilder, ServiceExt, buffer::Buffer}; -pub type IpfsService = Buffer, Error>>>; +pub type IpfsService = Buffer, Error>>>; + +#[derive(Debug, Clone, CheapClone)] +pub struct IpfsRequest { + pub ctx: IpfsContext, + pub path: ContentPath, +} pub fn ipfs_service( - client: IpfsClient, - max_file_size: u64, + client: Arc, + max_file_size: usize, timeout: Duration, - concurrency_and_rate_limit: u16, + rate_limit: u16, ) -> IpfsService { let ipfs = IpfsServiceInner { client, - max_file_size, timeout, + max_file_size, }; let svc = ServiceBuilder::new() - .rate_limit(concurrency_and_rate_limit.into(), Duration::from_secs(1)) - .concurrency_limit(concurrency_and_rate_limit as usize) + .rate_limit(rate_limit.into(), Duration::from_secs(1)) .service_fn(move |req| ipfs.cheap_clone().call_inner(req)) .boxed(); - // The `Buffer` makes it so the rate and concurrency limit are shared among clones. - Buffer::new(svc, 1) + // The `Buffer` makes it so the rate limit is shared among clones. + // Make it unbounded to avoid any risk of starvation. + Buffer::new(svc, u32::MAX as usize) } -#[derive(Clone)] +#[derive(Clone, CheapClone)] struct IpfsServiceInner { - client: IpfsClient, - max_file_size: u64, + client: Arc, timeout: Duration, -} - -impl CheapClone for IpfsServiceInner { - fn cheap_clone(&self) -> Self { - Self { - client: self.client.cheap_clone(), - max_file_size: self.max_file_size, - timeout: self.timeout, - } - } + max_file_size: usize, } impl IpfsServiceInner { - async fn call_inner(self, req: CidFile) -> Result, Error> { - let CidFile { cid, path } = req; - let multihash = cid.hash().code(); + async fn call_inner( + self, + IpfsRequest { ctx, path }: IpfsRequest, + ) -> Result, Error> { + let multihash = path.cid().hash().code(); if !SAFE_MULTIHASHES.contains(&multihash) { return Err(anyhow!("CID multihash {} is not allowed", multihash)); } - let cid_str = match path { - Some(path) => format!("{}/{}", cid, path), - None => cid.to_string(), - }; - - let size = match self + let res = self .client - .stat_size(StatApi::Files, cid_str.clone(), self.timeout) - .await - { - Ok(size) => size, - Err(e) => match e.status().map(|e| e.as_u16()) { - Some(GATEWAY_TIMEOUT) | Some(CLOUDFLARE_TIMEOUT) => return Ok(None), - _ if e.is_timeout() => return Ok(None), - _ => return Err(e.into()), - }, - }; - - if size > self.max_file_size { - return Err(anyhow!( - "IPFS file {} is too large. It can be at most {} bytes but is {} bytes", - cid_str, + .cat( + &ctx, + &path, self.max_file_size, - size - )); + Some(self.timeout), + RetryPolicy::None, + ) + .await; + + match res { + Ok(file_bytes) => Ok(Some(file_bytes)), + Err(err) if err.is_timeout() => { + // Timeouts in IPFS mean that the content is not available, so we return `None`. + Ok(None) + } + Err(err) => Err(err.into()), } - - Ok(self - .client - .cat_all(&cid_str, self.timeout) - .await - .map(Some)?) } } @@ -118,42 +101,109 @@ const SAFE_MULTIHASHES: [u64; 15] = [ #[cfg(test)] mod test { - use ipfs::IpfsApi; - use ipfs_api as ipfs; - use std::{fs, str::FromStr, time::Duration}; - use tower::ServiceExt; + use std::time::Duration; - use cid::Cid; - use graph::{ipfs_client::IpfsClient, tokio}; + use graph::components::link_resolver::ArweaveClient; + use graph::data::value::Word; + use graph::ipfs::test_utils::add_files_to_local_ipfs_node_for_testing; + use graph::ipfs::{IpfsContext, IpfsMetrics, IpfsRpcClient, ServerAddress}; + use graph::log::discard; + use tower::ServiceExt; + use wiremock::Mock; + use wiremock::MockServer; + use wiremock::ResponseTemplate; + use wiremock::matchers as m; - use uuid::Uuid; + use super::*; - #[tokio::test] + #[graph::test] async fn cat_file_in_folder() { - let path = "./tests/fixtures/ipfs_folder"; - let uid = Uuid::new_v4().to_string(); - fs::write(format!("{}/random.txt", path), &uid).unwrap(); - - let cl: ipfs::IpfsClient = ipfs::IpfsClient::default(); - - let rsp = cl.add_path(path).await.unwrap(); + let random_bytes = "One morning, when Gregor Samsa woke \ + from troubled dreams, he found himself transformed in his bed \ + into a horrible vermin" + .as_bytes() + .to_vec(); + let ipfs_file = ("dir/file.txt", random_bytes.clone()); + + let add_resp = add_files_to_local_ipfs_node_for_testing([ipfs_file]) + .await + .unwrap(); - let ipfs_folder = rsp.iter().find(|rsp| rsp.name == "ipfs_folder").unwrap(); + let dir_cid = add_resp.into_iter().find(|x| x.name == "dir").unwrap().hash; - let local = IpfsClient::localhost(); - let cid = Cid::from_str(&ipfs_folder.hash).unwrap(); - let file = "random.txt".to_string(); + let client = IpfsRpcClient::new_unchecked( + ServerAddress::test_rpc_api(), + IpfsMetrics::test(), + &graph::log::discard(), + ) + .unwrap(); - let svc = super::ipfs_service(local, 100000, Duration::from_secs(5), 10); + let svc = ipfs_service(Arc::new(client), 100000, Duration::from_secs(30), 10); + let path = ContentPath::new(format!("{dir_cid}/file.txt")).unwrap(); let content = svc - .oneshot(super::CidFile { - cid, - path: Some(file), + .oneshot(IpfsRequest { + ctx: IpfsContext::test(), + path, }) .await .unwrap() .unwrap(); - assert_eq!(content.to_vec(), uid.as_bytes().to_vec()); + + assert_eq!(content.to_vec(), random_bytes); + } + + // Ignored because arweave.net is unreliable right now + #[graph::test] + #[ignore] + async fn arweave_get() { + const ID: &str = "8APeQ5lW0-csTcBaGdPBDLAL2ci2AT9pTn2tppGPU_8"; + + let Some(body) = ArweaveClient::get_test(&Word::from(ID)).await else { + return; + }; + let body = String::from_utf8(body).unwrap(); + + let expected = r#" + {"name":"Arloader NFT #1","description":"Super dope, one of a kind NFT","collection":{"name":"Arloader NFT","family":"We AR"},"attributes":[{"trait_type":"cx","value":-0.4042198883730073},{"trait_type":"cy","value":0.5641681708263335},{"trait_type":"iters","value":44}],"properties":{"category":"image","files":[{"uri":"https://arweave.net/7gWCr96zc0QQCXOsn5Vk9ROVGFbMaA9-cYpzZI8ZMDs","type":"image/png"},{"uri":"https://arweave.net/URwQtoqrbYlc5183STNy3ZPwSCRY4o8goaF7MJay3xY/1.png","type":"image/png"}]},"image":"https://arweave.net/URwQtoqrbYlc5183STNy3ZPwSCRY4o8goaF7MJay3xY/1.png"} + "#.trim_start().trim_end(); + assert_eq!(expected, body); + } + + #[graph::test] + async fn no_client_retries_to_allow_polling_monitor_to_handle_retries_internally() { + const CID: &str = "QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn"; + + let server = MockServer::start().await; + let ipfs_client = + IpfsRpcClient::new_unchecked(server.uri(), IpfsMetrics::test(), &discard()).unwrap(); + let ipfs_service = ipfs_service(Arc::new(ipfs_client), 10, Duration::from_secs(1), 1); + let path = ContentPath::new(CID).unwrap(); + + Mock::given(m::method("POST")) + .and(m::path("/api/v0/cat")) + .and(m::query_param("arg", CID)) + .respond_with(ResponseTemplate::new(500)) + .up_to_n_times(1) + .expect(1) + .mount(&server) + .await; + + Mock::given(m::method("POST")) + .and(m::path("/api/v0/cat")) + .and(m::query_param("arg", CID)) + .respond_with(ResponseTemplate::new(200)) + .expect(..=1) + .mount(&server) + .await; + + // This means that we never reached the successful response. + ipfs_service + .oneshot(IpfsRequest { + ctx: IpfsContext::test(), + path, + }) + .await + .unwrap_err(); } } diff --git a/core/src/polling_monitor/metrics.rs b/core/src/polling_monitor/metrics.rs index 86d65790a7b..e296ddb8e00 100644 --- a/core/src/polling_monitor/metrics.rs +++ b/core/src/polling_monitor/metrics.rs @@ -5,6 +5,7 @@ use graph::{ prometheus::{Counter, Gauge}, }; +#[derive(Clone)] pub struct PollingMonitorMetrics { pub requests: Counter, pub errors: Counter, @@ -13,7 +14,7 @@ pub struct PollingMonitorMetrics { } impl PollingMonitorMetrics { - pub fn new(registry: Arc, subgraph_hash: &DeploymentHash) -> Self { + pub fn new(registry: Arc, subgraph_hash: &DeploymentHash) -> Self { let requests = registry .new_deployment_counter( "polling_monitor_requests", diff --git a/core/src/polling_monitor/mod.rs b/core/src/polling_monitor/mod.rs index e50979d39f2..f8a24db97d7 100644 --- a/core/src/polling_monitor/mod.rs +++ b/core/src/polling_monitor/mod.rs @@ -1,5 +1,7 @@ +mod arweave_service; mod ipfs_service; mod metrics; +mod request; use std::collections::HashMap; use std::fmt::Display; @@ -8,27 +10,29 @@ use std::sync::Arc; use std::task::Poll; use std::time::Duration; -use futures::future::BoxFuture; -use futures::stream::StreamExt; -use futures::{stream, Future, FutureExt, TryFutureExt}; use graph::cheap_clone::CheapClone; +use graph::env::ENV_VARS; +use graph::futures03::future::BoxFuture; +use graph::futures03::stream::StreamExt; +use graph::futures03::{Future, FutureExt, TryFutureExt, stream}; use graph::parking_lot::Mutex; use graph::prelude::tokio; use graph::prometheus::{Counter, Gauge}; -use graph::slog::{debug, Logger}; +use graph::slog::{Logger, debug}; use graph::util::monitored::MonitoredVecDeque as VecDeque; use tokio::sync::{mpsc, watch}; use tower::retry::backoff::{Backoff, ExponentialBackoff, ExponentialBackoffMaker, MakeBackoff}; use tower::util::rng::HasherRng; use tower::{Service, ServiceExt}; +use self::request::RequestId; + pub use self::metrics::PollingMonitorMetrics; -pub use ipfs_service::{ipfs_service, IpfsService}; +pub use arweave_service::{ArweaveService, arweave_service}; +pub use ipfs_service::{IpfsRequest, IpfsService, ipfs_service}; const MIN_BACKOFF: Duration = Duration::from_secs(5); -const MAX_BACKOFF: Duration = Duration::from_secs(600); - struct Backoffs { backoff_maker: ExponentialBackoffMaker, backoffs: HashMap, @@ -40,7 +44,7 @@ impl Backoffs { Self { backoff_maker: ExponentialBackoffMaker::new( MIN_BACKOFF, - MAX_BACKOFF, + ENV_VARS.mappings.fds_max_backoff, 1.0, HasherRng::new(), ) @@ -49,7 +53,7 @@ impl Backoffs { } } - fn next_backoff(&mut self, id: ID) -> impl Future { + fn next_backoff(&mut self, id: ID) -> impl Future + Send + use { self.backoffs .entry(id) .or_insert_with(|| self.backoff_maker.make_backoff()) @@ -96,15 +100,15 @@ impl Queue { /// /// The service returns the request ID along with errors or responses. The response is an /// `Option`, to represent the object not being found. -pub fn spawn_monitor( +pub fn spawn_monitor( service: S, - response_sender: mpsc::Sender<(ID, Res)>, + response_sender: mpsc::UnboundedSender<(Req, Res)>, logger: Logger, - metrics: PollingMonitorMetrics, -) -> PollingMonitor + metrics: Arc, +) -> PollingMonitor where - S: Service, Error = E> + Send + 'static, - ID: Display + Clone + Default + Eq + Send + Sync + Hash + 'static, + S: Service, Error = E> + Send + 'static, + Req: RequestId + Clone + Send + Sync + 'static, E: Display + Send + 'static, S::Future: Send, { @@ -124,13 +128,13 @@ where break None; } - let id = queue.pop_front(); - match id { - Some(id) => break Some((id, ())), + let req = queue.pop_front(); + match req { + Some(req) => break Some((req, ())), // Nothing on the queue, wait for a queue wake up or cancellation. None => { - futures::future::select( + graph::futures03::future::select( // Unwrap: `queue` holds a sender. queue_woken.changed().map(|r| r.unwrap()).boxed(), cancel_check.closed().boxed(), @@ -149,38 +153,43 @@ where let mut backoffs = Backoffs::new(); let mut responses = service.call_all(queue_to_stream).unordered().boxed(); while let Some(response) = responses.next().await { + // Note: Be careful not to `await` within this loop, as that could block requests in + // the `CallAll` from being polled. This can cause starvation as those requests may + // be holding on to resources such as slots for concurrent calls. match response { - Ok((id, Some(response))) => { - backoffs.remove(&id); - let send_result = response_sender.send((id, response)).await; + Ok((req, Some(response))) => { + backoffs.remove(req.request_id()); + let send_result = response_sender.send((req, response)); if send_result.is_err() { // The receiver has been dropped, cancel this task. break; } } - // Object not found, push the id to the back of the queue. - Ok((id, None)) => { + // Object not found, push the request to the back of the queue. + Ok((req, None)) => { + debug!(logger, "not found on polling"; "object_id" => req.request_id().to_string()); metrics.not_found.inc(); - queue.push_back(id); + + // We'll try again after a backoff. + backoff(req, &queue, &mut backoffs); } - // Error polling, log it and push the id to the back of the queue. - Err((id, e)) => { - debug!(logger, "error polling"; - "error" => format!("{:#}", e), - "object_id" => id.to_string()); + // Error polling, log it and push the request to the back of the queue. + Err((Some(req), e)) => { + debug!(logger, "error polling"; "error" => format!("{:#}", e), "object_id" => req.request_id().to_string()); metrics.errors.inc(); // Requests that return errors could mean there is a permanent issue with // fetching the given item, or could signal the endpoint is overloaded. // Either way a backoff makes sense. - let queue = queue.cheap_clone(); - let backoff = backoffs.next_backoff(id.clone()); - graph::spawn(async move { - backoff.await; - queue.push_back(id); - }); + backoff(req, &queue, &mut backoffs); + } + + // poll_ready call failure + Err((None, e)) => { + debug!(logger, "error polling"; "error" => format!("{:#}", e)); + metrics.errors.inc(); } } } @@ -190,16 +199,28 @@ where PollingMonitor { queue } } +fn backoff(req: Req, queue: &Arc>, backoffs: &mut Backoffs) +where + Req: RequestId + Send + Sync + 'static, +{ + let queue = queue.cheap_clone(); + let backoff = backoffs.next_backoff(req.request_id().clone()); + graph::spawn(async move { + backoff.await; + queue.push_back(req); + }); +} + /// Handle for adding objects to be monitored. -pub struct PollingMonitor { - queue: Arc>, +pub struct PollingMonitor { + queue: Arc>, } -impl PollingMonitor { - /// Add an object id to the polling queue. New requests have priority and are pushed to the +impl PollingMonitor { + /// Add a request to the polling queue. New requests have priority and are pushed to the /// front of the queue. - pub fn monitor(&self, id: ID) { - self.queue.push_front(id); + pub fn monitor(&self, req: Req) { + self.queue.push_front(req); } } @@ -210,17 +231,16 @@ struct ReturnRequest { impl Service for ReturnRequest where S: Service, - Req: Clone + Default + Send + Sync + 'static, + Req: Clone + Send + Sync + 'static, S::Error: Send, S::Future: Send + 'static, { type Response = (Req, S::Response); - type Error = (Req, S::Error); + type Error = (Option, S::Error); type Future = BoxFuture<'static, Result>; fn poll_ready(&mut self, cx: &mut std::task::Context<'_>) -> Poll> { - // `Req::default` is a value that won't be used since if `poll_ready` errors, the service is shot anyways. - self.service.poll_ready(cx).map_err(|e| (Req::default(), e)) + self.service.poll_ready(cx).map_err(|e| (None, e)) } fn call(&mut self, req: Req) -> Self::Future { @@ -228,7 +248,7 @@ where self.service .call(req.clone()) .map_ok(move |x| (req, x)) - .map_err(move |e| (req1, e)) + .map_err(move |e| (Some(req1), e)) .boxed() } } @@ -248,21 +268,26 @@ mod tests { fn setup() -> ( mock::Handle<&'static str, Option<&'static str>>, PollingMonitor<&'static str>, - mpsc::Receiver<(&'static str, &'static str)>, + mpsc::UnboundedReceiver<(&'static str, &'static str)>, ) { let (svc, handle) = mock::pair(); - let (tx, rx) = mpsc::channel(10); - let monitor = spawn_monitor(svc, tx, log::discard(), PollingMonitorMetrics::mock()); + let (tx, rx) = mpsc::unbounded_channel(); + let monitor = spawn_monitor( + svc, + tx, + log::discard(), + Arc::new(PollingMonitorMetrics::mock()), + ); (handle, monitor, rx) } - #[tokio::test] + #[graph::test] async fn polling_monitor_shared_svc() { let (svc, mut handle) = mock::pair(); let shared_svc = tower::buffer::Buffer::new(tower::limit::ConcurrencyLimit::new(svc, 1), 1); let make_monitor = |svc| { - let (tx, rx) = mpsc::channel(10); - let metrics = PollingMonitorMetrics::mock(); + let (tx, rx) = mpsc::unbounded_channel(); + let metrics = Arc::new(PollingMonitorMetrics::mock()); let monitor = spawn_monitor(svc, tx, log::discard(), metrics); (monitor, rx) }; @@ -278,7 +303,7 @@ mod tests { assert_eq!(rx1.recv().await, Some(("req-0", "res-0"))); } - #[tokio::test] + #[graph::test] async fn polling_monitor_simple() { let (mut handle, monitor, mut rx) = setup(); @@ -288,7 +313,7 @@ mod tests { assert_eq!(rx.recv().await, Some(("req-0", "res-0"))); } - #[tokio::test] + #[graph::test] async fn polling_monitor_unordered() { let (mut handle, monitor, mut rx) = setup(); @@ -305,7 +330,7 @@ mod tests { assert_eq!(rx.recv().await, Some(("req-1", "res-1"))); } - #[tokio::test] + #[graph::test] async fn polling_monitor_failed_push_to_back() { let (mut handle, monitor, mut rx) = setup(); @@ -329,8 +354,10 @@ mod tests { assert_eq!(rx.recv().await, Some(("req-1", "res-1"))); } - #[tokio::test] + #[graph::test] async fn polling_monitor_cancelation() { + const REQ: &str = "req-0"; + // Cancelation on receiver drop, no pending request. let (mut handle, _monitor, rx) = setup(); drop(rx); @@ -338,9 +365,20 @@ mod tests { // Cancelation on receiver drop, with pending request. let (mut handle, monitor, rx) = setup(); - monitor.monitor("req-0"); + monitor.monitor(REQ); drop(rx); - assert!(handle.next_request().await.is_none()); + let mut next = handle.next_request().await; + if let Some((req, responder)) = next { + // The request may or may not have been pulled from the queue + // before cancelation. + assert_eq!(REQ, req); + // Explicitly complete the request so the monitor task can process it + // and detect the cancellation. Dropping the responder without responding + // is racy in a multi-threaded runtime. + responder.send_error(anyhow!("cancelled")); + next = handle.next_request().await; + } + assert!(next.is_none()); // Cancelation on receiver drop, while queue is waiting. let (mut handle, _monitor, rx) = setup(); diff --git a/core/src/polling_monitor/request.rs b/core/src/polling_monitor/request.rs new file mode 100644 index 00000000000..42375fb38fb --- /dev/null +++ b/core/src/polling_monitor/request.rs @@ -0,0 +1,39 @@ +use std::fmt::Display; +use std::hash::Hash; + +use graph::{data_source::offchain::Base64, ipfs::ContentPath}; + +use crate::polling_monitor::ipfs_service::IpfsRequest; + +/// Request ID is used to create backoffs on request failures. +pub trait RequestId { + type Id: Clone + Display + Eq + Hash + Send + Sync + 'static; + + /// Returns the ID of the request. + fn request_id(&self) -> &Self::Id; +} + +impl RequestId for IpfsRequest { + type Id = ContentPath; + + fn request_id(&self) -> &ContentPath { + &self.path + } +} + +impl RequestId for Base64 { + type Id = Base64; + + fn request_id(&self) -> &Base64 { + self + } +} + +#[cfg(debug_assertions)] +impl RequestId for &'static str { + type Id = &'static str; + + fn request_id(&self) -> &Self::Id { + self + } +} diff --git a/core/src/subgraph/context.rs b/core/src/subgraph/context.rs deleted file mode 100644 index 121dfd637b5..00000000000 --- a/core/src/subgraph/context.rs +++ /dev/null @@ -1,236 +0,0 @@ -pub mod instance; - -use crate::polling_monitor::{spawn_monitor, IpfsService, PollingMonitor, PollingMonitorMetrics}; -use anyhow::{self, Error}; -use bytes::Bytes; -use graph::{ - blockchain::Blockchain, - components::{ - store::{DeploymentId, SubgraphFork}, - subgraph::{MappingError, SharedProofOfIndexing}, - }, - data_source::{offchain, CausalityRegion, DataSource, TriggerData}, - ipfs_client::CidFile, - prelude::{ - BlockNumber, BlockState, CancelGuard, CheapClone, DeploymentHash, MetricsRegistry, - RuntimeHostBuilder, SubgraphCountMetric, SubgraphInstanceMetrics, TriggerProcessor, - }, - slog::Logger, - tokio::sync::mpsc, -}; -use std::collections::HashMap; -use std::sync::{Arc, RwLock}; - -use self::instance::SubgraphInstance; - -#[derive(Clone, Debug)] -pub struct SubgraphKeepAlive { - alive_map: Arc>>, - manager_metrics: Arc, -} - -impl CheapClone for SubgraphKeepAlive { - fn cheap_clone(&self) -> Self { - self.clone() - } -} - -impl SubgraphKeepAlive { - pub fn new(metrics_registry: Arc) -> Self { - Self { - manager_metrics: Arc::new(SubgraphCountMetric::new(metrics_registry)), - alive_map: Arc::new(RwLock::new(HashMap::default())), - } - } - - pub fn remove(&self, deployment_id: &DeploymentId) { - self.alive_map.write().unwrap().remove(deployment_id); - self.manager_metrics.subgraph_count.dec(); - } - pub fn insert(&self, deployment_id: DeploymentId, guard: CancelGuard) { - self.alive_map.write().unwrap().insert(deployment_id, guard); - self.manager_metrics.subgraph_count.inc(); - } -} - -// The context keeps track of mutable in-memory state that is retained across blocks. -// -// Currently most of the changes are applied in `runner.rs`, but ideally more of that would be -// refactored into the context so it wouldn't need `pub` fields. The entity cache should probably -// also be moved here. -pub struct IndexingContext -where - T: RuntimeHostBuilder, - C: Blockchain, -{ - instance: SubgraphInstance, - pub instances: SubgraphKeepAlive, - pub filter: C::TriggerFilter, - pub offchain_monitor: OffchainMonitor, - trigger_processor: Box>, -} - -impl> IndexingContext { - pub fn new( - instance: SubgraphInstance, - instances: SubgraphKeepAlive, - filter: C::TriggerFilter, - offchain_monitor: OffchainMonitor, - trigger_processor: Box>, - ) -> Self { - Self { - instance, - instances, - filter, - offchain_monitor, - trigger_processor, - } - } - - pub async fn process_trigger( - &self, - logger: &Logger, - block: &Arc, - trigger: &TriggerData, - state: BlockState, - proof_of_indexing: &SharedProofOfIndexing, - causality_region: &str, - debug_fork: &Option>, - subgraph_metrics: &Arc, - ) -> Result, MappingError> { - self.process_trigger_in_hosts( - logger, - self.instance.hosts(), - block, - trigger, - state, - proof_of_indexing, - causality_region, - debug_fork, - subgraph_metrics, - ) - .await - } - - pub async fn process_trigger_in_hosts( - &self, - logger: &Logger, - hosts: &[Arc], - block: &Arc, - trigger: &TriggerData, - state: BlockState, - proof_of_indexing: &SharedProofOfIndexing, - causality_region: &str, - debug_fork: &Option>, - subgraph_metrics: &Arc, - ) -> Result, MappingError> { - self.trigger_processor - .process_trigger( - logger, - hosts, - block, - trigger, - state, - proof_of_indexing, - causality_region, - debug_fork, - subgraph_metrics, - ) - .await - } - - /// Removes data sources hosts with a creation block greater or equal to `reverted_block`, so - /// that they are no longer candidates for `process_trigger`. - /// - /// This does not currently affect the `offchain_monitor` or the `filter`, so they will continue - /// to include data sources that have been reverted. This is not ideal for performance, but it - /// does not affect correctness since triggers that have no matching host will be ignored by - /// `process_trigger`. - /// - /// File data sources that have been marked not done during this process will get re-queued - pub fn revert_data_sources(&mut self, reverted_block: BlockNumber) -> Result<(), Error> { - let removed = self.instance.revert_data_sources(reverted_block); - - removed - .into_iter() - .try_for_each(|source| self.offchain_monitor.add_source(source)) - } - - pub fn add_dynamic_data_source( - &mut self, - logger: &Logger, - data_source: DataSource, - ) -> Result>, Error> { - let source = data_source.as_offchain().map(|ds| ds.source.clone()); - let host = self.instance.add_dynamic_data_source(logger, data_source)?; - - if host.is_some() { - if let Some(source) = source { - self.offchain_monitor.add_source(source)?; - } - } - - Ok(host) - } - - pub fn causality_region_next_value(&mut self) -> CausalityRegion { - self.instance.causality_region_next_value() - } - - #[cfg(debug_assertions)] - pub fn instance(&self) -> &SubgraphInstance { - &self.instance - } -} - -pub struct OffchainMonitor { - ipfs_monitor: PollingMonitor, - ipfs_monitor_rx: mpsc::Receiver<(CidFile, Bytes)>, -} - -impl OffchainMonitor { - pub fn new( - logger: Logger, - registry: Arc, - subgraph_hash: &DeploymentHash, - ipfs_service: IpfsService, - ) -> Self { - let (ipfs_monitor_tx, ipfs_monitor_rx) = mpsc::channel(10); - let ipfs_monitor = spawn_monitor( - ipfs_service, - ipfs_monitor_tx, - logger, - PollingMonitorMetrics::new(registry, subgraph_hash), - ); - Self { - ipfs_monitor, - ipfs_monitor_rx, - } - } - - fn add_source(&mut self, source: offchain::Source) -> Result<(), Error> { - match source { - offchain::Source::Ipfs(cid_file) => self.ipfs_monitor.monitor(cid_file), - }; - Ok(()) - } - - pub fn ready_offchain_events(&mut self) -> Result, Error> { - use graph::tokio::sync::mpsc::error::TryRecvError; - - let mut triggers = vec![]; - loop { - match self.ipfs_monitor_rx.try_recv() { - Ok((cid_file, data)) => triggers.push(offchain::TriggerData { - source: offchain::Source::Ipfs(cid_file), - data: Arc::new(data), - }), - Err(TryRecvError::Disconnected) => { - anyhow::bail!("ipfs monitor unexpectedly terminated") - } - Err(TryRecvError::Empty) => break, - } - } - Ok(triggers) - } -} diff --git a/core/src/subgraph/context/instance.rs b/core/src/subgraph/context/instance.rs deleted file mode 100644 index d760dad1386..00000000000 --- a/core/src/subgraph/context/instance.rs +++ /dev/null @@ -1,214 +0,0 @@ -use futures01::sync::mpsc::Sender; -use graph::{ - blockchain::Blockchain, - data_source::{ - causality_region::CausalityRegionSeq, offchain, CausalityRegion, DataSource, - DataSourceTemplate, - }, - prelude::*, -}; -use std::collections::HashMap; - -use super::OffchainMonitor; - -pub struct SubgraphInstance> { - subgraph_id: DeploymentHash, - network: String, - host_builder: T, - templates: Arc>>, - host_metrics: Arc, - - /// Runtime hosts, one for each data source mapping. - /// - /// The runtime hosts are created and added in the same order the - /// data sources appear in the subgraph manifest. Incoming block - /// stream events are processed by the mappings in this same order. - hosts: Vec>, - - /// Maps the hash of a module to a channel to the thread in which the module is instantiated. - module_cache: HashMap<[u8; 32], Sender>, - - /// This manages the sequence of causality regions for the subgraph. - causality_region_seq: CausalityRegionSeq, -} - -impl SubgraphInstance -where - C: Blockchain, - T: RuntimeHostBuilder, -{ - pub fn from_manifest( - logger: &Logger, - manifest: SubgraphManifest, - host_builder: T, - host_metrics: Arc, - offchain_monitor: &mut OffchainMonitor, - causality_region_seq: CausalityRegionSeq, - ) -> Result { - let subgraph_id = manifest.id.clone(); - let network = manifest.network_name(); - let templates = Arc::new(manifest.templates); - - let mut this = SubgraphInstance { - host_builder, - subgraph_id, - network, - hosts: Vec::new(), - module_cache: HashMap::new(), - templates, - host_metrics, - causality_region_seq, - }; - - // Create a new runtime host for each data source in the subgraph manifest; - // we use the same order here as in the subgraph manifest to make the - // event processing behavior predictable - for ds in manifest.data_sources { - // TODO: This is duplicating code from `IndexingContext::add_dynamic_data_source` and - // `SubgraphInstance::add_dynamic_data_source`. Ideally this should be refactored into - // `IndexingContext`. - - let runtime = ds.runtime(); - let module_bytes = match runtime { - None => continue, - Some(ref module_bytes) => module_bytes, - }; - - if let DataSource::Offchain(ds) = &ds { - // monitor data source only if it's not processed. - if !ds.is_processed() { - offchain_monitor.add_source(ds.source.clone())?; - } - } - - let host = this.new_host(logger.cheap_clone(), ds, module_bytes)?; - this.hosts.push(Arc::new(host)); - } - - Ok(this) - } - - // module_bytes is the same as data_source.runtime().unwrap(), this is to ensure that this - // function is only called for data_sources for which data_source.runtime().is_some() is true. - fn new_host( - &mut self, - logger: Logger, - data_source: DataSource, - module_bytes: &Arc>, - ) -> Result { - let mapping_request_sender = { - let module_hash = tiny_keccak::keccak256(module_bytes.as_ref()); - if let Some(sender) = self.module_cache.get(&module_hash) { - sender.clone() - } else { - let sender = T::spawn_mapping( - module_bytes.as_ref(), - logger, - self.subgraph_id.clone(), - self.host_metrics.cheap_clone(), - )?; - self.module_cache.insert(module_hash, sender.clone()); - sender - } - }; - self.host_builder.build( - self.network.clone(), - self.subgraph_id.clone(), - data_source, - self.templates.cheap_clone(), - mapping_request_sender, - self.host_metrics.cheap_clone(), - ) - } - - pub(super) fn add_dynamic_data_source( - &mut self, - logger: &Logger, - data_source: DataSource, - ) -> Result>, Error> { - // Protect against creating more than the allowed maximum number of data sources - if self.hosts.len() >= ENV_VARS.subgraph_max_data_sources { - anyhow::bail!( - "Limit of {} data sources per subgraph exceeded", - ENV_VARS.subgraph_max_data_sources, - ); - } - - // `hosts` will remain ordered by the creation block. - // See also 8f1bca33-d3b7-4035-affc-fd6161a12448. - assert!( - self.hosts.last().and_then(|h| h.creation_block_number()) - <= data_source.creation_block() - ); - - let module_bytes = match &data_source.runtime() { - None => return Ok(None), - Some(ref module_bytes) => module_bytes.cheap_clone(), - }; - - let host = Arc::new(self.new_host(logger.clone(), data_source, &module_bytes)?); - - Ok(if self.hosts.contains(&host) { - None - } else { - self.hosts.push(host.clone()); - Some(host) - }) - } - - /// Reverts any DataSources that have been added from the block forwards (inclusively) - /// This function also reverts the done_at status if it was 'done' on this block or later. - /// It only returns the offchain::Source because we don't currently need to know which - /// DataSources were removed, the source is used so that the offchain DDS can be found again. - pub(super) fn revert_data_sources( - &mut self, - reverted_block: BlockNumber, - ) -> Vec { - self.revert_hosts_cheap(reverted_block); - - // The following code handles resetting offchain datasources so in most - // cases this is enough processing. - // At some point we prolly need to improve the linear search but for now this - // should be fine. *IT'S FINE* - // - // Any File DataSources (Dynamic Data Sources), will have their own causality region - // which currently is the next number of the sequence but that should be an internal detail. - // Regardless of the sequence logic, if the current causality region is ONCHAIN then there are - // no others and therefore the remaining code is a noop and we can just stop here. - if self.causality_region_seq.0 == CausalityRegion::ONCHAIN { - return vec![]; - } - - self.hosts - .iter() - .filter(|host| matches!(host.done_at(), Some(done_at) if done_at >= reverted_block)) - .map(|host| { - host.set_done_at(None); - // Safe to call unwrap() because only offchain DataSources have done_at = Some - host.data_source().as_offchain().unwrap().source.clone() - }) - .collect() - } - - /// Because hosts are ordered, removing them based on creation block is cheap and simple. - fn revert_hosts_cheap(&mut self, reverted_block: BlockNumber) { - // `hosts` is ordered by the creation block. - // See also 8f1bca33-d3b7-4035-affc-fd6161a12448. - while self - .hosts - .last() - .filter(|h| h.creation_block_number() >= Some(reverted_block)) - .is_some() - { - self.hosts.pop(); - } - } - - pub fn hosts(&self) -> &[Arc] { - &self.hosts - } - - pub(super) fn causality_region_next_value(&mut self) -> CausalityRegion { - self.causality_region_seq.next_val() - } -} diff --git a/core/src/subgraph/context/instance/hosts.rs b/core/src/subgraph/context/instance/hosts.rs new file mode 100644 index 00000000000..9c18e12ce1e --- /dev/null +++ b/core/src/subgraph/context/instance/hosts.rs @@ -0,0 +1,211 @@ +use std::{ + collections::{BTreeMap, HashMap}, + sync::Arc, +}; + +use graph::{ + blockchain::Blockchain, + cheap_clone::CheapClone, + components::{ + store::BlockNumber, + subgraph::{RuntimeHost, RuntimeHostBuilder}, + }, +}; + +/// This structure maintains a partition of the hosts by address, for faster trigger matching. This +/// partition uses the host's index in the main vec, to maintain the correct ordering. +pub(super) struct OnchainHosts> { + hosts: Vec>, + + // The `usize` is the index of the host in `hosts`. + hosts_by_address: HashMap, Vec>, + hosts_without_address: Vec, +} + +impl> OnchainHosts { + pub fn new() -> Self { + Self { + hosts: Vec::new(), + hosts_by_address: HashMap::new(), + hosts_without_address: Vec::new(), + } + } + + pub fn hosts(&self) -> &[Arc] { + &self.hosts + } + + pub fn contains(&self, other: &Arc) -> bool { + // Narrow down the host list by address, as an optimization. + let hosts = match other.data_source().address() { + Some(address) => self.hosts_by_address.get(address.as_slice()), + None => Some(&self.hosts_without_address), + }; + + hosts + .into_iter() + .flatten() + .any(|idx| &self.hosts[*idx] == other) + } + + pub fn last(&self) -> Option<&Arc> { + self.hosts.last() + } + + pub fn len(&self) -> usize { + self.hosts.len() + } + + pub fn push(&mut self, host: Arc) { + assert!(host.data_source().is_chain_based()); + + self.hosts.push(host.cheap_clone()); + let idx = self.hosts.len() - 1; + let address = host.data_source().address(); + match address { + Some(address) => { + self.hosts_by_address + .entry(address.into()) + .or_default() + .push(idx); + } + None => { + self.hosts_without_address.push(idx); + } + } + } + + pub fn pop(&mut self) { + let Some(host) = self.hosts.pop() else { return }; + let address = host.data_source().address(); + match address { + Some(address) => { + // Unwrap and assert: The same host we just popped must be the last one in `hosts_by_address`. + let hosts = self.hosts_by_address.get_mut(address.as_slice()).unwrap(); + let idx = hosts.pop().unwrap(); + assert_eq!(idx, self.hosts.len()); + } + None => { + // Unwrap and assert: The same host we just popped must be the last one in `hosts_without_address`. + let idx = self.hosts_without_address.pop().unwrap(); + assert_eq!(idx, self.hosts.len()); + } + } + } + + /// Returns an iterator over all hosts that match the given address, in the order they were inserted in `hosts`. + /// Note that this always includes the hosts without an address, since they match all addresses. + /// If no address is provided, returns an iterator over all hosts. + pub fn matches_by_address( + &self, + address: Option<&[u8]>, + ) -> Box + Send + '_> { + let Some(address) = address else { + return Box::new(self.hosts.iter().map(|host| host.as_ref())); + }; + + let mut matching_hosts: Vec = self + .hosts_by_address + .get(address) + .into_iter() + .flatten() // Flatten non-existing `address` into empty. + .copied() + .chain(self.hosts_without_address.iter().copied()) + .collect(); + matching_hosts.sort(); + Box::new( + matching_hosts + .into_iter() + .map(move |idx| self.hosts[idx].as_ref()), + ) + } +} + +/// Note that unlike `OnchainHosts`, this does not maintain the order of insertion. Ultimately, the +/// processing order should not matter because each offchain ds has its own causality region. +pub(super) struct OffchainHosts> { + // Indexed by creation block + by_block: BTreeMap, Vec>>, + // Indexed by `offchain::Source::address` + by_address: BTreeMap, Vec>>, + wildcard_address: Vec>, +} + +impl> OffchainHosts { + pub fn new() -> Self { + Self { + by_block: BTreeMap::new(), + by_address: BTreeMap::new(), + wildcard_address: Vec::new(), + } + } + + pub fn len(&self) -> usize { + self.by_block.values().map(Vec::len).sum() + } + + pub fn all(&self) -> impl Iterator> + Send + '_ { + self.by_block.values().flatten() + } + + pub fn contains(&self, other: &Arc) -> bool { + // Narrow down the host list by address, as an optimization. + let hosts = match other.data_source().address() { + Some(address) => self.by_address.get(address.as_slice()), + None => Some(&self.wildcard_address), + }; + + hosts.into_iter().flatten().any(|host| host == other) + } + + pub fn push(&mut self, host: Arc) { + assert!(host.data_source().as_offchain().is_some()); + + let block = host.creation_block_number(); + self.by_block + .entry(block) + .or_default() + .push(host.cheap_clone()); + + match host.data_source().address() { + Some(address) => self.by_address.entry(address).or_default().push(host), + None => self.wildcard_address.push(host), + } + } + + /// Removes all entries with block number >= block. + pub fn remove_ge_block(&mut self, block: BlockNumber) { + let removed = self.by_block.split_off(&Some(block)); + for (_, hosts) in removed { + for host in hosts { + match host.data_source().address() { + Some(address) => { + let hosts = self.by_address.get_mut(&address).unwrap(); + hosts.retain(|h| !Arc::ptr_eq(h, &host)); + } + None => { + self.wildcard_address.retain(|h| !Arc::ptr_eq(h, &host)); + } + } + } + } + } + + pub fn matches_by_address<'a>( + &'a self, + address: Option<&[u8]>, + ) -> Box + Send + 'a> { + let Some(address) = address else { + return Box::new(self.by_block.values().flatten().map(|host| host.as_ref())); + }; + + Box::new( + self.by_address + .get(address) + .into_iter() + .flatten() // Flatten non-existing `address` into empty. + .map(|host| host.as_ref()) + .chain(self.wildcard_address.iter().map(|host| host.as_ref())), + ) + } +} diff --git a/core/src/subgraph/context/instance/mod.rs b/core/src/subgraph/context/instance/mod.rs new file mode 100644 index 00000000000..494ae227e97 --- /dev/null +++ b/core/src/subgraph/context/instance/mod.rs @@ -0,0 +1,258 @@ +mod hosts; + +use anyhow::ensure; +use graph::futures01::sync::mpsc::Sender; +use graph::{ + blockchain::{Blockchain, TriggerData as _}, + data_source::{ + CausalityRegion, DataSource, DataSourceTemplate, TriggerData, + causality_region::CausalityRegionSeq, offchain, + }, + prelude::*, +}; +use hosts::{OffchainHosts, OnchainHosts}; +use std::collections::HashMap; + +pub(crate) struct SubgraphInstance> { + subgraph_id: DeploymentHash, + network: String, + host_builder: T, + pub templates: Arc>>, + /// The data sources declared in the subgraph manifest. This does not include dynamic data sources. + pub(super) static_data_sources: Arc>>, + host_metrics: Arc, + + /// The hosts represent the onchain data sources in the subgraph. There is one host per data source. + /// Data sources with no mappings (e.g. direct substreams) have no host. + /// + /// Onchain hosts must be created in increasing order of block number. `fn hosts_for_trigger` + /// will return the onchain hosts in the same order as they were inserted. + onchain_hosts: OnchainHosts, + + /// `subgraph_hosts` represent subgraph data sources declared in the manifest. These are a special + /// kind of data source that depends on the data from another source subgraph. + subgraph_hosts: OnchainHosts, + + offchain_hosts: OffchainHosts, + + /// Maps the hash of a module to a channel to the thread in which the module is instantiated. + module_cache: HashMap<[u8; 32], Sender>, + + /// This manages the sequence of causality regions for the subgraph. + causality_region_seq: CausalityRegionSeq, +} + +impl SubgraphInstance +where + C: Blockchain, + T: RuntimeHostBuilder, +{ + /// All onchain data sources that are part of this subgraph. This includes data sources + /// that are included in the subgraph manifest and dynamic data sources. + pub fn onchain_data_sources(&self) -> impl Iterator + Clone { + let host_data_sources = self + .onchain_hosts + .hosts() + .iter() + .map(|h| h.data_source().as_onchain().unwrap()); + + // Datasources that are defined in the subgraph manifest but does not correspond to any host + // in the subgraph. Currently these are only substreams data sources. + let substreams_data_sources = self + .static_data_sources + .iter() + .filter(|ds| ds.runtime().is_none()) + .filter_map(|ds| ds.as_onchain()); + + host_data_sources.chain(substreams_data_sources) + } + + pub fn new( + manifest: SubgraphManifest, + host_builder: T, + host_metrics: Arc, + causality_region_seq: CausalityRegionSeq, + ) -> Self { + let subgraph_id = manifest.id.clone(); + let network = manifest.network_name(); + let templates = Arc::new(manifest.templates); + + SubgraphInstance { + host_builder, + subgraph_id, + network, + static_data_sources: Arc::new(manifest.data_sources), + onchain_hosts: OnchainHosts::new(), + subgraph_hosts: OnchainHosts::new(), + offchain_hosts: OffchainHosts::new(), + module_cache: HashMap::new(), + templates, + host_metrics, + causality_region_seq, + } + } + + // If `data_source.runtime()` is `None`, returns `Ok(None)`. + fn new_host( + &mut self, + logger: Logger, + data_source: DataSource, + ) -> Result>, Error> { + let module_bytes = match &data_source.runtime() { + None => return Ok(None), + Some(module_bytes) => module_bytes.cheap_clone(), + }; + + let mapping_request_sender = { + let module_hash = alloy::primitives::keccak256(module_bytes.as_ref()).0; + if let Some(sender) = self.module_cache.get(&module_hash) { + sender.clone() + } else { + let sender = T::spawn_mapping( + module_bytes.as_ref(), + logger, + self.subgraph_id.clone(), + self.host_metrics.cheap_clone(), + )?; + self.module_cache.insert(module_hash, sender.clone()); + sender + } + }; + + let host = self.host_builder.build( + self.network.clone(), + self.subgraph_id.clone(), + data_source, + self.templates.cheap_clone(), + mapping_request_sender, + self.host_metrics.cheap_clone(), + )?; + Ok(Some(Arc::new(host))) + } + + pub(super) fn add_dynamic_data_source( + &mut self, + logger: &Logger, + data_source: DataSource, + ) -> Result>, Error> { + // Protect against creating more than the allowed maximum number of data sources + if self.hosts_len() >= ENV_VARS.subgraph_max_data_sources { + anyhow::bail!( + "Limit of {} data sources per subgraph exceeded", + ENV_VARS.subgraph_max_data_sources, + ); + } + + let Some(host) = self.new_host(logger.clone(), data_source)? else { + return Ok(None); + }; + + // Check for duplicates and add the host. + match host.data_source() { + DataSource::Onchain(_) => { + // `onchain_hosts` will remain ordered by the creation block. + // See also 8f1bca33-d3b7-4035-affc-fd6161a12448. + ensure!( + self.onchain_hosts + .last() + .and_then(|h| h.creation_block_number()) + <= host.data_source().creation_block(), + ); + + if self.onchain_hosts.contains(&host) { + Ok(None) + } else { + self.onchain_hosts.push(host.cheap_clone()); + Ok(Some(host)) + } + } + DataSource::Offchain(_) => { + if self.offchain_hosts.contains(&host) { + Ok(None) + } else { + self.offchain_hosts.push(host.cheap_clone()); + Ok(Some(host)) + } + } + DataSource::Subgraph(_) => { + if self.subgraph_hosts.contains(&host) { + Ok(None) + } else { + self.subgraph_hosts.push(host.cheap_clone()); + Ok(Some(host)) + } + } + DataSource::Amp(_) => unreachable!(), + } + } + + /// Reverts any DataSources that have been added from the block forwards (inclusively) + /// This function also reverts the done_at status if it was 'done' on this block or later. + /// It only returns the offchain::Source because we don't currently need to know which + /// DataSources were removed, the source is used so that the offchain DDS can be found again. + pub(super) fn revert_data_sources( + &mut self, + reverted_block: BlockNumber, + ) -> Vec { + self.revert_onchain_hosts(reverted_block); + self.offchain_hosts.remove_ge_block(reverted_block); + + // Any File DataSources (Dynamic Data Sources), will have their own causality region + // which currently is the next number of the sequence but that should be an internal detail. + // Regardless of the sequence logic, if the current causality region is ONCHAIN then there are + // no others and therefore the remaining code is a noop and we can just stop here. + if self.causality_region_seq.0 == CausalityRegion::ONCHAIN { + return vec![]; + } + + self.offchain_hosts + .all() + .filter(|host| matches!(host.done_at(), Some(done_at) if done_at >= reverted_block)) + .map(|host| { + host.set_done_at(None); + host.data_source().as_offchain().unwrap().source.clone() + }) + .collect() + } + + /// Because onchain hosts are ordered, removing them based on creation block is cheap and simple. + fn revert_onchain_hosts(&mut self, reverted_block: BlockNumber) { + // `onchain_hosts` is ordered by the creation block. + // See also 8f1bca33-d3b7-4035-affc-fd6161a12448. + while self + .onchain_hosts + .last() + .filter(|h| h.creation_block_number() >= Some(reverted_block)) + .is_some() + { + self.onchain_hosts.pop(); + } + } + + /// Returns all hosts which match the trigger's address. + /// This is a performance optimization to reduce the number of calls to `match_and_decode`. + pub fn hosts_for_trigger( + &self, + trigger: &TriggerData, + ) -> Box + Send + '_> { + match trigger { + TriggerData::Onchain(trigger) => self + .onchain_hosts + .matches_by_address(trigger.address_match()), + TriggerData::Offchain(trigger) => self + .offchain_hosts + .matches_by_address(trigger.source.address().as_deref()), + TriggerData::Subgraph(trigger) => self + .subgraph_hosts + .matches_by_address(Some(trigger.source.to_bytes().as_slice())), + } + } + + pub(super) fn causality_region_next_value(&mut self) -> CausalityRegion { + self.causality_region_seq.next_val() + } + + pub fn hosts_len(&self) -> usize { + self.onchain_hosts.len() + self.offchain_hosts.len() + } +} diff --git a/core/src/subgraph/context/mod.rs b/core/src/subgraph/context/mod.rs new file mode 100644 index 00000000000..affbc09aa0f --- /dev/null +++ b/core/src/subgraph/context/mod.rs @@ -0,0 +1,257 @@ +mod instance; + +use crate::polling_monitor::{ + ArweaveService, IpfsRequest, IpfsService, PollingMonitor, PollingMonitorMetrics, spawn_monitor, +}; +use anyhow::{self, Error}; +use bytes::Bytes; +use graph::{ + blockchain::Blockchain, + components::{store::DeploymentId, subgraph::HostMetrics}, + data::subgraph::SubgraphManifest, + data_source::{ + CausalityRegion, DataSource, DataSourceTemplate, + causality_region::CausalityRegionSeq, + offchain::{self, Base64}, + }, + derive::CheapClone, + ipfs::IpfsContext, + prelude::{ + BlockNumber, CancelGuard, CheapClone, DeploymentHash, MetricsRegistry, RuntimeHostBuilder, + SubgraphCountMetric, TriggerProcessor, + }, + slog::Logger, +}; +use std::collections::HashMap; +use std::sync::Arc; + +use graph::parking_lot::RwLock; +use tokio::sync::mpsc; + +use self::instance::SubgraphInstance; +use super::Decoder; + +#[derive(Clone, CheapClone, Debug)] +pub struct SubgraphKeepAlive { + alive_map: Arc>>, + sg_metrics: Arc, +} + +impl SubgraphKeepAlive { + pub fn new(sg_metrics: Arc) -> Self { + Self { + sg_metrics, + alive_map: Arc::new(RwLock::new(HashMap::default())), + } + } + + pub fn remove(&self, deployment_id: &DeploymentId) { + self.alive_map.write().remove(deployment_id); + self.sg_metrics.running_count.dec(); + } + pub fn insert(&self, deployment_id: DeploymentId, guard: CancelGuard) { + let old = self.alive_map.write().insert(deployment_id, guard); + if old.is_none() { + self.sg_metrics.running_count.inc(); + } + } + + pub fn contains(&self, deployment_id: &DeploymentId) -> bool { + self.alive_map.read().contains_key(deployment_id) + } +} + +// The context keeps track of mutable in-memory state that is retained across blocks. +// +// Currently most of the changes are applied in `runner.rs`, but ideally more of that would be +// refactored into the context so it wouldn't need `pub` fields. The entity cache should probably +// also be moved here. +pub struct IndexingContext +where + T: RuntimeHostBuilder, + C: Blockchain, +{ + pub(crate) instance: SubgraphInstance, + pub instances: SubgraphKeepAlive, + pub offchain_monitor: OffchainMonitor, + pub(crate) trigger_processor: Box>, + pub(crate) decoder: Box>, +} + +impl> IndexingContext { + pub fn new( + manifest: SubgraphManifest, + host_builder: T, + host_metrics: Arc, + causality_region_seq: CausalityRegionSeq, + instances: SubgraphKeepAlive, + offchain_monitor: OffchainMonitor, + trigger_processor: Box>, + decoder: Box>, + ) -> Self { + let instance = SubgraphInstance::new( + manifest, + host_builder, + host_metrics.clone(), + causality_region_seq, + ); + + Self { + instance, + instances, + offchain_monitor, + trigger_processor, + decoder, + } + } + + /// Removes data sources hosts with a creation block greater or equal to `reverted_block`, so + /// that they are no longer candidates for `process_trigger`. + /// + /// This does not currently affect the `offchain_monitor` or the `filter`, so they will continue + /// to include data sources that have been reverted. This is not ideal for performance, but it + /// does not affect correctness since triggers that have no matching host will be ignored by + /// `process_trigger`. + /// + /// File data sources that have been marked not done during this process will get re-queued + pub fn revert_data_sources(&mut self, reverted_block: BlockNumber) { + let removed = self.instance.revert_data_sources(reverted_block); + + removed + .into_iter() + .for_each(|source| self.offchain_monitor.add_source(source)) + } + + pub fn add_dynamic_data_source( + &mut self, + logger: &Logger, + data_source: DataSource, + ) -> Result>, Error> { + let offchain_fields = data_source + .as_offchain() + .map(|ds| (ds.source.clone(), ds.is_processed())); + let host = self.instance.add_dynamic_data_source(logger, data_source)?; + + if host.is_some() + && let Some((source, is_processed)) = offchain_fields + { + // monitor data source only if it has not yet been processed. + if !is_processed { + self.offchain_monitor.add_source(source); + } + } + + Ok(host) + } + + pub fn causality_region_next_value(&mut self) -> CausalityRegion { + self.instance.causality_region_next_value() + } + + pub fn hosts_len(&self) -> usize { + self.instance.hosts_len() + } + + pub fn onchain_data_sources(&self) -> impl Iterator + Clone { + self.instance.onchain_data_sources() + } + + pub fn static_data_sources(&self) -> &[DataSource] { + &self.instance.static_data_sources + } + + pub fn templates(&self) -> &[DataSourceTemplate] { + &self.instance.templates + } +} + +pub struct OffchainMonitor { + ipfs_monitor: PollingMonitor, + ipfs_monitor_rx: mpsc::UnboundedReceiver<(IpfsRequest, Bytes)>, + arweave_monitor: PollingMonitor, + arweave_monitor_rx: mpsc::UnboundedReceiver<(Base64, Bytes)>, + deployment_hash: DeploymentHash, + logger: Logger, +} + +impl OffchainMonitor { + pub fn new( + logger: Logger, + registry: Arc, + subgraph_hash: &DeploymentHash, + ipfs_service: IpfsService, + arweave_service: ArweaveService, + ) -> Self { + let metrics = Arc::new(PollingMonitorMetrics::new(registry, subgraph_hash)); + // The channel is unbounded, as it is expected that `fn ready_offchain_events` is called + // frequently, or at least with the same frequency that requests are sent. + let (ipfs_monitor_tx, ipfs_monitor_rx) = mpsc::unbounded_channel(); + let (arweave_monitor_tx, arweave_monitor_rx) = mpsc::unbounded_channel(); + + let ipfs_monitor = spawn_monitor( + ipfs_service, + ipfs_monitor_tx, + logger.cheap_clone(), + metrics.cheap_clone(), + ); + + let arweave_monitor = spawn_monitor( + arweave_service, + arweave_monitor_tx, + logger.cheap_clone(), + metrics, + ); + + Self { + ipfs_monitor, + ipfs_monitor_rx, + arweave_monitor, + arweave_monitor_rx, + deployment_hash: subgraph_hash.to_owned(), + logger, + } + } + + fn add_source(&mut self, source: offchain::Source) { + match source { + offchain::Source::Ipfs(path) => self.ipfs_monitor.monitor(IpfsRequest { + ctx: IpfsContext::new(&self.deployment_hash, &self.logger), + path, + }), + offchain::Source::Arweave(base64) => self.arweave_monitor.monitor(base64), + }; + } + + pub fn ready_offchain_events(&mut self) -> Result, Error> { + use tokio::sync::mpsc::error::TryRecvError; + + let mut triggers = vec![]; + loop { + match self.ipfs_monitor_rx.try_recv() { + Ok((req, data)) => triggers.push(offchain::TriggerData { + source: offchain::Source::Ipfs(req.path), + data: Arc::new(data), + }), + Err(TryRecvError::Disconnected) => { + anyhow::bail!("ipfs monitor unexpectedly terminated") + } + Err(TryRecvError::Empty) => break, + } + } + + loop { + match self.arweave_monitor_rx.try_recv() { + Ok((base64, data)) => triggers.push(offchain::TriggerData { + source: offchain::Source::Arweave(base64), + data: Arc::new(data), + }), + Err(TryRecvError::Disconnected) => { + anyhow::bail!("arweave monitor unexpectedly terminated") + } + Err(TryRecvError::Empty) => break, + } + } + + Ok(triggers) + } +} diff --git a/core/src/subgraph/error.rs b/core/src/subgraph/error.rs index b3131255aed..00413acb258 100644 --- a/core/src/subgraph/error.rs +++ b/core/src/subgraph/error.rs @@ -1,28 +1,191 @@ use graph::data::subgraph::schema::SubgraphError; -use graph::prelude::{thiserror, Error, StoreError}; +use graph::env::ENV_VARS; +use graph::prelude::{Error, StoreError, anyhow, thiserror}; +pub trait DeterministicError: std::fmt::Debug + std::fmt::Display + Send + Sync + 'static {} + +impl DeterministicError for SubgraphError {} + +impl DeterministicError for StoreError {} + +impl DeterministicError for anyhow::Error {} + +/// Classification of processing errors for unified error handling. +/// +/// This enum provides a consistent way to categorize errors and determine +/// the appropriate response. The error handling invariants are: +/// +/// - **Deterministic**: Stop processing the current block, persist PoI only. +/// The subgraph will be marked as failed. These errors are reproducible +/// and indicate a bug in the subgraph or a permanent data issue. +/// +/// - **NonDeterministic**: Retry with exponential backoff. These errors are +/// transient (network issues, temporary database problems) and may succeed +/// on retry. +/// +/// - **PossibleReorg**: Restart the block stream cleanly without persisting. +/// The block stream needs to be restarted to detect and handle a potential +/// blockchain reorganization. +/// +/// - **Canceled**: The subgraph was canceled (unassigned or shut down). +/// No error should be recorded; this is a clean shutdown. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ProcessingErrorKind { + /// Error is deterministic - stop processing, persist PoI only + Deterministic, + /// Error is non-deterministic - retry with backoff + NonDeterministic, + /// Possible blockchain reorg detected - restart block stream cleanly + PossibleReorg, + /// Processing was canceled - clean shutdown + Canceled, +} + +/// An error happened during processing and we need to classify errors into +/// deterministic and non-deterministic errors. This struct holds the result +/// of that classification #[derive(thiserror::Error, Debug)] -pub enum BlockProcessingError { +pub enum ProcessingError { #[error("{0:#}")] - Unknown(#[from] Error), + Unknown(Error), // The error had a deterministic cause but, for a possibly non-deterministic reason, we chose to // halt processing due to the error. #[error("{0}")] - Deterministic(SubgraphError), + Deterministic(Box), + + /// A possible blockchain reorganization was detected. + /// The block stream should be restarted to detect and handle the reorg. + #[error("possible reorg detected: {0:#}")] + PossibleReorg(Error), #[error("subgraph stopped while processing triggers")] Canceled, } -impl BlockProcessingError { +impl ProcessingError { + /// Classify the error into one of the defined error kinds. + /// + /// This method provides a unified way to determine how to handle an error: + /// - `Deterministic`: Stop processing, persist PoI only + /// - `NonDeterministic`: Retry with backoff + /// - `PossibleReorg`: Restart block stream cleanly + /// - `Canceled`: Clean shutdown, no error recording + pub fn kind(&self) -> ProcessingErrorKind { + match self { + ProcessingError::Unknown(_) => ProcessingErrorKind::NonDeterministic, + ProcessingError::Deterministic(_) => ProcessingErrorKind::Deterministic, + ProcessingError::PossibleReorg(_) => ProcessingErrorKind::PossibleReorg, + ProcessingError::Canceled => ProcessingErrorKind::Canceled, + } + } + + #[allow(dead_code)] pub fn is_deterministic(&self) -> bool { - matches!(self, BlockProcessingError::Deterministic(_)) + matches!(self.kind(), ProcessingErrorKind::Deterministic) + } + + /// Returns true if this error should stop processing the current block. + /// + /// Deterministic errors stop processing because continuing would produce + /// incorrect results. The PoI is still persisted for debugging purposes. + #[allow(dead_code)] + pub fn should_stop_processing(&self) -> bool { + matches!(self.kind(), ProcessingErrorKind::Deterministic) + } + + /// Returns true if this error requires a clean restart of the block stream. + /// + /// Possible reorgs require restarting to allow the block stream to detect + /// and properly handle the reorganization. No state should be persisted + /// in this case. + #[allow(dead_code)] + pub fn should_restart(&self) -> bool { + matches!(self.kind(), ProcessingErrorKind::PossibleReorg) + } + + /// Returns true if this error is retryable with exponential backoff. + /// + /// Non-deterministic errors (network issues, temporary failures) may + /// succeed on retry and should not immediately fail the subgraph. + #[allow(dead_code)] + pub fn is_retryable(&self) -> bool { + matches!(self.kind(), ProcessingErrorKind::NonDeterministic) + } + + /// Returns true if processing was canceled (clean shutdown). + /// + /// Canceled errors indicate the subgraph was unassigned or shut down + /// intentionally and should not be treated as failures. + #[allow(dead_code)] + pub fn is_canceled(&self) -> bool { + matches!(self.kind(), ProcessingErrorKind::Canceled) } + + pub fn detail(self, ctx: &str) -> ProcessingError { + match self { + ProcessingError::Unknown(e) => { + let x = e.context(ctx.to_string()); + ProcessingError::Unknown(x) + } + ProcessingError::Deterministic(e) => { + ProcessingError::Deterministic(Box::new(anyhow!("{e}").context(ctx.to_string()))) + } + ProcessingError::PossibleReorg(e) => { + ProcessingError::PossibleReorg(e.context(ctx.to_string())) + } + ProcessingError::Canceled => ProcessingError::Canceled, + } + } +} + +/// Similar to `anyhow::Context`, but for `Result`. We +/// call the method `detail` to avoid ambiguity with anyhow's `context` +/// method +pub trait DetailHelper { + fn detail(self, ctx: &str) -> Result; +} + +impl DetailHelper for Result { + fn detail(self, ctx: &str) -> Result { + self.map_err(|e| e.detail(ctx)) + } +} + +/// Implement this for errors that are always non-deterministic. +pub(crate) trait NonDeterministicErrorHelper { + fn non_deterministic(self) -> Result; +} + +impl NonDeterministicErrorHelper for Result { + fn non_deterministic(self) -> Result { + self.map_err(ProcessingError::Unknown) + } +} + +impl NonDeterministicErrorHelper for Result { + fn non_deterministic(self) -> Result { + self.map_err(|e| ProcessingError::Unknown(Error::from(e))) + } +} + +/// Implement this for errors where it depends on the details whether they +/// are deterministic or not. +pub(crate) trait ClassifyErrorHelper { + fn classify(self) -> Result; } -impl From for BlockProcessingError { - fn from(e: StoreError) -> Self { - BlockProcessingError::Unknown(e.into()) +impl ClassifyErrorHelper for Result { + fn classify(self) -> Result { + self.map_err(|e| { + if ENV_VARS.mappings.store_errors_are_nondeterministic { + // Old behavior, just in case the new behavior causes issues + ProcessingError::Unknown(Error::from(e)) + } else if e.is_deterministic() { + ProcessingError::Deterministic(Box::new(e)) + } else { + ProcessingError::Unknown(Error::from(e)) + } + }) } } diff --git a/core/src/subgraph/inputs.rs b/core/src/subgraph/inputs.rs index 191dc69cbf4..03a8cec29bc 100644 --- a/core/src/subgraph/inputs.rs +++ b/core/src/subgraph/inputs.rs @@ -1,7 +1,7 @@ use graph::{ - blockchain::{Blockchain, TriggersAdapter}, + blockchain::{Blockchain, block_stream::TriggersAdapterWrapper}, components::{ - store::{DeploymentLocator, SubgraphFork, WritableStore}, + store::{DeploymentLocator, SourceableStore, SubgraphFork, WritableStore}, subgraph::ProofOfIndexingVersion, }, data::subgraph::{SubgraphFeature, UnifiedMappingApiVersion}, @@ -15,10 +15,13 @@ pub struct IndexingInputs { pub deployment: DeploymentLocator, pub features: BTreeSet, pub start_blocks: Vec, + pub end_blocks: BTreeSet, + pub source_subgraph_stores: Vec>, pub stop_block: Option, + pub max_end_block: Option, pub store: Arc, pub debug_fork: Option>, - pub triggers_adapter: Arc>, + pub triggers_adapter: Arc>, pub chain: Arc, pub templates: Arc>>, pub unified_api_version: UnifiedMappingApiVersion, @@ -26,6 +29,58 @@ pub struct IndexingInputs { pub poi_version: ProofOfIndexingVersion, pub network: String, - // Correspondence between data source or template position in the manifest and name. - pub manifest_idx_and_name: Vec<(u32, String)>, + /// Whether to instrument trigger processing and log additional, + /// possibly expensive and noisy, information + pub instrument: bool, +} + +impl IndexingInputs { + pub fn with_store(&self, store: Arc) -> Self { + let IndexingInputs { + deployment, + features, + start_blocks, + end_blocks, + source_subgraph_stores, + stop_block, + max_end_block, + store: _, + debug_fork, + triggers_adapter, + chain, + templates, + unified_api_version, + static_filters, + poi_version, + network, + instrument, + } = self; + IndexingInputs { + deployment: deployment.clone(), + features: features.clone(), + start_blocks: start_blocks.clone(), + end_blocks: end_blocks.clone(), + source_subgraph_stores: source_subgraph_stores.clone(), + stop_block: *stop_block, + max_end_block: *max_end_block, + store, + debug_fork: debug_fork.clone(), + triggers_adapter: triggers_adapter.clone(), + chain: chain.clone(), + templates: templates.clone(), + unified_api_version: unified_api_version.clone(), + static_filters: *static_filters, + poi_version: *poi_version, + network: network.clone(), + instrument: *instrument, + } + } + + pub fn errors_are_non_fatal(&self) -> bool { + self.features.contains(&SubgraphFeature::NonFatalErrors) + } + + pub fn errors_are_fatal(&self) -> bool { + !self.features.contains(&SubgraphFeature::NonFatalErrors) + } } diff --git a/core/src/subgraph/instance_manager.rs b/core/src/subgraph/instance_manager.rs index 16913b37161..ea301ffc723 100644 --- a/core/src/subgraph/instance_manager.rs +++ b/core/src/subgraph/instance_manager.rs @@ -1,122 +1,127 @@ -use crate::polling_monitor::IpfsService; +use std::sync::atomic::AtomicU64; +use std::sync::atomic::Ordering; + +use crate::polling_monitor::{ArweaveService, IpfsService}; +use crate::subgraph::Decoder; use crate::subgraph::context::{IndexingContext, SubgraphKeepAlive}; use crate::subgraph::inputs::IndexingInputs; use crate::subgraph::loader::load_dynamic_data_sources; +use std::collections::BTreeSet; use crate::subgraph::runner::SubgraphRunner; -use graph::blockchain::block_stream::BlockStreamMetrics; -use graph::blockchain::Blockchain; -use graph::blockchain::NodeCapabilities; -use graph::blockchain::{BlockchainKind, TriggerFilter}; +use async_trait::async_trait; +use graph::amp; +use graph::blockchain::block_stream::{BlockStreamMetrics, TriggersAdapterWrapper}; +use graph::blockchain::{Blockchain, BlockchainKind, DataSource, NodeCapabilities}; +use graph::components::metrics::gas::GasMetrics; +use graph::components::metrics::subgraph::DeploymentStatusMetric; +use graph::components::store::SourceableStore; use graph::components::subgraph::ProofOfIndexingVersion; -use graph::data::subgraph::{UnresolvedSubgraphManifest, SPEC_VERSION_0_0_6}; +use graph::data::subgraph::{SPEC_VERSION_0_0_6, UnresolvedSubgraphManifest}; +use graph::data::value::Word; use graph::data_source::causality_region::CausalityRegionSeq; use graph::env::EnvVars; use graph::prelude::{SubgraphInstanceManager as SubgraphInstanceManagerTrait, *}; use graph::{blockchain::BlockchainMap, components::store::DeploymentLocator}; -use graph_runtime_wasm::module::ToAscPtr; use graph_runtime_wasm::RuntimeHostBuilder; +use graph_runtime_wasm::module::ToAscPtr; use tokio::task; -use super::context::OffchainMonitor; use super::SubgraphTriggerProcessor; +use super::context::OffchainMonitor; +use crate::{subgraph::runner::SubgraphRunnerError, subgraph_manifest}; #[derive(Clone)] -pub struct SubgraphInstanceManager { +pub struct SubgraphInstanceManager { logger_factory: LoggerFactory, subgraph_store: Arc, chains: Arc, - metrics_registry: Arc, + metrics_registry: Arc, instances: SubgraphKeepAlive, link_resolver: Arc, ipfs_service: IpfsService, + arweave_service: ArweaveService, + amp_client: Option>, static_filters: bool, env_vars: Arc, + + /// By design, there should be only one subgraph runner process per subgraph, but the current + /// implementation does not completely prevent multiple runners from being active at the same + /// time, and we have already had a [bug][0] due to this limitation. Investigating the problem + /// was quite complicated because there was no way to know that the logs were coming from two + /// different processes because all the logs looked the same. Ideally, the implementation + /// should be refactored to make it more strict, but until then, we keep this counter, which + /// is incremented each time a new runner is started, and the previous count is embedded in + /// each log of the started runner, to make debugging future issues easier. + /// + /// [0]: https://github.com/graphprotocol/graph-node/issues/5452 + subgraph_start_counter: Arc, } #[async_trait] -impl SubgraphInstanceManagerTrait for SubgraphInstanceManager { +impl SubgraphInstanceManagerTrait for SubgraphInstanceManager +where + NC: amp::Client + Send + Sync + 'static, +{ async fn start_subgraph( self: Arc, loc: DeploymentLocator, - manifest: serde_yaml::Mapping, stop_block: Option, ) { + let runner_index = self.subgraph_start_counter.fetch_add(1, Ordering::SeqCst); + let logger = self.logger_factory.subgraph_logger(&loc); + let logger = logger.new(o!("runner_index" => runner_index)); + let err_logger = logger.clone(); let instance_manager = self.cheap_clone(); - let subgraph_start_future = async move { - match BlockchainKind::from_manifest(&manifest)? { - BlockchainKind::Arweave => { - let runner = instance_manager - .build_subgraph_runner::( - logger.clone(), - self.env_vars.cheap_clone(), - loc.clone(), - manifest, - stop_block, - Box::new(SubgraphTriggerProcessor {}), - ) - .await?; - - self.start_subgraph_inner(logger, loc, runner).await - } - BlockchainKind::Ethereum => { - let runner = instance_manager - .build_subgraph_runner::( - logger.clone(), - self.env_vars.cheap_clone(), - loc.clone(), - manifest, - stop_block, - Box::new(SubgraphTriggerProcessor {}), - ) - .await?; - - self.start_subgraph_inner(logger, loc, runner).await - } - BlockchainKind::Near => { - let runner = instance_manager - .build_subgraph_runner::( - logger.clone(), - self.env_vars.cheap_clone(), - loc.clone(), - manifest, - stop_block, - Box::new(SubgraphTriggerProcessor {}), - ) - .await?; - - self.start_subgraph_inner(logger, loc, runner).await - } - BlockchainKind::Cosmos => { - let runner = instance_manager - .build_subgraph_runner::( - logger.clone(), - self.env_vars.cheap_clone(), - loc.clone(), - manifest, - stop_block, - Box::new(SubgraphTriggerProcessor {}), - ) - .await?; - - self.start_subgraph_inner(logger, loc, runner).await - } - BlockchainKind::Substreams => { - let runner = instance_manager - .build_subgraph_runner::( - logger.clone(), - self.env_vars.cheap_clone(), - loc.cheap_clone(), - manifest, - stop_block, - Box::new(graph_chain_substreams::TriggerProcessor::new(loc.clone())), - ) - .await?; - - self.start_subgraph_inner(logger, loc, runner).await + let deployment_status_metric = self.new_deployment_status_metric(&loc); + deployment_status_metric.starting(); + + let subgraph_start_future = { + let deployment_status_metric = deployment_status_metric.clone(); + + async move { + let raw_manifest = subgraph_manifest::load_raw_subgraph_manifest( + &logger, + &*instance_manager.subgraph_store, + &*instance_manager.link_resolver, + &loc.hash, + ) + .await?; + + match BlockchainKind::from_manifest(&raw_manifest)? { + BlockchainKind::Ethereum => { + let runner = instance_manager + .build_subgraph_runner::( + logger.clone(), + self.env_vars.cheap_clone(), + loc.clone(), + raw_manifest, + stop_block, + Box::new(SubgraphTriggerProcessor {}), + deployment_status_metric, + ) + .await?; + + self.start_subgraph_inner(logger, loc, runner).await + } + BlockchainKind::Near => { + let runner = instance_manager + .build_subgraph_runner::( + logger.clone(), + self.env_vars.cheap_clone(), + loc.clone(), + raw_manifest, + stop_block, + Box::new(SubgraphTriggerProcessor {}), + deployment_status_metric, + ) + .await?; + + self.start_subgraph_inner(logger, loc, runner).await + } } } }; @@ -129,12 +134,16 @@ impl SubgraphInstanceManagerTrait for SubgraphInstanceManager< graph::spawn(async move { match subgraph_start_future.await { Ok(()) => {} - Err(err) => error!( - err_logger, - "Failed to start subgraph"; - "error" => format!("{:#}", err), - "code" => LogCode::SubgraphStartFailure - ), + Err(err) => { + deployment_status_metric.failed(); + + error!( + err_logger, + "Failed to start subgraph"; + "error" => format!("{:#}", err), + "code" => LogCode::SubgraphStartFailure + ); + } } }); } @@ -155,15 +164,18 @@ impl SubgraphInstanceManagerTrait for SubgraphInstanceManager< } } -impl SubgraphInstanceManager { +impl SubgraphInstanceManager { pub fn new( logger_factory: &LoggerFactory, env_vars: Arc, subgraph_store: Arc, chains: Arc, - metrics_registry: Arc, + sg_metrics: Arc, + metrics_registry: Arc, link_resolver: Arc, ipfs_service: IpfsService, + arweave_service: ArweaveService, + amp_client: Option>, static_filters: bool, ) -> Self { let logger = logger_factory.component_logger("SubgraphInstanceManager", None); @@ -174,22 +186,79 @@ impl SubgraphInstanceManager { subgraph_store, chains, metrics_registry: metrics_registry.cheap_clone(), - instances: SubgraphKeepAlive::new(metrics_registry), + instances: SubgraphKeepAlive::new(sg_metrics), link_resolver, ipfs_service, + amp_client, static_filters, env_vars, + arweave_service, + subgraph_start_counter: Arc::new(AtomicU64::new(0)), } } + pub async fn get_sourceable_stores( + &self, + hashes: Vec, + is_runner_test: bool, + ) -> anyhow::Result>> { + if is_runner_test { + return Ok(Vec::new()); + } + + let mut sourceable_stores = Vec::new(); + let subgraph_store = self.subgraph_store.clone(); + + for hash in hashes { + let loc = subgraph_store + .active_locator(&hash) + .await? + .ok_or_else(|| anyhow!("no active deployment for hash {}", hash))?; + + let sourceable_store = subgraph_store.clone().sourceable(loc.id).await?; + sourceable_stores.push(sourceable_store); + } + + Ok(sourceable_stores) + } + pub async fn build_subgraph_runner( &self, logger: Logger, env_vars: Arc, deployment: DeploymentLocator, - manifest: serde_yaml::Mapping, + raw_manifest: serde_yaml::Mapping, + stop_block: Option, + tp: Box>>, + deployment_status_metric: DeploymentStatusMetric, + ) -> anyhow::Result>> + where + C: Blockchain, + ::MappingTrigger: ToAscPtr, + { + self.build_subgraph_runner_inner( + logger, + env_vars, + deployment, + raw_manifest, + stop_block, + tp, + deployment_status_metric, + false, + ) + .await + } + + pub async fn build_subgraph_runner_inner( + &self, + logger: Logger, + env_vars: Arc, + deployment: DeploymentLocator, + raw_manifest: serde_yaml::Mapping, stop_block: Option, tp: Box>>, + deployment_status_metric: DeploymentStatusMetric, + is_runner_test: bool, ) -> anyhow::Result>> where C: Blockchain, @@ -198,45 +267,79 @@ impl SubgraphInstanceManager { let subgraph_store = self.subgraph_store.cheap_clone(); let registry = self.metrics_registry.cheap_clone(); - let store = self - .subgraph_store - .cheap_clone() - .writable(logger.clone(), deployment.id) - .await?; - - let raw_yaml = serde_yaml::to_string(&manifest).unwrap(); - let manifest = UnresolvedSubgraphManifest::parse(deployment.hash.cheap_clone(), manifest)?; + let manifest = + UnresolvedSubgraphManifest::parse(deployment.hash.cheap_clone(), raw_manifest)?; // Allow for infinite retries for subgraph definition files. - let link_resolver = Arc::from(self.link_resolver.with_retries()); + let link_resolver = Arc::from( + self.link_resolver + .for_manifest(&deployment.hash.to_string()) + .map_err(SubgraphRegistrarError::Unknown)? + .with_retries(), + ); - // Make sure the `raw_yaml` is present on both this subgraph and the graft base. - self.subgraph_store - .set_manifest_raw_yaml(&deployment.hash, raw_yaml) + if let Some(graft) = &manifest.graft + && self.subgraph_store.is_deployed(&graft.base).await? + { + // Makes sure the raw manifest is cached in the subgraph store + let _raw_manifest = subgraph_manifest::load_raw_subgraph_manifest( + &logger, + &*self.subgraph_store, + &*self.link_resolver, + &graft.base, + ) .await?; - if let Some(graft) = &manifest.graft { - if self.subgraph_store.is_deployed(&graft.base)? { - let file_bytes = self - .link_resolver - .cat(&logger, &graft.base.to_ipfs_link()) - .await?; - let yaml = String::from_utf8(file_bytes)?; - - self.subgraph_store - .set_manifest_raw_yaml(&graft.base, yaml) - .await?; - } } - info!(logger, "Resolve subgraph files using IPFS"); + info!(logger, "Resolve subgraph files using IPFS"; + "n_data_sources" => manifest.data_sources.len(), + "n_templates" => manifest.templates.len(), + ); - let mut manifest = manifest - .resolve(&link_resolver, &logger, ENV_VARS.max_spec_version.clone()) + let manifest = manifest + .resolve( + &deployment.hash, + &link_resolver, + self.amp_client.cheap_clone(), + &logger, + ENV_VARS.max_spec_version.clone(), + ) .await?; - info!(logger, "Successfully resolved subgraph files using IPFS"); + { + let features = if manifest.features.is_empty() { + "ø".to_string() + } else { + manifest + .features + .iter() + .map(|f| f.to_string()) + .collect::>() + .join(", ") + }; + info!(logger, "Successfully resolved subgraph files using IPFS"; + "n_data_sources" => manifest.data_sources.len(), + "n_templates" => manifest.templates.len(), + "features" => features + ); + } - let manifest_idx_and_name: Vec<(u32, String)> = manifest.template_idx_and_name().collect(); + let store = self + .subgraph_store + .cheap_clone() + .writable( + logger.clone(), + deployment.id, + Arc::new(manifest.template_idx_and_name().collect()), + ) + .await?; + + // Create deployment features from the manifest + // Write it to the database + let deployment_features = manifest.deployment_features(); + self.subgraph_store + .create_subgraph_features(deployment_features) + .await?; // Start the subgraph deployment before reading dynamic data // sources; if the subgraph is a graft or a copy, starting it will @@ -244,38 +347,34 @@ impl SubgraphInstanceManager { // that is done store.start_subgraph_deployment(&logger).await?; - // Dynamic data sources are loaded by appending them to the manifest. - // - // Refactor: Preferrably we'd avoid any mutation of the manifest. - let (manifest, static_data_sources) = { - let data_sources = load_dynamic_data_sources(store.clone(), logger.clone(), &manifest) + let dynamic_data_sources = + load_dynamic_data_sources(store.clone(), logger.clone(), &manifest) .await .context("Failed to load dynamic data sources")?; - let static_data_sources = manifest.data_sources.clone(); - - // Add dynamic data sources to the subgraph - manifest.data_sources.extend(data_sources); + // Combine the data sources from the manifest with the dynamic data sources + let mut data_sources = manifest.data_sources.clone(); + data_sources.extend(dynamic_data_sources); - info!( - logger, - "Data source count at start: {}", - manifest.data_sources.len() - ); + info!(logger, "Data source count at start: {}", data_sources.len()); - (manifest, static_data_sources) - }; + let onchain_data_sources = data_sources + .iter() + .filter_map(|d| d.as_onchain().cloned()) + .collect::>(); - let static_filters = - self.static_filters || manifest.data_sources.len() >= ENV_VARS.static_filters_threshold; + let subgraph_data_sources = data_sources + .iter() + .filter_map(|d| d.as_subgraph()) + .collect::>(); - let onchain_data_sources = manifest - .data_sources + let subgraph_ds_source_deployments = subgraph_data_sources .iter() - .filter_map(|d| d.as_onchain().cloned()) + .map(|d| d.source.address()) .collect::>(); + let required_capabilities = C::NodeCapabilities::from_data_sources(&onchain_data_sources); - let network = manifest.network_name(); + let network: Word = manifest.network_name().into(); let chain = self .chains @@ -283,40 +382,44 @@ impl SubgraphInstanceManager { .with_context(|| format!("no chain configured for network {}", network))? .clone(); - // if static_filters is enabled, build a minimal filter with the static data sources and - // add the necessary filters based on templates. - // if not enabled we just stick to the filter based on all the data sources. - // This specifically removes dynamic data sources based filters because these can be derived - // from templates AND this reduces the cost of egress traffic by making the payloads smaller. - let filter = if static_filters { - if !self.static_filters { - info!(logger, "forcing subgraph to use static filters.") - } - - let onchain_data_sources = static_data_sources.iter().filter_map(|d| d.as_onchain()); - - let mut filter = C::TriggerFilter::from_data_sources(onchain_data_sources); + let start_blocks: Vec = data_sources + .iter() + .filter_map(|d| d.start_block()) + .collect(); - filter.extend_with_template( - manifest - .templates - .iter() - .filter_map(|ds| ds.as_onchain()) - .cloned(), - ); - filter + let end_blocks: BTreeSet = manifest + .data_sources + .iter() + .filter_map(|d| d.as_onchain().and_then(|d: &C::DataSource| d.end_block())) + .collect(); + + // We can set `max_end_block` to the maximum of `end_blocks` and stop the subgraph + // only when there are no dynamic data sources and no offchain data sources present. This is because: + // - Dynamic data sources do not have a defined `end_block`, so we can't determine + // when to stop processing them. + // - Offchain data sources might require processing beyond the end block of + // onchain data sources, so the subgraph needs to continue. + // + // Note: we explicitly check each data source rather than comparing lengths, because + // `end_blocks` is a BTreeSet and deduplicates equal values, which would cause the + // length comparison to fail when multiple data sources share the same `end_block`. + let max_end_block: Option = if data_sources.iter().all(|d| { + d.as_onchain() + .and_then(|d: &C::DataSource| d.end_block()) + .is_some() + }) { + end_blocks.iter().max().cloned() } else { - C::TriggerFilter::from_data_sources(onchain_data_sources.iter()) + None }; - let start_blocks = manifest.start_blocks(); - let templates = Arc::new(manifest.templates.clone()); // Obtain the debug fork from the subgraph store let debug_fork = self .subgraph_store - .debug_fork(&deployment.hash, logger.clone())?; + .debug_fork(&deployment.hash, logger.clone()) + .await?; // Create a subgraph instance from the manifest; this moves // ownership of the manifest and host builder into the new instance @@ -325,8 +428,11 @@ impl SubgraphInstanceManager { deployment.hash.clone(), "process", self.metrics_registry.clone(), + store.shard().to_string(), ); + let gas_metrics = GasMetrics::new(deployment.hash.clone(), self.metrics_registry.clone()); + let unified_mapping_api_version = manifest.unified_mapping_api_version()?; let triggers_adapter = chain.triggers_adapter(&deployment, &required_capabilities, unified_mapping_api_version).map_err(|e| anyhow!( @@ -338,12 +444,14 @@ impl SubgraphInstanceManager { registry.cheap_clone(), deployment.hash.as_str(), stopwatch_metrics.clone(), + gas_metrics.clone(), )); let subgraph_metrics = Arc::new(SubgraphInstanceMetrics::new( registry.cheap_clone(), deployment.hash.as_str(), stopwatch_metrics.clone(), + deployment_status_metric, )); let block_stream_metrics = Arc::new(BlockStreamMetrics::new( @@ -354,11 +462,12 @@ impl SubgraphInstanceManager { stopwatch_metrics, )); - let mut offchain_monitor = OffchainMonitor::new( + let offchain_monitor = OffchainMonitor::new( logger.cheap_clone(), registry.cheap_clone(), &manifest.id, self.ipfs_service.clone(), + self.arweave_service.clone(), ); // Initialize deployment_head with current deployment head. Any sort of trouble in @@ -366,8 +475,9 @@ impl SubgraphInstanceManager { let deployment_head = store.block_ptr().map(|ptr| ptr.number).unwrap_or(0) as f64; block_stream_metrics.deployment_head.set(deployment_head); + let (runtime_adapter, decoder_hook) = chain.runtime().await?; let host_builder = graph_runtime_wasm::RuntimeHostBuilder::new( - chain.runtime_adapter(), + runtime_adapter, self.link_resolver.cheap_clone(), subgraph_store.ens_lookup(), ); @@ -383,40 +493,58 @@ impl SubgraphInstanceManager { let causality_region_seq = CausalityRegionSeq::from_current(store.causality_region_curr_val().await?); - let instance = super::context::instance::SubgraphInstance::from_manifest( - &logger, - manifest, - host_builder, - host_metrics.clone(), - &mut offchain_monitor, - causality_region_seq, - )?; + let instrument = self.subgraph_store.instrument(&deployment).await?; + + let decoder = Box::new(Decoder::new(decoder_hook)); + + let subgraph_data_source_stores = self + .get_sourceable_stores::(subgraph_ds_source_deployments, is_runner_test) + .await?; + + let triggers_adapter = Arc::new(TriggersAdapterWrapper::new( + triggers_adapter, + subgraph_data_source_stores.clone(), + )); let inputs = IndexingInputs { deployment: deployment.clone(), features, start_blocks, + end_blocks, + source_subgraph_stores: subgraph_data_source_stores, stop_block, + max_end_block, store, debug_fork, triggers_adapter, chain, templates, unified_api_version, - static_filters, - manifest_idx_and_name, + static_filters: self.static_filters, poi_version, - network, + network: network.to_string(), + instrument, }; - // The subgraph state tracks the state of the subgraph instance over time - let ctx = IndexingContext::new( - instance, - self.instances.cheap_clone(), - filter, - offchain_monitor, - tp, - ); + // Initialize the indexing context, including both static and dynamic data sources. + // The order of inclusion is the order of processing when a same trigger matches + // multiple data sources. + let ctx = { + let mut ctx = IndexingContext::new( + manifest, + host_builder, + host_metrics.clone(), + causality_region_seq, + self.instances.cheap_clone(), + offchain_monitor, + tp, + decoder, + ); + for data_source in data_sources { + ctx.add_dynamic_data_source(&logger, data_source)?; + } + ctx + }; let metrics = RunnerMetrics { subgraph: subgraph_metrics, @@ -443,7 +571,7 @@ impl SubgraphInstanceManager { ::MappingTrigger: ToAscPtr, { let registry = self.metrics_registry.cheap_clone(); - let subgraph_metrics_unregister = runner.metrics.subgraph.cheap_clone(); + let subgraph_metrics = runner.metrics.subgraph.cheap_clone(); // Keep restarting the subgraph until it terminates. The subgraph // will usually only run once, but is restarted whenever a block @@ -459,16 +587,31 @@ impl SubgraphInstanceManager { // it has a dedicated OS thread so the OS will handle the preemption. See // https://github.com/tokio-rs/tokio/issues/3493. graph::spawn_thread(deployment.to_string(), move || { - if let Err(e) = graph::block_on(task::unconstrained(runner.run())) { - error!( - &logger, - "Subgraph instance failed to run: {}", - format!("{:#}", e) - ); + match graph::block_on(task::unconstrained(runner.run())) { + Ok(()) => { + subgraph_metrics.deployment_status.stopped(); + } + Err(SubgraphRunnerError::Duplicate) => { + // We do not need to unregister metrics because they are unique per subgraph + // and another runner is still active. + return; + } + Err(err) => { + error!(&logger, "Subgraph instance failed to run: {:#}", err); + subgraph_metrics.deployment_status.failed(); + } } - subgraph_metrics_unregister.unregister(registry); + + subgraph_metrics.unregister(registry); }); Ok(()) } + + pub fn new_deployment_status_metric( + &self, + deployment: &DeploymentLocator, + ) -> DeploymentStatusMetric { + DeploymentStatusMetric::register(&self.metrics_registry, deployment) + } } diff --git a/core/src/subgraph/mod.rs b/core/src/subgraph/mod.rs index 45f8d5b98ef..8f6bc932daa 100644 --- a/core/src/subgraph/mod.rs +++ b/core/src/subgraph/mod.rs @@ -3,7 +3,6 @@ mod error; mod inputs; mod instance_manager; mod loader; -mod provider; mod registrar; mod runner; mod state; @@ -11,7 +10,6 @@ mod stream; mod trigger_processor; pub use self::instance_manager::SubgraphInstanceManager; -pub use self::provider::SubgraphAssignmentProvider; pub use self::registrar::SubgraphRegistrar; pub use self::runner::SubgraphRunner; pub use self::trigger_processor::*; diff --git a/core/src/subgraph/provider.rs b/core/src/subgraph/provider.rs deleted file mode 100644 index 4d3a0cab51d..00000000000 --- a/core/src/subgraph/provider.rs +++ /dev/null @@ -1,88 +0,0 @@ -use std::collections::HashSet; -use std::sync::Mutex; - -use async_trait::async_trait; - -use graph::{ - components::store::{DeploymentId, DeploymentLocator}, - prelude::{SubgraphAssignmentProvider as SubgraphAssignmentProviderTrait, *}, -}; - -pub struct SubgraphAssignmentProvider { - logger_factory: LoggerFactory, - subgraphs_running: Arc>>, - link_resolver: Arc, - instance_manager: Arc, -} - -impl SubgraphAssignmentProvider { - pub fn new( - logger_factory: &LoggerFactory, - link_resolver: Arc, - instance_manager: I, - ) -> Self { - let logger = logger_factory.component_logger("SubgraphAssignmentProvider", None); - let logger_factory = logger_factory.with_parent(logger.clone()); - - // Create the subgraph provider - SubgraphAssignmentProvider { - logger_factory, - subgraphs_running: Arc::new(Mutex::new(HashSet::new())), - link_resolver: link_resolver.with_retries().into(), - instance_manager: Arc::new(instance_manager), - } - } -} - -#[async_trait] -impl SubgraphAssignmentProviderTrait for SubgraphAssignmentProvider { - async fn start( - &self, - loc: DeploymentLocator, - stop_block: Option, - ) -> Result<(), SubgraphAssignmentProviderError> { - let logger = self.logger_factory.subgraph_logger(&loc); - - // If subgraph ID already in set - if !self.subgraphs_running.lock().unwrap().insert(loc.id) { - info!(logger, "Subgraph deployment is already running"); - - return Err(SubgraphAssignmentProviderError::AlreadyRunning( - loc.hash.clone(), - )); - } - - let file_bytes = self - .link_resolver - .cat(&logger, &loc.hash.to_ipfs_link()) - .await - .map_err(SubgraphAssignmentProviderError::ResolveError)?; - - let raw: serde_yaml::Mapping = serde_yaml::from_slice(&file_bytes) - .map_err(|e| SubgraphAssignmentProviderError::ResolveError(e.into()))?; - - self.instance_manager - .cheap_clone() - .start_subgraph(loc, raw, stop_block) - .await; - - Ok(()) - } - - async fn stop( - &self, - deployment: DeploymentLocator, - ) -> Result<(), SubgraphAssignmentProviderError> { - // If subgraph ID was in set - if self - .subgraphs_running - .lock() - .unwrap() - .remove(&deployment.id) - { - // Shut down subgraph processing - self.instance_manager.stop_subgraph(deployment).await; - } - Ok(()) - } -} diff --git a/core/src/subgraph/registrar.rs b/core/src/subgraph/registrar.rs index b8d0f408e23..e9422671e25 100644 --- a/core/src/subgraph/registrar.rs +++ b/core/src/subgraph/registrar.rs @@ -1,36 +1,45 @@ use std::collections::HashSet; -use std::time::Instant; use async_trait::async_trait; -use graph::blockchain::Blockchain; -use graph::blockchain::BlockchainKind; -use graph::blockchain::BlockchainMap; -use graph::components::store::{DeploymentId, DeploymentLocator, SubscriptionManager}; -use graph::data::subgraph::schema::DeploymentCreate; -use graph::data::subgraph::Graft; -use graph::prelude::{ - CreateSubgraphResult, SubgraphAssignmentProvider as SubgraphAssignmentProviderTrait, - SubgraphRegistrar as SubgraphRegistrarTrait, *, +use graph::amp; +use graph::blockchain::{Blockchain, BlockchainKind, BlockchainMap}; +use graph::components::{ + link_resolver::LinkResolverContext, + network_provider::AmpChainNames, + store::{DeploymentId, DeploymentLocator, SubscriptionManager}, + subgraph::Settings, }; +use graph::data::{ + subgraph::{Graft, schema::DeploymentCreate}, + value::Word, +}; +use graph::futures03::{self, Stream, StreamExt, future::TryFutureExt}; +use graph::prelude::{CreateSubgraphResult, SubgraphRegistrar as SubgraphRegistrarTrait, *}; +use graph::util::futures::{RETRY_DEFAULT_LIMIT, retry_strategy}; +use tokio_retry::Retry; -pub struct SubgraphRegistrar { +pub struct SubgraphRegistrar { logger: Logger, logger_factory: LoggerFactory, resolver: Arc, provider: Arc

, store: Arc, subscription_manager: Arc, + amp_client: Option>, chains: Arc, node_id: NodeId, version_switching_mode: SubgraphVersionSwitchingMode, assignment_event_stream_cancel_guard: CancelGuard, // cancels on drop + settings: Arc, + amp_chain_names: Arc, } -impl SubgraphRegistrar +impl SubgraphRegistrar where - P: SubgraphAssignmentProviderTrait, + P: graph::components::subgraph::SubgraphInstanceManager, S: SubgraphStore, SM: SubscriptionManager, + AC: amp::Client + Send + Sync + 'static, { pub fn new( logger_factory: &LoggerFactory, @@ -38,9 +47,12 @@ where provider: Arc

, store: Arc, subscription_manager: Arc, + amp_client: Option>, chains: Arc, node_id: NodeId, version_switching_mode: SubgraphVersionSwitchingMode, + settings: Arc, + amp_chain_names: Arc, ) -> Self { let logger = logger_factory.component_logger("SubgraphRegistrar", None); let logger_factory = logger_factory.with_parent(logger.clone()); @@ -54,21 +66,17 @@ where provider, store, subscription_manager, + amp_client, chains, node_id, version_switching_mode, assignment_event_stream_cancel_guard: CancelGuard::new(), + settings, + amp_chain_names, } } - pub fn start(&self) -> impl Future { - let logger_clone1 = self.logger.clone(); - let logger_clone2 = self.logger.clone(); - let provider = self.provider.clone(); - let node_id = self.node_id.clone(); - let assignment_event_stream_cancel_handle = - self.assignment_event_stream_cancel_guard.handle(); - + pub async fn start(self: Arc) -> Result<(), Error> { // The order of the following three steps is important: // - Start assignment event stream // - Read assignments table and start assigned subgraphs @@ -83,178 +91,153 @@ where // // The discrepancy between the start time of the event stream and the table read can result // in some extraneous events on start up. Examples: - // - The event stream sees an Add event for subgraph A, but the table query finds that + // - The event stream sees an 'set' event for subgraph A, but the table query finds that // subgraph A is already in the table. - // - The event stream sees a Remove event for subgraph B, but the table query finds that + // - The event stream sees a 'removed' event for subgraph B, but the table query finds that // subgraph B has already been removed. - // The `handle_assignment_events` function handles these cases by ignoring AlreadyRunning - // (on subgraph start) which makes the operations idempotent. Subgraph stop is already idempotent. + // The `change_assignment` function handles these cases by ignoring + // such cases which makes the operations idempotent // Start event stream - let assignment_event_stream = self.assignment_events(); + let assignment_event_stream = self.cheap_clone().assignment_events().await; // Deploy named subgraphs found in store - self.start_assigned_subgraphs().and_then(move |()| { - // Spawn a task to handle assignment events. - // Blocking due to store interactions. Won't be blocking after #905. - graph::spawn_blocking( - assignment_event_stream - .compat() - .map_err(SubgraphAssignmentProviderError::Unknown) - .map_err(CancelableError::Error) - .cancelable(&assignment_event_stream_cancel_handle, || { - Err(CancelableError::Cancel) - }) - .compat() - .for_each(move |assignment_event| { - assert_eq!(assignment_event.node_id(), &node_id); - handle_assignment_event( - assignment_event, - provider.clone(), - logger_clone1.clone(), - ) - .boxed() - .compat() - }) - .map_err(move |e| match e { - CancelableError::Cancel => panic!("assignment event stream canceled"), - CancelableError::Error(e) => { - error!(logger_clone2, "Assignment event stream failed: {}", e); - panic!("assignment event stream failed: {}", e); - } - }) - .compat(), - ); + self.start_assigned_subgraphs().await?; + + let cancel_handle = self.assignment_event_stream_cancel_guard.handle(); + + // Spawn a task to handle assignment events. + let fut = assignment_event_stream.for_each({ + move |event| { + // The assignment stream should run forever. If it gets + // cancelled, that probably indicates a serious problem and + // we panic + if cancel_handle.is_canceled() { + panic!("assignment event stream canceled"); + } - Ok(()) - }) + let this = self.cheap_clone(); + async move { + this.change_assignment(event).await; + } + } + }); + + graph::spawn(fut); + Ok(()) } - pub fn assignment_events(&self) -> impl Stream + Send { - let store = self.store.clone(); - let node_id = self.node_id.clone(); - let logger = self.logger.clone(); + /// Start/stop subgraphs as needed, considering the current assignment + /// state in the database, ignoring changes that do not affect this + /// node, do not require anything to change, or for which we can not + /// find the assignment status from the database + async fn change_assignment(&self, change: AssignmentChange) { + let (deployment, operation) = change.into_parts(); - self.subscription_manager - .subscribe(FromIterator::from_iter([SubscriptionFilter::Assignment])) - .map_err(|()| anyhow!("Entity change stream failed")) - .map(|event| { - // We're only interested in the SubgraphDeploymentAssignment change; we - // know that there is at least one, as that is what we subscribed to - let filter = SubscriptionFilter::Assignment; - let assignments = event - .changes - .iter() - .filter(|change| filter.matches(change)) - .map(|change| match change { - EntityChange::Data { .. } => unreachable!(), - EntityChange::Assignment { - deployment, - operation, - } => (deployment.clone(), operation.clone()), - }) - .collect::>(); - stream::iter_ok(assignments) - }) - .flatten() - .and_then( - move |(deployment, operation)| -> Result + Send>, _> { - trace!(logger, "Received assignment change"; - "deployment" => %deployment, - "operation" => format!("{:?}", operation), - ); - - match operation { - EntityChangeOperation::Set => { - store - .assigned_node(&deployment) - .map_err(|e| { - anyhow!("Failed to get subgraph assignment entity: {}", e) - }) - .map(|assigned| -> Box + Send> { - if let Some(assigned) = assigned { - if assigned == node_id { - // Start subgraph on this node - debug!(logger, "Deployment assignee is this node, broadcasting add event"; "assigned_to" => assigned, "node_id" => &node_id); - Box::new(stream::once(Ok(AssignmentEvent::Add { - deployment, - node_id: node_id.clone(), - }))) - } else { - // Ensure it is removed from this node - debug!(logger, "Deployment assignee is not this node, broadcasting remove event"; "assigned_to" => assigned, "node_id" => &node_id); - Box::new(stream::once(Ok(AssignmentEvent::Remove { - deployment, - node_id: node_id.clone(), - }))) - } - } else { - // Was added/updated, but is now gone. - debug!(logger, "Deployment has not assignee, we will get a separate remove event later"; "node_id" => &node_id); - Box::new(stream::empty()) - } - }) - } - EntityChangeOperation::Removed => { - // Send remove event without checking node ID. - // If node ID does not match, then this is a no-op when handled in - // assignment provider. - Ok(Box::new(stream::once(Ok(AssignmentEvent::Remove { - deployment, - node_id: node_id.clone(), - })))) + trace!(self.logger, "Received assignment change"; + "deployment" => %deployment, + "operation" => format!("{:?}", operation), + ); + + match operation { + AssignmentOperation::Set => { + let assigned = match self.store.assignment_status(&deployment).await { + Ok(assigned) => assigned, + Err(e) => { + error!( + self.logger, + "Failed to get subgraph assignment entity"; "deployment" => deployment, "error" => e.to_string() + ); + return; + } + }; + + let logger = self.logger.new(o!("subgraph_id" => deployment.hash.to_string(), "node_id" => self.node_id.to_string())); + if let Some((assigned, is_paused)) = assigned { + if assigned == self.node_id { + if is_paused { + // Subgraph is paused, so we don't start it + debug!(logger, "Deployment assignee is this node"; "assigned_to" => assigned, "paused" => is_paused, "action" => "ignore"); + return; } + + // Start subgraph on this node + debug!(logger, "Deployment assignee is this node"; "assigned_to" => assigned, "action" => "add"); + self.provider + .cheap_clone() + .start_subgraph(deployment, None) + .await; + } else { + // Ensure it is removed from this node + debug!(logger, "Deployment assignee is not this node"; "assigned_to" => assigned, "action" => "remove"); + self.provider.stop_subgraph(deployment).await } - }, - ) + } else { + // Was added/updated, but is now gone. + debug!(self.logger, "Deployment assignee not found in database"; "action" => "ignore"); + } + } + AssignmentOperation::Removed => { + self.provider.stop_subgraph(deployment).await; + } + } + } + + pub async fn assignment_events(self: Arc) -> impl Stream + Send { + self.subscription_manager + .subscribe() + .map(|event| futures03::stream::iter(event.changes.clone())) .flatten() } - fn start_assigned_subgraphs(&self) -> impl Future { - let provider = self.provider.clone(); + async fn start_assigned_subgraphs(&self) -> Result<(), Error> { let logger = self.logger.clone(); let node_id = self.node_id.clone(); - future::result(self.store.assignments(&self.node_id)) - .map_err(|e| anyhow!("Error querying subgraph assignments: {}", e)) - .and_then(move |deployments| { - // This operation should finish only after all subgraphs are - // started. We wait for the spawned tasks to complete by giving - // each a `sender` and waiting for all of them to be dropped, so - // the receiver terminates without receiving anything. - let deployments = HashSet::::from_iter(deployments); - let deployments_len = deployments.len(); - let (sender, receiver) = futures01::sync::mpsc::channel::<()>(1); - for id in deployments { - let sender = sender.clone(); - let logger = logger.clone(); - - graph::spawn( - start_subgraph(id, provider.clone(), logger).map(move |()| drop(sender)), - ); - } - drop(sender); - receiver.collect().then(move |_| { - info!(logger, "Started all assigned subgraphs"; - "count" => deployments_len, "node_id" => &node_id); - future::ok(()) - }) - }) + let deployments = self + .store + .active_assignments(&self.node_id) + .await + .map_err(|e| anyhow!("Error querying subgraph assignments: {}", e))?; + // This operation should finish only after all subgraphs are + // started. We wait for the spawned tasks to complete by giving + // each a `sender` and waiting for all of them to be dropped, so + // the receiver terminates without receiving anything. + let deployments = HashSet::::from_iter(deployments); + let deployments_len = deployments.len(); + debug!(logger, "Starting all assigned subgraphs"; + "count" => deployments_len, "node_id" => &node_id); + let (sender, receiver) = futures03::channel::mpsc::channel::<()>(1); + for id in deployments { + let sender = sender.clone(); + let provider = self.provider.cheap_clone(); + + graph::spawn(async move { + provider.start_subgraph(id, None).await; + drop(sender) + }); + } + drop(sender); + let _: Vec<_> = receiver.collect().await; + info!(logger, "Started all assigned subgraphs"; + "count" => deployments_len, "node_id" => &node_id); + Ok(()) } } #[async_trait] -impl SubgraphRegistrarTrait for SubgraphRegistrar +impl SubgraphRegistrarTrait for SubgraphRegistrar where - P: SubgraphAssignmentProviderTrait, + P: graph::components::subgraph::SubgraphInstanceManager, S: SubgraphStore, SM: SubscriptionManager, + AC: amp::Client + Send + Sync + 'static, { async fn create_subgraph( &self, name: SubgraphName, ) -> Result { - let id = self.store.create_subgraph(name.clone())?; + let id = self.store.create_subgraph(name.clone()).await?; debug!(self.logger, "Created subgraph"; "subgraph_name" => name.to_string()); @@ -269,6 +252,8 @@ where debug_fork: Option, start_block_override: Option, graft_block_override: Option, + history_blocks: Option, + ignore_graft_base: bool, ) -> Result { // We don't have a location for the subgraph yet; that will be // assigned when we deploy for real. For logging purposes, make up a @@ -277,45 +262,48 @@ where .logger_factory .subgraph_logger(&DeploymentLocator::new(DeploymentId(0), hash.clone())); - let raw: serde_yaml::Mapping = { - let file_bytes = self - .resolver - .cat(&logger, &hash.to_ipfs_link()) - .await - .map_err(|e| { - SubgraphRegistrarError::ResolveError( - SubgraphManifestResolveError::ResolveError(e), + let resolver: Arc = Arc::from( + self.resolver + .for_manifest(&hash.to_string()) + .map_err(SubgraphRegistrarError::Unknown)?, + ); + + let raw = { + let mut raw: serde_yaml::Mapping = { + let file_bytes = resolver + .cat( + &LinkResolverContext::new(&hash, &logger), + &hash.to_ipfs_link(), ) - })?; + .await + .map_err(|e| { + SubgraphRegistrarError::ResolveError( + SubgraphManifestResolveError::ResolveError(e), + ) + })?; + + serde_yaml::from_slice(&file_bytes) + .map_err(|e| SubgraphRegistrarError::ResolveError(e.into()))? + }; - serde_yaml::from_slice(&file_bytes) - .map_err(|e| SubgraphRegistrarError::ResolveError(e.into()))? + if ignore_graft_base { + raw.remove("graft"); + } + + raw }; let kind = BlockchainKind::from_manifest(&raw).map_err(|e| { SubgraphRegistrarError::ResolveError(SubgraphManifestResolveError::ResolveError(e)) })?; + // Give priority to deployment specific history_blocks value. + let history_blocks = + history_blocks.or(self.settings.for_name(&name).map(|c| c.history_blocks)); + let deployment_locator = match kind { - BlockchainKind::Arweave => { - create_subgraph_version::( - &logger, - self.store.clone(), - self.chains.cheap_clone(), - name.clone(), - hash.cheap_clone(), - start_block_override, - graft_block_override, - raw, - node_id, - debug_fork, - self.version_switching_mode, - &self.resolver, - ) - .await? - } BlockchainKind::Ethereum => { - create_subgraph_version::( + create_subgraph_version::( &logger, self.store.clone(), self.chains.cheap_clone(), @@ -327,12 +315,15 @@ where node_id, debug_fork, self.version_switching_mode, - &self.resolver, + &resolver, + self.amp_client.cheap_clone(), + history_blocks, + &self.amp_chain_names, ) .await? } BlockchainKind::Near => { - create_subgraph_version::( + create_subgraph_version::( &logger, self.store.clone(), self.chains.cheap_clone(), @@ -344,41 +335,10 @@ where node_id, debug_fork, self.version_switching_mode, - &self.resolver, - ) - .await? - } - BlockchainKind::Cosmos => { - create_subgraph_version::( - &logger, - self.store.clone(), - self.chains.cheap_clone(), - name.clone(), - hash.cheap_clone(), - start_block_override, - graft_block_override, - raw, - node_id, - debug_fork, - self.version_switching_mode, - &self.resolver, - ) - .await? - } - BlockchainKind::Substreams => { - create_subgraph_version::( - &logger, - self.store.clone(), - self.chains.cheap_clone(), - name.clone(), - hash.cheap_clone(), - start_block_override, - graft_block_override, - raw, - node_id, - debug_fork, - self.version_switching_mode, - &self.resolver, + &resolver, + self.amp_client.cheap_clone(), + history_blocks, + &self.amp_chain_names, ) .await? } @@ -395,9 +355,12 @@ where } async fn remove_subgraph(&self, name: SubgraphName) -> Result<(), SubgraphRegistrarError> { - self.store.clone().remove_subgraph(name.clone())?; + use itertools::Itertools; + + let hashes = self.store.clone().remove_subgraph(name.clone()).await?; + let hashes = hashes.into_iter().join(", "); - debug!(self.logger, "Removed subgraph"; "subgraph_name" => name.to_string()); + debug!(self.logger, "Removed subgraph"; "subgraph_name" => name.to_string(), "deployments" => format!("[{}]", hashes)); Ok(()) } @@ -411,73 +374,33 @@ where hash: &DeploymentHash, node_id: &NodeId, ) -> Result<(), SubgraphRegistrarError> { - let locator = self.store.active_locator(hash)?; + let locator = self.store.active_locator(hash).await?; let deployment = locator.ok_or_else(|| SubgraphRegistrarError::DeploymentNotFound(hash.to_string()))?; - self.store.reassign_subgraph(&deployment, node_id)?; + self.store.reassign_subgraph(&deployment, node_id).await?; Ok(()) } -} -async fn handle_assignment_event( - event: AssignmentEvent, - provider: Arc, - logger: Logger, -) -> Result<(), CancelableError> { - let logger = logger.clone(); + async fn pause_subgraph(&self, hash: &DeploymentHash) -> Result<(), SubgraphRegistrarError> { + let locator = self.store.active_locator(hash).await?; + let deployment = + locator.ok_or_else(|| SubgraphRegistrarError::DeploymentNotFound(hash.to_string()))?; - debug!(logger, "Received assignment event: {:?}", event); + self.store.pause_subgraph(&deployment).await?; - match event { - AssignmentEvent::Add { - deployment, - node_id: _, - } => { - start_subgraph(deployment, provider.clone(), logger).await; - Ok(()) - } - AssignmentEvent::Remove { - deployment, - node_id: _, - } => match provider.stop(deployment).await { - Ok(()) => Ok(()), - Err(e) => Err(CancelableError::Error(e)), - }, + Ok(()) } -} -async fn start_subgraph( - deployment: DeploymentLocator, - provider: Arc, - logger: Logger, -) { - let logger = logger - .new(o!("subgraph_id" => deployment.hash.to_string(), "sgd" => deployment.id.to_string())); - - trace!(logger, "Start subgraph"); - - let start_time = Instant::now(); - let result = provider.start(deployment.clone(), None).await; + async fn resume_subgraph(&self, hash: &DeploymentHash) -> Result<(), SubgraphRegistrarError> { + let locator = self.store.active_locator(hash).await?; + let deployment = + locator.ok_or_else(|| SubgraphRegistrarError::DeploymentNotFound(hash.to_string()))?; - debug!( - logger, - "Subgraph started"; - "start_ms" => start_time.elapsed().as_millis() - ); + self.store.resume_subgraph(&deployment).await?; - match result { - Ok(()) => (), - Err(SubgraphAssignmentProviderError::AlreadyRunning(_)) => (), - Err(e) => { - // Errors here are likely an issue with the subgraph. - error!( - logger, - "Subgraph instance failed to start"; - "error" => e.to_string() - ); - } + Ok(()) } } @@ -497,15 +420,18 @@ async fn resolve_start_block( .expect("cannot identify minimum start block because there are no data sources") { 0 => Ok(None), - min_start_block => chain - .block_pointer_from_number(logger, min_start_block - 1) - .await - .map(Some) - .map_err(move |_| { - SubgraphRegistrarError::ManifestValidationError(vec![ - SubgraphManifestValidationError::BlockNotFound(min_start_block.to_string()), - ]) - }), + min_start_block => Retry::start(retry_strategy(Some(2), RETRY_DEFAULT_LIMIT), move || { + chain + .block_pointer_from_number(logger, min_start_block - 1) + .inspect_err(move |e| warn!(&logger, "Failed to get block number: {}", e)) + }) + .await + .map(Some) + .map_err(move |_| { + SubgraphRegistrarError::ManifestValidationError(vec![ + SubgraphManifestValidationError::BlockNotFound(min_start_block.to_string()), + ]) + }), } } @@ -528,7 +454,7 @@ async fn resolve_graft_block( }) } -async fn create_subgraph_version( +async fn create_subgraph_version( logger: &Logger, store: Arc, chains: Arc, @@ -541,27 +467,41 @@ async fn create_subgraph_version( debug_fork: Option, version_switching_mode: SubgraphVersionSwitchingMode, resolver: &Arc, + amp_client: Option>, + history_blocks_override: Option, + amp_chain_names: &AmpChainNames, ) -> Result { let raw_string = serde_yaml::to_string(&raw).unwrap(); + let unvalidated = UnvalidatedSubgraphManifest::::resolve( - deployment, + deployment.clone(), raw, resolver, + amp_client, logger, ENV_VARS.max_spec_version.clone(), ) .map_err(SubgraphRegistrarError::ResolveError) .await?; - + // Determine if the graft_base should be validated. + // Validate the graft_base if there is a pending graft, ensuring its presence. + // If the subgraph is new (indicated by DeploymentNotFound), the graft_base should be validated. + // If the subgraph already exists and there is no pending graft, graft_base validation is not required. + let should_validate = match store.graft_pending(&deployment).await { + Ok(graft_pending) => graft_pending, + Err(StoreError::DeploymentNotFound(_)) => true, + Err(e) => return Err(SubgraphRegistrarError::StoreError(e)), + }; let manifest = unvalidated - .validate(store.cheap_clone(), true) + .validate(store.cheap_clone(), should_validate) .await .map_err(SubgraphRegistrarError::ManifestValidationError)?; - let network_name = manifest.network_name(); + let network_name: Word = manifest.network_name().into(); + let resolved_name = amp_chain_names.resolve(&network_name); let chain = chains - .get::(network_name.clone()) + .get::(resolved_name.clone()) .map_err(SubgraphRegistrarError::NetworkNotSupported)? .cheap_clone(); @@ -569,7 +509,7 @@ async fn create_subgraph_version( let store = store.clone(); let deployment_store = store.clone(); - if !store.subgraph_exists(&name)? { + if !store.subgraph_exists(&name).await? { debug!( logger, "Subgraph not found, could not create_subgraph_version"; @@ -626,19 +566,24 @@ async fn create_subgraph_version( // Apply the subgraph versioning and deployment operations, // creating a new subgraph deployment if one doesn't exist. - let deployment = DeploymentCreate::new(raw_string, &manifest, start_block) + let mut deployment = DeploymentCreate::new(raw_string, &manifest, start_block) .graft(base_block) .debug(debug_fork) .entities_with_causality_region(needs_causality_region); + if let Some(history_blocks) = history_blocks_override { + deployment = deployment.with_history_blocks_override(history_blocks); + } + deployment_store .create_subgraph_deployment( name, &manifest.schema, deployment, node_id, - network_name, + resolved_name.into(), version_switching_mode, ) + .await .map_err(SubgraphRegistrarError::SubgraphDeploymentError) } diff --git a/core/src/subgraph/runner.rs b/core/src/subgraph/runner.rs deleted file mode 100644 index 521e68efe0c..00000000000 --- a/core/src/subgraph/runner.rs +++ /dev/null @@ -1,1069 +0,0 @@ -use crate::subgraph::context::IndexingContext; -use crate::subgraph::error::BlockProcessingError; -use crate::subgraph::inputs::IndexingInputs; -use crate::subgraph::state::IndexingState; -use crate::subgraph::stream::new_block_stream; -use atomic_refcell::AtomicRefCell; -use graph::blockchain::block_stream::{BlockStreamEvent, BlockWithTriggers, FirehoseCursor}; -use graph::blockchain::{Block, Blockchain, DataSource as _, TriggerFilter as _}; -use graph::components::store::{EmptyStore, EntityKey, StoredDynamicDataSource}; -use graph::components::{ - store::ModificationsAndCache, - subgraph::{MappingError, PoICausalityRegion, ProofOfIndexing, SharedProofOfIndexing}, -}; -use graph::data::store::scalar::Bytes; -use graph::data::subgraph::{ - schema::{SubgraphError, SubgraphHealth, POI_OBJECT}, - SubgraphFeature, -}; -use graph::data_source::{ - offchain, CausalityRegion, DataSource, DataSourceCreationError, DataSourceTemplate, TriggerData, -}; -use graph::env::EnvVars; -use graph::prelude::*; -use graph::util::{backoff::ExponentialBackoff, lfu_cache::LfuCache}; -use std::sync::Arc; -use std::time::{Duration, Instant}; - -const MINUTE: Duration = Duration::from_secs(60); - -const SKIP_PTR_UPDATES_THRESHOLD: Duration = Duration::from_secs(60 * 5); - -pub struct SubgraphRunner -where - C: Blockchain, - T: RuntimeHostBuilder, -{ - ctx: IndexingContext, - state: IndexingState, - inputs: Arc>, - logger: Logger, - pub metrics: RunnerMetrics, -} - -impl SubgraphRunner -where - C: Blockchain, - T: RuntimeHostBuilder, -{ - pub fn new( - inputs: IndexingInputs, - ctx: IndexingContext, - logger: Logger, - metrics: RunnerMetrics, - env_vars: Arc, - ) -> Self { - Self { - inputs: Arc::new(inputs), - ctx, - state: IndexingState { - should_try_unfail_non_deterministic: true, - synced: false, - skip_ptr_updates_timer: Instant::now(), - backoff: ExponentialBackoff::new( - (MINUTE * 2).min(env_vars.subgraph_error_retry_ceil), - env_vars.subgraph_error_retry_ceil, - ), - entity_lfu_cache: LfuCache::new(), - }, - logger, - metrics, - } - } - - /// Revert the state to a previous block. When handling revert operations - /// or failed block processing, it is necessary to remove part of the existing - /// in-memory state to keep it constent with DB changes. - /// During block processing new dynamic data sources are added directly to the - /// SubgraphInstance of the runner. This means that if, for whatever reason, - /// the changes don;t complete then the remnants of that block processing must - /// be removed. The same thing also applies to the block cache. - /// This function must be called before continuing to process in order to avoid - /// duplicated host insertion and POI issues with dirty entity changes. - fn revert_state(&mut self, block_number: BlockNumber) -> Result<(), Error> { - self.state.entity_lfu_cache = LfuCache::new(); - - // 1. Revert all hosts(created by DDS) up to block_number inclusively. - // 2. Unmark any offchain data sources that were marked done on the blocks being removed. - // When no offchain datasources are present, 2. should be a noop. - self.ctx.revert_data_sources(block_number)?; - Ok(()) - } - - #[cfg(debug_assertions)] - pub fn context(&self) -> &IndexingContext { - &self.ctx - } - - #[cfg(debug_assertions)] - pub async fn run_for_test(self, break_on_restart: bool) -> Result { - self.run_inner(break_on_restart).await - } - - pub async fn run(self) -> Result { - self.run_inner(false).await - } - - async fn run_inner(mut self, break_on_restart: bool) -> Result { - // If a subgraph failed for deterministic reasons, before start indexing, we first - // revert the deployment head. It should lead to the same result since the error was - // deterministic. - if let Some(current_ptr) = self.inputs.store.block_ptr() { - if let Some(parent_ptr) = self - .inputs - .triggers_adapter - .parent_ptr(¤t_ptr) - .await? - { - // This reverts the deployment head to the parent_ptr if - // deterministic errors happened. - // - // There's no point in calling it if we have no current or parent block - // pointers, because there would be: no block to revert to or to search - // errors from (first execution). - let _outcome = self - .inputs - .store - .unfail_deterministic_error(¤t_ptr, &parent_ptr) - .await?; - } - } - - loop { - debug!(self.logger, "Starting or restarting subgraph"); - - let block_stream_canceler = CancelGuard::new(); - let block_stream_cancel_handle = block_stream_canceler.handle(); - - let mut block_stream = - new_block_stream(&self.inputs, &self.ctx.filter, &self.metrics.subgraph) - .await? - .map_err(CancelableError::Error) - .cancelable(&block_stream_canceler, || Err(CancelableError::Cancel)); - - // Keep the stream's cancel guard around to be able to shut it down when the subgraph - // deployment is unassigned - self.ctx - .instances - .insert(self.inputs.deployment.id, block_stream_canceler); - - debug!(self.logger, "Starting block stream"); - - // Process events from the stream as long as no restart is needed - loop { - let event = { - let _section = self.metrics.stream.stopwatch.start_section("scan_blocks"); - - block_stream.next().await - }; - - // TODO: move cancel handle to the Context - // This will require some code refactor in how the BlockStream is created - match self - .handle_stream_event(event, &block_stream_cancel_handle) - .await? - { - Action::Continue => continue, - Action::Stop => { - info!(self.logger, "Stopping subgraph"); - self.inputs.store.flush().await?; - return Ok(self); - } - Action::Restart if break_on_restart => { - info!(self.logger, "Stopping subgraph on break"); - self.inputs.store.flush().await?; - return Ok(self); - } - Action::Restart => break, - }; - } - } - } - - /// Processes a block and returns the updated context and a boolean flag indicating - /// whether new dynamic data sources have been added to the subgraph. - async fn process_block( - &mut self, - block_stream_cancel_handle: &CancelHandle, - block: BlockWithTriggers, - firehose_cursor: FirehoseCursor, - ) -> Result { - let triggers = block.trigger_data; - let block = Arc::new(block.block); - let block_ptr = block.ptr(); - - let logger = self.logger.new(o!( - "block_number" => format!("{:?}", block_ptr.number), - "block_hash" => format!("{}", block_ptr.hash) - )); - - if triggers.len() == 1 { - debug!(&logger, "1 candidate trigger in this block"); - } else { - debug!( - &logger, - "{} candidate triggers in this block", - triggers.len() - ); - } - - let proof_of_indexing = if self.inputs.store.supports_proof_of_indexing().await? { - Some(Arc::new(AtomicRefCell::new(ProofOfIndexing::new( - block_ptr.number, - self.inputs.poi_version, - )))) - } else { - None - }; - - // Causality region for onchain triggers. - let causality_region = PoICausalityRegion::from_network(&self.inputs.network); - - // Process events one after the other, passing in entity operations - // collected previously to every new event being processed - let mut block_state = match self - .process_triggers( - &proof_of_indexing, - &block, - triggers.into_iter().map(TriggerData::Onchain), - &causality_region, - ) - .await - { - // Triggers processed with no errors or with only deterministic errors. - Ok(block_state) => block_state, - - // Some form of unknown or non-deterministic error ocurred. - Err(MappingError::Unknown(e)) => return Err(BlockProcessingError::Unknown(e)), - Err(MappingError::PossibleReorg(e)) => { - info!(logger, - "Possible reorg detected, retrying"; - "error" => format!("{:#}", e), - ); - - // In case of a possible reorg, we want this function to do nothing and restart the - // block stream so it has a chance to detect the reorg. - // - // The state is unchanged at this point, except for having cleared the entity cache. - // Losing the cache is a bit annoying but not an issue for correctness. - // - // See also b21fa73b-6453-4340-99fb-1a78ec62efb1. - return Ok(Action::Restart); - } - }; - - // If new data sources have been created, and static filters are not in use, it is necessary - // to restart the block stream with the new filters. - let needs_restart = block_state.has_created_data_sources() && !self.inputs.static_filters; - - // This loop will: - // 1. Instantiate created data sources. - // 2. Process those data sources for the current block. - // Until no data sources are created or MAX_DATA_SOURCES is hit. - - // Note that this algorithm processes data sources spawned on the same block _breadth - // first_ on the tree implied by the parent-child relationship between data sources. Only a - // very contrived subgraph would be able to observe this. - while block_state.has_created_data_sources() { - // Instantiate dynamic data sources, removing them from the block state. - let (data_sources, runtime_hosts) = - self.create_dynamic_data_sources(block_state.drain_created_data_sources())?; - - let filter = C::TriggerFilter::from_data_sources( - data_sources.iter().filter_map(DataSource::as_onchain), - ); - - let block: Arc = if self.inputs.chain.is_refetch_block_required() { - Arc::new( - self.inputs - .chain - .refetch_firehose_block(&logger, firehose_cursor.clone()) - .await?, - ) - } else { - block.cheap_clone() - }; - - // Reprocess the triggers from this block that match the new data sources - let block_with_triggers = self - .inputs - .triggers_adapter - .triggers_in_block(&logger, block.as_ref().clone(), &filter) - .await?; - - let triggers = block_with_triggers.trigger_data; - - if triggers.len() == 1 { - info!( - &logger, - "1 trigger found in this block for the new data sources" - ); - } else if triggers.len() > 1 { - info!( - &logger, - "{} triggers found in this block for the new data sources", - triggers.len() - ); - } - - // Add entity operations for the new data sources to the block state - // and add runtimes for the data sources to the subgraph instance. - self.persist_dynamic_data_sources(&mut block_state, data_sources); - - // Process the triggers in each host in the same order the - // corresponding data sources have been created. - for trigger in triggers { - block_state = self - .ctx - .process_trigger_in_hosts( - &logger, - &runtime_hosts, - &block, - &TriggerData::Onchain(trigger), - block_state, - &proof_of_indexing, - &causality_region, - &self.inputs.debug_fork, - &self.metrics.subgraph, - ) - .await - .map_err(|e| { - // This treats a `PossibleReorg` as an ordinary error which will fail the subgraph. - // This can cause an unnecessary subgraph failure, to fix it we need to figure out a - // way to revert the effect of `create_dynamic_data_sources` so we may return a - // clean context as in b21fa73b-6453-4340-99fb-1a78ec62efb1. - match e { - MappingError::PossibleReorg(e) | MappingError::Unknown(e) => { - BlockProcessingError::Unknown(e) - } - } - })?; - } - } - - let has_errors = block_state.has_errors(); - let is_non_fatal_errors_active = self - .inputs - .features - .contains(&SubgraphFeature::NonFatalErrors); - - // Apply entity operations and advance the stream - - // Avoid writing to store if block stream has been canceled - if block_stream_cancel_handle.is_canceled() { - return Err(BlockProcessingError::Canceled); - } - - if let Some(proof_of_indexing) = proof_of_indexing { - let proof_of_indexing = Arc::try_unwrap(proof_of_indexing).unwrap().into_inner(); - update_proof_of_indexing( - proof_of_indexing, - &self.metrics.host.stopwatch, - &mut block_state.entity_cache, - ) - .await?; - } - - let section = self - .metrics - .host - .stopwatch - .start_section("as_modifications"); - let ModificationsAndCache { - modifications: mut mods, - entity_lfu_cache: cache, - } = block_state - .entity_cache - .as_modifications() - .map_err(|e| BlockProcessingError::Unknown(e.into()))?; - section.end(); - - // Check for offchain events and process them, including their entity modifications in the - // set to be transacted. - let offchain_events = self.ctx.offchain_monitor.ready_offchain_events()?; - let (offchain_mods, processed_data_sources) = self - .handle_offchain_triggers(offchain_events, &block) - .await?; - mods.extend(offchain_mods); - - // Put the cache back in the state, asserting that the placeholder cache was not used. - assert!(self.state.entity_lfu_cache.is_empty()); - self.state.entity_lfu_cache = cache; - - if !mods.is_empty() { - info!(&logger, "Applying {} entity operation(s)", mods.len()); - } - - let err_count = block_state.deterministic_errors.len(); - for (i, e) in block_state.deterministic_errors.iter().enumerate() { - let message = format!("{:#}", e).replace('\n', "\t"); - error!(&logger, "Subgraph error {}/{}", i + 1, err_count; - "error" => message, - "code" => LogCode::SubgraphSyncingFailure - ); - } - - // Transact entity operations into the store and update the - // subgraph's block stream pointer - let _section = self.metrics.host.stopwatch.start_section("transact_block"); - let start = Instant::now(); - - let store = &self.inputs.store; - - // If a deterministic error has happened, make the PoI to be the only entity that'll be stored. - if has_errors && !is_non_fatal_errors_active { - let is_poi_entity = - |entity_mod: &EntityModification| entity_mod.entity_ref().entity_type.is_poi(); - mods.retain(is_poi_entity); - // Confidence check - assert!( - mods.len() == 1, - "There should be only one PoI EntityModification" - ); - } - - let BlockState { - deterministic_errors, - persisted_data_sources, - .. - } = block_state; - - let first_error = deterministic_errors.first().cloned(); - - store - .transact_block_operations( - block_ptr, - firehose_cursor, - mods, - &self.metrics.host.stopwatch, - persisted_data_sources, - deterministic_errors, - self.inputs.manifest_idx_and_name.clone(), - processed_data_sources, - ) - .await - .context("Failed to transact block operations")?; - - // For subgraphs with `nonFatalErrors` feature disabled, we consider - // any error as fatal. - // - // So we do an early return to make the subgraph stop processing blocks. - // - // In this scenario the only entity that is stored/transacted is the PoI, - // all of the others are discarded. - if has_errors && !is_non_fatal_errors_active { - // Only the first error is reported. - return Err(BlockProcessingError::Deterministic(first_error.unwrap())); - } - - let elapsed = start.elapsed().as_secs_f64(); - self.metrics - .subgraph - .block_ops_transaction_duration - .observe(elapsed); - - // To prevent a buggy pending version from replacing a current version, if errors are - // present the subgraph will be unassigned. - if has_errors && !ENV_VARS.disable_fail_fast && !store.is_deployment_synced().await? { - store - .unassign_subgraph() - .map_err(|e| BlockProcessingError::Unknown(e.into()))?; - - // Use `Canceled` to avoiding setting the subgraph health to failed, an error was - // just transacted so it will be already be set to unhealthy. - return Err(BlockProcessingError::Canceled); - } - - match needs_restart { - true => Ok(Action::Restart), - false => Ok(Action::Continue), - } - } - - async fn process_triggers( - &mut self, - proof_of_indexing: &SharedProofOfIndexing, - block: &Arc, - triggers: impl Iterator>, - causality_region: &str, - ) -> Result, MappingError> { - let mut block_state = BlockState::new( - self.inputs.store.clone(), - std::mem::take(&mut self.state.entity_lfu_cache), - ); - - for trigger in triggers { - block_state = self - .ctx - .process_trigger( - &self.logger, - block, - &trigger, - block_state, - proof_of_indexing, - causality_region, - &self.inputs.debug_fork, - &self.metrics.subgraph, - ) - .await - .map_err(move |mut e| { - let error_context = trigger.error_context(); - if !error_context.is_empty() { - e = e.context(error_context); - } - e.context("failed to process trigger".to_string()) - })?; - } - Ok(block_state) - } - - fn create_dynamic_data_sources( - &mut self, - created_data_sources: Vec>, - ) -> Result<(Vec>, Vec>), Error> { - let mut data_sources = vec![]; - let mut runtime_hosts = vec![]; - - for info in created_data_sources { - // Try to instantiate a data source from the template - - let data_source = { - let res = match info.template { - DataSourceTemplate::Onchain(_) => C::DataSource::from_template_info(info) - .map(DataSource::Onchain) - .map_err(DataSourceCreationError::from), - DataSourceTemplate::Offchain(_) => offchain::DataSource::from_template_info( - info, - self.ctx.causality_region_next_value(), - ) - .map(DataSource::Offchain), - }; - match res { - Ok(ds) => ds, - Err(e @ DataSourceCreationError::Ignore(..)) => { - warn!(self.logger, "{}", e.to_string()); - continue; - } - Err(DataSourceCreationError::Unknown(e)) => return Err(e), - } - }; - - // Try to create a runtime host for the data source - let host = self - .ctx - .add_dynamic_data_source(&self.logger, data_source.clone())?; - - match host { - Some(host) => { - data_sources.push(data_source); - runtime_hosts.push(host); - } - None => { - warn!( - self.logger, - "no runtime host created, there is already a runtime host instantiated for \ - this data source"; - "name" => &data_source.name(), - "address" => &data_source.address() - .map(hex::encode) - .unwrap_or("none".to_string()), - ) - } - } - } - - Ok((data_sources, runtime_hosts)) - } - - fn persist_dynamic_data_sources( - &mut self, - block_state: &mut BlockState, - data_sources: Vec>, - ) { - if !data_sources.is_empty() { - debug!( - self.logger, - "Creating {} dynamic data source(s)", - data_sources.len() - ); - } - - // Add entity operations to the block state in order to persist - // the dynamic data sources - for data_source in data_sources.iter() { - debug!( - self.logger, - "Persisting data_source"; - "name" => &data_source.name(), - "address" => &data_source.address().map(hex::encode).unwrap_or("none".to_string()), - ); - block_state.persist_data_source(data_source.as_stored_dynamic_data_source()); - } - - // Merge filters from data sources into the block stream builder - self.ctx - .filter - .extend(data_sources.iter().filter_map(|ds| ds.as_onchain())); - } -} - -impl SubgraphRunner -where - C: Blockchain, - T: RuntimeHostBuilder, -{ - async fn handle_stream_event( - &mut self, - event: Option, CancelableError>>, - cancel_handle: &CancelHandle, - ) -> Result { - let action = match event { - Some(Ok(BlockStreamEvent::ProcessBlock(block, cursor))) => { - self.handle_process_block(block, cursor, cancel_handle) - .await? - } - Some(Ok(BlockStreamEvent::Revert(revert_to_ptr, cursor))) => { - self.handle_revert(revert_to_ptr, cursor).await? - } - // Log and drop the errors from the block_stream - // The block stream will continue attempting to produce blocks - Some(Err(e)) => self.handle_err(e, cancel_handle).await?, - // If the block stream ends, that means that there is no more indexing to do. - // Typically block streams produce indefinitely, but tests are an example of finite block streams. - None => Action::Stop, - }; - - Ok(action) - } - - async fn handle_offchain_triggers( - &mut self, - triggers: Vec, - block: &Arc, - ) -> Result<(Vec, Vec), Error> { - let mut mods = vec![]; - let mut processed_data_sources = vec![]; - - for trigger in triggers { - // Using an `EmptyStore` and clearing the cache for each trigger is a makeshift way to - // get causality region isolation. - let schema = self.inputs.store.input_schema(); - let mut block_state = BlockState::::new(EmptyStore::new(schema), LfuCache::new()); - - // PoI ignores offchain events. - // See also: poi-ignores-offchain - let proof_of_indexing = None; - let causality_region = ""; - - block_state = self - .ctx - .process_trigger( - &self.logger, - block, - &TriggerData::Offchain(trigger), - block_state, - &proof_of_indexing, - causality_region, - &self.inputs.debug_fork, - &self.metrics.subgraph, - ) - .await - .map_err(move |err| { - let err = match err { - // Ignoring `PossibleReorg` isn't so bad since the subgraph will retry - // non-deterministic errors. - MappingError::PossibleReorg(e) | MappingError::Unknown(e) => e, - }; - err.context("failed to process trigger".to_string()) - })?; - - anyhow::ensure!( - !block_state.has_created_data_sources(), - "Attempted to create data source in offchain data source handler. This is not yet supported.", - ); - - // This propagates any deterministic error as a non-deterministic one. Which might make - // sense considering offchain data sources are non-deterministic. - if let Some(err) = block_state.deterministic_errors.into_iter().next() { - return Err(anyhow!("{}", err.to_string())); - } - - mods.extend(block_state.entity_cache.as_modifications()?.modifications); - processed_data_sources.extend(block_state.processed_data_sources); - } - - Ok((mods, processed_data_sources)) - } -} - -#[derive(Debug)] -enum Action { - Continue, - Stop, - Restart, -} - -#[async_trait] -trait StreamEventHandler { - async fn handle_process_block( - &mut self, - block: BlockWithTriggers, - cursor: FirehoseCursor, - cancel_handle: &CancelHandle, - ) -> Result; - async fn handle_revert( - &mut self, - revert_to_ptr: BlockPtr, - cursor: FirehoseCursor, - ) -> Result; - async fn handle_err( - &mut self, - err: CancelableError, - cancel_handle: &CancelHandle, - ) -> Result; -} - -#[async_trait] -impl StreamEventHandler for SubgraphRunner -where - C: Blockchain, - T: RuntimeHostBuilder, -{ - async fn handle_process_block( - &mut self, - block: BlockWithTriggers, - cursor: FirehoseCursor, - cancel_handle: &CancelHandle, - ) -> Result { - let block_ptr = block.ptr(); - self.metrics - .stream - .deployment_head - .set(block_ptr.number as f64); - - if block.trigger_count() > 0 { - self.metrics - .subgraph - .block_trigger_count - .observe(block.trigger_count() as f64); - } - - if block.trigger_count() == 0 - && self.state.skip_ptr_updates_timer.elapsed() <= SKIP_PTR_UPDATES_THRESHOLD - && !self.state.synced - && !close_to_chain_head( - &block_ptr, - self.inputs.chain.chain_store().cached_head_ptr().await?, - // The "skip ptr updates timer" is ignored when a subgraph is at most 1000 blocks - // behind the chain head. - 1000, - ) - { - return Ok(Action::Continue); - } else { - self.state.skip_ptr_updates_timer = Instant::now(); - } - - let start = Instant::now(); - - let res = self.process_block(cancel_handle, block, cursor).await; - - let elapsed = start.elapsed().as_secs_f64(); - self.metrics - .subgraph - .block_processing_duration - .observe(elapsed); - - match res { - Ok(action) => { - // Once synced, no need to try to update the status again. - if !self.state.synced - && close_to_chain_head( - &block_ptr, - self.inputs.chain.chain_store().cached_head_ptr().await?, - // We consider a subgraph synced when it's at most 1 block behind the - // chain head. - 1, - ) - { - // Updating the sync status is an one way operation. - // This state change exists: not synced -> synced - // This state change does NOT: synced -> not synced - self.inputs.store.deployment_synced()?; - - // Stop trying to update the sync status. - self.state.synced = true; - - // Stop recording time-to-sync metrics. - self.metrics.stream.stopwatch.disable(); - } - - // Keep trying to unfail subgraph for everytime it advances block(s) until it's - // health is not Failed anymore. - if self.state.should_try_unfail_non_deterministic { - // If the deployment head advanced, we can unfail - // the non-deterministic error (if there's any). - let outcome = self - .inputs - .store - .unfail_non_deterministic_error(&block_ptr)?; - - if let UnfailOutcome::Unfailed = outcome { - // Stop trying to unfail. - self.state.should_try_unfail_non_deterministic = false; - self.metrics.stream.deployment_failed.set(0.0); - self.state.backoff.reset(); - } - } - - if let Some(stop_block) = &self.inputs.stop_block { - if block_ptr.number >= *stop_block { - info!(self.logger, "stop block reached for subgraph"); - return Ok(Action::Stop); - } - } - - if matches!(action, Action::Restart) { - // Cancel the stream for real - self.ctx.instances.remove(&self.inputs.deployment.id); - - // And restart the subgraph - return Ok(Action::Restart); - } - - return Ok(Action::Continue); - } - Err(BlockProcessingError::Canceled) => { - debug!(self.logger, "Subgraph block stream shut down cleanly"); - return Ok(Action::Stop); - } - - // Handle unexpected stream errors by marking the subgraph as failed. - Err(e) => { - self.metrics.stream.deployment_failed.set(1.0); - self.revert_state(block_ptr.block_number())?; - - let message = format!("{:#}", e).replace('\n', "\t"); - let err = anyhow!("{}, code: {}", message, LogCode::SubgraphSyncingFailure); - let deterministic = e.is_deterministic(); - - let error = SubgraphError { - subgraph_id: self.inputs.deployment.hash.clone(), - message, - block_ptr: Some(block_ptr), - handler: None, - deterministic, - }; - - match deterministic { - true => { - // Fail subgraph: - // - Change status/health. - // - Save the error to the database. - self.inputs - .store - .fail_subgraph(error) - .await - .context("Failed to set subgraph status to `failed`")?; - - return Err(err); - } - false => { - // Shouldn't fail subgraph if it's already failed for non-deterministic - // reasons. - // - // If we don't do this check we would keep adding the same error to the - // database. - let should_fail_subgraph = - self.inputs.store.health().await? != SubgraphHealth::Failed; - - if should_fail_subgraph { - // Fail subgraph: - // - Change status/health. - // - Save the error to the database. - self.inputs - .store - .fail_subgraph(error) - .await - .context("Failed to set subgraph status to `failed`")?; - } - - // Retry logic below: - - // Cancel the stream for real. - self.ctx.instances.remove(&self.inputs.deployment.id); - - let message = format!("{:#}", e).replace('\n', "\t"); - error!(self.logger, "Subgraph failed with non-deterministic error: {}", message; - "attempt" => self.state.backoff.attempt, - "retry_delay_s" => self.state.backoff.delay().as_secs()); - - // Sleep before restarting. - self.state.backoff.sleep_async().await; - - self.state.should_try_unfail_non_deterministic = true; - - // And restart the subgraph. - return Ok(Action::Restart); - } - } - } - } - } - - async fn handle_revert( - &mut self, - revert_to_ptr: BlockPtr, - cursor: FirehoseCursor, - ) -> Result { - // Current deployment head in the database / WritableAgent Mutex cache. - // - // Safe unwrap because in a Revert event we're sure the subgraph has - // advanced at least once. - let subgraph_ptr = self.inputs.store.block_ptr().unwrap(); - if revert_to_ptr.number >= subgraph_ptr.number { - info!(&self.logger, "Block to revert is higher than subgraph pointer, nothing to do"; "subgraph_ptr" => &subgraph_ptr, "revert_to_ptr" => &revert_to_ptr); - return Ok(Action::Continue); - } - - info!(&self.logger, "Reverting block to get back to main chain"; "subgraph_ptr" => &subgraph_ptr, "revert_to_ptr" => &revert_to_ptr); - - if let Err(e) = self - .inputs - .store - .revert_block_operations(revert_to_ptr, cursor) - .await - { - error!(&self.logger, "Could not revert block. Retrying"; "error" => %e); - - // Exit inner block stream consumption loop and go up to loop that restarts subgraph - return Ok(Action::Restart); - } - - self.metrics - .stream - .reverted_blocks - .set(subgraph_ptr.number as f64); - self.metrics - .stream - .deployment_head - .set(subgraph_ptr.number as f64); - - self.revert_state(subgraph_ptr.number)?; - - Ok(Action::Continue) - } - - async fn handle_err( - &mut self, - err: CancelableError, - cancel_handle: &CancelHandle, - ) -> Result { - if cancel_handle.is_canceled() { - debug!(&self.logger, "Subgraph block stream shut down cleanly"); - return Ok(Action::Stop); - } - - debug!( - &self.logger, - "Block stream produced a non-fatal error"; - "error" => format!("{}", err), - ); - - Ok(Action::Continue) - } -} - -/// Transform the proof of indexing changes into entity updates that will be -/// inserted when as_modifications is called. -async fn update_proof_of_indexing( - proof_of_indexing: ProofOfIndexing, - stopwatch: &StopwatchMetrics, - entity_cache: &mut EntityCache, -) -> Result<(), Error> { - let _section_guard = stopwatch.start_section("update_proof_of_indexing"); - - let mut proof_of_indexing = proof_of_indexing.take(); - - for (causality_region, stream) in proof_of_indexing.drain() { - // Create the special POI entity key specific to this causality_region - let entity_key = EntityKey { - entity_type: POI_OBJECT.to_owned(), - - // There are two things called causality regions here, one is the causality region for - // the poi which is a string and the PoI entity id. The other is the data source - // causality region to which the PoI belongs as an entity. Currently offchain events do - // not affect PoI so it is assumed to be `ONCHAIN`. - // See also: poi-ignores-offchain - entity_id: causality_region.into(), - causality_region: CausalityRegion::ONCHAIN, - }; - - // Grab the current digest attribute on this entity - let prev_poi = - entity_cache - .get(&entity_key) - .map_err(Error::from)? - .map(|entity| match entity.get("digest") { - Some(Value::Bytes(b)) => b.clone(), - _ => panic!("Expected POI entity to have a digest and for it to be bytes"), - }); - - // Finish the POI stream, getting the new POI value. - let updated_proof_of_indexing = stream.pause(prev_poi.as_deref()); - let updated_proof_of_indexing: Bytes = (&updated_proof_of_indexing[..]).into(); - - // Put this onto an entity with the same digest attribute - // that was expected before when reading. - let new_poi_entity = entity! { - id: entity_key.entity_id.to_string(), - digest: updated_proof_of_indexing, - }; - - entity_cache.set(entity_key, new_poi_entity)?; - } - - Ok(()) -} - -/// Checks if the Deployment BlockPtr is at least X blocks behind to the chain head. -fn close_to_chain_head( - deployment_head_ptr: &BlockPtr, - chain_head_ptr: Option, - n: BlockNumber, -) -> bool { - matches!((deployment_head_ptr, &chain_head_ptr), (b1, Some(b2)) if b1.number >= (b2.number - n)) -} - -#[test] -fn test_close_to_chain_head() { - let offset = 1; - - let block_0 = BlockPtr::try_from(( - "bd34884280958002c51d3f7b5f853e6febeba33de0f40d15b0363006533c924f", - 0, - )) - .unwrap(); - let block_1 = BlockPtr::try_from(( - "8511fa04b64657581e3f00e14543c1d522d5d7e771b54aa3060b662ade47da13", - 1, - )) - .unwrap(); - let block_2 = BlockPtr::try_from(( - "b98fb783b49de5652097a989414c767824dff7e7fd765a63b493772511db81c1", - 2, - )) - .unwrap(); - - assert!(!close_to_chain_head(&block_0, None, offset)); - assert!(!close_to_chain_head(&block_2, None, offset)); - - assert!(!close_to_chain_head( - &block_0, - Some(block_2.clone()), - offset - )); - - assert!(close_to_chain_head(&block_1, Some(block_2.clone()), offset)); - assert!(close_to_chain_head(&block_2, Some(block_2.clone()), offset)); -} diff --git a/core/src/subgraph/runner/mod.rs b/core/src/subgraph/runner/mod.rs new file mode 100644 index 00000000000..026be3dfc27 --- /dev/null +++ b/core/src/subgraph/runner/mod.rs @@ -0,0 +1,1822 @@ +mod state; +mod trigger_runner; + +use crate::subgraph::context::IndexingContext; +use crate::subgraph::error::{ + ClassifyErrorHelper as _, DetailHelper as _, NonDeterministicErrorHelper as _, ProcessingError, + ProcessingErrorKind, +}; +use crate::subgraph::inputs::IndexingInputs; +use crate::subgraph::state::IndexingState; +use crate::subgraph::stream::new_block_stream; +use anyhow::Context as _; +use graph::blockchain::block_stream::{ + BlockStream, BlockStreamEvent, BlockWithTriggers, FirehoseCursor, +}; +use graph::blockchain::{ + Block, BlockTime, Blockchain, DataSource as _, SubgraphFilter, Trigger, TriggerFilter as _, + TriggerFilterWrapper, +}; +use graph::components::store::{ + EmptyStore, GetScope, ReadStore, SeqGenerator, StoredDynamicDataSource, +}; +use graph::components::subgraph::InstanceDSTemplate; +use graph::components::trigger_processor::RunnableTriggers; +use graph::components::{ + store::ModificationsAndCache, + subgraph::{MappingError, PoICausalityRegion, ProofOfIndexing, SharedProofOfIndexing}, +}; +use graph::data::store::scalar::Bytes; +use graph::data::subgraph::schema::SubgraphError; +use graph::data_source::{ + CausalityRegion, DataSource, DataSourceCreationError, TriggerData, offchain, +}; +use graph::env::EnvVars; +use graph::ext::futures::Cancelable; +use graph::futures03::stream::StreamExt; +use graph::prelude::{ + BlockNumber, BlockPtr, BlockState, CancelGuard, CancelHandle, CancelToken as _, + CheapClone as _, ENV_VARS, EntityCache, EntityModification, Error, InstanceDSTemplateInfo, + LogCode, RunnerMetrics, RuntimeHostBuilder, StopwatchMetrics, StoreError, StreamExtension, + UnfailOutcome, Value, anyhow, hex, retry, thiserror, +}; +use graph::schema::EntityKey; +use graph::slog::{Logger, debug, error, info, o, trace, warn}; +use graph::util::lfu_cache::EvictStats; +use graph::util::{backoff::ExponentialBackoff, lfu_cache::LfuCache}; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use std::vec; + +use self::state::{RestartReason, RunnerState, StopReason}; +use self::trigger_runner::TriggerRunner; + +const MINUTE: Duration = Duration::from_secs(60); + +const SKIP_PTR_UPDATES_THRESHOLD: Duration = Duration::from_secs(60 * 5); +const HANDLE_REVERT_SECTION_NAME: &str = "handle_revert"; +const PROCESS_BLOCK_SECTION_NAME: &str = "process_block"; +const PROCESS_TRIGGERS_SECTION_NAME: &str = "process_triggers"; +const HANDLE_CREATED_DS_SECTION_NAME: &str = "handle_new_data_sources"; + +pub struct SubgraphRunner +where + C: Blockchain, + T: RuntimeHostBuilder, +{ + ctx: IndexingContext, + state: IndexingState, + inputs: Arc>, + logger: Logger, + pub metrics: RunnerMetrics, + cancel_handle: Option, + /// The current state in the runner's state machine. + /// This field drives the main loop of the runner. + runner_state: RunnerState, +} + +#[derive(Debug, thiserror::Error)] +pub enum SubgraphRunnerError { + #[error("subgraph runner terminated because a newer one was active")] + Duplicate, + + #[error(transparent)] + Unknown(#[from] Error), +} + +impl SubgraphRunner +where + C: Blockchain, + T: RuntimeHostBuilder, +{ + pub fn new( + inputs: IndexingInputs, + ctx: IndexingContext, + logger: Logger, + metrics: RunnerMetrics, + env_vars: Arc, + ) -> Self { + Self { + inputs: Arc::new(inputs), + ctx, + state: IndexingState { + should_try_unfail_non_deterministic: true, + skip_ptr_updates_timer: Instant::now(), + backoff: ExponentialBackoff::with_jitter( + (MINUTE * 2).min(env_vars.subgraph_error_retry_ceil), + env_vars.subgraph_error_retry_ceil, + env_vars.subgraph_error_retry_jitter, + ), + entity_lfu_cache: LfuCache::new(), + cached_head_ptr: None, + postponed_indexes_created: false, + }, + logger, + metrics, + cancel_handle: None, + runner_state: RunnerState::Initializing, + } + } + + /// Revert the state to a previous block. When handling revert operations + /// or failed block processing, it is necessary to remove part of the existing + /// in-memory state to keep it constent with DB changes. + /// During block processing new dynamic data sources are added directly to the + /// IndexingContext of the runner. This means that if, for whatever reason, + /// the changes don;t complete then the remnants of that block processing must + /// be removed. The same thing also applies to the block cache. + /// This function must be called before continuing to process in order to avoid + /// duplicated host insertion and POI issues with dirty entity changes. + fn revert_state_to(&mut self, block_number: BlockNumber) { + self.state.entity_lfu_cache = LfuCache::new(); + + // 1. Revert all hosts(created by DDS) at a block higher than `block_number`. + // 2. Unmark any offchain data sources that were marked done on the blocks being removed. + // When no offchain datasources are present, 2. should be a noop. + self.ctx.revert_data_sources(block_number + 1); + } + + #[cfg(debug_assertions)] + pub fn context(&self) -> &IndexingContext { + &self.ctx + } + + #[cfg(debug_assertions)] + pub async fn run_for_test(self, break_on_restart: bool) -> Result { + self.run_inner(break_on_restart).await.map_err(Into::into) + } + + fn is_static_filters_enabled(&self) -> bool { + self.inputs.static_filters || self.ctx.hosts_len() > ENV_VARS.static_filters_threshold + } + + fn build_filter(&self) -> TriggerFilterWrapper { + let current_ptr = self.inputs.store.block_ptr(); + let static_filters = self.is_static_filters_enabled(); + + // Filter out data sources that have reached their end block + let end_block_filter = |ds: &&C::DataSource| match current_ptr.as_ref() { + // We filter out datasources for which the current block is at or past their end block. + Some(block) => ds.end_block().is_none_or(|end| block.number < end), + // If there is no current block, we keep all datasources. + None => true, + }; + + let data_sources = self.ctx.static_data_sources(); + + let subgraph_filter = data_sources + .iter() + .filter_map(|ds| ds.as_subgraph()) + .map(|ds| SubgraphFilter { + subgraph: ds.source.address(), + start_block: ds.source.start_block, + entities: ds + .mapping + .handlers + .iter() + .map(|handler| handler.entity.clone()) + .collect(), + manifest_idx: ds.manifest_idx, + }) + .collect::>(); + + // if static_filters is not enabled we just stick to the filter based on all the data sources. + if !static_filters { + return TriggerFilterWrapper::new( + C::TriggerFilter::from_data_sources( + self.ctx.onchain_data_sources().filter(end_block_filter), + ), + subgraph_filter, + ); + } + + // if static_filters is enabled, build a minimal filter with the static data sources and + // add the necessary filters based on templates. + // This specifically removes dynamic data sources based filters because these can be derived + // from templates AND this reduces the cost of egress traffic by making the payloads smaller. + + if !self.inputs.static_filters { + info!(self.logger, "forcing subgraph to use static filters.") + } + + let data_sources = self.ctx.static_data_sources(); + + let mut filter = C::TriggerFilter::from_data_sources( + data_sources + .iter() + .filter_map(|ds| ds.as_onchain()) + // Filter out data sources that have reached their end block if the block is final. + .filter(end_block_filter), + ); + + let templates = self.ctx.templates(); + + filter.extend_with_template(templates.iter().filter_map(|ds| ds.as_onchain()).cloned()); + + TriggerFilterWrapper::new(filter, subgraph_filter) + } + + #[cfg(debug_assertions)] + pub fn build_filter_for_test(&self) -> TriggerFilterWrapper { + self.build_filter() + } + + async fn start_block_stream(&mut self) -> Result>>, Error> { + let block_stream_canceler = CancelGuard::new(); + let block_stream_cancel_handle = block_stream_canceler.handle(); + // TriggerFilter needs to be rebuilt eveytime the blockstream is restarted + let filter = self.build_filter(); + + let block_stream = new_block_stream(&self.inputs, filter, &self.metrics.subgraph) + .await? + .cancelable(&block_stream_canceler); + + self.cancel_handle = Some(block_stream_cancel_handle); + + // Keep the stream's cancel guard around to be able to shut it down when the subgraph + // deployment is unassigned + self.ctx + .instances + .insert(self.inputs.deployment.id, block_stream_canceler); + + Ok(block_stream) + } + + fn is_canceled(&self) -> bool { + if let Some(ref cancel_handle) = self.cancel_handle { + cancel_handle.is_canceled() + } else { + false + } + } + + /// Initialize the runner by performing pre-loop setup. + /// + /// This method handles: + /// - Updating the deployment synced metric + /// - Attempting to unfail deterministic errors from the previous run + /// - Checking if the subgraph has already reached its max end block + /// + /// Returns the next state to transition to: + /// - `Restarting` to start the block stream (normal case) + /// - `Stopped` if the max end block was already reached + async fn initialize(&self) -> Result, SubgraphRunnerError> { + self.update_deployment_synced_metric(); + + // If a subgraph failed for deterministic reasons, before start indexing, we first + // revert the deployment head. It should lead to the same result since the error was + // deterministic. + if let Some(current_ptr) = self.inputs.store.block_ptr() { + if let Some(parent_ptr) = self + .inputs + .triggers_adapter + .parent_ptr(¤t_ptr) + .await? + { + // This reverts the deployment head to the parent_ptr if + // deterministic errors happened. + // + // There's no point in calling it if we have no current or parent block + // pointers, because there would be: no block to revert to or to search + // errors from (first execution). + // + // We attempt to unfail deterministic errors to mitigate deterministic + // errors caused by wrong data being consumed from the providers. It has + // been a frequent case in the past so this helps recover on a larger scale. + let _outcome = self + .inputs + .store + .unfail_deterministic_error(¤t_ptr, &parent_ptr) + .await?; + } + + // Stop subgraph when we reach maximum endblock. + if let Some(max_end_block) = self.inputs.max_end_block + && max_end_block <= current_ptr.block_number() + { + info!(self.logger, "Stopping subgraph as we reached maximum endBlock"; + "max_end_block" => max_end_block, + "current_block" => current_ptr.block_number()); + self.inputs.store.flush().await?; + return Ok(RunnerState::Stopped { + reason: StopReason::MaxEndBlockReached, + }); + } + } + + // Normal case: proceed to start the block stream + Ok(RunnerState::Restarting { + reason: RestartReason::StoreError, // Initial start uses the same path as restart + }) + } + + /// Await the next block stream event and transition to the appropriate state. + /// + /// This method waits for the next event from the block stream and determines + /// which state the runner should transition to: + /// - `ProcessingBlock` for new blocks to process + /// - `Reverting` for revert events + /// - `Stopped` when the stream ends or is canceled + /// - Returns back to `AwaitingBlock` for non-fatal errors that allow continuation + async fn await_block( + &mut self, + mut block_stream: Cancelable>>, + ) -> Result, SubgraphRunnerError> { + let event = { + let _section = self.metrics.stream.stopwatch.start_section("scan_blocks"); + block_stream.next().await + }; + + if self.is_canceled() { + return self.cancel(); + } + + match event { + Some(Ok(BlockStreamEvent::ProcessBlock(block, cursor))) => { + Ok(RunnerState::ProcessingBlock { + block_stream, + block, + cursor, + }) + } + Some(Ok(BlockStreamEvent::Revert(to_ptr, cursor))) => Ok(RunnerState::Reverting { + block_stream, + to_ptr, + cursor, + }), + // Log and drop the errors from the block_stream + // The block stream will continue attempting to produce blocks + Some(Err(e)) => { + // Log error and continue waiting for blocks + debug!( + &self.logger, + "Block stream produced a non-fatal error"; + "error" => format!("{}", e), + ); + Ok(RunnerState::AwaitingBlock { block_stream }) + } + // If the block stream ends, that means that there is no more indexing to do. + None => Ok(RunnerState::Stopped { + reason: StopReason::StreamEnded, + }), + } + } + + fn cancel(&mut self) -> Result, SubgraphRunnerError> { + if self.ctx.instances.contains(&self.inputs.deployment.id) { + warn!( + self.logger, + "Terminating the subgraph runner because a newer one is active. \ + Possible reassignment detected while the runner was in a non-cancellable pending state", + ); + return Err(SubgraphRunnerError::Duplicate); + } + warn!( + self.logger, + "Terminating the subgraph runner because subgraph was unassigned", + ); + Ok(RunnerState::Stopped { + reason: StopReason::Unassigned, + }) + } + + /// Construct a SubgraphError and mark the subgraph as failed in the store. + async fn fail_subgraph( + &mut self, + message: String, + block_ptr: Option, + deterministic: bool, + ) -> Result<(), SubgraphRunnerError> { + let error = SubgraphError { + subgraph_id: self.inputs.deployment.hash.clone(), + message, + block_ptr, + handler: None, + deterministic, + }; + self.inputs + .store + .fail_subgraph(error) + .await + .context("Failed to set subgraph status to `failed`")?; + Ok(()) + } + + /// Handle a restart by potentially restarting the store and starting a new block stream. + /// + /// This method handles: + /// - Restarting the store if there were errors (to clear error state) + /// - Reverting state to the last good block if the store was restarted + /// - Starting a new block stream with updated filters + /// + /// Returns the next state to transition to: + /// - `AwaitingBlock` with the new block stream (normal case) + async fn restart( + &mut self, + reason: RestartReason, + ) -> Result, SubgraphRunnerError> { + debug!(self.logger, "Starting or restarting subgraph"; "reason" => ?reason); + + // If restarting due to store error, try to restart the store + if matches!(reason, RestartReason::StoreError) { + let store = self.inputs.store.cheap_clone(); + if let Some(store) = store.restart().await? { + let last_good_block = store.block_ptr().map(|ptr| ptr.number).unwrap_or(0); + self.revert_state_to(last_good_block); + self.inputs = Arc::new(self.inputs.with_store(store)); + } + } + + let block_stream = self.start_block_stream().await?; + + debug!(self.logger, "Started block stream"); + self.metrics.subgraph.deployment_status.running(); + self.update_deployment_synced_metric(); + + Ok(RunnerState::AwaitingBlock { block_stream }) + } + + /// Finalize the runner when it reaches a terminal state. + /// + /// This method handles cleanup tasks when the runner stops: + /// - Flushing the store to ensure all changes are persisted + /// - Logging the stop reason + async fn finalize(self, reason: StopReason) -> Result { + match reason { + StopReason::MaxEndBlockReached => { + info!(self.logger, "Stopping subgraph - max end block reached"); + } + StopReason::Canceled => { + info!(self.logger, "Stopping subgraph - canceled"); + } + StopReason::Unassigned => { + info!(self.logger, "Stopping subgraph - unassigned"); + } + StopReason::StreamEnded => { + info!(self.logger, "Stopping subgraph - stream ended"); + } + } + + self.inputs.store.flush().await?; + Ok(self) + } + + pub async fn run(self) -> Result<(), SubgraphRunnerError> { + self.run_inner(false).await.map(|_| ()) + } + + /// Main state machine loop for the subgraph runner. + /// + /// This method drives the runner through its state machine, transitioning + /// between states based on events and actions. The state machine replaces + /// the previous nested loop structure with explicit state transitions. + /// + /// ## State Machine + /// + /// The runner starts in `Initializing` and transitions through states: + /// - `Initializing` → `Restarting` (or `Stopped` if max end block reached) + /// - `Restarting` → `AwaitingBlock` + /// - `AwaitingBlock` → `ProcessingBlock`, `Reverting`, or `Stopped` + /// - `ProcessingBlock` → `AwaitingBlock` or `Restarting` + /// - `Reverting` → `AwaitingBlock` or `Restarting` + /// - `Stopped` → terminal (returns) + async fn run_inner(mut self, break_on_restart: bool) -> Result { + // Start in Initializing state + self.runner_state = RunnerState::Initializing; + + // Track whether we've started processing blocks (not just initialized). + // This is used for break_on_restart logic - we should only stop on restart + // after we've actually started processing, not on the initial "restart" + // which is really the first start of the block stream. + let mut has_processed_blocks = false; + + loop { + self.runner_state = match std::mem::take(&mut self.runner_state) { + RunnerState::Initializing => self.initialize().await?, + + RunnerState::Restarting { reason } => { + if break_on_restart && has_processed_blocks { + // In test mode, stop on restart after first block processing + info!(self.logger, "Stopping subgraph on break"); + RunnerState::Stopped { + reason: StopReason::Canceled, + } + } else { + self.restart(reason).await? + } + } + + RunnerState::AwaitingBlock { block_stream } => { + self.await_block(block_stream).await? + } + + RunnerState::ProcessingBlock { + block_stream, + block, + cursor, + } => { + has_processed_blocks = true; + self.process_block_state(block_stream, block, cursor) + .await? + } + + RunnerState::Reverting { + block_stream, + to_ptr, + cursor, + } => self.handle_revert_state(block_stream, to_ptr, cursor).await, + + RunnerState::Stopped { reason } => { + return self.finalize(reason).await; + } + }; + } + } + + /// Process a block and determine the next state. + /// + /// This is the state machine wrapper around `process_block` that handles + /// the block processing action and determines state transitions. + async fn process_block_state( + &mut self, + block_stream: Cancelable>>, + block: BlockWithTriggers, + cursor: FirehoseCursor, + ) -> Result, SubgraphRunnerError> { + let block_ptr = block.ptr(); + self.metrics + .stream + .deployment_head + .set(block_ptr.number as f64); + + if block.trigger_count() > 0 { + self.metrics + .subgraph + .block_trigger_count + .observe(block.trigger_count() as f64); + } + + // Check if we should skip this block (optimization for blocks without triggers). + // Do not skip if max_end_block has been reached — fall through to process_block so the + // block pointer is persisted and the existing max_end_block check in handle_action fires. + let max_end_block_reached = self + .inputs + .max_end_block + .is_some_and(|max| block_ptr.number >= max); + if block.trigger_count() == 0 + && self.state.skip_ptr_updates_timer.elapsed() <= SKIP_PTR_UPDATES_THRESHOLD + && !self.inputs.store.is_deployment_synced() + && !close_to_chain_head(&block_ptr, &self.inputs.chain.chain_head_ptr().await?, 1000) + && !max_end_block_reached + { + // Skip this block and continue with the same stream + return Ok(RunnerState::AwaitingBlock { block_stream }); + } else { + self.state.skip_ptr_updates_timer = Instant::now(); + } + + let block_start = Instant::now(); + + let action = { + let stopwatch = &self.metrics.stream.stopwatch; + let _section = stopwatch.start_section(PROCESS_BLOCK_SECTION_NAME); + self.process_block(block, cursor).await + }; + + let action = self.handle_action(block_start, block_ptr, action).await?; + + self.update_deployment_synced_metric(); + + if self.is_canceled() { + return self.cancel(); + } + + self.metrics + .subgraph + .observe_block_processed(block_start.elapsed(), action.block_finished()); + + // Convert Action to RunnerState + match action { + Action::Continue => Ok(RunnerState::AwaitingBlock { block_stream }), + Action::Restart => Ok(RunnerState::Restarting { + reason: RestartReason::DynamicDataSourceCreated, + }), + Action::Stop => Ok(RunnerState::Stopped { + reason: StopReason::MaxEndBlockReached, + }), + } + } + + /// Handle a revert event and determine the next state. + /// + /// This is the state machine wrapper around `handle_revert` that handles + /// the revert action and determines state transitions. + async fn handle_revert_state( + &mut self, + block_stream: Cancelable>>, + revert_to_ptr: BlockPtr, + cursor: FirehoseCursor, + ) -> RunnerState { + let stopwatch = &self.metrics.stream.stopwatch; + let _section = stopwatch.start_section(HANDLE_REVERT_SECTION_NAME); + + let action = self.handle_revert(revert_to_ptr, cursor).await; + + match action { + Action::Continue => RunnerState::AwaitingBlock { block_stream }, + Action::Restart => RunnerState::Restarting { + reason: RestartReason::StoreError, + }, + Action::Stop => RunnerState::Stopped { + reason: StopReason::Canceled, + }, + } + } + + async fn transact_block_state( + &mut self, + logger: &Logger, + block_ptr: BlockPtr, + firehose_cursor: FirehoseCursor, + block_time: BlockTime, + block_state: BlockState, + proof_of_indexing: SharedProofOfIndexing, + offchain_mods: Vec, + processed_offchain_data_sources: Vec, + ) -> Result<(), ProcessingError> { + fn log_evict_stats(logger: &Logger, evict_stats: &EvictStats) { + trace!(logger, "Entity cache statistics"; + "weight" => evict_stats.new_weight, + "evicted_weight" => evict_stats.evicted_weight, + "count" => evict_stats.new_count, + "evicted_count" => evict_stats.evicted_count, + "stale_update" => evict_stats.stale_update, + "hit_rate" => format!("{:.0}%", evict_stats.hit_rate_pct()), + "accesses" => evict_stats.accesses, + "evict_time_ms" => evict_stats.evict_time.as_millis()); + } + + let BlockState { + deterministic_errors, + persisted_data_sources, + metrics: block_state_metrics, + mut entity_cache, + .. + } = block_state; + let first_error = deterministic_errors.first().cloned(); + let has_errors = first_error.is_some(); + + // Avoid writing to store if block stream has been canceled + if self.is_canceled() { + return Err(ProcessingError::Canceled); + } + + if let Some(proof_of_indexing) = proof_of_indexing.into_inner() { + update_proof_of_indexing( + proof_of_indexing, + block_time, + &self.metrics.host.stopwatch, + &mut entity_cache, + ) + .await + .non_deterministic()?; + } + + let section = self + .metrics + .host + .stopwatch + .start_section("as_modifications"); + let ModificationsAndCache { + modifications: mut mods, + entity_lfu_cache: cache, + evict_stats, + } = entity_cache + .as_modifications(block_ptr.number, &self.metrics.host.stopwatch) + .await + .classify()?; + section.end(); + + log_evict_stats(&self.logger, &evict_stats); + + mods.extend(offchain_mods); + + // Put the cache back in the state, asserting that the placeholder cache was not used. + assert!(self.state.entity_lfu_cache.is_empty()); + self.state.entity_lfu_cache = cache; + + if !mods.is_empty() { + info!(&logger, "Applying {} entity operation(s)", mods.len()); + } + + let err_count = deterministic_errors.len(); + for (i, e) in deterministic_errors.iter().enumerate() { + let message = format!("{:#}", e).replace('\n', "\t"); + error!(&logger, "Subgraph error {}/{}", i + 1, err_count; + "error" => message, + "code" => LogCode::SubgraphSyncingFailure + ); + } + + // Transact entity operations into the store and update the + // subgraph's block stream pointer + let _section = self.metrics.host.stopwatch.start_section("transact_block"); + let start = Instant::now(); + + // If a deterministic error has happened, make the PoI to be the only entity that'll be stored. + if has_errors && self.inputs.errors_are_fatal() { + let is_poi_entity = + |entity_mod: &EntityModification| entity_mod.key().entity_type.is_poi(); + mods.retain(is_poi_entity); + // Confidence check + assert!( + mods.len() == 1, + "There should be only one PoI EntityModification" + ); + } + + let is_caught_up = self.is_caught_up(&block_ptr).await.non_deterministic()?; + + if !self.state.postponed_indexes_created + && close_to_chain_head( + &block_ptr, + &self.state.cached_head_ptr, + ENV_VARS.postpone_indexes_creation_threshold, + ) + { + self.state.postponed_indexes_created = true; + self.inputs + .store + .create_postponed_indexes() + .await + .non_deterministic()?; + } + + self.inputs + .store + .transact_block_operations( + block_ptr.clone(), + block_time, + firehose_cursor, + mods, + &self.metrics.host.stopwatch, + persisted_data_sources, + deterministic_errors, + processed_offchain_data_sources, + self.inputs.errors_are_non_fatal(), + is_caught_up, + ) + .await + .classify() + .detail("Failed to transact block operations")?; + + // For subgraphs with `nonFatalErrors` feature disabled, we consider + // any error as fatal. + // + // So we do an early return to make the subgraph stop processing blocks. + // + // In this scenario the only entity that is stored/transacted is the PoI, + // all of the others are discarded. + if has_errors && self.inputs.errors_are_fatal() { + if let Err(e) = self.inputs.store.flush().await { + error!(logger, "Failed to flush store after fatal errors"; "error" => format!("{:#}", e)); + } + // Only the first error is reported. + return Err(ProcessingError::Deterministic(Box::new( + first_error.unwrap(), + ))); + } + + let elapsed = start.elapsed().as_secs_f64(); + self.metrics + .subgraph + .block_ops_transaction_duration + .observe(elapsed); + + block_state_metrics + .flush_metrics_to_store(logger, block_ptr, self.inputs.deployment.id) + .non_deterministic()?; + + if has_errors { + self.maybe_cancel().await?; + } + + Ok(()) + } + + /// Cancel the subgraph if `disable_fail_fast` is not set and it is not + /// synced + async fn maybe_cancel(&self) -> Result<(), ProcessingError> { + // To prevent a buggy pending version from replacing a current version, if errors are + // present the subgraph will be unassigned. + let store = &self.inputs.store; + if !ENV_VARS.disable_fail_fast && !store.is_deployment_synced() { + store + .pause_subgraph() + .await + .map_err(|e| ProcessingError::Unknown(e.into()))?; + + // Use `Canceled` to avoiding setting the subgraph health to failed, an error was + // just transacted so it will be already be set to unhealthy. + Err(ProcessingError::Canceled) + } else { + Ok(()) + } + } + + async fn match_and_decode_many<'a, F>( + &'a self, + logger: &Logger, + block: &Arc, + triggers: Vec>, + hosts_filter: F, + ) -> Result>, MappingError> + where + F: Fn(&TriggerData) -> Box + Send + 'a>, + { + let triggers = triggers.into_iter().map(|t| match t { + Trigger::Chain(t) => TriggerData::Onchain(t), + Trigger::Subgraph(t) => TriggerData::Subgraph(t), + }); + + self.ctx + .decoder + .match_and_decode_many( + logger, + block, + triggers, + hosts_filter, + &self.metrics.subgraph, + ) + .await + } + + // ========================================================================= + // Pipeline Stage Methods + // ========================================================================= + // + // The following methods implement the block processing pipeline stages. + // Each stage handles a specific phase of block processing: + // + // 1. match_triggers: Match and decode triggers against hosts + // 2. execute_triggers: Execute the matched triggers + // 3. process_dynamic_data_sources: Handle dynamically created data sources + // 4. (process_offchain_triggers): Existing handle_offchain_triggers method + // 5. (persist_block_state): Existing transact_block_state method + // + // ========================================================================= + + /// Pipeline Stage 1: Match triggers to hosts and decode them. + /// + /// Takes raw triggers from a block and matches them against all registered + /// hosts, returning runnable triggers ready for execution. + async fn match_triggers<'a>( + &'a self, + logger: &Logger, + block: &Arc, + triggers: Vec>, + ) -> Result>, MappingError> { + let hosts_filter = |trigger: &TriggerData| self.ctx.instance.hosts_for_trigger(trigger); + self.match_and_decode_many(logger, block, triggers, hosts_filter) + .await + } + + /// Pipeline Stage 2: Execute matched triggers. + /// + /// Takes runnable triggers and executes them using the TriggerRunner, + /// accumulating state changes in the block state. + async fn execute_triggers( + &self, + block: &Arc, + runnables: Vec>, + block_state: BlockState, + proof_of_indexing: &SharedProofOfIndexing, + causality_region: &str, + ) -> Result { + let trigger_runner = TriggerRunner::new( + self.ctx.trigger_processor.as_ref(), + &self.logger, + &self.metrics.subgraph, + &self.inputs.debug_fork, + self.inputs.instrument, + ); + trigger_runner + .execute( + block, + runnables, + block_state, + proof_of_indexing, + causality_region, + ) + .await + } + + /// Pipeline Stage 3: Process dynamically created data sources. + /// + /// This loop processes data sources created during trigger execution: + /// 1. Instantiate the created data sources + /// 2. Reprocess triggers from this block that match the new data sources + /// 3. Repeat until no more data sources are created + /// + /// Note: This algorithm processes data sources spawned on the same block + /// _breadth first_ on the tree implied by the parent-child relationship + /// between data sources. + async fn process_dynamic_data_sources( + &mut self, + logger: &Logger, + block: &Arc, + firehose_cursor: &FirehoseCursor, + mut block_state: BlockState, + proof_of_indexing: &SharedProofOfIndexing, + causality_region: &str, + ) -> Result { + fn log_triggers_found(logger: &Logger, triggers: &[Trigger]) { + if triggers.len() == 1 { + info!(logger, "1 trigger found in this block"); + } else if triggers.len() > 1 { + info!(logger, "{} triggers found in this block", triggers.len()); + } + } + + let _section = self + .metrics + .stream + .stopwatch + .start_section(HANDLE_CREATED_DS_SECTION_NAME); + + while block_state.has_created_data_sources() { + // Instantiate dynamic data sources, removing them from the block state. + let (data_sources, runtime_hosts) = + self.create_dynamic_data_sources(block_state.drain_created_data_sources())?; + + let filter = &Arc::new(TriggerFilterWrapper::new( + C::TriggerFilter::from_data_sources( + data_sources.iter().filter_map(DataSource::as_onchain), + ), + vec![], + )); + + // TODO: We have to pass a reference to `block` to + // `refetch_block`, otherwise the call to + // handle_offchain_triggers below gets an error that `block` + // has moved. That is extremely fishy since it means that + // `handle_offchain_triggers` uses the non-refetched block + // + // It's also not clear why refetching needs to happen inside + // the loop; will firehose really return something diffrent + // each time even though the cursor doesn't change? + let block = self.refetch_block(logger, block, firehose_cursor).await?; + + // Reprocess the triggers from this block that match the new data sources + let block_with_triggers = self + .inputs + .triggers_adapter + .triggers_in_block(logger, block.as_ref().clone(), filter) + .await + .non_deterministic()?; + + let triggers = block_with_triggers.trigger_data; + log_triggers_found::(logger, &triggers); + + // Add entity operations for the new data sources to the block state + // and add runtimes for the data sources to the subgraph instance. + self.persist_dynamic_data_sources(&mut block_state, data_sources); + + // Process the triggers in each host in the same order the + // corresponding data sources have been created. + let hosts_filter = |_: &'_ TriggerData| -> Box + Send> { + Box::new(runtime_hosts.iter().map(Arc::as_ref)) + }; + let runnables = self + .match_and_decode_many(logger, &block, triggers, hosts_filter) + .await; + + let trigger_runner = TriggerRunner::new( + self.ctx.trigger_processor.as_ref(), + &self.logger, + &self.metrics.subgraph, + &self.inputs.debug_fork, + self.inputs.instrument, + ); + let res = match runnables { + Ok(runnables) => { + trigger_runner + .execute( + &block, + runnables, + block_state, + proof_of_indexing, + causality_region, + ) + .await + } + Err(e) => Err(e), + }; + + block_state = res.map_err(|e| { + // This treats a `PossibleReorg` as an ordinary error which will fail the subgraph. + // This can cause an unnecessary subgraph failure, to fix it we need to figure out a + // way to revert the effect of `create_dynamic_data_sources` so we may return a + // clean context as in b21fa73b-6453-4340-99fb-1a78ec62efb1. + match e { + MappingError::PossibleReorg(e) | MappingError::Unknown(e) => { + ProcessingError::Unknown(e) + } + } + })?; + } + + Ok(block_state) + } + + /// Pipeline Stage 4: Process offchain triggers. + /// + /// Retrieves ready offchain events and processes them, returning entity + /// modifications and processed data sources to be included in the transaction. + async fn process_offchain_triggers( + &mut self, + block: &Arc, + block_state: &mut BlockState, + ) -> Result<(Vec, Vec), ProcessingError> { + let offchain_events = self + .ctx + .offchain_monitor + .ready_offchain_events() + .non_deterministic()?; + + let vid_gen = block_state.seq_gen(); + let (offchain_mods, processed_offchain_data_sources, persisted_off_chain_data_sources) = + self.handle_offchain_triggers(offchain_events, block, vid_gen) + .await + .non_deterministic()?; + + block_state + .persisted_data_sources + .extend(persisted_off_chain_data_sources); + + Ok((offchain_mods, processed_offchain_data_sources)) + } + + /// Processes a block and returns the updated context and a boolean flag indicating + /// whether new dynamic data sources have been added to the subgraph. + /// + /// ## Pipeline Stages + /// + /// Block processing follows a pipeline of stages: + /// 1. **match_triggers**: Match and decode triggers against hosts + /// 2. **execute_triggers**: Execute the matched triggers via TriggerRunner + /// 3. **process_dynamic_data_sources**: Handle dynamically created data sources + /// 4. **process_offchain_triggers**: Process offchain events + /// 5. **persist_block**: Persist block state to the store + async fn process_block( + &mut self, + block: BlockWithTriggers, + firehose_cursor: FirehoseCursor, + ) -> Result { + let triggers = block.trigger_data; + let block = Arc::new(block.block); + let block_ptr = block.ptr(); + + let logger = self.logger.new(o!( + "block_number" => format!("{:?}", block_ptr.number), + "block_hash" => format!("{}", block_ptr.hash) + )); + + info!(logger, "Start processing block"; + "triggers" => triggers.len()); + + let proof_of_indexing = + SharedProofOfIndexing::new(block_ptr.number, self.inputs.poi_version); + + // Causality region for onchain triggers. + let causality_region = PoICausalityRegion::from_network(&self.inputs.network); + + let vid_gen = SeqGenerator::new(block_ptr.number); + let mut block_state = BlockState::new( + self.inputs.store.clone(), + std::mem::take(&mut self.state.entity_lfu_cache), + vid_gen, + ); + + let _section = self + .metrics + .stream + .stopwatch + .start_section(PROCESS_TRIGGERS_SECTION_NAME); + + // Stage 1: Match triggers to hosts and decode + let runnables = self.match_triggers(&logger, &block, triggers).await; + + // Stage 2: Execute triggers + let res = match runnables { + Ok(runnables) => { + self.execute_triggers( + &block, + runnables, + block_state, + &proof_of_indexing, + &causality_region, + ) + .await + } + Err(e) => Err(e), + }; + + match res { + // Triggers processed with no errors or with only deterministic errors. + Ok(state) => block_state = state, + + // Some form of unknown or non-deterministic error ocurred. + Err(MappingError::Unknown(e)) => return Err(ProcessingError::Unknown(e)), + + // Possible blockchain reorg detected - signal restart via ProcessingError::PossibleReorg. + // See also b21fa73b-6453-4340-99fb-1a78ec62efb1. + Err(MappingError::PossibleReorg(e)) => return Err(ProcessingError::PossibleReorg(e)), + } + + // Check if there are any datasources that have expired in this block. ie: the end_block + // of that data source is equal to the block number of the current block. + let has_expired_data_sources = self.inputs.end_blocks.contains(&block_ptr.number); + + // If new onchain data sources have been created, and static filters are not in use, it is necessary + // to restart the block stream with the new filters. + let created_data_sources_needs_restart = + !self.is_static_filters_enabled() && block_state.has_created_on_chain_data_sources(); + + // Determine if the block stream needs to be restarted due to newly created on-chain data sources + // or data sources that have reached their end block. + let needs_restart = created_data_sources_needs_restart || has_expired_data_sources; + + // Checkpoint before dynamic DS processing for potential rollback scenarios. + // This captures the current state so it can be restored if dynamic data source + // processing fails in a way that requires partial rollback. + let _checkpoint = block_state.checkpoint(); + + // Stage 3: Process dynamic data sources + block_state = self + .process_dynamic_data_sources( + &logger, + &block, + &firehose_cursor, + block_state, + &proof_of_indexing, + &causality_region, + ) + .await?; + + // Stage 4: Process offchain triggers + let (offchain_mods, processed_offchain_data_sources) = self + .process_offchain_triggers(&block, &mut block_state) + .await?; + + // Stage 5: Persist block state + self.transact_block_state( + &logger, + block_ptr, + firehose_cursor, + block.timestamp(), + block_state, + proof_of_indexing, + offchain_mods, + processed_offchain_data_sources, + ) + .await?; + + match needs_restart { + true => Ok(Action::Restart), + false => Ok(Action::Continue), + } + } + + /// Refetch the block if it that is needed. Otherwise return the block as is. + async fn refetch_block( + &mut self, + logger: &Logger, + block: &Arc, + firehose_cursor: &FirehoseCursor, + ) -> Result, ProcessingError> { + if !self.inputs.chain.is_refetch_block_required() { + return Ok(block.cheap_clone()); + } + + let cur = firehose_cursor.clone(); + let log = logger.cheap_clone(); + let chain = self.inputs.chain.cheap_clone(); + let block = retry( + "refetch firehose block after dynamic datasource was added", + logger, + ) + .limit(5) + .no_timeout() + .run(move || { + let cur = cur.clone(); + let log = log.cheap_clone(); + let chain = chain.cheap_clone(); + async move { chain.refetch_firehose_block(&log, cur).await } + }) + .await + .non_deterministic()?; + Ok(Arc::new(block)) + } + + fn create_dynamic_data_sources( + &mut self, + created_data_sources: Vec, + ) -> Result<(Vec>, Vec>), ProcessingError> { + let mut data_sources = vec![]; + let mut runtime_hosts = vec![]; + + for info in created_data_sources { + let manifest_idx = info + .template + .manifest_idx() + .ok_or_else(|| anyhow!("Expected template to have an idx")) + .non_deterministic()?; + let created_ds_template = self + .inputs + .templates + .iter() + .find(|t| t.manifest_idx() == manifest_idx) + .ok_or_else(|| anyhow!("Expected to find a template for this dynamic data source")) + .non_deterministic()?; + + // Try to instantiate a data source from the template + let data_source = { + let res = match info.template { + InstanceDSTemplate::Onchain(_) => { + C::DataSource::from_template_info(info, created_ds_template) + .map(DataSource::Onchain) + .map_err(DataSourceCreationError::from) + } + InstanceDSTemplate::Offchain(_) => offchain::DataSource::from_template_info( + info, + self.ctx.causality_region_next_value(), + ) + .map(DataSource::Offchain), + }; + match res { + Ok(ds) => ds, + Err(e @ DataSourceCreationError::Ignore(..)) => { + warn!(self.logger, "{}", e.to_string()); + continue; + } + Err(DataSourceCreationError::Unknown(e)) => return Err(e).non_deterministic(), + } + }; + + // Try to create a runtime host for the data source + let host = self + .ctx + .add_dynamic_data_source(&self.logger, data_source.clone()) + .non_deterministic()?; + + match host { + Some(host) => { + data_sources.push(data_source); + runtime_hosts.push(host); + } + None => { + warn!( + self.logger, + "no runtime host created, there is already a runtime host instantiated for \ + this data source"; + "name" => &data_source.name(), + "address" => &data_source.address() + .map(hex::encode) + .unwrap_or("none".to_string()), + ) + } + } + } + + Ok((data_sources, runtime_hosts)) + } + + async fn handle_action( + &mut self, + start: Instant, + block_ptr: BlockPtr, + action: Result, + ) -> Result { + self.state.skip_ptr_updates_timer = Instant::now(); + + let elapsed = start.elapsed().as_secs_f64(); + self.metrics + .subgraph + .block_processing_duration + .observe(elapsed); + + match action { + Ok(action) => { + // Keep trying to unfail subgraph for everytime it advances block(s) until it's + // health is not Failed anymore. + if self.state.should_try_unfail_non_deterministic { + // If the deployment head advanced, we can unfail + // the non-deterministic error (if there's any). + let outcome = self + .inputs + .store + .unfail_non_deterministic_error(&block_ptr) + .await?; + + // Stop trying unless we're still behind the error block. + if outcome != UnfailOutcome::BehindErrorBlock { + self.state.should_try_unfail_non_deterministic = false; + + if outcome == UnfailOutcome::Unfailed { + self.metrics.subgraph.deployment_status.running(); + self.state.backoff.reset(); + } + } + } + + if let Some(stop_block) = self.inputs.stop_block + && block_ptr.number >= stop_block + { + info!(self.logger, "Stop block reached for subgraph"); + return Ok(Action::Stop); + } + + if let Some(max_end_block) = self.inputs.max_end_block + && block_ptr.number >= max_end_block + { + info!( + self.logger, + "Stopping subgraph as maximum endBlock reached"; + "max_end_block" => max_end_block, + "current_block" => block_ptr.number + ); + return Ok(Action::Stop); + } + + Ok(action) + } + // Handle errors based on their kind using the unified error classification. + // + // Error handling invariants: + // - Deterministic: Stop processing, persist PoI only, fail subgraph + // - NonDeterministic: Retry with backoff, may succeed on retry + // - PossibleReorg: Restart cleanly without persisting (don't fail subgraph) + // - Canceled: Clean shutdown, no error recording + Err(e) => match e.kind() { + ProcessingErrorKind::Canceled => { + debug!(self.logger, "Subgraph block stream shut down cleanly"); + Ok(Action::Stop) + } + + ProcessingErrorKind::PossibleReorg => { + // Possible reorg detected - restart the block stream cleanly. + // Don't persist anything and don't mark subgraph as failed. + // The block stream restart will allow detection of the actual reorg. + info!(self.logger, + "Possible reorg detected, restarting block stream"; + "error" => format!("{:#}", e), + ); + + // Revert in-memory state to last good block but don't touch the store + let last_good_block = self + .inputs + .store + .block_ptr() + .map(|ptr| ptr.number) + .unwrap_or(0); + self.revert_state_to(last_good_block); + + Ok(Action::Restart) + } + + ProcessingErrorKind::Deterministic => { + // Deterministic error - fail the subgraph permanently. + self.metrics.subgraph.deployment_status.failed(); + let last_good_block = self + .inputs + .store + .block_ptr() + .map(|ptr| ptr.number) + .unwrap_or(0); + self.revert_state_to(last_good_block); + + let message = format!("{:#}", e).replace('\n', "\t"); + let err = anyhow!("{}, code: {}", message, LogCode::SubgraphSyncingFailure); + + self.fail_subgraph(message, Some(block_ptr), true).await?; + + Err(err) + } + + ProcessingErrorKind::NonDeterministic => { + // Non-deterministic error - retry with backoff. + self.metrics.subgraph.deployment_status.failed(); + let last_good_block = self + .inputs + .store + .block_ptr() + .map(|ptr| ptr.number) + .unwrap_or(0); + self.revert_state_to(last_good_block); + + let message = format!("{:#}", e).replace('\n', "\t"); + + error!(self.logger, "Subgraph failed with non-deterministic error: {}", message; + "attempt" => self.state.backoff.attempt, + "retry_delay_s" => self.state.backoff.delay().as_secs()); + + // Shouldn't fail subgraph if it's already failed for non-deterministic + // reasons. + // + // If we don't do this check we would keep adding the same error to the + // database. + if !self.inputs.store.health().await?.is_failed() { + self.fail_subgraph(message, Some(block_ptr), false).await?; + } + + // Sleep before restarting. + self.state.backoff.sleep_async().await; + + self.state.should_try_unfail_non_deterministic = true; + + // And restart the subgraph. + Ok(Action::Restart) + } + }, + } + } + + fn persist_dynamic_data_sources( + &mut self, + block_state: &mut BlockState, + data_sources: Vec>, + ) { + if !data_sources.is_empty() { + debug!( + self.logger, + "Creating {} dynamic data source(s)", + data_sources.len() + ); + } + + // Add entity operations to the block state in order to persist + // the dynamic data sources + for data_source in data_sources.iter() { + debug!( + self.logger, + "Persisting data_source"; + "name" => &data_source.name(), + "address" => &data_source.address().map(hex::encode).unwrap_or("none".to_string()), + ); + block_state.persist_data_source(data_source.as_stored_dynamic_data_source()); + } + } + + /// We consider a subgraph caught up when it's at most 10 blocks behind the chain head. + async fn is_caught_up(&mut self, block_ptr: &BlockPtr) -> Result { + const CAUGHT_UP_DISTANCE: BlockNumber = 10; + + // Ensure that `state.cached_head_ptr` has a value since it could be `None` on the first + // iteration of loop. If the deployment head has caught up to the `cached_head_ptr`, update + // it so that we are up to date when checking if synced. + let cached_head_ptr = self.state.cached_head_ptr.cheap_clone(); + if cached_head_ptr.is_none() + || close_to_chain_head(block_ptr, &cached_head_ptr, CAUGHT_UP_DISTANCE) + { + self.state.cached_head_ptr = self.inputs.chain.chain_head_ptr().await?; + } + let is_caught_up = + close_to_chain_head(block_ptr, &self.state.cached_head_ptr, CAUGHT_UP_DISTANCE); + if is_caught_up { + // Stop recording time-to-sync metrics. + self.metrics.stream.stopwatch.disable(); + } + Ok(is_caught_up) + } +} + +impl SubgraphRunner +where + C: Blockchain, + T: RuntimeHostBuilder, +{ + async fn handle_offchain_triggers( + &mut self, + triggers: Vec, + block: &Arc, + vid_gen: SeqGenerator, + ) -> Result< + ( + Vec, + Vec, + Vec, + ), + Error, + > { + let mut mods = vec![]; + let mut processed_data_sources = vec![]; + let mut persisted_data_sources = vec![]; + + for trigger in triggers { + // Using an `EmptyStore` and clearing the cache for each trigger is a makeshift way to + // get causality region isolation. + let schema = ReadStore::input_schema(&self.inputs.store); + let mut block_state = BlockState::new( + EmptyStore::new(schema), + LfuCache::new(), + vid_gen.cheap_clone(), + ); + + // PoI ignores offchain events. + // See also: poi-ignores-offchain + let proof_of_indexing = SharedProofOfIndexing::ignored(); + let causality_region = ""; + + let trigger = TriggerData::Offchain(trigger); + let process_res = { + let hosts = self.ctx.instance.hosts_for_trigger(&trigger); + let triggers_res = self.ctx.decoder.match_and_decode( + &self.logger, + block, + trigger, + hosts, + &self.metrics.subgraph, + ); + match triggers_res { + Ok(runnable) => { + self.ctx + .trigger_processor + .process_trigger( + &self.logger, + runnable.hosted_triggers, + block, + block_state, + &proof_of_indexing, + causality_region, + &self.inputs.debug_fork, + &self.metrics.subgraph, + self.inputs.instrument, + ) + .await + } + Err(e) => Err(e), + } + }; + match process_res { + Ok(state) => block_state = state, + Err(err) => { + let err = match err { + // Ignoring `PossibleReorg` isn't so bad since the subgraph will retry + // non-deterministic errors. + MappingError::PossibleReorg(e) | MappingError::Unknown(e) => e, + }; + return Err(err.context("failed to process trigger".to_string())); + } + } + + anyhow::ensure!( + !block_state.has_created_on_chain_data_sources(), + "Attempted to create on-chain data source in offchain data source handler. This is not yet supported.", + ); + + let (data_sources, _) = + self.create_dynamic_data_sources(block_state.drain_created_data_sources())?; + + // Add entity operations for the new data sources to the block state + // and add runtimes for the data sources to the subgraph instance. + self.persist_dynamic_data_sources(&mut block_state, data_sources); + + // This propagates any deterministic error as a non-deterministic one. Which might make + // sense considering offchain data sources are non-deterministic. + if let Some(err) = block_state.deterministic_errors.into_iter().next() { + return Err(anyhow!("{}", err)); + } + + mods.extend( + block_state + .entity_cache + .as_modifications(block.number(), &self.metrics.subgraph.stopwatch) + .await? + .modifications, + ); + processed_data_sources.extend(block_state.processed_data_sources); + persisted_data_sources.extend(block_state.persisted_data_sources) + } + + Ok((mods, processed_data_sources, persisted_data_sources)) + } + + fn update_deployment_synced_metric(&self) { + self.metrics + .subgraph + .deployment_synced + .record(self.inputs.store.is_deployment_synced()); + } +} + +#[derive(Debug)] +enum Action { + Continue, + Stop, + Restart, +} + +impl Action { + /// Return `true` if the action indicates that we are done with a block + fn block_finished(&self) -> bool { + match self { + Action::Restart => false, + Action::Continue | Action::Stop => true, + } + } +} + +impl SubgraphRunner +where + C: Blockchain, + T: RuntimeHostBuilder, +{ + async fn handle_revert(&mut self, revert_to_ptr: BlockPtr, cursor: FirehoseCursor) -> Action { + // Current deployment head in the database / WritableAgent Mutex cache. + // + // Safe unwrap because in a Revert event we're sure the subgraph has + // advanced at least once. + let subgraph_ptr = self.inputs.store.block_ptr().unwrap(); + if revert_to_ptr.number >= subgraph_ptr.number { + info!(&self.logger, "Block to revert is higher than subgraph pointer, nothing to do"; "subgraph_ptr" => &subgraph_ptr, "revert_to_ptr" => &revert_to_ptr); + return Action::Continue; + } + + info!(&self.logger, "Reverting block to get back to main chain"; "subgraph_ptr" => &subgraph_ptr, "revert_to_ptr" => &revert_to_ptr); + + if let Err(e) = self + .inputs + .store + .revert_block_operations(revert_to_ptr.clone(), cursor) + .await + { + error!(&self.logger, "Could not revert block. Retrying"; "error" => %e); + + // Exit inner block stream consumption loop and go up to loop that restarts subgraph + return Action::Restart; + } + + self.metrics + .stream + .reverted_blocks + .set(subgraph_ptr.number as f64); + self.metrics + .stream + .deployment_head + .set(subgraph_ptr.number as f64); + + self.revert_state_to(revert_to_ptr.number); + + let needs_restart: bool = self.needs_restart(revert_to_ptr, subgraph_ptr); + + if needs_restart { + Action::Restart + } else { + Action::Continue + } + } + + /// Determines if the subgraph needs to be restarted. + /// Currently returns true when there are data sources that have reached their end block + /// in the range between `revert_to_ptr` and `subgraph_ptr`. + fn needs_restart(&self, revert_to_ptr: BlockPtr, subgraph_ptr: BlockPtr) -> bool { + self.inputs + .end_blocks + .range(revert_to_ptr.number..=subgraph_ptr.number) + .next() + .is_some() + } +} + +impl From for SubgraphRunnerError { + fn from(err: StoreError) -> Self { + Self::Unknown(err.into()) + } +} + +/// Transform the proof of indexing changes into entity updates that will be +/// inserted when as_modifications is called. +async fn update_proof_of_indexing( + proof_of_indexing: ProofOfIndexing, + block_time: BlockTime, + stopwatch: &StopwatchMetrics, + entity_cache: &mut EntityCache, +) -> Result<(), Error> { + // Helper to store the digest as a PoI entity in the cache + async fn store_poi_entity( + entity_cache: &mut EntityCache, + key: EntityKey, + digest: Bytes, + block_time: BlockTime, + ) -> Result<(), Error> { + let digest_name = entity_cache.schema.poi_digest(); + let mut data = vec![ + ( + graph::data::store::ID.clone(), + Value::from(key.entity_id.to_string()), + ), + (digest_name, Value::from(digest)), + ]; + if entity_cache.schema.has_aggregations() { + let block_time = Value::Int8(block_time.as_secs_since_epoch()); + data.push((entity_cache.schema.poi_block_time(), block_time)); + } + let poi = entity_cache.make_entity(data)?; + entity_cache.set(key, poi, None).await + } + + let _section_guard = stopwatch.start_section("update_proof_of_indexing"); + + let mut proof_of_indexing = proof_of_indexing.take(); + + for (causality_region, stream) in proof_of_indexing.drain() { + // Create the special POI entity key specific to this causality_region + // There are two things called causality regions here, one is the causality region for + // the poi which is a string and the PoI entity id. The other is the data source + // causality region to which the PoI belongs as an entity. Currently offchain events do + // not affect PoI so it is assumed to be `ONCHAIN`. + // See also: poi-ignores-offchain + let entity_key = entity_cache + .schema + .poi_type() + .key_in(causality_region, CausalityRegion::ONCHAIN); + + // Grab the current digest attribute on this entity + let poi_digest = entity_cache.schema.poi_digest().clone(); + let prev_poi = entity_cache + .get(&entity_key, GetScope::Store) + .await + .map_err(Error::from)? + .map(|entity| match entity.get(poi_digest.as_str()) { + Some(Value::Bytes(b)) => b.clone(), + _ => panic!("Expected POI entity to have a digest and for it to be bytes"), + }); + + // Finish the POI stream, getting the new POI value. + let updated_proof_of_indexing = stream.pause(prev_poi.as_deref()); + let updated_proof_of_indexing: Bytes = (&updated_proof_of_indexing[..]).into(); + + // Put this onto an entity with the same digest attribute + // that was expected before when reading. + store_poi_entity( + entity_cache, + entity_key, + updated_proof_of_indexing, + block_time, + ) + .await?; + } + + Ok(()) +} + +/// Checks if the Deployment BlockPtr is within N blocks of the chain head or ahead. +fn close_to_chain_head( + deployment_head_ptr: &BlockPtr, + chain_head_ptr: &Option, + n: BlockNumber, +) -> bool { + matches!((deployment_head_ptr, &chain_head_ptr), (b1, Some(b2)) if b1.number >= (b2.number - n)) +} + +#[test] +fn test_close_to_chain_head() { + let offset = 1; + + let block_0 = BlockPtr::try_from(( + "bd34884280958002c51d3f7b5f853e6febeba33de0f40d15b0363006533c924f", + 0, + )) + .unwrap(); + let block_1 = BlockPtr::try_from(( + "8511fa04b64657581e3f00e14543c1d522d5d7e771b54aa3060b662ade47da13", + 1, + )) + .unwrap(); + let block_2 = BlockPtr::try_from(( + "b98fb783b49de5652097a989414c767824dff7e7fd765a63b493772511db81c1", + 2, + )) + .unwrap(); + + assert!(!close_to_chain_head(&block_0, &None, offset)); + assert!(!close_to_chain_head(&block_2, &None, offset)); + + assert!(!close_to_chain_head( + &block_0, + &Some(block_2.clone()), + offset + )); + + assert!(close_to_chain_head( + &block_1, + &Some(block_2.clone()), + offset + )); + assert!(close_to_chain_head( + &block_2, + &Some(block_2.clone()), + offset + )); +} diff --git a/core/src/subgraph/runner/state.rs b/core/src/subgraph/runner/state.rs new file mode 100644 index 00000000000..d6401842522 --- /dev/null +++ b/core/src/subgraph/runner/state.rs @@ -0,0 +1,96 @@ +//! State machine types for SubgraphRunner. +//! +//! This module defines the explicit state machine that controls the runner's lifecycle, +//! replacing the previous nested loop structure with clear state transitions. + +use graph::blockchain::Blockchain; +use graph::blockchain::block_stream::{BlockStream, BlockWithTriggers, FirehoseCursor}; +use graph::ext::futures::Cancelable; +use graph::prelude::BlockPtr; + +/// The current state of the SubgraphRunner's lifecycle. +/// +/// The runner transitions through these states as it processes blocks, +/// handles reverts, and responds to errors or cancellation signals. +/// +/// ## State Transitions +/// +/// ```text +/// Initializing ───────────────────────────────────┐ +/// │ │ +/// v │ +/// AwaitingBlock ◄────────────────────────────────┤ +/// │ │ +/// ├── ProcessBlock event ──► ProcessingBlock │ +/// │ │ │ +/// │ ├── success ┼──► AwaitingBlock +/// │ │ │ +/// │ └── restart ┼──► Restarting +/// │ │ +/// ├── Revert event ──────────► Reverting ────┤ +/// │ │ +/// ├── Error ─────────────────► Restarting ───┤ +/// │ │ +/// └── Cancel/MaxBlock ───────► Stopped │ +/// │ +/// Restarting ─────────────────────────────────────┘ +/// ``` +#[derive(Default)] +pub enum RunnerState { + /// Initial state, ready to start block stream. + #[default] + Initializing, + + /// Block stream active, waiting for next event. + AwaitingBlock { + block_stream: Cancelable>>, + }, + + /// Processing a block through the pipeline. + /// The block stream is kept alive to continue processing after this block. + ProcessingBlock { + block_stream: Cancelable>>, + block: BlockWithTriggers, + cursor: FirehoseCursor, + }, + + /// Handling a revert event. + /// The block stream is kept alive to continue processing after the revert. + Reverting { + block_stream: Cancelable>>, + to_ptr: BlockPtr, + cursor: FirehoseCursor, + }, + + /// Restarting block stream (new filters, store restart, etc.). + Restarting { reason: RestartReason }, + + /// Terminal state. + Stopped { reason: StopReason }, +} + +/// Reasons for restarting the block stream. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RestartReason { + /// New dynamic data source was created that requires filter updates. + DynamicDataSourceCreated, + /// Store error occurred and store needs to be restarted. + StoreError, + /// Possible reorg detected, need to restart to detect it. + /// NOTE: Currently unused but reserved for future error handling consolidation (Phase 5). + #[allow(dead_code)] + PossibleReorg, +} + +/// Reasons for stopping the runner. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum StopReason { + /// The maximum end block was reached. + MaxEndBlockReached, + /// The runner was canceled (unassigned or shutdown). + Canceled, + /// The subgraph was unassigned while this runner was active. + Unassigned, + /// The block stream ended (typically in tests). + StreamEnded, +} diff --git a/core/src/subgraph/runner/trigger_runner.rs b/core/src/subgraph/runner/trigger_runner.rs new file mode 100644 index 00000000000..6c89052e5dc --- /dev/null +++ b/core/src/subgraph/runner/trigger_runner.rs @@ -0,0 +1,79 @@ +use std::sync::Arc; + +use graph::blockchain::Blockchain; +use graph::components::store::SubgraphFork; +use graph::components::subgraph::{MappingError, SharedProofOfIndexing}; +use graph::components::trigger_processor::RunnableTriggers; +use graph::prelude::{BlockState, RuntimeHostBuilder, SubgraphInstanceMetrics, TriggerProcessor}; +use graph::slog::Logger; + +/// Handles the execution of triggers against runtime hosts, accumulating state. +/// +/// This component unifies the trigger processing loop that was previously duplicated +/// for initial triggers and dynamically created data source triggers. +pub struct TriggerRunner<'a, C: Blockchain, T: RuntimeHostBuilder> { + processor: &'a dyn TriggerProcessor, + logger: &'a Logger, + metrics: &'a Arc, + debug_fork: &'a Option>, + instrument: bool, +} + +impl<'a, C, T> TriggerRunner<'a, C, T> +where + C: Blockchain, + T: RuntimeHostBuilder, +{ + /// Create a new TriggerRunner with the given dependencies. + pub fn new( + processor: &'a dyn TriggerProcessor, + logger: &'a Logger, + metrics: &'a Arc, + debug_fork: &'a Option>, + instrument: bool, + ) -> Self { + Self { + processor, + logger, + metrics, + debug_fork, + instrument, + } + } + + /// Execute a sequence of runnable triggers, accumulating state changes. + /// + /// Processes each trigger in order. If any trigger fails with a non-deterministic + /// error, processing stops and the error is returned. Deterministic errors are + /// accumulated in the block state. + pub async fn execute( + &self, + block: &Arc, + runnables: Vec>, + block_state: BlockState, + proof_of_indexing: &SharedProofOfIndexing, + causality_region: &str, + ) -> Result { + let mut state = block_state; + + for runnable in runnables { + state = self + .processor + .process_trigger( + self.logger, + runnable.hosted_triggers, + block, + state, + proof_of_indexing, + causality_region, + self.debug_fork, + self.metrics, + self.instrument, + ) + .await + .map_err(|e| e.add_trigger_context(&runnable.trigger))?; + } + + Ok(state) + } +} diff --git a/core/src/subgraph/state.rs b/core/src/subgraph/state.rs index 0d5edd84b65..c1da4b43199 100644 --- a/core/src/subgraph/state.rs +++ b/core/src/subgraph/state.rs @@ -1,15 +1,11 @@ use graph::{ - components::store::EntityKey, - prelude::Entity, - util::{backoff::ExponentialBackoff, lfu_cache::LfuCache}, + components::store::EntityLfuCache, prelude::BlockPtr, util::backoff::ExponentialBackoff, }; use std::time::Instant; pub struct IndexingState { /// `true` -> `false` on the first run pub should_try_unfail_non_deterministic: bool, - /// `false` -> `true` once it reaches chain head - pub synced: bool, /// Backoff used for the retry mechanism on non-deterministic errors pub backoff: ExponentialBackoff, /// Related to field above `backoff` @@ -18,5 +14,9 @@ pub struct IndexingState { /// - The time THRESHOLD is passed /// - Or the subgraph has triggers for the block pub skip_ptr_updates_timer: Instant, - pub entity_lfu_cache: LfuCache>, + pub entity_lfu_cache: EntityLfuCache, + pub cached_head_ptr: Option, + /// Set to `true` once postponed indexes have been created. This + /// ensures we only trigger index creation once per subgraph run. + pub postponed_indexes_created: bool, } diff --git a/core/src/subgraph/stream.rs b/core/src/subgraph/stream.rs index 791a9becc92..5547543f13d 100644 --- a/core/src/subgraph/stream.rs +++ b/core/src/subgraph/stream.rs @@ -1,50 +1,38 @@ use crate::subgraph::inputs::IndexingInputs; +use anyhow::bail; use graph::blockchain::block_stream::{BlockStream, BufferedBlockStream}; -use graph::blockchain::Blockchain; -use graph::prelude::{Error, SubgraphInstanceMetrics}; +use graph::blockchain::{Blockchain, TriggerFilterWrapper}; +use graph::prelude::{CheapClone, Error, SubgraphInstanceMetrics}; use std::sync::Arc; -const BUFFERED_BLOCK_STREAM_SIZE: usize = 100; -const BUFFERED_FIREHOSE_STREAM_SIZE: usize = 1; - pub async fn new_block_stream( inputs: &IndexingInputs, - filter: &C::TriggerFilter, + filter: TriggerFilterWrapper, metrics: &SubgraphInstanceMetrics, ) -> Result>, Error> { let is_firehose = inputs.chain.chain_client().is_firehose(); - let buffer_size = match is_firehose { - true => BUFFERED_FIREHOSE_STREAM_SIZE, - false => BUFFERED_BLOCK_STREAM_SIZE, - }; - - let current_ptr = inputs.store.block_ptr(); - - let block_stream = match is_firehose { - true => inputs.chain.new_firehose_block_stream( + match inputs + .chain + .new_block_stream( inputs.deployment.clone(), - inputs.store.block_cursor(), + inputs.store.cheap_clone(), inputs.start_blocks.clone(), - current_ptr, + inputs.source_subgraph_stores.clone(), Arc::new(filter.clone()), inputs.unified_api_version.clone(), - ), - false => inputs.chain.new_polling_block_stream( - inputs.deployment.clone(), - inputs.start_blocks.clone(), - current_ptr, - Arc::new(filter.clone()), - inputs.unified_api_version.clone(), - ), + ) + .await + { + Ok(block_stream) => Ok(BufferedBlockStream::spawn_from_stream( + block_stream.buffer_size_hint(), + block_stream, + )), + Err(e) => { + if is_firehose { + metrics.firehose_connection_errors.inc(); + } + bail!(e); + } } - .await; - if is_firehose && block_stream.is_err() { - metrics.firehose_connection_errors.inc(); - } - - Ok(BufferedBlockStream::spawn_from_stream( - block_stream?, - buffer_size, - )) } diff --git a/core/src/subgraph/trigger_processor.rs b/core/src/subgraph/trigger_processor.rs index 2eeb8275500..c3123e87268 100644 --- a/core/src/subgraph/trigger_processor.rs +++ b/core/src/subgraph/trigger_processor.rs @@ -1,14 +1,16 @@ use async_trait::async_trait; -use graph::blockchain::{Block, Blockchain}; +use graph::blockchain::{Block, Blockchain, DecoderHook as _}; use graph::cheap_clone::CheapClone; use graph::components::store::SubgraphFork; use graph::components::subgraph::{MappingError, SharedProofOfIndexing}; -use graph::data_source::{MappingTrigger, TriggerData, TriggerWithHandler}; +use graph::components::trigger_processor::{HostedTrigger, RunnableTriggers}; +use graph::data_source::TriggerData; use graph::prelude::tokio::time::Instant; use graph::prelude::{ BlockState, RuntimeHost, RuntimeHostBuilder, SubgraphInstanceMetrics, TriggerProcessor, }; use graph::slog::Logger; +use std::marker::PhantomData; use std::sync::Arc; pub struct SubgraphTriggerProcessor {} @@ -19,58 +21,40 @@ where C: Blockchain, T: RuntimeHostBuilder, { - async fn process_trigger( - &self, + async fn process_trigger<'a>( + &'a self, logger: &Logger, - hosts: &[Arc], + triggers: Vec>, block: &Arc, - trigger: &TriggerData, - mut state: BlockState, + mut state: BlockState, proof_of_indexing: &SharedProofOfIndexing, causality_region: &str, debug_fork: &Option>, subgraph_metrics: &Arc, - ) -> Result, MappingError> { + instrument: bool, + ) -> Result { let error_count = state.deterministic_errors.len(); - let mut host_mapping: Vec<(&T::Host, TriggerWithHandler>)> = vec![]; - - { - let _section = subgraph_metrics.stopwatch.start_section("match_and_decode"); - - for host in hosts { - let mapping_trigger = match host.match_and_decode(trigger, block, logger)? { - // Trigger matches and was decoded as a mapping trigger. - Some(mapping_trigger) => mapping_trigger, - - // Trigger does not match, do not process it. - None => continue, - }; - - host_mapping.push((host, mapping_trigger)); - } - } - - if host_mapping.is_empty() { + if triggers.is_empty() { return Ok(state); } - if let Some(proof_of_indexing) = proof_of_indexing { - proof_of_indexing - .borrow_mut() - .start_handler(causality_region); - } + proof_of_indexing.start_handler(causality_region); - for (host, mapping_trigger) in host_mapping { + for HostedTrigger { + host, + mapping_trigger, + } in triggers + { let start = Instant::now(); state = host .process_mapping_trigger( logger, - mapping_trigger.block_ptr(), mapping_trigger, state, proof_of_indexing.cheap_clone(), debug_fork, + instrument, ) .await?; let elapsed = start.elapsed().as_secs_f64(); @@ -85,18 +69,113 @@ where } } - if let Some(proof_of_indexing) = proof_of_indexing { - if state.deterministic_errors.len() != error_count { - assert!(state.deterministic_errors.len() == error_count + 1); + if state.deterministic_errors.len() != error_count { + assert!(state.deterministic_errors.len() == error_count + 1); - // If a deterministic error has happened, write a new - // ProofOfIndexingEvent::DeterministicError to the SharedProofOfIndexing. - proof_of_indexing - .borrow_mut() - .write_deterministic_error(logger, causality_region); - } + // If a deterministic error has happened, write a new + // ProofOfIndexingEvent::DeterministicError to the SharedProofOfIndexing. + proof_of_indexing.write_deterministic_error(logger, causality_region); } Ok(state) } } + +/// A helper for taking triggers as `TriggerData` (usually from the block +/// stream) and turning them into `HostedTrigger`s that are ready to run. +/// +/// The output triggers will be run in the order in which they are returned. +pub struct Decoder +where + C: Blockchain, + T: RuntimeHostBuilder, +{ + hook: C::DecoderHook, + _builder: PhantomData, +} + +impl Decoder +where + C: Blockchain, + T: RuntimeHostBuilder, +{ + pub fn new(hook: C::DecoderHook) -> Self { + Decoder { + hook, + _builder: PhantomData, + } + } +} + +impl> Decoder { + fn match_and_decode_inner<'a>( + &'a self, + logger: &Logger, + block: &Arc, + trigger: &TriggerData, + hosts: Box + Send + 'a>, + subgraph_metrics: &Arc, + ) -> Result>, MappingError> { + let mut host_mapping = vec![]; + + { + let _section = subgraph_metrics.stopwatch.start_section("match_and_decode"); + + for host in hosts { + let mapping_trigger = match host.match_and_decode(trigger, block, logger)? { + // Trigger matches and was decoded as a mapping trigger. + Some(mapping_trigger) => mapping_trigger, + + // Trigger does not match, do not process it. + None => continue, + }; + + host_mapping.push(HostedTrigger { + host, + mapping_trigger, + }); + } + } + Ok(host_mapping) + } + + pub(crate) fn match_and_decode<'a>( + &'a self, + logger: &Logger, + block: &Arc, + trigger: TriggerData, + hosts: Box + Send + 'a>, + subgraph_metrics: &Arc, + ) -> Result, MappingError> { + self.match_and_decode_inner(logger, block, &trigger, hosts, subgraph_metrics) + .map_err(|e| e.add_trigger_context(&trigger)) + .map(|hosted_triggers| RunnableTriggers { + trigger, + hosted_triggers, + }) + } + + pub(crate) async fn match_and_decode_many<'a, F>( + &'a self, + logger: &Logger, + block: &Arc, + triggers: impl Iterator>, + hosts_filter: F, + metrics: &Arc, + ) -> Result>, MappingError> + where + F: Fn(&TriggerData) -> Box + Send + 'a>, + { + let mut runnables = vec![]; + for trigger in triggers { + let hosts = hosts_filter(&trigger); + match self.match_and_decode(logger, block, trigger, hosts, metrics) { + Ok(runnable_triggers) => runnables.push(runnable_triggers), + Err(e) => return Err(e), + } + } + self.hook + .after_decode(logger, &block.ptr(), runnables, metrics) + .await + } +} diff --git a/core/src/subgraph_manifest.rs b/core/src/subgraph_manifest.rs new file mode 100644 index 00000000000..ddc01e0bc69 --- /dev/null +++ b/core/src/subgraph_manifest.rs @@ -0,0 +1,93 @@ +use graph::{ + cheap_clone::CheapClone as _, + components::{ + link_resolver::{LinkResolver, LinkResolverContext}, + store::{StoreError, SubgraphStore}, + }, + data::subgraph::DeploymentHash, +}; +use slog::{Logger, debug}; + +pub(super) async fn load_raw_subgraph_manifest( + logger: &Logger, + subgraph_store: &dyn SubgraphStore, + link_resolver: &dyn LinkResolver, + hash: &DeploymentHash, +) -> Result { + if let Some(raw_manifest) = + subgraph_store + .raw_manifest(hash) + .await + .map_err(|e| Error::LoadManifest { + hash: hash.cheap_clone(), + source: anyhow::Error::from(e), + })? + { + debug!(logger, "Loaded raw manifest from the subgraph store"); + return Ok(raw_manifest); + } + + debug!(logger, "Loading raw manifest using link resolver"); + + let link_resolver = + link_resolver + .for_manifest(&hash.to_string()) + .map_err(|e| Error::CreateLinkResolver { + hash: hash.cheap_clone(), + source: e, + })?; + + let file_bytes = link_resolver + .cat( + &LinkResolverContext::new(hash, logger), + &hash.to_ipfs_link(), + ) + .await + .map_err(|e| Error::LoadManifest { + hash: hash.cheap_clone(), + source: e, + })?; + + let raw_manifest: serde_yaml::Mapping = + serde_yaml::from_slice(&file_bytes).map_err(|e| Error::ParseManifest { + hash: hash.cheap_clone(), + source: e, + })?; + + subgraph_store + .set_raw_manifest_once(hash, &raw_manifest) + .await + .map_err(|e| Error::StoreManifest { + hash: hash.cheap_clone(), + source: e, + })?; + + Ok(raw_manifest) +} + +#[derive(Debug, thiserror::Error)] +pub(super) enum Error { + #[error("failed to create link resolver for '{hash}': {source:#}")] + CreateLinkResolver { + hash: DeploymentHash, + source: anyhow::Error, + }, + + #[error("failed to load manifest for '{hash}': {source:#}")] + LoadManifest { + hash: DeploymentHash, + source: anyhow::Error, + }, + + #[error("failed to parse manifest for '{hash}': {source:#}")] + ParseManifest { + hash: DeploymentHash, + source: serde_yaml::Error, + }, + + #[error("failed to store manifest for '{hash}': {source:#}")] + StoreManifest { + hash: DeploymentHash, + source: StoreError, + }, +} diff --git a/core/src/subgraph_provider.rs b/core/src/subgraph_provider.rs new file mode 100644 index 00000000000..7e479dd9495 --- /dev/null +++ b/core/src/subgraph_provider.rs @@ -0,0 +1,346 @@ +use std::{collections::HashMap, sync::Arc, time::Instant}; + +use graph::{ + amp, + cheap_clone::CheapClone as _, + components::{ + link_resolver::LinkResolver, + metrics::subgraph::SubgraphCountMetric, + store::{DeploymentLocator, SubgraphStore}, + subgraph::SubgraphInstanceManager, + }, + log::factory::LoggerFactory, +}; +use itertools::Itertools as _; +use parking_lot::RwLock; +use slog::{debug, error}; +use tokio_util::sync::CancellationToken; + +use super::subgraph_manifest; + +/// Starts and stops subgraph deployments. +/// +/// For each subgraph deployment, checks the subgraph processing kind +/// and finds the appropriate subgraph instance manager to handle the +/// processing of the subgraph deployment. +/// +/// This is required to support both trigger-based subgraphs and Amp-powered subgraphs, +/// which have separate runners. +pub struct SubgraphProvider { + logger_factory: LoggerFactory, + count_metrics: Arc, + subgraph_store: Arc, + link_resolver: Arc, + + /// Stops active subgraph start request tasks. + /// + /// When a subgraph deployment start request is processed, a background task is created + /// to load the subgraph manifest and determine the subgraph processing kind. The processing + /// kind is then used to find the appropriate subgraph instance manager. This token stops + /// all tasks that are still loading manifests or waiting for subgraphs to start. + cancel_token: CancellationToken, + + /// Contains the enabled subgraph instance managers. + /// + /// Only subgraphs for which there is an appropriate instance manager will be started. + instance_managers: SubgraphInstanceManagers, + + /// Maintains a list of started subgraphs with their processing kinds. + /// + /// Used to forward subgraph deployment stop requests to the appropriate subgraph instance manager. + assignments: SubgraphAssignments, +} + +impl SubgraphProvider { + /// Creates a new subgraph provider. + /// + /// # Arguments + /// - `logger_factory`: Creates loggers for each subgraph deployment start/stop request + /// - `count_metrics`: Tracks the number of started subgraph deployments + /// - `subgraph_store`: Loads subgraph manifests to determine the subgraph processing kinds + /// - `link_resolver`: Loads subgraph manifests to determine the subgraph processing kinds + /// - `cancel_token`: Stops active subgraph start request tasks + /// - `instance_managers`: Contains the enabled subgraph instance managers + pub fn new( + logger_factory: &LoggerFactory, + count_metrics: Arc, + subgraph_store: Arc, + link_resolver: Arc, + cancel_token: CancellationToken, + instance_managers: SubgraphInstanceManagers, + ) -> Self { + let logger = logger_factory.component_logger("SubgraphProvider", None); + let logger_factory = logger_factory.with_parent(logger.cheap_clone()); + + debug!(logger, "Creating subgraph provider"; + "enabled_subgraph_processing_kinds" => instance_managers.0.keys().join(", ") + ); + + Self { + logger_factory, + count_metrics, + subgraph_store, + link_resolver, + cancel_token, + instance_managers, + assignments: SubgraphAssignments::new(), + } + } + + /// Starts a subgraph deployment with the appropriate subgraph instance manager. + /// + /// Loads the subgraph manifest for the specified deployment locator, determines + /// the subgraph processing kind, finds the required instance manager, and forwards + /// the start request to that instance manager. Keeps the subgraph processing kind + /// in memory for handling the stop requests. + async fn assign_and_start_subgraph( + &self, + loc: DeploymentLocator, + stop_block: Option, + ) -> Result<(), Error> { + let logger = self.logger_factory.subgraph_logger(&loc); + + let raw_manifest = subgraph_manifest::load_raw_subgraph_manifest( + &logger, + &*self.subgraph_store, + &*self.link_resolver, + &loc.hash, + ) + .await + .map_err(|e| Error::LoadManifest { + loc: loc.cheap_clone(), + source: e, + })?; + + let subgraph_kind = SubgraphProcessingKind::from_manifest(&raw_manifest); + self.assignments.set_subgraph_kind(&loc, subgraph_kind); + + let Some(instance_manager) = self.instance_managers.get(&subgraph_kind) else { + return Err(Error::GetManager { loc, subgraph_kind }); + }; + + instance_manager.start_subgraph(loc, stop_block).await; + Ok(()) + } +} + +#[async_trait::async_trait] +impl SubgraphInstanceManager for SubgraphProvider { + async fn start_subgraph(self: Arc, loc: DeploymentLocator, stop_block: Option) { + let logger = self + .logger_factory + .subgraph_logger(&loc) + .new(slog::o!("method" => "start_subgraph")); + + if self.assignments.is_assigned(&loc) { + debug!(logger, "Subgraph is already started"); + return; + } + + self.count_metrics.deployment_count.inc(); + + let handle = tokio::spawn({ + let provider = self.cheap_clone(); + let loc = loc.cheap_clone(); + let start_instant = Instant::now(); + + async move { + debug!(logger, "Starting subgraph"); + + let fut = provider.assign_and_start_subgraph(loc, stop_block); + match provider.cancel_token.run_until_cancelled(fut).await { + Some(Ok(())) => { + debug!(logger, "Subgraph started"; + "duration_ms" => start_instant.elapsed().as_millis() + ); + } + Some(Err(e)) => { + error!(logger, "Subgraph failed to start"; + "e" => ?e + ); + } + None => { + debug!(logger, "Subgraph start cancelled"); + } + } + } + }); + + self.assignments.add( + loc, + SubgraphAssignment { + handle, + subgraph_kind: None, + }, + ) + } + + async fn stop_subgraph(&self, loc: DeploymentLocator) { + let logger = self + .logger_factory + .subgraph_logger(&loc) + .new(slog::o!("method" => "stop_subgraph")); + + debug!(logger, "Stopping subgraph"); + + let Some(SubgraphAssignment { + handle, + subgraph_kind, + }) = self.assignments.take(&loc) + else { + debug!(logger, "Subgraph is not started"); + return; + }; + + handle.abort(); + self.count_metrics.deployment_count.dec(); + + let Some(subgraph_kind) = subgraph_kind else { + debug!(logger, "Unknown subgraph kind"); + return; + }; + + let Some(instance_manager) = self.instance_managers.get(&subgraph_kind) else { + debug!(logger, "Missing instance manager"); + return; + }; + + instance_manager.stop_subgraph(loc).await; + debug!(logger, "Subgraph stopped"); + } +} + +/// Enumerates all possible errors of the subgraph provider. +#[derive(Debug, thiserror::Error)] +enum Error { + #[error("failed to load manifest for '{loc}': {source:#}")] + LoadManifest { + loc: DeploymentLocator, + source: subgraph_manifest::Error, + }, + + #[error("failed to get instance manager for '{loc}' with kind '{subgraph_kind}'")] + GetManager { + loc: DeploymentLocator, + subgraph_kind: SubgraphProcessingKind, + }, +} + +/// Contains a mapping of enabled subgraph instance managers by subgraph processing kinds. +/// +/// Before starting a subgraph, its processing kind is determined from the subgraph manifest. +/// Then, the appropriate instance manager is loaded from this mapping. +#[derive(Default)] +pub struct SubgraphInstanceManagers( + HashMap>, +); + +impl SubgraphInstanceManagers { + /// Creates a new empty subgraph instance manager mapping. + pub fn new() -> Self { + Self(HashMap::new()) + } + + /// Adds a new subgraph instance manager for all subgraphs of the specified processing kind. + pub fn add( + &mut self, + subgraph_kind: SubgraphProcessingKind, + instance_manager: Arc, + ) { + self.0.insert(subgraph_kind, instance_manager); + } + + /// Returns the subgraph instance manager for the specified processing kind. + pub fn get( + &self, + subgraph_kind: &SubgraphProcessingKind, + ) -> Option> { + self.0 + .get(subgraph_kind) + .map(|instance_manager| instance_manager.cheap_clone()) + } +} + +/// Enumerates the supported subgraph processing kinds. +/// +/// Subgraphs may have different processing requirements, and this enum helps to map them +/// to the appropriate instance managers. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, strum::Display)] +#[strum(serialize_all = "snake_case")] +pub enum SubgraphProcessingKind { + /// Represents trigger-based subgraphs. + Trigger, + + /// Represents Amp-powered subgraphs. + Amp, +} + +impl SubgraphProcessingKind { + /// Determines the subgraph processing kind from the subgraph manifest. + fn from_manifest(raw_manifest: &serde_yaml::Mapping) -> Self { + use serde_yaml::Value; + + let is_amp_manifest = raw_manifest + .get("dataSources") + .and_then(Value::as_sequence) + .and_then(|seq| { + seq.iter() + .filter_map(Value::as_mapping) + .filter_map(|map| map.get("kind")) + .filter_map(Value::as_str) + .find(|kind| *kind == amp::manifest::DataSource::KIND) + }) + .is_some(); + + if is_amp_manifest { + return Self::Amp; + } + + Self::Trigger + } +} + +/// Maintains a list of started subgraph deployments with details required for stopping them. +struct SubgraphAssignments(RwLock>); + +impl SubgraphAssignments { + /// Creates a new empty list of started subgraph deployments. + fn new() -> Self { + Self(RwLock::new(HashMap::new())) + } + + /// Adds a new subgraph deployment to the list of started subgraph deployments. + fn add(&self, loc: DeploymentLocator, subgraph_assignment: SubgraphAssignment) { + self.0.write().insert(loc, subgraph_assignment); + } + + /// Updates the started subgraph deployment with the specified subgraph processing kind. + fn set_subgraph_kind(&self, loc: &DeploymentLocator, subgraph_kind: SubgraphProcessingKind) { + if let Some(subgraph_assignment) = self.0.write().get_mut(loc) { + subgraph_assignment.subgraph_kind = Some(subgraph_kind); + } + } + + /// Checks if the subgraph deployment is started. + fn is_assigned(&self, loc: &DeploymentLocator) -> bool { + self.0.read().contains_key(loc) + } + + /// Removes the subgraph deployment from the list of started subgraph deployments and returns its details. + fn take(&self, loc: &DeploymentLocator) -> Option { + self.0.write().remove(loc) + } +} + +/// Contains the details of a started subgraph deployment. +struct SubgraphAssignment { + /// The handle to the background task that starts this subgraph deployment. + handle: tokio::task::JoinHandle<()>, + + /// The subgraph processing kind of this subgraph deployment. + /// + /// Used to get the appropriate subgraph instance manager to forward the stop request to. + /// + /// Set to `None` until the subgraph manifest is loaded and parsed. + subgraph_kind: Option, +} diff --git a/core/tests/README.md b/core/tests/README.md new file mode 100644 index 00000000000..261623bcccf --- /dev/null +++ b/core/tests/README.md @@ -0,0 +1,5 @@ +Put integration tests for this crate into `store/test-store/tests/core`. +This avoids cyclic dev-dependencies which make rust-analyzer nearly +unusable. Once [this +issue](https://github.com/rust-lang/rust-analyzer/issues/14167) has been +fixed, we can move tests back here diff --git a/docker/Dockerfile b/docker/Dockerfile index 8c0a8e19919..29d8e142db0 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -4,7 +4,7 @@ # by running something like the following # docker build --target STAGE -f docker/Dockerfile . -FROM golang:bullseye as envsubst +FROM golang:bookworm AS envsubst # v1.2.0 ARG ENVSUBST_COMMIT_SHA=16035fe3571ad42c7796bf554f978bb2df64231b @@ -13,7 +13,7 @@ ARG ENVSUBST_COMMIT_SHA=16035fe3571ad42c7796bf554f978bb2df64231b RUN go install github.com/a8m/envsubst/cmd/envsubst@$ENVSUBST_COMMIT_SHA \ && strip -g /go/bin/envsubst -FROM rust:bullseye as graph-node-build +FROM rust:bookworm AS graph-node-build ARG COMMIT_SHA=unknown ARG REPO_NAME=unknown @@ -22,20 +22,18 @@ ARG TAG_NAME=unknown ADD . /graph-node -RUN apt-get update \ - && apt-get install -y cmake protobuf-compiler && \ - cd /graph-node && \ - RUSTFLAGS="-g" cargo build --release --package graph-node \ +RUN apt-get update && apt-get install -y cmake protobuf-compiler + +RUN cd /graph-node \ + && RUSTFLAGS="-g -C force-frame-pointers=yes" cargo build --release --package graph-node \ && cp target/release/graph-node /usr/local/bin/graph-node \ && cp target/release/graphman /usr/local/bin/graphman \ - # Reduce the size of the layer by removing unnecessary files. && cargo clean \ - && objcopy --only-keep-debug /usr/local/bin/graph-node /usr/local/bin/graph-node.debug \ - && strip -g /usr/local/bin/graph-node \ - && strip -g /usr/local/bin/graphman \ - && cd /usr/local/bin \ - && objcopy --add-gnu-debuglink=graph-node.debug graph-node \ - && echo "REPO_NAME='$REPO_NAME'" > /etc/image-info \ + && cp /usr/local/bin/graph-node /usr/local/bin/graph-node-debug \ + && cp /usr/local/bin/graphman /usr/local/bin/graphman-debug \ + && strip -g /usr/local/bin/graph-node /usr/local/bin/graphman + +RUN echo "REPO_NAME='$REPO_NAME'" > /etc/image-info \ && echo "TAG_NAME='$TAG_NAME'" >> /etc/image-info \ && echo "BRANCH_NAME='$BRANCH_NAME'" >> /etc/image-info \ && echo "COMMIT_SHA='$COMMIT_SHA'" >> /etc/image-info \ @@ -43,48 +41,41 @@ RUN apt-get update \ && echo "RUST_VERSION='$(rustc --version)'" >> /etc/image-info \ && echo "CARGO_DEV_BUILD='$CARGO_DEV_BUILD'" >> /etc/image-info -# Debug image to access core dumps -FROM graph-node-build as graph-node-debug -RUN apt-get update \ - && apt-get install -y curl gdb postgresql-client - -COPY docker/Dockerfile /Dockerfile -COPY docker/bin/* /usr/local/bin/ - # The graph-node runtime image with only the executable -FROM debian:bullseye-slim as graph-node -ENV RUST_LOG "" -ENV GRAPH_LOG "" -ENV EARLY_LOG_CHUNK_SIZE "" -ENV ETHEREUM_RPC_PARALLEL_REQUESTS "" -ENV ETHEREUM_BLOCK_CHUNK_SIZE "" - -ENV postgres_host "" -ENV postgres_user "" -ENV postgres_pass "" -ENV postgres_db "" +FROM debian:bookworm-20241111-slim AS graph-node +ENV RUST_LOG="" +ENV GRAPH_LOG="" +ENV EARLY_LOG_CHUNK_SIZE="" +ENV ETHEREUM_RPC_PARALLEL_REQUESTS="" +ENV ETHEREUM_BLOCK_CHUNK_SIZE="" + +ENV postgres_host="" +ENV postgres_user="" +ENV postgres_pass="" +ENV postgres_db="" +ENV postgres_args="sslmode=prefer" # The full URL to the IPFS node -ENV ipfs "" +ENV ipfs="" # The etherum network(s) to connect to. Set this to a space-separated # list of the networks where each entry has the form NAME:URL -ENV ethereum "" +ENV ethereum="" # The role the node should have, one of index-node, query-node, or # combined-node -ENV node_role "combined-node" +ENV node_role="combined-node" # The name of this node -ENV node_id "default" +ENV node_id="default" # The ethereum network polling interval (in milliseconds) -ENV ethereum_polling_interval "" +ENV ethereum_polling_interval="" # The location of an optional configuration file for graph-node, as # described in ../docs/config.md # Using a configuration file is experimental, and the file format may # change in backwards-incompatible ways -ENV GRAPH_NODE_CONFIG "" +ENV GRAPH_NODE_CONFIG="" # Disable core dumps; this is useful for query nodes with large caches. Set # this to anything to disable coredumps (via 'ulimit -c 0') -ENV disable_core_dumps "" +ENV disable_core_dumps="" # HTTP port EXPOSE 8000 @@ -96,7 +87,7 @@ EXPOSE 8020 EXPOSE 8030 RUN apt-get update \ - && apt-get install -y libpq-dev ca-certificates netcat + && apt-get install -y libpq-dev ca-certificates netcat-openbsd ADD docker/wait_for docker/start /usr/local/bin/ COPY --from=graph-node-build /usr/local/bin/graph-node /usr/local/bin/graphman /usr/local/bin/ @@ -104,3 +95,13 @@ COPY --from=graph-node-build /etc/image-info /etc/image-info COPY --from=envsubst /go/bin/envsubst /usr/local/bin/ COPY docker/Dockerfile /Dockerfile CMD ["start"] + +# Debug and profiling image +# Build with: docker build --target graph-node-debug -f docker/Dockerfile . +FROM graph-node AS graph-node-debug +COPY --from=graph-node-build /usr/local/bin/graph-node-debug /usr/local/bin/graph-node +COPY --from=graph-node-build /usr/local/bin/graphman-debug /usr/local/bin/graphman +RUN apt-get update \ + && apt-get install -y curl gdb postgresql-client linux-perf procps binutils +COPY docker/Dockerfile /Dockerfile +COPY docker/bin/* /usr/local/bin/ diff --git a/docker/README.md b/docker/README.md index 326a3535e9f..6ea02f70b0f 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,34 +1,9 @@ -# Graph Node Docker Image +# Running prebuilt `graph-node` images -Preconfigured Docker image for running a Graph Node. - -## Usage - -```sh -docker run -it \ - -e postgres_host= \ - -e postgres_port= \ - -e postgres_user= \ - -e postgres_pass= \ - -e postgres_db= \ - -e ipfs=: \ - -e ethereum=: \ - graphprotocol/graph-node:latest -``` - -### Example usage - -```sh -docker run -it \ - -e postgres_host=host.docker.internal \ - -e postgres_port=5432 \ - -e postgres_user=graph-node \ - -e postgres_pass=oh-hello \ - -e postgres_db=graph-node \ - -e ipfs=host.docker.internal:5001 \ - -e ethereum=mainnet:http://localhost:8545/ \ - graphprotocol/graph-node:latest -``` +You can run the `graph-node` docker image either in a [complete +setup](#docker-compose) controlled by Docker Compose, or, if you already +have an IPFS and Postgres server, [by +itself](#running-with-existing-ipfs-and-postgres). ## Docker Compose @@ -59,7 +34,7 @@ can access these via: - `postgresql://graph-node:let-me-in@localhost:5432/graph-node` Once this is up and running, you can use -[`graph-cli`](https://github.com/graphprotocol/graph-cli) to create and +[`graph-cli`](https://github.com/graphprotocol/graph-tooling/tree/main/packages/cli) to create and deploy your subgraph to the running Graph Node. ### Running Graph Node on an Macbook M1 @@ -77,3 +52,17 @@ docker rmi graphprotocol/graph-node:latest # Tag the newly created image docker tag graph-node graphprotocol/graph-node:latest ``` + +## Running with existing IPFS and Postgres + +```sh +docker run -it \ + -e postgres_host= \ + -e postgres_port= \ + -e postgres_user= \ + -e postgres_pass= \ + -e postgres_db= \ + -e ipfs=: \ + -e ethereum=: \ + graphprotocol/graph-node:latest +``` diff --git a/docker/cloudbuild.yaml b/docker/cloudbuild.yaml index 39cf2856e62..0bf800cddad 100644 --- a/docker/cloudbuild.yaml +++ b/docker/cloudbuild.yaml @@ -1,5 +1,5 @@ options: - machineType: "N1_HIGHCPU_32" + machineType: "E2_HIGHCPU_32" timeout: 1800s steps: - name: 'gcr.io/cloud-builders/docker' diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 742de12649d..c78c2eb2194 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -22,11 +22,11 @@ services: ethereum: 'mainnet:http://host.docker.internal:8545' GRAPH_LOG: info ipfs: - image: ipfs/go-ipfs:v0.10.0 + image: ipfs/kubo:v0.17.0 ports: - '5001:5001' volumes: - - ./data/ipfs:/data/ipfs + - ./data/ipfs:/data/ipfs:Z postgres: image: postgres ports: @@ -34,7 +34,8 @@ services: command: [ "postgres", - "-cshared_preload_libraries=pg_stat_statements" + "-cshared_preload_libraries=pg_stat_statements", + "-cmax_connections=200" ] environment: POSTGRES_USER: graph-node @@ -46,4 +47,4 @@ services: PGDATA: "/var/lib/postgresql/data" POSTGRES_INITDB_ARGS: "-E UTF8 --locale=C" volumes: - - ./data/postgres:/var/lib/postgresql/data + - ./data/postgres:/var/lib/postgresql/data:Z diff --git a/docker/start b/docker/start index bbeabd166a3..f1e4106363e 100755 --- a/docker/start +++ b/docker/start @@ -42,6 +42,7 @@ wait_for_ipfs() { then [ "$proto" = "https" ] && port=443 || port=80 fi + echo "Waiting for IPFS ($host:$port)" wait_for "$host:$port" -t 120 else echo "invalid IPFS URL: $1" @@ -64,9 +65,10 @@ run_graph_node() { else unset GRAPH_NODE_CONFIG postgres_port=${postgres_port:-5432} - postgres_url="postgresql://$postgres_user:$postgres_pass@$postgres_host:$postgres_port/$postgres_db?sslmode=prefer" + postgres_url="postgresql://$postgres_user:$postgres_pass@$postgres_host:$postgres_port/$postgres_db?$postgres_args" wait_for_ipfs "$ipfs" + echo "Waiting for Postgres ($postgres_host:$postgres_port)" wait_for "$postgres_host:$postgres_port" -t 120 sleep 5 diff --git a/docker/tag.sh b/docker/tag.sh index 032ab54417b..1abafa95afa 100644 --- a/docker/tag.sh +++ b/docker/tag.sh @@ -25,4 +25,7 @@ tag_and_push "$SHORT_SHA" # Builds for tags vN.N.N become the 'latest' [[ "$TAG_NAME" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] && tag_and_push latest +# If the build is from the master branch, tag it as 'nightly' +[ "$BRANCH_NAME" = "master" ] && tag_and_push nightly + exit 0 diff --git a/docs/aggregations.md b/docs/aggregations.md new file mode 100644 index 00000000000..3e90d7ee62f --- /dev/null +++ b/docs/aggregations.md @@ -0,0 +1,253 @@ +# Timeseries and aggregations + +_This feature is available from spec version 1.1.0 onwards_ + +## Overview + +Aggregations are declared in the subgraph schema through two types: one that +stores the raw data points for the time series, and one that defines how raw +data points are to be aggregated. A very simple aggregation can be declared like this: + +```graphql +type Data @entity(timeseries: true) { + id: Int8! + timestamp: Timestamp! + price: BigDecimal! +} + +type Stats @aggregation(intervals: ["hour", "day"], source: "Data") { + id: Int8! + timestamp: Timestamp! + sum: BigDecimal! @aggregate(fn: "sum", arg: "price") +} +``` + +Mappings for this schema will add data points by creating `Data` entities +just as they would for normal entities. `graph-node` will then automatically +populate the `Stats` aggregations whenever a given hour or day ends. + +The type for the raw data points is defined with an `@entity(timeseries: +true)` annotation. Timeseries types are immutable, and must have an `id` +field and a `timestamp` field. The `id` must be of type `Int8` and is set +automatically so that ids are increasing in insertion order. The `timestamp` +is set automatically by `graph-node` to the timestamp of the current block; +if mappings set this field, it is silently overridden when the entity is +saved. + +Aggregations are declared with an `@aggregation` annotation instead of an +`@entity` annotation. They must have an `id` field and a `timestamp` field. +Both fields are set automatically by `graph-node`. The `timestamp` is set to +the beginning of the time period that that aggregation instance represents, +for example, to the beginning of the hour for an hourly aggregation. The +`id` field is set to the `id` of one of the raw data points that went into +the aggregation. Which one is chosen is not specified and should not be +relied on. + +**TODO**: figure out whether we should just automatically add `id` and +`timestamp` and have validation just check that these fields don't exist + +Aggregations can also contain _dimensions_, which are fields that are not +aggregated but are used to group the data points. For example, the +`TokenStats` aggregation below has a `token` field that is used to group the +data points by token: + +```graphql +# Normal entity +type Token @entity { .. } + +# Raw data points +type TokenData @entity(timeseries: true) { + id: Bytes! + timestamp: Timestamp! + token: Token! + amount: BigDecimal! + priceUSD: BigDecimal! +} + +# Aggregations over TokenData +type TokenStats @aggregation(intervals: ["hour", "day"], source: "TokenData") { + id: Int8! + timestamp: Timestamp! + token: Token! + totalVolume: BigDecimal! @aggregate(fn: "sum", arg: "amount") + priceUSD: BigDecimal! @aggregate(fn: "last", arg: "priceUSD") + count: Int8! @aggregate(fn: "count", cumulative: true) +} +``` + +Fields in aggregations without the `@aggregate` directive are called +_dimensions_, and fields with the `@aggregate` directive are called +_aggregates_. A timeseries type really represents many timeseries, one for +each combination of values for the dimensions. + +The same timeseries can be used for multiple aggregations. For example, the +`Stats` aggregation could also be formed by aggregating over the `TokenData` +timeseries. Since `Stats` doesn't have a `token` dimension, all aggregates +will be formed across all tokens. + +Each `@aggregate` by default starts at 0 for each new bucket and therefore +just aggregates over the time interval for the bucket. The `@aggregate` +directive also accepts a boolean flag `cumulative` that indicates whether +the aggregation should be cumulative. Cumulative aggregations aggregate over +the entire timeseries up to the end of the time interval for the bucket. + +## Specification + +### Timeseries + +A timeseries is an entity type with the annotation `@entity(timeseries: +true)`. It must have an `id` attribute of type `Int8` and a `timestamp` +attribute of type `Timestamp`. It must not also be annotated with +`immutable: false` as timeseries are always immutable. + +### Aggregations + +An aggregation is defined with an `@aggregation` annotation. The annotation +must have two arguments: + +- `intervals`: a non-empty array of intervals; currently, only `hour` and `day` + are supported +- `source`: the name of a timeseries type. Aggregates are computed based on + the attributes of the timeseries type. + +The aggregation type must have an `id` attribute of type `Int8` and a +`timestamp` attribute of type `Timestamp`. + +The aggregation type must have at least one attribute with the `@aggregate` +annotation. These attributes must be of a numeric type (`Int`, `Int8`, +`BigInt`, or `BigDecimal`) The annotation must have two arguments: + +- `fn`: the name of an aggregation function +- `arg`: the name of an attribute in the timeseries type, or an expression + using only constants and attributes of the timeseries type + +#### Aggregation functions + +The following aggregation functions are currently supported: + +| Name | Description | +| ------- | ----------------- | +| `sum` | Sum of all values | +| `count` | Number of values | +| `min` | Minimum value | +| `max` | Maximum value | +| `first` | First value | +| `last` | Last value | + +The `first` and `last` aggregation function calculate the first and last +value in an interval by sorting the data by `id`; `graph-node` enforces +correctness here by automatically setting the `id` for timeseries entities. + +#### Aggregation expressions + +The `arg` can be the name of any attribute in the timeseries type, or an +expression using only constants and attributes of the timeseries type such +as `price * amount` or `greatest(amount0, amount1)`. Expressions use SQL +syntax and support a subset of builtin SQL functions, operators, and other +constructs. + +Supported operators are `+`, `-`, `*`, `/`, `%`, `^`, `=`, `!=`, `<`, `<=`, +`>`, `>=`, `<->`, `and`, `or`, and `not`. In addition the operators `is +[not] {null|true|false}`, and `is [not] distinct from` are supported. + +The supported SQL functions are the [math +functions](https://www.postgresql.org/docs/current/functions-math.html) +`abs`, `ceil`, `ceiling`, `div`, `floor`, `gcd`, `lcm`, `mod`, `power`, +`sign`, and the [conditional +functions](https://www.postgresql.org/docs/current/functions-conditional.html) +`coalesce`, `nullif`, `greatest`, and `least`. + +The +[statement](https://www.postgresql.org/docs/current/functions-conditional.html#FUNCTIONS-CASE) +`case when .. else .. end` is also supported. + +Some examples of valid expressions, assuming the underlying timeseries +contains the mentioned fields: + +- Aggregate the value of a token: `@aggregate(fn: "sum", arg: "priceUSD * amount")` +- Aggregate the maximum positive amount of two different amounts: + `@aggregate(fn: "max", arg: "greatest(amount0, amount1, 0)")` +- Conditionally sum an amount: `@aggregate(fn: "sum", arg: "case when amount0 > amount1 then amount0 else 0 end")` + +## Querying + +We create a toplevel query field for each aggregation. That query field +accepts the following arguments: + +- For each dimension, an optional filter to test for equality of that + dimension +- A mandatory `interval` +- An optional `current` to indicate whether to include the current, + partially filled bucket in the response. Can be either `exclude` (the + default) or `include` +- Optional `timestamp_{gte|gt|lt|lte|eq|in}` filters to restrict the range + of timestamps to return. The timestamp to filter by must be a string + containing microseconds since the epoch. The value `"1704164640000000"` + corresponds to `2024-01-02T03:04Z` +- Timeseries are sorted by `timestamp` and `id` in descending order by + default + +```graphql +token_stats(interval: "hour", + current: exclude, + where: { + token: "0x1234", + timestamp_gte: 1234567890, + timestamp_lt: 1234567890 }) { + id + timestamp + token + totalVolume + avgVolume +} +``` + +### Current Bucket + +By default, aggregation queries return only completed, rolled-up buckets +(`current: exclude`). These are buckets whose time interval has ended and +whose data has been fully aggregated by `graph-node`'s rollup process. + +Setting `current: include` adds an additional, partially filled bucket that +is computed on-the-fly from the unrolled timeseries data that has been +inserted since the last rollup. This current bucket aggregates raw data +points from the source timeseries table that have not yet been rolled up +into the aggregation table. It covers the time period from the end of the +last completed bucket up to the most recent data point. + +- `current: exclude` (default) — return only completed, rolled-up buckets +- `current: include` — also return the in-progress bucket computed from + unrolled source data + +The current bucket is useful when you need near-real-time aggregation data +without waiting for the next rollup cycle to complete. + +#### Nested Aggregation Queries + +The `current` argument also works on nested aggregation fields accessed +through a parent entity. For example, if a `Token` entity has a derived +aggregation field `tokenStats`, you can query the current bucket for each +token: + +```graphql +{ + tokens { + id + name + tokenStats(interval: "hour", current: include) { + timestamp + totalVolume + } + } +} +``` + +This returns both the completed rolled-up hourly buckets and the current +in-progress bucket for each token's stats. + +#### Limitations + +Current bucket support for nested aggregation fields is only available when +the field references a single aggregation type. It is not supported when the +aggregation field is accessed through an interface with multiple +implementations. diff --git a/docs/amp-powered-subgraphs.md b/docs/amp-powered-subgraphs.md new file mode 100644 index 00000000000..60bccdde449 --- /dev/null +++ b/docs/amp-powered-subgraphs.md @@ -0,0 +1,582 @@ +# Amp-powered subgraphs + +> [!WARNING] +> This feature is experimental and may change in future releases. + +> [!NOTE] +> This feature is available starting from spec version `1.5.0` + +Amp-powered subgraphs are a new kind of subgraphs with SQL data sources that query and index data from the Amp servers. +They are significantly more efficient than the standard subgraphs, and the indexing time can be reduced from days and weeks, +to minutes and hours in most cases. + +## Prerequisites + +To enable Amp-powered subgraphs, the `GRAPH_AMP_FLIGHT_SERVICE_ADDRESS` ENV variable must be set to a valid Amp Flight gRPC service address. + +Additionally, if authentication is required for the Amp Flight gRPC service, the `GRAPH_AMP_FLIGHT_SERVICE_TOKEN` ENV variable must contain a valid authentication token. + +## Subgraph manifest + +Amp-powered subgraphs introduce a new structure for defining Amp subgraph data sources within the manifest. + +### Spec version + +The minimum spec version for Amp-powered subgraphs is `1.5.0`. + +

+Example YAML: + +```diff ++ specVersion: 1.5.0 + dataSources: + - kind: amp + name: Transfers + network: ethereum-mainnet + source: + dataset: edgeandnode/ethereum_mainnet + tables: + - blocks + - transactions + transformer: + apiVersion: 0.0.1 + tables: + - name: Transfer + file: +``` + +
+ +### Data source structure + +### `kind` + +Every Amp data source must have the `kind` set to `amp`, and Amp-powered subgraphs must contain only Amp data sources. +This is used to assign the subgraph to the appropriate indexing process. + +
+Example YAML: + +```diff + specVersion: 1.5.0 ++ dataSources: ++ - kind: amp + name: Transfers + network: ethereum-mainnet + source: + dataset: edgeandnode/ethereum_mainnet + tables: + - blocks + - transactions + transformer: + apiVersion: 0.0.1 + tables: + - name: Transfer + file: +``` + +
+ +### `name` + +Every Amp data source must have the `name` set to a non-empty string, containing only numbers, letters, hypens, or underscores. +This name is used for observability purposes and to identify progress and potential errors produced by the data source. + +
+Example YAML: + +```diff + specVersion: 1.5.0 ++ dataSources: + - kind: amp ++ name: Transfers + network: ethereum-mainnet + source: + dataset: edgeandnode/ethereum_mainnet + tables: + - blocks + - transactions + transformer: + apiVersion: 0.0.1 + tables: + - name: Transfer + file: +``` + +
+ +### `network` + +Every Amp data source must have the `network` field set to a valid network name. +This is used to validate that the SQL queries for this data source produce results for the expected network. + +> [!NOTE] +> Currently, the SQL queries are required to produce results for a single network in order to maintain compatibility with non-Amp subgraphs. + +
+Example YAML: + +```diff + specVersion: 1.5.0 ++ dataSources: + - kind: amp + name: Transfers ++ network: ethereum-mainnet + source: + dataset: edgeandnode/ethereum_mainnet + tables: + - blocks + - transactions + transformer: + apiVersion: 0.0.1 + tables: + - name: Transfer + file: +``` + +
+ +### `source` + +Every Amp data source must have a valid `source` that describes the behavior of SQL queries from this data source. + +### `source.dataset` + +Contains the name of the dataset that can be queried by SQL queries in this data source. +This is used to validate that the SQL queries for this data source only query the expected dataset. + +
+Example YAML: + +```diff + specVersion: 1.5.0 ++ dataSources: + - kind: amp + name: Transfers + network: ethereum-mainnet ++ source: ++ dataset: edgeandnode/ethereum_mainnet + tables: + - blocks + - transactions + transformer: + apiVersion: 0.0.1 + tables: + - name: Transfer + file: +``` + +
+ +### `source.tables` + +Contains the names of the tables that can be queried by SQL queries in this data source. +This is used to validate that the SQL queries for this data source only query the expected tables. + +
+Example YAML: + +```diff + specVersion: 1.5.0 ++ dataSources: + - kind: amp + name: Transfers + network: ethereum-mainnet ++ source: + dataset: edgeandnode/ethereum_mainnet ++ tables: ++ - blocks ++ - transactions + transformer: + apiVersion: 0.0.1 + tables: + - name: Transfer + file: +``` + +
+ +### Optional `source.address` + +Contains the contract address with which SQL queries in the data source interact. + +Enables SQL query reuse through `sg_source_address()` calls instead of hard-coding the contract address. +SQL queries resolve `sg_source_address()` calls to this contract address. + +
+Example YAML: + +```diff + specVersion: 1.5.0 ++ dataSources: + - kind: amp + name: Transfers + network: ethereum-mainnet ++ source: ++ address: "0xc944E90C64B2c07662A292be6244BDf05Cda44a7" + dataset: edgeandnode/ethereum_mainnet + tables: + - blocks + - transactions + transformer: + apiVersion: 0.0.1 + tables: + - name: Transfer + file: +``` + +
+ +### Optional `source.startBlock` + +Contains the minimum block number that SQL queries in the data source can query. +This is used as a starting point for the indexing process. + +_When not provided, defaults to block number `0`._ + +
+Example YAML: + +```diff + specVersion: 1.5.0 ++ dataSources: + - kind: amp + name: Transfers + network: ethereum-mainnet ++ source: ++ startBlock: 11446769 + dataset: edgeandnode/ethereum_mainnet + tables: + - blocks + - transactions + transformer: + apiVersion: 0.0.1 + tables: + - name: Transfer + file: +``` + +
+ +### Optional `source.endBlock` + +Contains the maximum block number that SQL queries in the data source can query. +Reaching this block number will complete the indexing process. + +_When not provided, defaults to the maximum possible block number._ + +
+Example YAML: + +```diff + specVersion: 1.5.0 ++ dataSources: + - kind: amp + name: Transfers + network: ethereum-mainnet ++ source: ++ endBlock: 23847939 + dataset: edgeandnode/ethereum_mainnet + tables: + - blocks + - transactions + transformer: + apiVersion: 0.0.1 + tables: + - name: Transfer + file: +``` + +
+ +### `transformer` + +Every Amp data source must have a valid `transformer` that describes the transformations of source tables indexed by the Amp-powered subgraph. + +### `transformer.apiVersion` + +Represents the version of this transformer. Each version may contain a different set of features. + +> [!NOTE] +> Currently, only the version `0.0.1` is available. + +
+Example YAML: + +```diff + specVersion: 1.5.0 ++ dataSources: + - kind: amp + name: Transfers + network: ethereum-mainnet + source: + endBlock: 23847939 + dataset: edgeandnode/ethereum_mainnet + tables: + - blocks + - transactions ++ transformer: ++ apiVersion: 0.0.1 + tables: + - name: Transfers + file: +``` + +
+ +### Optional `transformer.abis` + +Contains a list of ABIs that SQL queries can reference to extract event signatures. + +Enables the use of `sg_event_signature('CONTRACT_NAME', 'EVENT_NAME')` calls in the +SQL queries which are resolved to full event signatures based on this list. + +_When not provided, defaults to an empty list._ + +
+Example YAML: + +```diff + specVersion: 1.5.0 ++ dataSources: + - kind: amp + name: Transfers + network: ethereum-mainnet + source: + endBlock: 23847939 + dataset: edgeandnode/ethereum_mainnet + tables: + - blocks + - transactions ++ transformer: ++ abis: ++ - name: ERC721 # The name of the contract ++ file: + apiVersion: 0.0.1 + tables: + - name: Transfer + file: +``` + +
+ +### `transformer.tables` + +Contains a list of transformed tables that extract data from source tables into subgraph entities. + +### Transformer table structure + +### `transformer.tables[i].name` + +Represents the name of the transformed table. Must reference a valid entity name from the subgraph schema. + +
+Example: + +**GraphQL schema:** + +```graphql +type Block @entity(immutable: true) { + # .. entity fields ... +} +``` + +**YAML manifest:** + +```diff + specVersion: 1.5.0 ++ dataSources: + - kind: amp + name: Blocks + network: ethereum-mainnet + source: + endBlock: 23847939 + dataset: edgeandnode/ethereum_mainnet + tables: + - blocks + - transactions ++ transformer: + apiVersion: 0.0.1 ++ tables: ++ - name: Block + file: +``` + +
+ +### `transformer.tables[i].query` + +Contains an inline SQL query that executes on the Amp server. +This is useful for simple SQL queries like `SELECT * FROM "edgeandnode/ethereum_mainnet".blocks;`. +For more complex cases, a separate file containing the SQL query can be used in the `file` field. + +The data resulting from this SQL query execution transforms into subgraph entities. + +_When not provided, the `file` field is used instead._ + +
+Example YAML: + +```diff + specVersion: 1.5.0 ++ dataSources: + - kind: amp + name: Blocks + network: ethereum-mainnet + source: + endBlock: 23847939 + dataset: edgeandnode/ethereum_mainnet + tables: + - blocks + - transactions ++ transformer: + apiVersion: 0.0.1 ++ tables: + - name: Block ++ query: SELECT * FROM "edgeandnode/ethereum_mainnet".blocks; +``` + +
+ +### `transformer.tables[i].file` + +Contains the IPFS link to the SQL query that executes on the Amp server. + +The data resulting from this SQL query execution transforms into subgraph entities. + +_Ignored when the `query` field is provided._ +_When not provided, the `query` field is used instead._ + +
+Example YAML: + +```diff + specVersion: 1.5.0 ++ dataSources: + - kind: amp + name: Blocks + network: ethereum-mainnet + source: + endBlock: 23847939 + dataset: edgeandnode/ethereum_mainnet + tables: + - blocks + - transactions ++ transformer: + apiVersion: 0.0.1 ++ tables: + - name: Block ++ file: +``` + +
+ +### Amp-powered subgraph examples + +Complete examples on how to create, deploy and query Amp-powered subgraphs are available in a separate repository: +https://github.com/edgeandnode/amp-subgraph-examples + +## SQL query requirements + +### Names + +The names of tables, columns, and aliases must not start with `amp_` as this +prefix is reserved for internal use. + +### Block numbers + +Every SQL query in Amp-powered subgraphs must return the block number for every row. +This is required because subgraphs rely on this information for storing subgraph entities. + +Graph-node will look for block numbers in the following columns: +`_block_num`, `block_num`, `blockNum`, `block`, `block_number`, `blockNumber`. + +Example SQL query: `SELECT _block_num, /* .. other projections .. */ FROM "edgeandnode/ethereum_mainnet".blocks;` + +### Block hashes + +Every SQL query in Amp-powered subgraphs is expected to return the block hash for every row. +This is required because subgraphs rely on this information for storing subgraph entities. + +When a SQL query does not have the block hash projection, graph-node will attempt to get it from the +source tables specified in the subgraph manifest. + +Graph-node will look for block hashes in the following columns: +`hash`, `block_hash`, `blockHash`. + +Example SQL query: `SELECT hash, /* .. other projections .. */ FROM "edgeandnode/ethereum_mainnet".blocks;` + +> [!NOTE] +> If a table does not contain the block hash column, it can be retrieved by joining that table with another that contains the column on the `_block_num` column. + +### Block timestamps + +> [!NOTE] +> Only required for Amp-powered subgraphs that use subgraph aggregations. + +Every SQL query in Amp-powered subgraphs is expected to return the block timestamps for every row. +This is required because subgraphs rely on this information for storing subgraph entities. + +When a SQL query does not have the block timestamps projection, graph-node will attempt to get it from the +source tables specified in the subgraph manifest. + +Graph-node will look for block timestamps in the following columns: +`timestamp`, `block_timestamp`, `blockTimestamp`. + +Example SQL query: `SELECT timestamp, /* .. other projections .. */ FROM "edgeandnode/ethereum_mainnet".blocks;` + +> [!NOTE] +> If a table does not contain the block timestamp column, it can be retrieved by joining that table with another that contains the column on the `_block_num` column. + +## Type conversions + +Amp core SQL data types are converted intuitively to compatible subgraph entity types. + +## Schema generation + +Amp-powered subgraphs support the generation of GraphQL schemas based on the schemas of SQL queries referenced in the subgraph manifest. +This is useful when indexing entities that do not rely on complex relationships, such as contract events. + +The generated subgraph entities are immutable. + +To enable schema generation, simply remove the `schema` field from the subgraph manifest. + +> [!NOTE] +> For more flexibility and control over the schema, a manually created GraphQL schema is preferred. + +## Aggregations + +Amp-powered subgraphs fully support the subgraph aggregations feature. +This allows having complex aggregations on top of data indexed from the Amp servers. + +For more information on using the powerful subgraph aggregations feature, +refer to the [documentation](https://github.com/graphprotocol/graph-node/blob/master/docs/aggregations.md). + +## Composition + +Amp-powered subgraphs fully support the subgraph composition feature. +This allows applying complex subgraph mappings on top of data indexed from the Amp servers. + +For more information on using the powerful subgraph composition feature, +refer to the [documentation](https://github.com/graphprotocol/example-composable-subgraph). + +## ENV variables + +Amp-powered subgraphs feature introduces the following new ENV variables: + +- `GRAPH_AMP_FLIGHT_SERVICE_ADDRESS` – The address of the Amp Flight gRPC service. _Defaults to `None`, which disables support for Amp-powered subgraphs._ +- `GRAPH_AMP_FLIGHT_SERVICE_TOKEN` – Token used to authenticate Amp Flight gRPC service requests. _Defaults to `None`, which disables authentication._ +- `GRAPH_AMP_BUFFER_SIZE` – Maximum number of response batches to buffer in memory per stream for each SQL query. _Defaults to `1,000`._ +- `GRAPH_AMP_BLOCK_RANGE` – Maximum number of blocks to request per stream for each SQL query. _Defaults to `100,000`._ +- `GRAPH_AMP_QUERY_RETRY_MIN_DELAY_SECONDS` – Minimum time to wait before retrying a failed SQL query to the Amp server. _Defaults to `1` second._ +- `GRAPH_AMP_QUERY_RETRY_MAX_DELAY_SECONDS` – Maximum time to wait before retrying a failed SQL query to the Amp server. _Defaults to `600` seconds._ + +## Metrics + +In addition to reporting updates to the existing `deployment_status`, `deployment_head`, `deployment_synced` and `deployment_blocks_processed_count` +metrics, Amp-powered subgraphs feature introduces the following new metrics: + +- `deployment_target` – Tracks the maximum block number currently available for indexing within a deployment. +- `deployment_indexing_duration_seconds` – Tracks the total duration in seconds of deployment indexing. + +Additionally, the `deployment_sync_secs` is extended with new sections specific to the Amp indexing process. diff --git a/docs/config.md b/docs/config.md index 53a9299efed..db5fc083a5f 100644 --- a/docs/config.md +++ b/docs/config.md @@ -5,11 +5,11 @@ CLI. The location of the file is passed with the `--config` command line switch. configuration file, it is not possible to use the options `--postgres-url`, `--postgres-secondary-hosts`, and `--postgres-host-weights`. -The TOML file consists of four sections: -* `[chains]` sets the endpoints to blockchain clients. -* `[store]` describes the available databases. -* `[ingestor]` sets the name of the node responsible for block ingestion. -* `[deployment]` describes how to place newly deployed subgraphs. +The TOML file consists of three sections: + +- [`[chains]`](#configuring-chains) lists the available chains and how to access them. +- [`[store]`](#configuring-multiple-databases) describes the available databases. +- [`[deployment]`](#controlling-deployment) describes how to place newly deployed subgraphs. Some of these sections support environment variable expansion out of the box, most notably Postgres connection strings. The official `graph-node` Docker image @@ -99,48 +99,162 @@ time the configuration is changed to make sure that the connection pools are what is expected. Here, `$all_nodes` should be a list of all the node names that will use this configuration file. -## Configuring Ethereum Providers +## Configuring Chains -The `[chains]` section controls the ethereum providers that `graph-node` +The `[chains]` section controls the providers that `graph-node` connects to, and where blocks and other metadata for each chain are -stored. The section consists of the name of the node doing block ingestion -(currently not used), and a list of chains. The configuration for a chain -`name` is specified in the section `[chains.]`, and consists of the -`shard` where chain data is stored and a list of providers for that -chain. For each provider, the following information must be given: - -* `label`: a label that is used when logging information about that - provider (not implemented yet) -* `transport`: one of `rpc`, `ws`, and `ipc`. Defaults to `rpc`. -* `url`: the URL for the provider -* `features`: an array of features that the provider supports, either empty - or any combination of `traces` and `archive` -* `headers`: HTTP headers to be added on every request. Defaults to none. -* `limit`: the maximum number of subgraphs that can use this provider. +stored. The section consists of the name of the node responsible for block +ingestion and a list of chains. Block ingestion only runs on the node +whose `--node-id` matches the `ingestor` value. The +`--disable-block-ingestor` flag (or `DISABLE_BLOCK_INGESTOR` env var) +acts as a hard override that always prevents ingestion regardless of +the config. + +The section-level setting `cache_size` controls the default number of +blocks from the chain head for which block data is kept cached. Individual +chains can override this value. The default is 500. When the environment +variable `GRAPH_STORE_IGNORE_BLOCK_CACHE` is set, blocks older than +`cache_size` are treated as if they have no data. The value must be greater +than the reorg threshold. + +The configuration for a chain `name` is specified in the section +`[chains.]`, with the following: + +- `shard`: where chain data is stored +- `protocol`: the protocol type being indexed, default `ethereum` + (alternatively `near`, `cosmos`,`arweave`,`starknet`) +- `amp`: the network name used by AMP for this chain; defaults to the chain name. + Set this when AMP uses a different name than graph-node (e.g., `amp = "ethereum-mainnet"` + on a chain named `mainnet`). +- `cache_size`: number of blocks from the chain head for which to keep + block data cached. Defaults to the section-level `cache_size`. +- `provider`: a list of providers for that chain + +Additionally, Ethereum chains support per-chain RPC tuning settings. When +omitted, each setting falls back to its corresponding environment variable +default (see [environment-variables.md](environment-variables.md) for +details): + +- `polling_interval`: block ingestor polling interval in milliseconds. + Default: `ETHEREUM_POLLING_INTERVAL` (1000ms). +- `json_rpc_timeout`: timeout for JSON-RPC requests in seconds. + Default: `GRAPH_ETHEREUM_JSON_RPC_TIMEOUT` (180s). +- `request_retries`: number of times to retry failed JSON-RPC requests. + Default: `GRAPH_ETHEREUM_REQUEST_RETRIES` (10). +- `max_block_range_size`: maximum number of blocks to scan for triggers per + request. Default: `GRAPH_ETHEREUM_MAX_BLOCK_RANGE_SIZE` (1000). +- `block_batch_size`: number of blocks to request in parallel. + Default: `ETHEREUM_BLOCK_BATCH_SIZE` (10). +- `block_ptr_batch_size`: number of block pointers to request in parallel. + Default: `ETHEREUM_BLOCK_PTR_BATCH_SIZE` (100). +- `max_event_only_range`: maximum range for `eth_getLogs` requests that + don't filter on contract address. + Default: `GRAPH_ETHEREUM_MAX_EVENT_ONLY_RANGE` (500). +- `target_triggers_per_block_range`: ideal number of triggers per batch. + Default: `GRAPH_ETHEREUM_TARGET_TRIGGERS_PER_BLOCK_RANGE` (100). +- `get_logs_max_contracts`: maximum contracts per `eth_getLogs` call. + Default: `GRAPH_ETH_GET_LOGS_MAX_CONTRACTS` (2000). +- `block_ingestor_max_concurrent_json_rpc_calls`: maximum concurrent + JSON-RPC calls for transaction receipts during block ingestion. + Default: `GRAPH_ETHEREUM_BLOCK_INGESTOR_MAX_CONCURRENT_JSON_RPC_CALLS_FOR_TXN_RECEIPTS` (1000). +- `genesis_block_number`: genesis block number for this chain. + Default: `GRAPH_ETHEREUM_GENESIS_BLOCK_NUMBER` (0). + +A `provider` is an object with the following characteristics: + +- `label`: the name of the provider, which will appear in logs +- `details`: provider details + +`details` includes the following: + +- `type`: one of `web3` (default), `firehose`, or `web3call` +- `transport`: one of `rpc`, `ws`, and `ipc`. Defaults to `rpc`. +- `url`: the URL for the provider +- `features`: an array of features that the provider supports, either empty + or any combination of the following for Web3 providers: + - `traces`: provider supports `debug_traceBlockByNumber` for call tracing + - `archive`: provider is an archive node with full historical state + - `no_eip1898`: provider doesn't support EIP-1898 (block parameter by hash/number object) + - `no_eip2718`: provider doesn't return the `type` field in transaction receipts + (pre-EIP-2718 chains). When set, receipts are patched to add + `"type": "0x0"` for legacy transaction compatibility. + - `compression/`: provider supports compression for RPC requests, + where `` is the compression method supported by the provider, + one of `gzip`, `brotli`, or `deflate`. The default is no compression. + + For Firehose providers: `compression` and `filters` + +- `headers`: HTTP headers to be added on every request. Defaults to none. +- `limit`: the maximum number of subgraphs that can use this provider. Defaults to unlimited. At least one provider should be unlimited, otherwise `graph-node` might not be able to handle all subgraphs. The tracking for this is approximate, and a small amount of deviation from this value should be expected. The deviation will be less than 10. +- `token`: bearer token, for Firehose providers +- `key`: API key for Firehose providers when using key-based authentication + +Note that for backwards compatibility, Web3 provider `details` can be specified at the "top level" of +the `provider`. -The following example configures two chains, `mainnet` and `kovan`, where -blocks for `mainnet` are stored in the `vip` shard and blocks for `kovan` +The following example configures three chains, `mainnet`, `sepolia` and `near-mainnet`, where +blocks for `mainnet` are stored in the `vip` shard and blocks for `sepolia` are stored in the primary shard. The `mainnet` chain can use two different -providers, whereas `kovan` only has one provider. +providers, whereas `sepolia` only has one provider. The `near-mainnet` chain expects data from +the `near` protocol via a Firehose, where the Firehose offers the `compression` and `filters` +optimisations. ```toml [chains] ingestor = "block_ingestor_node" +cache_size = 500 [chains.mainnet] shard = "vip" +amp = "ethereum-mainnet" +# Per-chain RPC tuning (all optional — omitted fields use env var defaults) +json_rpc_timeout = 300 +request_retries = 15 +max_block_range_size = 2000 provider = [ { label = "mainnet1", url = "http://..", features = [], headers = { Authorization = "Bearer foo" } }, { label = "mainnet2", url = "http://..", features = [ "archive", "traces" ] } ] -[chains.kovan] +[chains.sepolia] shard = "primary" -provider = [ { label = "kovan", url = "http://..", features = [] } ] +provider = [ { label = "sepolia", url = "http://..", features = [] } ] + +[chains.near-mainnet] +shard = "blocks_b" +protocol = "near" +provider = [ { label = "near", details = { type = "firehose", url = "https://..", key = "", features = ["compression", "filters"] } } ] ``` +### Block ingestor failover + +When the block ingestor's `do_poll()` call fails (after all internal per-request retries are +exhausted), `graph-node` automatically attempts to switch to a healthier provider. The logic +is: + +1. **Probe the current provider first.** `do_poll()` can fail for reasons unrelated to RPC + availability (e.g. a database error or a chain reorg). If the current provider still + responds to `eth_blockNumber`, the failure was not caused by the provider — no switch + occurs. +2. **Probe all alternatives in parallel.** If the current provider is unreachable, all other + validated providers are probed simultaneously via `eth_blockNumber` to minimise wait time + when providers are timing out. +3. **Switch to the first reachable provider.** The first provider to respond successfully + to the probe is selected as the new provider for the ingestor. + The remaining probes are cancelled at this point. +4. **If all providers are unreachable**, the ingestor stays on the current provider and + re-probes on the next `do_poll()` failure. + +There is no automatic return to the original provider. Once the ingestor switches, it stays on +the new provider until that provider fails, at which point the same probe-and-switch logic +applies. + +Only validated providers are eligible as failover candidates. A provider in a temporary failure +state (e.g. unreachable at startup, pending re-validation) is excluded until it passes +validation again. + ### Controlling the number of subgraphs using a provider **This feature is experimental and might be removed in a future release** @@ -151,13 +265,13 @@ approximate and can differ from the true number by a small amount (generally less than 10) The limit is set through rules that match on the node name. If a node's -name does not match any rule, the corresponding provider will be disabled -for that node. +name does not match any rule, the corresponding provider will be disabled +for that node. If the match property is omitted then the provider will be unlimited on every -node. +node. -It is recommended that at least one provider is generally unlimited. +It is recommended that at least one provider is generally unlimited. The limit is set in the following way: ```toml @@ -174,7 +288,7 @@ provider = [ Nodes named `some_node_.*` will use `mainnet-1` for at most 10 subgraphs, and `mainnet-0` for everything else, nodes named `other_node_.*` will never use `mainnet-1` and always `mainnet-0`. Any node whose name does not match -one of these patterns will not be able to use and `mainnet-1`. +one of these patterns will not be able to use and `mainnet-1`. ## Controlling Deployment @@ -237,6 +351,7 @@ indexers = [ Nodes can be configured to explicitly be query nodes by including the following in the configuration file: + ```toml [general] query = "" @@ -250,6 +365,7 @@ try to connect to any of the configured Ethereum providers. The following file is equivalent to using the `--postgres-url` command line option: + ```toml [store] [store.primary] @@ -261,21 +377,26 @@ indexers = [ "<.. list of all indexing nodes ..>" ] ## Validating configuration files -A configuration file can be checked for validity by passing the `--check-config` -flag to `graph-node`. The command +A configuration file can be checked for validity with the `config check` +command. Running + ```shell -graph-node --config $CONFIG_FILE --check-config +graph-node --config $CONFIG_FILE config check ``` -will read the configuration file and print information about syntax errors or, for -valid files, a JSON representation of the configuration. + +will read the configuration file and print information about syntax errors +and some internal inconsistencies, for example, when a shard that is not +declared as a store is used in a deployment rule. ## Simulating deployment placement Given a configuration file, placement of newly deployed subgraphs can be simulated with + ```shell graphman --config $CONFIG_FILE config place some/subgraph mainnet ``` + The command will not make any changes, but simply print where that subgraph would be placed. The output will indicate the database shard that will hold the subgraph's data, and a list of indexing nodes that could be used for diff --git a/docs/dump.md b/docs/dump.md new file mode 100644 index 00000000000..c8a3d32372b --- /dev/null +++ b/docs/dump.md @@ -0,0 +1,311 @@ +## Dump Format + +The `graphman dump` command exports all entity data and metadata for a +single subgraph deployment into a self-contained directory of Parquet files +and JSON metadata. The resulting dump can be used to restore the deployment +into a different `graph-node` instance via `graphman restore`. Dumps are +consistent snapshots of the deployment's state at a specific point in time. + +**WARNING**: The dump and restore commands are experimental and can not +replace proper database backups at this point. In particular, there is no +guarantee that a dump will be restorable. Having said that, we encourage +users to try out the dump and restore commands in non-production +environments and report any issues they encounter. + +**WARNING**: Dumping happens in a single transaction and can put significant +load on the database for large subgraphs. Use with caution on production +instances. + +**WARNING**: Restoring a dump will currently create all the default indexes +that a new deployment gets, and ignores the indexes that might have been +carefully curated for the original deployment, even though they are recorded +in the dump. This can lead to very long restore times for large subgraphs. +The restore process will be optimized in the future to only create indexes +that are present in the dump's metadata. + +### Usage + +#### Dumping a deployment + +```bash +graphman dump +``` + +`` identifies the subgraph deployment to dump. It can be a +subgraph name, a deployment hash (`Qm...`), or a database namespace +(`sgdNNN`). `` is the path where the dump will be written; it +will be created if it does not exist. + +Running `graphman dump` against an existing dump directory performs an +**incremental dump**: only rows added since the last dump are exported, and +new chunk files are appended rather than rewriting existing ones. + +```bash +# Full dump +graphman dump my-subgraph /backups/my-subgraph + +# Incremental update of the same dump +graphman dump my-subgraph /backups/my-subgraph +``` + +#### Restoring a deployment + +```bash +graphman restore [options] +``` + +`` is the path to a dump previously created with `graphman +dump`. + +| Option | Description | +| ----------- | --------------------------------------------------------------------------------------------------- | +| `--shard` | Target database shard. Uses deployment rules (or primary shard) when omitted. Required with `--add` | +| `--name` | Subgraph name for deployment rule matching and node assignment. Falls back to an existing name | +| `--replace` | Drop and recreate if the deployment already exists in the target shard | +| `--add` | Create a copy in a shard that doesn't already have this deployment (requires `--shard`) | +| `--force` | Replace if the deployment exists in the target shard, add if it doesn't | + +`--replace`, `--add`, and `--force` are mutually exclusive. When none is +given, restore fails if the deployment already exists in the target shard. + +```bash +# Restore into the default shard +graphman restore /backups/my-subgraph + +# Restore into a specific shard, replacing if it already exists +graphman restore /backups/my-subgraph --shard shard1 --replace + +# Force-restore (replace or add as needed) +graphman restore /backups/my-subgraph --force +``` + +### Directory layout + +A dump directory has the following structure: + +``` +/ + metadata.json -- deployment metadata + per-table state + schema.graphql -- raw GraphQL schema text + subgraph.yaml -- raw subgraph manifest YAML (optional) + / + chunk_000000.parquet -- rows ordered by vid + chunk_000001.parquet -- incremental append (future chunks) + ... + data_sources$/ + chunk_000000.parquet -- dynamic data sources +``` + +Each entity type defined in the GraphQL schema gets its own subdirectory, +named after the entity type exactly as it appears in the schema (e.g. +`Token/`, `Pool/`). The Proof of Indexing appears as a regular entity +directory name `Poi$`. The special `data_sources$` directory holds dynamic +data sources created at runtime. + +Within each directory, data is stored in numbered chunk files +(`chunk_000000.parquet`, `chunk_000001.parquet`, ...). A fresh dump +produces a single `chunk_000000.parquet` per table. Incremental dumps +append new chunks rather than rewriting existing ones. + +The GraphQL schema and subgraph manifest are stored as separate plain-text +files `schema.graphql` and `subgraph.yaml`. + +### metadata.json + +The top-level `metadata.json` contains everything needed to reconstruct the +deployment's table structure, plus diagnostic information captured at dump +time. Its structure is: + +```json +{ + "version": 1, + "deployment": "Qm...", + "network": "mainnet", + + "manifest": { + "spec_version": "1.0.0", + "description": "Optional subgraph description", + "repository": "https://github.com/...", + "features": ["..."], + "entities_with_causality_region": ["EntityType1"], + "history_blocks": 2147483647 + }, + + "earliest_block_number": 12345, + "start_block": { "number": 12345, "hash": "0xabc..." }, + "head_block": { "number": 99999, "hash": "0xdef..." }, + "entity_count": 150000, + + "graft_base": null, + "graft_block": null, + "debug_fork": null, + + "health": { + "failed": false, + "health": "healthy", + "fatal_error": null, + "non_fatal_errors": [] + }, + + "indexes": { + "token": [ + "CREATE INDEX CONCURRENTLY IF NOT EXISTS attr_0_0_id ON sgd.token USING btree (id)" + ] + }, + + "tables": { + "Token": { + "immutable": true, + "has_causality_region": false, + "chunks": [ + { + "file": "Token/chunk_000000.parquet", + "min_vid": 0, + "max_vid": 50000, + "row_count": 50000 + } + ], + "max_vid": 50000 + }, + "data_sources$": { + "immutable": false, + "has_causality_region": true, + "chunks": [ + { + "file": "data_sources$/chunk_000000.parquet", + "min_vid": 0, + "max_vid": 100, + "row_count": 100 + } + ], + "max_vid": 100 + } + } +} +``` + +**Field descriptions:** + +| Field | Description | +| ----------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------- | +| `version` | Format version. Must be `1`. | +| `deployment` | Deployment hash (`Qm...`). | +| `network` | The blockchain network (e.g. `mainnet`, `goerli`). | +| `manifest` | Manifest metadata extracted from `subgraphs.subgraph_manifest`. | +| `manifest.spec_version` | Subgraph API version. Required to parse `schema.graphql`. | +| `manifest.entities_with_causality_region` | Entity types that have a `causality_region` column. | +| `manifest.history_blocks` | How many blocks of entity version history are retained. | +| `earliest_block_number` | Earliest block for which data exists (accounts for pruning). | +| `start_block` | The block where indexing started. Null if not set. | +| `head_block` | The latest indexed block at dump time. | +| `entity_count` | Total entity count across all tables. | +| `graft_base` | Deployment hash of the graft base, if any. | +| `graft_block` | Block pointer of the graft point, if any. | +| `debug_fork` | Debug fork deployment hash, if any. | +| `health` | Point-in-time health snapshot. Not used during restore. | +| `indexes` | Point-in-time index definitions as SQL. Not used during restore (indexes are auto-created by `Layout::create_relational_schema()`). | +| `tables` | Per-table metadata keyed by entity type name (or `data_sources$`). | + +Each entry in `tables` contains: + +| Field | Description | +| ---------------------- | ------------------------------------------------------------------------------ | +| `immutable` | Whether the entity type is immutable (uses `block$` instead of `block_range`). | +| `has_causality_region` | Whether rows have a `causality_region` column. | +| `chunks` | Ordered list of Parquet chunk files for this table. | +| `chunks[].file` | Relative path from the dump directory. | +| `chunks[].min_vid` | Minimum `vid` value in this chunk. | +| `chunks[].max_vid` | Maximum `vid` value in this chunk. | +| `chunks[].row_count` | Number of rows in this chunk. | +| `max_vid` | Maximum `vid` across all chunks. `-1` if the table is empty. | + +### Parquet schema: entity tables + +Each entity table's Parquet file uses an Arrow schema derived from the +entity's GraphQL definition. Columns are ordered as follows: + +1. **System columns** (always present, in this order): + - `vid` (Int64, non-nullable) -- row version ID + - Block tracking (one of): + - Immutable entities: `block$` (Int32, non-nullable) + - Mutable entities: `block_range_start` (Int32, non-nullable), + `block_range_end` (Int32, nullable -- null means unbounded/current) + - `causality_region` (Int32, non-nullable) -- only if the entity has one + +2. **Data columns** in GraphQL declaration order, skipping fulltext + (`TSVector`) columns which are generated and rebuilt on restore. + +The PostgreSQL `int4range` type used for `block_range` is decomposed into +two scalar columns (`block_range_start`, `block_range_end`) in the Parquet +representation. This avoids the need for a custom range type in Arrow. + +#### Type mapping + +GraphQL/PostgreSQL column types map to Arrow data types as follows: + +| ColumnType | Arrow DataType | Notes | +| --------------- | ------------------------------ | -------------------------------------------------------------- | +| `Boolean` | `Boolean` | | +| `Int` | `Int32` | | +| `Int8` | `Int64` | | +| `Bytes` | `Binary` | Raw bytes, no hex encoding | +| `BigInt` | `Utf8` | Stored as decimal string for arbitrary precision | +| `BigDecimal` | `Utf8` | Stored as decimal string for arbitrary precision | +| `Timestamp` | `Timestamp(Microsecond, None)` | Microseconds since epoch, no timezone | +| `String` | `Utf8` | | +| `Enum(...)` | `Utf8` | Enum variant as string (cast from PG enum to text during dump) | +| `TSVector(...)` | _skipped_ | Fulltext index columns are generated; rebuilt on restore | + +**Array columns:** A GraphQL list field (e.g. `tags: [String!]!`) is +stored as `List` where `T` is the base Arrow type from the table +above. Whether a column is a list is determined by the GraphQL field type, +not by `ColumnType`. For example, `[String!]!` becomes `List` and +`[Int!]` becomes `List`. + +**Nullability** follows the GraphQL schema: non-null fields produce +non-nullable Arrow columns; optional fields produce nullable columns. List +elements within list columns are always marked nullable in the Arrow schema. + +### Parquet schema: data_sources$ + +The `data_sources$` table has a fixed schema independent of the GraphQL +definition: + +| Column | Arrow DataType | Nullable | Description | +| ------------------- | -------------- | -------- | -------------------------------------------------- | +| `vid` | `Int64` | no | Row version ID | +| `block_range_start` | `Int32` | no | Lower bound of `block_range` | +| `block_range_end` | `Int32` | yes | Upper bound (null = unbounded) | +| `causality_region` | `Int32` | no | Causality region | +| `manifest_idx` | `Int32` | no | Index into the manifest's data source list | +| `parent` | `Int32` | yes | Self-referencing parent data source | +| `id` | `Binary` | yes | Data source identifier | +| `param` | `Binary` | yes | Data source parameter | +| `context` | `Utf8` | yes | JSON context | +| `done_at` | `Int32` | yes | Block number where the data source was marked done | + +### Compression + +All Parquet files use ZSTD compression (default level). + +### Row ordering + +Within each Parquet chunk file, rows are ordered by `vid` (ascending). +This matches the primary key ordering in PostgreSQL and enables efficient +sequential reads during restore. + +### Incremental dumps + +An incremental dump reads the existing `metadata.json`, determines the +`max_vid` for each table, and queries only rows with `vid > max_vid`. New +rows are written to new chunk files (e.g. `chunk_000001.parquet`) and the +metadata is updated atomically (write to a temp file, then rename). + +### Atomicity + +The `metadata.json` file is always written atomically: the dump writes to +`metadata.json.tmp` first, then renames it to `metadata.json`. This +ensures that a reader never sees a partially-written metadata file. If the +dump process crashes mid-write, the previous `metadata.json` remains +intact. The Parquet chunk files are written before `metadata.json` is +updated, so chunk files referenced by `metadata.json` are always complete. diff --git a/docs/environment-variables.md b/docs/environment-variables.md index 2bed9a09ab4..7a601e32e64 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -10,10 +10,15 @@ those. ## JSON-RPC configuration for EVM chains +> **Note**: Many of these settings can now be overridden per-chain in the TOML +> configuration file under `[chains.]`. When a per-chain value is set in +> the config file it takes precedence over the environment variable. See +> [config.md](config.md#configuring-chains) for details. + - `ETHEREUM_REORG_THRESHOLD`: Maximum expected reorg size, if a larger reorg happens, subgraphs might process inconsistent data. Defaults to 250. - `ETHEREUM_POLLING_INTERVAL`: how often to poll Ethereum for new blocks (in ms, - defaults to 500ms) + defaults to 1000ms) - `GRAPH_ETHEREUM_TARGET_TRIGGERS_PER_BLOCK_RANGE`: The ideal amount of triggers to be processed in a batch. If this is too small it may cause too many requests to the ethereum node, if it is too large it may cause unreasonably expensive @@ -25,11 +30,11 @@ those. - `DISABLE_BLOCK_INGESTOR`: set to `true` to disable block ingestion. Leave unset or set to `false` to leave block ingestion enabled. - `ETHEREUM_BLOCK_BATCH_SIZE`: number of Ethereum blocks to request in parallel. - Also limits other parallel requests such such as trace_filter. Defaults to 10. + Also limits other parallel requests such as trace_filter. Defaults to 10. - `GRAPH_ETHEREUM_MAX_BLOCK_RANGE_SIZE`: Maximum number of blocks to scan for triggers in each request (defaults to 1000). - `GRAPH_ETHEREUM_MAX_EVENT_ONLY_RANGE`: Maximum range size for `eth.getLogs` - requests that dont filter on contract address, only event signature (defaults to 500). + requests that don't filter on contract address, only event signature (defaults to 500). - `GRAPH_ETHEREUM_JSON_RPC_TIMEOUT`: Timeout for Ethereum JSON-RPC requests. - `GRAPH_ETHEREUM_REQUEST_RETRIES`: Number of times to retry JSON-RPC requests made against Ethereum. This is used for requests that will not fail the @@ -53,6 +58,13 @@ those. be used if the store uses more than one shard. - `GRAPH_ETHEREUM_GENESIS_BLOCK_NUMBER`: Specify genesis block number. If the flag is not set, the default value will be `0`. +- `GRAPH_ETH_GET_LOGS_MAX_CONTRACTS`: Maximum number of contracts to query in a single `eth_getLogs` request. + Defaults to 2000. + +## Firehose configuration + +- `GRAPH_NODE_FIREHOSE_MAX_DECODE_SIZE`: Maximum size of a message that can be + decoded by the firehose. Defaults to 25MB. ## Running mapping handlers @@ -70,7 +82,8 @@ those. - `GRAPH_IPFS_TIMEOUT`: timeout for IPFS, which includes requests for manifest files and from mappings (in seconds, default is 60). -- `GRAPH_MAX_IPFS_FILE_BYTES`: maximum size for a file that can be retrieved (in bytes, default is 256 MiB). +- `GRAPH_MAX_IPFS_FILE_BYTES`: maximum size for a file that can be retrieved by an `ipfs cat` call. + This affects both subgraph definition files and `file/ipfs` data sources. In bytes, default is 25 MiB. - `GRAPH_MAX_IPFS_MAP_FILE_SIZE`: maximum size of files that can be processed with `ipfs.map`. When a file is processed through `ipfs.map`, the entities generated from that are kept in memory until the entire file is done @@ -78,8 +91,17 @@ those. may use (in bytes, defaults to 256MB). - `GRAPH_MAX_IPFS_CACHE_SIZE`: maximum number of files cached (defaults to 50). - `GRAPH_MAX_IPFS_CACHE_FILE_SIZE`: maximum size of each cached file (in bytes, defaults to 1MiB). -- `GRAPH_IPFS_REQUEST_LIMIT`: Limits both concurrent and per second requests to IPFS for file data - sources. Defaults to 100. +- `GRAPH_IPFS_REQUEST_LIMIT`: Limits the number of requests per second to IPFS for file data sources. Defaults to 100. +- `GRAPH_IPFS_MAX_ATTEMPTS`: This limits the IPFS retry requests in case of a + file not found or logical issue working as a safety mechanism to + prevent infinite spamming of IPFS servers and network congestion + (default: 100 000). +- `GRAPH_IPFS_CACHE_LOCATION`: When set, files retrieved from IPFS will be + cached in that location; future accesses to the same file will be served + from cache rather than IPFS. This can either be a URL starting with + `redis://`, in which case there must be a Redis instance running at that + URL, or an absolute file system path which must be a directory writable + by the `graph-node` process (experimental) ## GraphQL @@ -104,23 +126,17 @@ those. result is checked while the response is being constructed, so that execution does not take more memory than what is configured. The default value for both is unlimited. -- `GRAPH_GRAPHQL_MAX_OPERATIONS_PER_CONNECTION`: maximum number of GraphQL - operations per WebSocket connection. Any operation created after the limit - will return an error to the client. Default: 1000. - `GRAPH_GRAPHQL_HTTP_PORT` : Port for the GraphQL HTTP server -- `GRAPH_GRAPHQL_WS_PORT` : Port for the GraphQL WebSocket server - `GRAPH_SQL_STATEMENT_TIMEOUT`: the maximum number of seconds an individual SQL query is allowed to take during GraphQL execution. Default: unlimited -- `GRAPH_DISABLE_SUBSCRIPTION_NOTIFICATIONS`: disables the internal - mechanism that is used to trigger updates on GraphQL subscriptions. When - this variable is set to any value, `graph-node` will still accept GraphQL - subscriptions, but they won't receive any updates. - `ENABLE_GRAPHQL_VALIDATIONS`: enables GraphQL validations, based on the GraphQL specification. - This will validate and ensure every query executes follows the execution rules. + This will validate and ensure every query executes follows the execution + rules. Default: `false` - `SILENT_GRAPHQL_VALIDATIONS`: If `ENABLE_GRAPHQL_VALIDATIONS` is enabled, you are also able to just silently print the GraphQL validation errors, without failing the actual query. Note: queries - might still fail as part of the later stage validations running, during GraphQL engine execution. + might still fail as part of the later stage validations running, during + GraphQL engine execution. Default: `true` - `GRAPH_GRAPHQL_DISABLE_BOOL_FILTERS`: disables the ability to use AND/OR filters. This is useful if we want to disable filters because of performance reasons. @@ -139,6 +155,10 @@ those. - `GRAPH_QUERY_CACHE_BLOCKS`: How many recent blocks per network should be kept in the query cache. This should be kept small since the lookup time and the cache memory usage are proportional to this value. Set to 0 to disable the cache. Defaults to 1. - `GRAPH_QUERY_CACHE_MAX_MEM`: Maximum total memory to be used by the query cache, in MB. The total amount of memory used for caching will be twice this value - once for recent blocks, divided evenly among the `GRAPH_QUERY_CACHE_BLOCKS`, and once for frequent queries against older blocks. The default is plenty for most loads, particularly if `GRAPH_QUERY_CACHE_BLOCKS` is kept small. Defaults to 1000, which corresponds to 1GB. - `GRAPH_QUERY_CACHE_STALE_PERIOD`: Number of queries after which a cache entry can be considered stale. Defaults to 100. +- `GRAPH_QUERY_CACHE_MAX_ENTRY_RATIO`: Limits the maximum size of a cache + entry. Query results larger than the size of a cache shard divided by this + value will not be cached. The default is 3. A value of 0 means that there + is no limit on the size of a cache entry. ## Miscellaneous @@ -157,6 +177,8 @@ those. - `THEGRAPH_STORE_POSTGRES_DIESEL_URL`: postgres instance used when running tests. Set to `postgresql://:@:/` - `GRAPH_KILL_IF_UNRESPONSIVE`: If set, the process will be killed if unresponsive. +- `GRAPH_KILL_IF_UNRESPONSIVE_TIMEOUT_SECS`: Timeout in seconds before killing + the node if `GRAPH_KILL_IF_UNRESPONSIVE` is true. The default value is 10s. - `GRAPH_LOG_QUERY_TIMING`: Control whether the process logs details of processing GraphQL and SQL queries. The value is a comma separated list of `sql`,`gql`, and `cache`. If `gql` is present in the list, each @@ -169,11 +191,10 @@ those. query, and the `query_id` of the GraphQL query that caused the SQL query. These SQL queries are marked with `component: GraphQlRunner` There are additional SQL queries that get logged when `sql` is given. These are - queries caused by mappings when processing blocks for a subgraph, and - queries caused by subscriptions. If `cache` is present in addition to - `gql`, also logs information for each toplevel GraphQL query field - whether that could be retrieved from cache or not. Defaults to no - logging. + queries caused by mappings when processing blocks for a subgraph. If + `cache` is present in addition to `gql`, also logs information for each + toplevel GraphQL query field whether that could be retrieved from cache + or not. Defaults to no logging. - `GRAPH_LOG_TIME_FORMAT`: Custom log time format.Default value is `%b %d %H:%M:%S%.3f`. More information [here](https://docs.rs/chrono/latest/chrono/#formatting-and-parsing). - `STORE_CONNECTION_POOL_SIZE`: How many simultaneous connections to allow to the store. Due to implementation details, this value may not be strictly adhered to. Defaults to 10. @@ -200,6 +221,14 @@ those. decisions. Set to `true` to turn simulation on, defaults to `false` - `GRAPH_STORE_CONNECTION_TIMEOUT`: How long to wait to connect to a database before assuming the database is down in ms. Defaults to 5000ms. +- `GRAPH_STORE_SETUP_TIMEOUT`: Timeout for database setup operations + (migrations, schema creation) in milliseconds. Defaults to 30000ms (30s). + Setup operations can legitimately take longer than normal runtime operations. +- `GRAPH_STORE_CONNECTION_UNAVAILABLE_RETRY`: When a database shard is marked + unavailable due to connection timeouts, this controls how often to allow a + single probe request through to check if the database has recovered. Only one + request per interval will attempt a connection; all others fail instantly. + Value is in seconds and defaults to 2s. - `EXPERIMENTAL_SUBGRAPH_VERSION_SWITCHING_MODE`: default is `instant`, set to `synced` to only switch a named subgraph to a new deployment once it has synced, making the new deployment the "Pending" version. @@ -216,7 +245,125 @@ those. copying or grafting should take. This limits how long transactions for such long running operations will be, and therefore helps control bloat in other tables. Value is in seconds and defaults to 180s. +- `GRAPH_STORE_BATCH_TIMEOUT`: How long a batch operation during copying, + grafting, or pruning is allowed to take at most. This is meant to guard + against batches that are catastrophically big and should be set to a + small multiple of `GRAPH_STORE_BATCH_TARGET_DURATION`, like 10 times that + value, and needs to be at least 2 times that value when set. If this + timeout is hit, the batch size is reset to 1 so we can be sure that + batches stay below `GRAPH_STORE_BATCH_TARGET_DURATION` and the smaller + batch is retried. Value is in seconds and defaults to unlimited. +- `GRAPH_STORE_BATCH_WORKERS`: The number of workers to use for batch + operations. If there are idle connectiosn, each subgraph copy operation + will use up to this many workers to copy tables in parallel. Defaults + to 1 and must be at least 1 - `GRAPH_START_BLOCK`: block hash:block number where the forked subgraph will start indexing at. - `GRAPH_FORK_BASE`: api url for where the graph node will fork from, use `https://api.thegraph.com/subgraphs/id/` for the hosted service. - `GRAPH_DEBUG_FORK`: the IPFS hash id of the subgraph to fork. +- `GRAPH_STORE_HISTORY_SLACK_FACTOR`: How much history a subgraph with + limited history can accumulate before it will be pruned. Setting this to + 1.1 means that the subgraph will be pruned every time it contains 10% + more history (in blocks) than its history limit. The default value is 1.2 + and the value must be at least 1.01 +- `GRAPH_STORE_HISTORY_REBUILD_THRESHOLD`, + `GRAPH_STORE_HISTORY_DELETE_THRESHOLD`: when pruning, prune by copying + the entities we will keep to new tables if we estimate that we will + remove more than a factor of `REBUILD_THRESHOLD` of the deployment's + history. If we estimate to remove a factor between `REBUILD_THRESHOLD` + and `DELETE_THRESHOLD`, prune by deleting from the existing tables of the + deployment. If we estimate to remove less than `DELETE_THRESHOLD` + entities, do not change the table. Both settings are floats, and default + to 0.5 for the `REBUILD_THRESHOLD` and 0.05 for the `DELETE_THRESHOLD`; + they must be between 0 and 1, and `REBUILD_THRESHOLD` must be bigger than + `DELETE_THRESHOLD`. +- `GRAPH_STORE_WRITE_BATCH_DURATION`: how long to accumulate changes during + syncing into a batch before a write has to happen in seconds. The default + is 300s. Setting this to 0 disables write batching. +- `GRAPH_STORE_WRITE_BATCH_SIZE`: how many changes to accumulate during + syncing in kilobytes before a write has to happen. The default is 10_000 + which corresponds to 10MB. Setting this to 0 disables write batching. +- `GRAPH_MIN_HISTORY_BLOCKS`: Specifies the minimum number of blocks to + retain for subgraphs with historyBlocks set to auto. The default value is 2 times the reorg threshold. +- `GRAPH_ETHEREUM_BLOCK_RECEIPTS_CHECK_TIMEOUT`: Timeout for checking + `eth_getBlockReceipts` support during chain startup, if this times out + individual transaction receipts will be fetched instead. Defaults to 10s. +- `GRAPH_POSTPONE_ATTRIBUTE_INDEX_CREATION`: During the coping of a subgraph + postponing creation of certain indexes (btree, attribute based ones), would + speed up syncing +- `GRAPH_STORE_INSERT_EXTRA_COLS`: Makes it possible to work around bugs in + the subgraph writing code that manifest as Postgres errors saying 'number + of parameters must be between 0 and 65535' Such errors are always + graph-node bugs, but since it is hard to work around them, setting this + variable to something like 10 makes it possible to work around such a bug + while it is being fixed (default: 0) +- `GRAPH_ENABLE_SQL_QUERIES`: Enable the experimental [SQL query + interface](implementation/sql-interface.md). + (default: false) +- `GRAPH_STORE_ACCOUNT_LIKE_SCAN_INTERVAL_HOURS`: If set, enables an experimental job that + periodically scans for entity tables that may benefit from an [account-like optimization](https://thegraph.com/docs/en/indexing/tooling/graph-node/#account-like-optimisation) and marks them with an + account-like flag. The value is the interval in hours at which the job + should run. The job reads data from the `info.table_stats` materialized view, which refreshes every six hours. + Expects an integer value, e.g., 24. Requires also setting + `GRAPH_STORE_ACCOUNT_LIKE_MIN_VERSIONS_COUNT` and `GRAPH_STORE_ACCOUNT_LIKE_MAX_UNIQUE_RATIO`. +- `GRAPH_STORE_ACCOUNT_LIKE_MIN_VERSIONS_COUNT`: Sets the minimum total number of versions a table must have + to be considered for account-like flagging. Expects a positive integer value. No default value. +- `GRAPH_STORE_ACCOUNT_LIKE_MAX_UNIQUE_RATIO`: Sets the maximum unique entities to version ratio + (e.g., 0.01 ≈ 1:100 entity-to-version ratio). +- `GRAPH_STORE_DISABLE_CALL_CACHE`: Disables storing or reading `eth_call` results from the store call cache. + This option may be useful for indexers who are running their own RPC nodes. + Disabling the store call cache may significantly impact performance; the actual impact depends on + the average execution time of an `eth_call` compared to the cost of a database lookup for a cached result. + (default: false) + +## Log Store Configuration + +`graph-node` supports storing and querying subgraph logs through multiple backends: Elasticsearch, Loki, local files, or disabled. + +**For complete log store documentation**, including detailed configuration, querying examples, and choosing the right backend, see the **[Log Store Guide](log-store.md)**. + +### Quick Reference + +**Backend selection:** +- `GRAPH_LOG_STORE_BACKEND`: `disabled` (default), `elasticsearch`, `loki`, or `file` + +**Elasticsearch:** +- `GRAPH_LOG_STORE_ELASTICSEARCH_URL`: Elasticsearch endpoint URL (required) +- `GRAPH_LOG_STORE_ELASTICSEARCH_USER`: Username (optional) +- `GRAPH_LOG_STORE_ELASTICSEARCH_PASSWORD`: Password (optional) +- `GRAPH_LOG_STORE_ELASTICSEARCH_INDEX`: Index name (default: `subgraph`) + +**Loki:** +- `GRAPH_LOG_STORE_LOKI_URL`: Loki endpoint URL (required) +- `GRAPH_LOG_STORE_LOKI_TENANT_ID`: Tenant ID (optional) + +**File-based:** +- `GRAPH_LOG_STORE_FILE_DIR`: Log directory (required) +- `GRAPH_LOG_STORE_FILE_MAX_SIZE`: Max file size in bytes (default: 104857600 = 100MB) +- `GRAPH_LOG_STORE_FILE_RETENTION_DAYS`: Retention period (default: 30) + +**Deprecated variables** (will be removed in future versions): +- `GRAPH_ELASTICSEARCH_URL` → use `GRAPH_LOG_STORE_ELASTICSEARCH_URL` +- `GRAPH_ELASTICSEARCH_USER` → use `GRAPH_LOG_STORE_ELASTICSEARCH_USER` +- `GRAPH_ELASTICSEARCH_PASSWORD` → use `GRAPH_LOG_STORE_ELASTICSEARCH_PASSWORD` +- `GRAPH_ELASTIC_SEARCH_INDEX` → use `GRAPH_LOG_STORE_ELASTICSEARCH_INDEX` + +### Example: File-based Logs for Local Development + +```bash +mkdir -p ./graph-logs +export GRAPH_LOG_STORE_BACKEND=file +export GRAPH_LOG_STORE_FILE_DIR=./graph-logs + +graph-node \ + --postgres-url postgresql://graph:pass@localhost/graph-node \ + --ethereum-rpc mainnet:https://... \ + --ipfs 127.0.0.1:5001 +``` + +See the **[Log Store Guide](log-store.md)** for: +- Detailed configuration for all backends +- How log stores work internally +- GraphQL query examples +- Choosing the right backend for your use case +- Best practices and troubleshooting diff --git a/docs/getting-started.md b/docs/getting-started.md deleted file mode 100644 index e7ea53a7ca1..00000000000 --- a/docs/getting-started.md +++ /dev/null @@ -1,486 +0,0 @@ -# Getting Started -> **Note:** This project is heavily a WIP, and until it reaches v1.0, the API is subject to change in breaking ways without notice. - -## 0 Introduction - -This page explains everything you need to know to run a local Graph Node, including links to other reference pages. First, we describe what The Graph is and then explain how to get started. - -### 0.1 What Is The Graph? - -The Graph is a decentralized protocol for indexing and querying data from blockchains, which makes it possible to query for data that is difficult or impossible to do directly. Currently, we only work with Ethereum. - -For example, with the popular Cryptokitties decentralized application (dApp) that implements the [ERC-721 Non-Fungible Token (NFT)](https://github.com/ethereum/eips/issues/721) standard, it is relatively straightforward to ask the following questions: -> *How many cryptokitties does a specific Ethereum account own?* -> *When was a particular cryptokitty born?* - -These read patterns are directly supported by the methods exposed by the [contract](https://github.com/dapperlabs/cryptokitties-bounty/blob/master/contracts/KittyCore.sol): the [`balanceOf`](https://github.com/dapperlabs/cryptokitties-bounty/blob/master/contracts/KittyOwnership.sol#L64) and [`getKitty`](https://github.com/dapperlabs/cryptokitties-bounty/blob/master/contracts/KittyCore.sol#L91) methods for these two examples. - -However, other questions are more difficult to answer: -> *Who are the owners of the cryptokitties born between January and February of 2018?* - -To answer this question, you need to process all [`Birth` events](https://github.com/dapperlabs/cryptokitties-bounty/blob/master/contracts/KittyBase.sol#L15) and then call the [`ownerOf` method](https://github.com/dapperlabs/cryptokitties-bounty/blob/master/contracts/KittyOwnership.sol#L144) for each cryptokitty born. An alternate approach could involve processing all (`Transfer` events) and filtering based on the most recent transfer for each cryptokitty. - -Even for this relatively simple question, it would take hours or even days for a dApp running in a browser to find an answer. Indexing and caching data off blockchains is hard. There are also edge cases around finality, chain reorganizations, uncled blocks, etc., which make it even more difficult to display deterministic data to the end user. - -The Graph solves this issue by providing an open source node implementation, [Graph Node](../README.md), which handles indexing and caching of blockchain data. The entire community can contribute to and utilize this tool. In the current implementation, it exposes functionality through a GraphQL API for end users. - -### 0.2 How Does It Work? - -The Graph must be run alongside a running IPFS node, Ethereum node, and a store (Postgres, in this initial implementation). - -![Data Flow Diagram](images/TheGraph_DataFlowDiagram.png) - -The high-level dataflow for a dApp using The Graph is as follows: -1. The dApp creates/modifies data on Ethereum through a transaction to a smart contract. -2. The smart contract emits one or more events (logs) while processing this transaction. -3. The Graph Node listens for specific events and triggers handlers in a user-defined mapping. -4. The mapping is a WASM module that runs in a WASM runtime. It creates one or more store transactions in response to Ethereum events. -5. The store is updated along with the indexes. -6. The dApp queries the Graph Node for data indexed from the blockchain using the node's [GraphQL endpoint](https://graphql.org/learn/). The Graph Node, in turn, translates the GraphQL queries into queries for its underlying store to fetch this data. This makes use of the store's indexing capabilities. -7. The dApp displays this data in a user-friendly format, which an end-user leverages when making new transactions against the Ethereum blockchain. -8. And, this cycle repeats. - -### 0.3 What's Needed to Build a Graph Node? -Three repositories are relevant to building on The Graph: -1. [Graph Node](../README.md) – A server implementation for indexing, caching, and serving queries against data from Ethereum. -2. [Graph CLI](https://github.com/graphprotocol/graph-cli) – A CLI for building and compiling projects that are deployed to the Graph Node. -3. [Graph TypeScript Library](https://github.com/graphprotocol/graph-ts) – TypeScript/AssemblyScript library for writing subgraph mappings to be deployed to The Graph. - -### 0.4 Getting Started Overview -Below, we outline the required steps to build a subgraph from scratch, which will serve queries from a GraphQL endpoint. The three major steps are: - -1. [Define the subgraph](#1-define-the-subgraph) - 1. [Define the data sources and create a manifest](#11-define-the-data-sources-and-create-a-manifest) - - 2. [Create the GraphQL schema](#12-create-the-graphql-schema-for-the-data-source) - - 3. [Create a subgraph project and generate types](#13-create-a-subgraph-project-and-generate-types) - - 4. [Write the mappings](#14-writing-mappings) -2. Deploy the subgraph - 1. [Start up an IPFS node](#21-start-up-ipfs) - - 2. [Create the Postgres database](#22-create-the-postgres-db) - - 3. [Start the Graph Node and Connect to an Etheruem node](#23-starting-the-graph-node-and-connecting-to-an-etheruem-node) - - 4. [Deploy the subgraph](#24-deploying-the-subgraph) -3. Query the subgraph - 1. [Query the newly deployed GraphQL API](#3-query-the-local-graph-node) - -Now, let's dig in! - -## 1 Define the Subgraph -When we refer to a subgraph, we reference the entire project that is indexing a chosen set of data. - -To start, create a repository for this project. - -### 1.1 Define the Data Sources and Create a Manifest - -When building a subgraph, you must first decide what blockchain data you want the Graph Node to index. These are known as `dataSources`, which are datasets derived from a blockchain, i.e., an Ethereum smart contract. - -The subgraph is defined by a YAML file known as the **subgraph manifest**. This file should always be named `subgraph.yaml`. View the full specification for the subgraph manifest [here](subgraph-manifest.md). It contains a schema, data sources, and mappings that are used to deploy the GraphQL endpoint. - -Let's go through an example to display what a subgraph manifest looks like. In this case, we use the common ERC721 contract and look at the `Transfer` event because it is familiar to many developers. Below, we define a subgraph manifest with one contract under `dataSources`, which is a smart contract implementing the ERC721 interface: -```yaml -specVersion: 0.0.1 -description: ERC-721 Example -repository: https://github.com//erc721-example -schema: - file: ./schema.graphql -dataSources: -- kind: ethereum/contract - name: MyERC721Contract - network: mainnet - source: - address: "0x06012c8cf97BEaD5deAe237070F9587f8E7A266d" - abi: ERC721 - mapping: - kind: ethereum/events - apiVersion: 0.0.1 - language: wasm/assemblyscript - entities: - - Token - abis: - - name: ERC721 - file: ./abis/ERC721ABI.json - eventHandlers: - - event: Transfer(address,address,uint256) - handler: handleTransfer - file: ./mapping.ts -``` -We point out a few important facts from this example to supplement the [subgraph manifest spec](subgraph-manifest.md): - -* The name `ERC721` under `source > abi` must match the name displayed underneath `abis > name`. -* The event `Transfer(address,address,uint256)` under `eventHandlers` must match what is in the ABI. The name `handleTransfer` under `eventHandlers > handler` must match the name of the mapping function, which we explain in section 1.4. -* Ensure that you have the correct contract address under `source > address`. This is also the case when indexing testnet contracts as well because you might switch back and forth. -* You can define multiple data sources under dataSources. Within a datasource, you can also have multiple `entities` and `events`. See [this subgraph](https://github.com/graphprotocol/decentraland-subgraph/blob/master/subgraph.yaml) for an example. -* If at any point the Graph CLI outputs 'Failed to copy subgraph files', it probably means you have a typo in the manifest. - -#### 1.1.1 Obtain the Contract ABIs -The ABI JSON file must contain the correct ABI to source all the events or any contract state you wish to ingest into the Graph Node. There are a few ways to obtain an ABI for the contract: -* If you are building your own project, you likely have access to your most current ABIs of your smart contracts. -* If you are building a subgraph for a public project, you can download that project to your computer and generate the ABI by using [`truffle compile`](https://truffleframework.com/docs/truffle/overview) or `solc` to compile. This creates the ABI files that you can then transfer to your subgraph `/abi` folder. -* Sometimes, you can also find the ABI on [Etherscan](https://etherscan.io), but this is not always reliable because the uploaded ABI may be out of date. Make sure you have the correct ABI. Otherwise, you will not be able to start a Graph Node. - -If you run into trouble here, double-check the ABI and ensure that the event signatures exist *exactly* as you expect them by examining the smart contract code you are sourcing. Also, note with the ABI, you only need the array for the ABI. Compiling the contracts locally results in a `.json` file that contains the complete ABI nested within the `.json` file under the key `abi`. - -An example `abi` for the `Transfer` event is shown below and would be stored in the `/abi` folder with the name `ERC721ABI.json`: - -```json - [{ - "anonymous": false, - "inputs": [ - { - "indexed": true, - "name": "_from", - "type": "address" - }, - { - "indexed": true, - "name": "_to", - "type": "address" - }, - { - "indexed": true, - "name": "_tokenId", - "type": "uint256" - } - ], - "name": "Transfer", - "type": "event" - }] - ``` - -Once you create this `subgraph.yaml` file, move to the next section. - -### 1.2 Create the GraphQL Schema for the Data Source -GraphQL schemas are defined using the GraphQL interface definition language (IDL). If you have never written a GraphQL schema, we recommend checking out a [quick primer](https://graphql.org/learn/schema/#type-language) on the GraphQL type system. - -With The Graph, rather than defining the top-level `Query` type, you simply define entity types. Then, the Graph Node will generate top-level fields for querying single instances and collections of that entity type. Each entity type is required to be annotated with an `@entity` directive. - -As you see in the example `subgraph.yaml` manifest above, it contains one entity named `Token`. Let's define what that would look like for the GraphQL schema: - -Define a Token entity type: -```graphql -type Token @entity { - id: ID! - currentOwner: Address! -} -``` - -This `entity` tracks a single ERC721 token on Ethereum by its ID and the current owner. The **`ID` field is required** and stores values of the ID type, which are strings. The `ID` must be a unique value so that it can be placed into the store. For an ERC721 token, the unique ID could be the token ID because that value is unique to that coin. - -The exclamation mark represents the fact that that field must be set when the entity is stored in the database, i.e., it cannot be `null`. See the [Schema API](https://github.com/graphprotocol/docs/blob/main/pages/en/querying/graphql-api.mdx#schema) for a complete reference on defining the schema for The Graph. - -When you complete the schema, add its path to the top-level `schema` key in the subgraph manifest. See the code below for an example: - -```yaml -specVersion: 0.0.1 -schema: - file: ./schema.graphql -``` - -### 1.3 Create a Subgraph Project and Generate Types -Once you have the `subgraph.yaml` manifest and the `./schema.graphql` file, you are ready to use the Graph CLI to set up the subgraph directory. The Graph CLI is a command-line tool that contains helpful commands for deploying the subgraphs. Before continuing with this guide, please go to the [Graph CLI README](https://github.com/graphprotocol/graph-cli/) and follow the instructions up to Step 7 for setting up the subgraph directory. - -Once you run `yarn codegen` as outlined in the [Graph CLI README](https://github.com/graphprotocol/graph-cli/), you are ready to create the mappings. - -`yarn codegen` looks at the contract ABIs defined in the subgraph manifest and generates TypeScript classes for the smart contracts the mappings script will interface with, which includes the types of public methods and events. In reality, the classes are AssemblyScript but more on that later. - -Classes are also generated based on the types defined in the GraphQL schema. These generated classes are incredibly useful for writing correct mappings. This allows you to autocomplete Ethererum events as well as improve developer productivity using the TypeScript language support in your favorite editor or IDE. - -### 1.4 Write the Mappings - -The mappings that you write will perform transformations on the Ethereum data you are sourcing, and it will dictate how this data is loaded into the Graph Node. Mappings can be very simple but can become complex. It depends on how much abstraction you want between the data and the underlying Ethereum contract. - -Mappings are written in a subset of TypeScript called AssemblyScript, which can be compiled down to WASM. AssemblyScript is stricter than normal TypeScript but follows the same backbone. A few TypeScript/JavaScript features that are not supported in AssemblyScript include plain old Javascript objects (POJOs), untyped arrays, untyped maps, union types, the `any` type, and variadic functions. In addition, `switch` statements also work differently. See the [AssemblyScript wiki](https://github.com/AssemblyScript/assemblyscript/wiki) for a full reference on AssemblyScript features. - -In the mapping file, create export functions named after the event handlers in the subgraph manifest. Each handler should accept a single parameter called `event` with a type corresponding to the name of the event that is being handled. This type was generated for you in the previous step, 1.3. - -```typescript -export function handleTransfer(event: Transfer): void { - // Event handler logic goes here -} -``` - -As mentioned, AssemblyScript does not have untyped maps or POJOs, so classes are generated to represent the types defined in the GraphQL schema. The generated type classes handle property type conversions for you, so AssemblyScript's requirement of strictly typed functions is satisfied without the extra work of converting each property explicitly. - -Let's look at an example. Continuing with our previous token example, let's write a mapping that tracks the owner of a particular ERC721 token. - -```typescript - -// This is an example event type generated by `graph-cli` -// from an Ethereum smart contract ABI -import { Transfer } from './types/abis/SomeContract' - -// This is an example of an entity type generated from a -// subgraph's GraphQL schema -import { Token } from './types/schema' - -export function handleTransfer(event: Transfer): void { - let tokenID = event.params.tokenID.toHex() - let token = new Token(tokenID) - token.currentOwner = event.params.to - - token.save() -} -``` -A few things to note from this code: -* We create a new entity named `token`, which is stored in the Graph Node database. -* We create an ID for that token, which must be unique, and then create an entity with `new Token(tokenID)`. We get the token ID from the event emitted by Ethereum, which was turned into an AssemblyScript type by the [Graph TypeScript Library](https://github.com/graphprotocol/graph-ts). We access it at `event.params.tokenId`. Note that you must set `ID` as a string and call `toHex()` on the `tokenID` to turn it into a hex string. -* This entity is updated by the `Transfer` event emitted by the ERC721 contract. -* The current owner is gathered from the event with `event.params.to`. It is set as an Address by the Token class. -* Event handlers functions always return `void`. -* `token.save()` is used to set the Token entity. `.save()` comes from `graph-ts` just like the entity type (`Token` in this example). It is used for setting the value(s) of a particular entity's attribute(s) in the store. There is also a `.load()` function, which will be explained in 1.4.1. - -#### 1.4.1 Use the `save`, `load`, and `remove` entity functions - -The only way that entities may be added to The Graph is by calling `.save()`, which may be called multiple times in an event handler. `.save()` will only set the entity attributes that have explicitly been set on the `entity`. Attributes that are not explicitly set or are unset by calling `Entity.unset()` will not be overwritten. This means you can safely update one field of an entity and not worry about overwriting other fields not referenced in the mapping. - -The definition for `.save()` is: - -```typescript -entity.save() // Entity is representative of the entity type being updated. In our example above, it is Token. -``` - - `.load()` expects the entity type and ID of the entity. Use `.load()` to retrieve information previously added with `.save()`. - -The definition for `.load()` is: - - ```typescript -entity.load() // Entity is representative of the entity type being updated. In our example above, it is Token. -``` - -Once again, all these functions come from the [Graph TypeScript Library](https://github.com/graphprotocol/graph-ts). - -Let's look at the ERC721 token as an example for using `token.load()`. Above, we showed how to use `token.save()`. Now, let's consider that you have another event handler that needs to retrieve the currentOwner of an ERC721 token. To do this within an event handler, you would write the following: - -```typescript - let token = token.load(tokenID.toHex()) - if (token !== null) { - let owner = token.currentOwner - } -``` - -You now have the `owner` data, and you can use that in the mapping to set the owner value to a new entity. - -There is also `.remove()`, which allows you to erase an entry that exists in the store. You simply pass the entity and ID: - -```typescript -entity.remove(ID) -``` - -#### 1.4.2 Call into the Contract Storage to Get Data - -You can also obtain data that is stored in one of the included ABI contracts. Any state variable that is marked `public` or any `view` function can be accessed. Below shows how you obtain the token -symbol of an ERC721 token, which is a state variable of the smart contract. You would add this inside of the event handler function. - -```typescript - let tokenContract = ERC721.bind(event.address); - let tokenSymbol = tokenContract.symbol(); -``` - -Note, we are using an ERC721 class generated from the ABI, which we call bind on. This is gathered from the subgraph manifest here: -```yaml - source: - address: "0x06012c8cf97BEaD5deAe237070F9587f8E7A266d" - abi: ERC721 -``` - -The class is imported from the ABI's TypeScript file generated via `yarn codegen`. - -## 2 Deploy the Subgraph - -### 2.1 Start Up an IPFS Node -To deploy the subgraph to the Graph Node, the subgraph will first need to be built and stored on IPFS, along with all linked files. - -To run an IPFS daemon locally, execute the following: -1. Download and install IPFS. -2. Run `ipfs init`. -3. Run `ipfs daemon`. - -If you encounter problems, follow the instructions from the [IPFS website](https://ipfs.io/docs/getting-started/). - -To confirm the subgraph is stored on IPFS, pass that subgraph ID into `ipfs cat` to view the subgraph manifest with file paths replaced by IPLD links. - -### 2.2 Create the Postgres database - -Ensure that you have Postgres installed. Navigate to a location where you want to save the `.postgres` folder. The desktop is fine since this folder can be used for many different subgraphs. Then, run the following commands: - -``` -initdb -D .postgres -pg_ctl -D .postgres -l logfile start -createdb -``` -Name the database something relevant to the project so that you always know how to access it. - -### 2.3 Start the Graph Node and Connect to an Ethereum Node - -When you start the Graph Node, you need to specify which Ethereum network it should connect to. There are three common ways to do this: - * Infura - * A local Ethereum node - * Ganache - -The Ethereum Network (Mainnet, Ropsten, Rinkeby, etc.) must be passed as a flag in the command that starts the Graph Node as laid out in the following subsections. - -#### 2.3.1 Infura - -[Infura](https://infura.io/) is supported and is the simplest way to connect to an Ethereum node because you do not have to set up your own geth or parity node. However, it does sync slower than being connected to your own node. The following flags are passed to start the Graph Node and indicate you want to use Infura: - -```sh -cargo run -p graph-node --release -- \ - --postgres-url postgresql://<:PASSWORD>@localhost:5432/ \ - --ethereum-rpc :https://mainnet.infura.io \ - --ipfs 127.0.0.1:5001 \ - --debug -``` - -Also, note that the Postgres database may not have a password at all. If that is the case, the Postgres connection URL can be passed as follows: - -` --postgres-url postgresql://@localhost:5432/ \ ` - -#### 2.3.2 Local Geth or Parity Node - -This is the speediest way to get mainnet or testnet data. The problem is that if you do not already have a synced [geth](https://geth.ethereum.org/docs/getting-started) or [parity](https://github.com/paritytech/parity-ethereum) node, you will have to sync one, which takes a very long time and takes up a lot of space. Additionally, note that geth `fast sync` works. So, if you are starting from scratch, this is the fastest way to get caught up, but expect at least 12 hours of syncing on a modern laptop with a good internet connection to sync geth. Normal mode geth or parity will take much longer. Use the following geth command to start syncing: - -`geth --syncmode "fast" --rpc --ws --wsorigins="*" --rpcvhosts="*" --cache 1024` - -Once you have the local node fully synced, run the following command: - -```sh -cargo run -p graph-node --release -- \ - --postgres-url postgresql://<:PASSWORD>@localhost:5432/ \ - --ethereum-rpc :127.0.0.1:8545 \ - --ipfs 127.0.0.1:5001 \ - --debug -``` - -This assumes the local node is on the default `8545` port. If you are on a different port, change it. - -Switching back and forth between sourcing data from Infura and your own local nodes is fine. The Graph Node picks up where it left off. - -#### 2.3.3 Ganache - -**IMPORTANT: Ganache fixed the [issue](https://github.com/trufflesuite/ganache/issues/907) that prevented things from working properly. However, it did not release the new version. Follow the steps in this [issue](https://github.com/graphprotocol/graph-node/issues/375) to run the fixed version locally.** - -[Ganache](https://github.com/trufflesuite/ganache-cli) can be used as well and is preferable for quick testing. This might be an option if you are simply testing out the contracts for quick iterations. Of course, if you close Ganache, then the Graph Node will no longer have any data to source. Ganache is best for short-term projects such as hackathons. Also, it is useful for testing to see that the schema and mappings are working properly before working on the mainnet. - -You can connect the Graph Node to Ganache the same way you connected to a local geth or parity node in the previous section, 2.3.2. Note, however, that Ganache normally runs on port `9545` instead of `8545`. - -#### 2.3.4 Local Parity Testnet - -To set up a local testnet that will allow you to rapidly test the project, download the parity software if you do not already have it. - -This command will work for a one-line install: - -`bash <(curl https://get.parity.io -L)` - -Next, you want to make an account that you can unlock and make transactions on for the parity dev chain. Run the following command: - -`parity account new --chain dev` - -Create a password that you will remember. Take note of the account that gets output. Now, you also have to make that password a text file and pass it into the next command. The desktop is a good location for it. If the password is `123`, only put the numbers in the text file. Do not include any quotes. - -Then, run this command: - -`parity --config dev --unsafe-expose --jsonrpc-cors="all" --unlock --password ~/Desktop/password.txt` - -The chain should start and will be accessible by default on `localhost:8545`. It is a chain with 0 block time and instant transactions, making testing very fast. Passing `unsafe-expose` and `--jsonrpc-cors="all"` as flags allows MetaMask to connect. The `unlock` flag gives parity the ability to send transactions with that account. You can also import the account to MetaMask, which allows you to interact with the test chain directly in your browser. With MetaMask, you need to import the account with the private testnet Ether. The base account that the normal configuration of parity gives you is -`0x00a329c0648769A73afAc7F9381E08FB43dBEA72`. - -The private key is: -``` -4d5db4107d237df6a3d58ee5f70ae63d73d7658d4026f2eefd2f204c81682cb7 (note this is the private key given along with the parity dev chain, so it is okay to share) -``` -Use MetaMask ---> import account ---> private key. - -All the extra information for customization of a parity dev chain is located [here](https://wiki.parity.io/Private-development-chain#customizing-the-development-chain). - -You now have an Ethereum account with a ton of Ether and should be able to set up the migrations on this network and use Truffle. Now, send some Ether to the previous account that was created and unlocked. This way, you can run `truffle migrate` with this account. - -#### 2.3.5 Syncing with a Public Testnet - -If you want to sync using a public testnet such as Kovan, Rinkeby, or Ropsten, just make sure the local node is a testnet node or that you are hitting the correct Infura testnet endpoint. - -### 2.4 Deploy the Subgraph - -When you deploy the subgraph to the Graph Node, it will start ingesting all the subgraph events from the blockchain, transforming that data with the subgraph mappings and storing it in the Graph Node. Note that a running subgraph can safely be stopped and restarted, picking up where it left off. - -Now that the infrastructure is set up, you can run `yarn create-subgraph` and then `yarn deploy` in the subgraph directory. These commands should have been added to `package.json` in section 1.3 when we took a moment to go through the set up for [Graph CLI documentation](https://github.com/graphprotocol/graph-cli). This builds the subgraph and creates the WASM files in the `dist/` folder. Next, it uploads the `dist/ -` files to IPFS and deploys it to the Graph Node. The subgraph is now fully running. - -The `watch` flag allows the subgraph to continually restart every time you save an update to the `manifest`, `schema`, or `mappings`. If you are making many edits or have a subgraph that has been syncing for a few hours, leave this flag off. - -Depending on how many events have been emitted by your smart contracts, it could take less than a minute to get fully caught up. If it is a large contract, it could take hours. For example, ENS takes about 12 to 14 hours to register every single ENS domain. - -## 3 Query the Local Graph Node -With the subgraph deployed to the locally running Graph Node, visit http://127.0.0.1:8000/ to open up a [GraphiQL](https://github.com/graphql/graphiql) interface where you can explore the deployed GraphQL API for the subgraph by issuing queries and viewing the schema. - -We provide a few simple examples below, but please see the [Query API](https://github.com/graphprotocol/docs/blob/main/pages/en/querying/graphql-api.mdx#queries) for a complete reference on how to query the subgraph's entities. - -Query the `Token` entities: -```graphql -{ - tokens(first: 100) { - id - currentOwner - } -} -``` -Notice that `tokens` is plural and that it will return at most 100 entities. - -Later, when you have deployed the subgraph with this entity, you can query for a specific value, such as the token ID: - -```graphql -{ - token(first: 100, id: "c2dac230ed4ced84ad0ca5dfb3ff8592d59cef7ff2983450113d74a47a12") { - currentOwner - } -} -``` - -You can also sort, filter, or paginate query results. The query below would organize all tokens by their ID and return the current owner of each token. - -```graphql -{ - tokens(first: 100, orderBy: id) { - currentOwner - } -} -``` - -GraphQL provides a ton of functionality. Once again, check out the [Query API](graphql-api.md#1-queries) to find out how to use all supported query features. - -## 4 Changing the Schema, Mappings, and Manifest, and Launching a New Subgraph - -When you first start building the subgraph, it is likely that you will make a few changes to the manifest, mappings, or schema. If you update any of them, rerun `yarn codegen` and `yarn deploy`. This will post the new files on IPFS and deploy the new subgraph. Note that the Graph Node can track multiple subgraphs, so you can do this as many times as you like. - -## 5 Common Patterns for Building Subgraphs - -### 5.1 Removing Elements of an Array in a Subgraph - -Using the AssemblyScript built-in functions for arrays is the way to go. Find the source code [here](https://github.com/AssemblyScript/assemblyscript/blob/18826798074c9fb02243dff76b1a938570a8eda7/std/assembly/array.ts). Using `.indexOf()` to find the element and then using `.splice()` is one way to do so. See this [file](https://github.com/graphprotocol/aragon-subgraph/blob/master/individual-dao-subgraph/mappings/ACL.ts) from the Aragon subgraph for a working implementation. - -### 5.2 Getting Data from Multiple Versions of Your Contracts - -If you have launched multiple versions of your smart contracts onto Ethereum, it is very easy to source data from all of them. This simply requires you to add all versions of the contracts to the `subgraph.yaml` file and handle the events from each contract. Design your schema to consider both versions, and handle any changes to the event signatures that are emitted from each version. See the [0x Subgraph](https://github.com/graphprotocol/0x-subgraph/tree/master/src/mappings) for an implementation of multiple versions of smart contracts being ingested by a subgraph. - -## 5 Example Subgraphs - -Here is a list of current subgraphs that we have open sourced: -* https://github.com/graphprotocol/ens-subgraph -* https://github.com/graphprotocol/decentraland-subgraph -* https://github.com/graphprotocol/adchain-subgraph -* https://github.com/graphprotocol/0x-subgraph -* https://github.com/graphprotocol/aragon-subgraph -* https://github.com/graphprotocol/dharma-subgraph -* https://github.com/daostack/subgraph -* https://github.com/graphprotocol/dydx-subgraph -* https://github.com/livepeer/livepeerjs/tree/master/packages/subgraph -* https://github.com/graphprotocol/augur-subgraph - -## Contributions - -All feedback and contributions in the form of issues and pull requests are welcome! - diff --git a/docs/graphman-graphql-api.md b/docs/graphman-graphql-api.md new file mode 100644 index 00000000000..486bee6090d --- /dev/null +++ b/docs/graphman-graphql-api.md @@ -0,0 +1,213 @@ +# Graphman GraphQL API + +The graphman API provides functionality to manage various aspects of `graph-node` through GraphQL operations. It is only +started when the environment variable `GRAPHMAN_SERVER_AUTH_TOKEN` is set. The token is used to authenticate graphman +GraphQL requests. Even with the token, the server should not be exposed externally as it provides operations that an +attacker can use to severely impede the functioning of an indexer. The server listens on the port `GRAPHMAN_PORT`, port +`8050` by default. + +Environment variables to control the graphman API: + +- `GRAPHMAN_SERVER_AUTH_TOKEN` - The token is used to authenticate graphman GraphQL requests. +- `GRAPHMAN_PORT` - The port for the graphman GraphQL server (Defaults to `8050`) + +## GraphQL playground + +When the graphman GraphQL server is running the GraphQL playground is available at the following +address: http://127.0.0.1:8050 + +**Note:** The port might be different. + +Please make sure to set the authorization header to be able to use the playground: + +```json +{ + "Authorization": "Bearer GRAPHMAN_SERVER_AUTH_TOKEN" +} +``` + +**Note:** There is a headers section at the bottom of the playground page. + +## Supported commands + +The playground is the best place to see the full schema, the latest available queries and mutations, and their +documentation. Below, we will briefly describe some supported commands and example queries. + +At the time of writing, the following graphman commands are available via the GraphQL API: + +### Deployment Info + +Returns the available information about one, multiple, or all deployments. + +**Example query:** + +```text +query { + deployment { + info(deployment: { hash: "Qm..." }) { + status { + isPaused + } + } + } +} +``` + +**Example response:** + +```json +{ + "data": { + "deployment": { + "info": [ + { + "status": { + "isPaused": false + } + } + ] + } + } +} +``` + +### Pause Deployment + +Pauses a deployment that is not already paused. + +**Example query:** + +```text +mutation { + deployment { + pause(deployment: { hash: "Qm..." }) { + success + } + } +} +``` + +**Example response:** + +```json +{ + "data": { + "deployment": { + "pause": { + "success": true + } + } + } +} +``` + +### Resume Deployment + +Resumes a deployment that has been previously paused. + +**Example query:** + +```text +mutation { + deployment { + resume(deployment: { hash: "Qm..." }) { + success + } + } +} +``` + +**Example response:** + +```json +{ + "data": { + "deployment": { + "resume": { + "success": true + } + } + } +} +``` + +### Restart Deployment + +Pauses a deployment and resumes it after a delay. + +**Example query:** + +```text +mutation { + deployment { + restart(deployment: { hash: "Qm..." }) { + id + } + } +} +``` + +**Example response:** + +```json +{ + "data": { + "deployment": { + "restart": { + "id": "UNIQUE_EXECUTION_ID" + } + } + } +} +``` + +This is a long-running command because the default delay before resuming the deployment is 20 seconds. Long-running +commands are executed in the background. For long-running commands, the GraphQL API will return a unique execution ID. + +The ID can be used to query the execution status and the output of the command: + +```text +query { + execution { + info(id: "UNIQUE_EXECUTION_ID") { + status + errorMessage + } + } +} +``` + +**Example response when execution is in-progress:** + +```json +{ + "data": { + "execution": { + "info": { + "status": "RUNNING", + "errorMessage": null + } + } + } +} +``` + +**Example response when execution is completed:** + +```json +{ + "data": { + "execution": { + "info": { + "status": "SUCCEEDED", + "errorMessage": null + } + } + } +} +``` + +## Other commands + +GraphQL support for other graphman commands will be added over time, so please make sure to check the GraphQL playground +for the full schema and the latest available queries and mutations. diff --git a/docs/graphman.md b/docs/graphman.md index 0964efc6051..8c857703dda 100644 --- a/docs/graphman.md +++ b/docs/graphman.md @@ -52,7 +52,7 @@ By default, it shows the following attributes for the deployment: - **name** - **status** *(`pending` or `current`)* - **id** *(the `Qm...` identifier for the deployment's subgraph)* -- **namespace** *(The database schema which contain's that deployment data tables)* +- **namespace** *(The database schema which contains that deployment data tables)* - **shard** - **active** *(If there are multiple entries for the same subgraph, only one of them will be active. That's the one we use for querying)* - **chain** @@ -169,7 +169,7 @@ primary shard. No indexed data is lost as a result of this command. -This sub-command is used as previus step towards removing all data from unused subgraphs, followed by +This sub-command is used as previous step towards removing all data from unused subgraphs, followed by `graphman unused remove`. A deployment is unused if it fulfills all of these criteria: @@ -236,7 +236,7 @@ Remove a specific unused deployment ### SYNOPSIS - Delete a deployment and all it's indexed data + Delete a deployment and all its indexed data The deployment can be specified as either a subgraph name, an IPFS hash `Qm..`, or the database namespace `sgdNNN`. Since the same IPFS hash can be deployed in multiple shards, it is possible to @@ -288,7 +288,7 @@ Stop, unassign and delete all indexed data from a specific deployment by its dep Stop, unassign and delete all indexed data from a specific deployment by its subgraph name - graphman --config config.toml drop autor/subgraph-name + graphman --config config.toml drop author/subgraph-name
# ⌘ Check Blocks @@ -322,7 +322,7 @@ is useful to diagnose the integrity of cached blocks and eventually fix them. ### OPTIONS -Blocks can be selected by different methods. The `check-blocks` command let's you use the block hash, a single +Blocks can be selected by different methods. The `check-blocks` command lets you use the block hash, a single number or a number range to refer to which blocks it should verify: #### `by-hash` @@ -338,7 +338,7 @@ number or a number range to refer to which blocks it should verify: graphman --config chain check-blocks by-range [-f|--from ] [-t|--to ] [--delete-duplicates] The `by-range` method lets you scan for numeric block ranges and offers the `--from` and `--to` options for -you to define the search bounds. If one of those options is ommited, `graphman` will consider an open bound +you to define the search bounds. If one of those options is omitted, `graphman` will consider an open bound and will scan all blocks up to or after that number. Over time, it can happen that a JSON RPC provider offers different blocks for the same block number. In those @@ -371,21 +371,30 @@ Inspect all blocks after block `13000000`: Remove the call cache of the specified chain. -If block numbers are not mentioned in `--from` and `--to`, then all the call cache will be removed. +Either remove entries in the range `--from` and `--to`, remove stale contracts which have not been accessed for a specified duration `--ttl_days`, or remove the entire cache with `--remove-entire-cache`. Removing the entire cache can reduce indexing performance significantly and should generally be avoided. -USAGE: - graphman chain call-cache remove [OPTIONS] + Usage: graphman chain call-cache remove [OPTIONS] -OPTIONS: - -f, --from - Starting block number + Options: + --remove-entire-cache + Remove the entire cache + + --ttl-days + Remove stale contracts based on call_meta table - -h, --help - Print help information + --ttl-max-contracts + Limit the number of contracts to consider for stale contract removal + + -f, --from + Starting block number - -t, --to + -t, --to Ending block number + -h, --help + Print help (see a summary with '-h') + + ### DESCRIPTION Remove the call cache of a specified chain. @@ -404,6 +413,15 @@ the first block number will be used as the starting block number. The `to` option is used to specify the ending block number of the block range. In the absence of `to` option, the last block number will be used as the ending block number. +#### `--remove-entire-cache` +The `--remove-entire-cache` option is used to remove the entire call cache of the specified chain. + +#### `--ttl-days ` +The `--ttl-days` option is used to remove stale contracts based on the `call_meta.accessed_at` field. For example, if `--ttl-days` is set to 7, all calls to a contract that has not been accessed in the last 7 days will be removed from the call cache. + +#### `--ttl-max-contracts ` +The `--ttl-max-contracts` option is used to limit the maximum number of contracts to be removed when using the `--ttl-days` option. For example, if `--ttl-max-contracts` is set to 100, at most 100 contracts will be removed from the call cache even if more contracts meet the TTL criteria. + ### EXAMPLES Remove the call cache for all blocks numbered from 10 to 20: @@ -412,5 +430,12 @@ Remove the call cache for all blocks numbered from 10 to 20: Remove all the call cache of the specified chain: - graphman --config config.toml chain call-cache ethereum remove + graphman --config config.toml chain call-cache ethereum remove --remove-entire-cache + +Remove stale contracts from the call cache that have not been accessed in the last 7 days: + + graphman --config config.toml chain call-cache ethereum remove --ttl-days 7 + +Remove stale contracts from the call cache that have not been accessed in the last 7 days, limiting the removal to a maximum of 100 contracts: + graphman --config config.toml chain call-cache ethereum remove --ttl-days 7 --ttl-max-contracts 100 diff --git a/docs/implementation/README.md b/docs/implementation/README.md index 441c5f279aa..d54a39babbe 100644 --- a/docs/implementation/README.md +++ b/docs/implementation/README.md @@ -9,3 +9,5 @@ the code should go into comments. * [Time-travel Queries](./time-travel.md) * [SQL Query Generation](./sql-query-generation.md) * [Adding support for a new chain](./add-chain.md) +* [Pruning](./pruning.md) +* [Dump Format](./dump.md) diff --git a/docs/implementation/add-chain.md b/docs/implementation/add-chain.md deleted file mode 100644 index eea61687910..00000000000 --- a/docs/implementation/add-chain.md +++ /dev/null @@ -1,279 +0,0 @@ -# Adding support for a new chain - -## Context - -`graph-node` started as a project that could only index EVM compatible chains, eg: `ethereum`, `xdai`, etc. - -It was known from the start that with growth we would like `graph-node` to be able to index other chains like `NEAR`, `Solana`, `Cosmos`, list goes on... - -However to do it, several refactors were necessary, because the code had a great amount of assumptions based of how Ethereum works. - -At first there was a [RFC](https://github.com/graphprotocol/rfcs/blob/10aaae30fdf82f0dd2ccdf4bbecf7ec6bbfb703b/rfcs/0005-multi-blockchain-support.md) for a design overview, then actual PRs such as: - -- https://github.com/graphprotocol/graph-node/pull/2272 -- https://github.com/graphprotocol/graph-node/pull/2292 -- https://github.com/graphprotocol/graph-node/pull/2399 -- https://github.com/graphprotocol/graph-node/pull/2411 -- https://github.com/graphprotocol/graph-node/pull/2453 -- https://github.com/graphprotocol/graph-node/pull/2463 -- https://github.com/graphprotocol/graph-node/pull/2755 - -All new chains, besides the EVM compatible ones, are integrated using [StreamingFast](https://www.streamingfast.io/)'s [Firehose](https://firehose.streamingfast.io/). The integration consists of chain specific `protobuf` files with the type definitions. - -## How to do it? - -The `graph-node` repository contains multiple Rust crates in it, this section will be divided in each of them that needs to be modified/created. - -> It's important to remember that this document is static and may not be up to date with the current implementation. Be aware too that it won't contain all that's needed, it's mostly listing the main areas that need change. - -### chain - -You'll need to create a new crate in the [chain folder](https://github.com/graphprotocol/graph-node/tree/1cd7936f9143f317feb51be1fc199122761fcbb1/chain) with an appropriate name and the same `version` as the rest of the other ones. - -> Note: you'll probably have to add something like `graph-chain-{{CHAIN_NAME}} = { path = "../chain/{{CHAIN_NAME}}" }` to the `[dependencies]` section of a few other `Cargo.toml` files - -It's here that you add the `protobuf` definitions with the specific types for the chain you're integrating with. Examples: - -- [Ethereum](https://github.com/graphprotocol/graph-node/blob/1cd7936f9143f317feb51be1fc199122761fcbb1/chain/ethereum/proto/codec.proto) -- [NEAR](https://github.com/graphprotocol/graph-node/blob/1cd7936f9143f317feb51be1fc199122761fcbb1/chain/near/proto/codec.proto) -- [Cosmos](https://github.com/graphprotocol/graph-node/blob/caa54c1039d3c282ac31bb0e96cb277dbf82f793/chain/cosmos/proto/type.proto) - -To compile those we use a crate called `tonic`, it will require a [`build.rs` file](https://doc.rust-lang.org/cargo/reference/build-scripts.html) like the one in the other folders/chains, eg: - -```rust -fn main() { - println!("cargo:rerun-if-changed=proto"); - tonic_build::configure() - .out_dir("src/protobuf") - .compile(&["proto/codec.proto"], &["proto"]) - .expect("Failed to compile Firehose CoolChain proto(s)"); -} -``` - -You'll also need a `src/codec.rs` to extract the data from the generated Rust code, much like [this one](https://github.com/graphprotocol/graph-node/blob/caa54c1039d3c282ac31bb0e96cb277dbf82f793/chain/cosmos/src/codec.rs). - -Besides this source file, there should also be a `TriggerFilter`, `NodeCapabilities` and `RuntimeAdapter`, here are a few empty examples: - -`src/adapter.rs` -```rust -use crate::capabilities::NodeCapabilities; -use crate::{data_source::DataSource, Chain}; -use graph::blockchain as bc; -use graph::prelude::*; - -#[derive(Clone, Debug, Default)] -pub struct TriggerFilter {} - -impl bc::TriggerFilter for TriggerFilter { - fn extend<'a>(&mut self, _data_sources: impl Iterator + Clone) {} - - fn node_capabilities(&self) -> NodeCapabilities { - NodeCapabilities {} - } - - fn extend_with_template( - &mut self, - _data_source: impl Iterator::DataSourceTemplate>, - ) { - } - - fn to_firehose_filter(self) -> Vec { - vec![] - } -} -``` - -`src/capabilities.rs` -```rust -use std::cmp::PartialOrd; -use std::fmt; -use std::str::FromStr; - -use anyhow::Error; -use graph::impl_slog_value; - -use crate::DataSource; - -#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd)] -pub struct NodeCapabilities {} - -impl FromStr for NodeCapabilities { - type Err = Error; - - fn from_str(_s: &str) -> Result { - Ok(NodeCapabilities {}) - } -} - -impl fmt::Display for NodeCapabilities { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - f.write_str("{{CHAIN_NAME}}") - } -} - -impl_slog_value!(NodeCapabilities, "{}"); - -impl graph::blockchain::NodeCapabilities for NodeCapabilities { - fn from_data_sources(_data_sources: &[DataSource]) -> Self { - NodeCapabilities {} - } -} -``` - -`src/runtime/runtime_adapter.rs` -```rust -use crate::{Chain, DataSource}; -use anyhow::Result; -use blockchain::HostFn; -use graph::blockchain; - -pub struct RuntimeAdapter {} - -impl blockchain::RuntimeAdapter for RuntimeAdapter { - fn host_fns(&self, _ds: &DataSource) -> Result> { - Ok(vec![]) - } -} -``` - -The chain specific type definitions should also be available for the `runtime`. Since it comes mostly from the `protobuf` files, there's a [generation tool](https://github.com/streamingfast/graph-as-to-rust) made by StreamingFast that you can use to create the `src/runtime/generated.rs`. - -You'll also have to implement `ToAscObj` for those types, that usually is made in a `src/runtime/abi.rs` file. - -Another thing that will be needed is the `DataSource` types for the [subgraph manifest](https://thegraph.com/docs/en/developer/create-subgraph-hosted/#the-subgraph-manifest). - -`src/data_source.rs` -```rust -#[derive(Clone, Debug)] -pub struct DataSource { - // example fields: - pub kind: String, - pub network: Option, - pub name: String, - pub source: Source, - pub mapping: Mapping, - pub context: Arc>, - pub creation_block: Option, - /*...*/ -} - -impl blockchain::DataSource for DataSource { /*...*/ } - -#[derive(Clone, Debug, Eq, PartialEq, Deserialize)] -pub struct UnresolvedDataSource { - pub kind: String, - pub network: Option, - pub name: String, - pub source: Source, - pub mapping: UnresolvedMapping, - pub context: Option, -} - -#[async_trait] -impl blockchain::UnresolvedDataSource for UnresolvedDataSource { /*...*/ } - -#[derive(Clone, Debug, Default, Hash, Eq, PartialEq, Deserialize)] -pub struct BaseDataSourceTemplate { - pub kind: String, - pub network: Option, - pub name: String, - pub mapping: M, -} - -pub type UnresolvedDataSourceTemplate = BaseDataSourceTemplate; -pub type DataSourceTemplate = BaseDataSourceTemplate; - -#[async_trait] -impl blockchain::UnresolvedDataSourceTemplate for UnresolvedDataSourceTemplate { /*...*/ } - -impl blockchain::DataSourceTemplate for DataSourceTemplate { /*...*/ } -``` - -And at last, the type that will glue them all, the `Chain` itself. - -`src/chain.rs` -```rust -pub struct Chain { /*...*/ } - -#[async_trait] -impl Blockchain for Chain { - const KIND: BlockchainKind = BlockchainKind::CoolChain; - - type Block = codec::...; - - type DataSource = DataSource; - - // ... - - type TriggerFilter = TriggerFilter; - - type NodeCapabilities = NodeCapabilities; - - type RuntimeAdapter = RuntimeAdapter; -} - -pub struct TriggersAdapter { /*...*/ } - -#[async_trait] -impl TriggersAdapterTrait for TriggersAdapter { /*...*/ } - -pub struct FirehoseMapper { - endpoint: Arc, -} - -#[async_trait] -impl FirehoseMapperTrait for FirehoseMapper { /*...*/ } -``` - -### node - -The `src/main.rs` file should be able to handle the connection to the new chain via Firehose for the startup, similar to [this](https://github.com/graphprotocol/graph-node/blob/1cd7936f9143f317feb51be1fc199122761fcbb1/node/src/main.rs#L255). - -### graph - -Two changes are required here: - -1. [BlockchainKind](https://github.com/graphprotocol/graph-node/blob/1cd7936f9143f317feb51be1fc199122761fcbb1/graph/src/blockchain/mod.rs#L309) needs to have a new variant for the chain you're integrating with. -2. And the [IndexForAscTypeId](https://github.com/graphprotocol/graph-node/blob/1cd7936f9143f317feb51be1fc199122761fcbb1/graph/src/runtime/mod.rs#L147) should have the new variants for the chain specific types of the `runtime`. - -### server - -You'll just have to handle the new `BlockchainKind` in the [index-node/src/resolver.rs](https://github.com/graphprotocol/graph-node/blob/1cd7936f9143f317feb51be1fc199122761fcbb1/server/index-node/src/resolver.rs#L361). - -### core - -Just like in the `server` crate, you'll just have to handle the new `BlockchainKind` in the [SubgraphInstanceManager](https://github.com/graphprotocol/graph-node/blob/1cd7936f9143f317feb51be1fc199122761fcbb1/core/src/subgraph/instance_manager.rs#L41). - -## Example Integrations (PRs) - -- NEAR by StreamingFast - - https://github.com/graphprotocol/graph-node/pull/2820 -- Cosmos by Figment - - https://github.com/graphprotocol/graph-node/pull/3212 - - https://github.com/graphprotocol/graph-node/pull/3543 -- Solana by StreamingFast - - https://github.com/graphprotocol/graph-node/pull/3210 - -## What else? - -Besides making `graph-node` support the new chain, [graph-cli](https://github.com/graphprotocol/graph-cli) and [graph-ts](https://github.com/graphprotocol/graph-ts) should also include the new types and enable the new functionality so that subgraph developers can use it. - -For now this document doesn't include how to do that integration, here are a few PRs that might help you with that: - -- NEAR - - `graph-cli` - - https://github.com/graphprotocol/graph-cli/pull/760 - - https://github.com/graphprotocol/graph-cli/pull/783 - - `graph-ts` - - https://github.com/graphprotocol/graph-ts/pull/210 - - https://github.com/graphprotocol/graph-ts/pull/217 -- Cosmos - - `graph-cli` - - https://github.com/graphprotocol/graph-cli/pull/827 - - https://github.com/graphprotocol/graph-cli/pull/851 - - https://github.com/graphprotocol/graph-cli/pull/888 - - `graph-ts` - - https://github.com/graphprotocol/graph-ts/pull/250 - - https://github.com/graphprotocol/graph-ts/pull/273 - -Also this document doesn't include the multi-blockchain part required for The Graph Network, which at this current moment is in progress, for now the network only supports Ethereum `mainnet`. diff --git a/docs/implementation/metadata.md b/docs/implementation/metadata.md index ee54fb30361..1cf3c189c6c 100644 --- a/docs/implementation/metadata.md +++ b/docs/implementation/metadata.md @@ -7,7 +7,7 @@ List of all known subgraph names. Maintained in the primary, but there is a background job that periodically copies the table from the primary to all other shards. Those copies are used for queries when the primary is down. | Column | Type | Use | -|-------------------|--------------|-------------------------------------------| +| ----------------- | ------------ | ----------------------------------------- | | `id` | `text!` | primary key, UUID | | `name` | `text!` | user-chosen name | | `current_version` | `text` | `subgraph_version.id` for current version | @@ -18,13 +18,12 @@ List of all known subgraph names. Maintained in the primary, but there is a back The `id` is used by the hosted explorer to reference the subgraph. - ### `subgraphs.subgraph_version` Mapping of subgraph names from `subgraph` to IPFS hashes. Maintained in the primary, but there is a background job that periodically copies the table from the primary to all other shards. Those copies are used for queries when the primary is down. | Column | Type | Use | -|---------------|--------------|-------------------------| +| ------------- | ------------ | ----------------------- | | `id` | `text!` | primary key, UUID | | `subgraph` | `text!` | `subgraph.id` | | `deployment` | `text!` | IPFS hash of deployment | @@ -32,15 +31,14 @@ Mapping of subgraph names from `subgraph` to IPFS hashes. Maintained in the prim | `vid` | `int8!` | unused | | `block_range` | `int4range!` | unused | - ## Managing a deployment Directory of all deployments. Maintained in the primary, but there is a background job that periodically copies the table from the primary to all other shards. Those copies are used for queries when the primary is down. -### `deployment_schemas` +### `public.deployment_schemas` | Column | Type | Use | -|--------------|----------------|----------------------------------------------| +| ------------ | -------------- | -------------------------------------------- | | `id` | `int4!` | primary key | | `subgraph` | `text!` | IPFS hash of deployment | | `name` | `text!` | name of `sgdNNN` schema | @@ -52,49 +50,66 @@ Directory of all deployments. Maintained in the primary, but there is a backgrou There can be multiple copies of the same deployment, but at most one per shard. The `active` flag indicates which of these copies will be used for queries; `graph-node` makes sure that there is always exactly one for each IPFS hash. -### `subgraph_deployment` +### `subgraphs.head` + +Details about a deployment that change on every block. Maintained in the +shard alongside the deployment's data in `sgdNNN`. + +| Column | Type | Use | +| ----------------- | ---------- | -------------------------------------------- | +| `id` | `integer!` | primary key, same as `deployment_schemas.id` | +| `block_hash` | `bytea` | current subgraph head | +| `block_number` | `numeric` | | +| `entity_count` | `numeric!` | total number of entities | +| `firehose_cursor` | `text` | | + +The head block pointer in `block_number` and `block_hash` is the latest +block that has been fully processed by the deployment. It will be `null` +until the deployment is fully initialized, and only set when the deployment +processes the first block. For deployments that are grafted or being copied, +the head block pointer will be `null` until the graft/copy has finished +which can take considerable time. + +### `subgraphs.deployment` Details about a deployment to track sync progress etc. Maintained in the shard alongside the deployment's data in `sgdNNN`. The table should only -contain frequently changing data, but for historical reasons contains also -static data. - -| Column | Type | Use | -|--------------------------------------|------------|----------------------------------------------| -| `id` | `integer!` | primary key, same as `deployment_schemas.id` | -| `deployment` | `text!` | IPFS hash | -| `failed` | `boolean!` | | -| `synced` | `boolean!` | | -| `earliest_block_number` | `integer!` | earliest block for which we have data | -| `latest_ethereum_block_hash` | `bytea` | current subgraph head | -| `latest_ethereum_block_number` | `numeric` | | -| `entity_count` | `numeric!` | total number of entities | -| `graft_base` | `text` | IPFS hash of graft base | -| `graft_block_hash` | `bytea` | graft block | -| `graft_block_number` | `numeric` | | -| `reorg_count` | `integer!` | | -| `current_reorg_depth` | `integer!` | | -| `max_reorg_depth` | `integer!` | | -| `fatal_error` | `text` | | -| `non_fatal_errors` | `text[]` | | -| `health` | `health!` | | -| `last_healthy_ethereum_block_hash` | `bytea` | | -| `last_healthy_ethereum_block_number` | `numeric` | | -| `firehose_cursor` | `text` | | -| `debug_fork` | `text` | | +contain data that changes fairly infrequently, but for historical reasons +contains also static data. + +| Column | Type | Use | +| ------------------------------------ | ------------- | ---------------------------------------------------- | +| `id` | `integer!` | primary key, same as `deployment_schemas.id` | +| `subgraph` | `text!` | IPFS hash | +| `earliest_block_number` | `integer!` | earliest block for which we have data | +| `health` | `health!` | | +| `failed` | `boolean!` | | +| `fatal_error` | `text` | | +| `non_fatal_errors` | `text[]` | | +| `graft_base` | `text` | IPFS hash of graft base | +| `graft_block_hash` | `bytea` | graft block | +| `graft_block_number` | `numeric` | | +| `reorg_count` | `integer!` | | +| `current_reorg_depth` | `integer!` | | +| `max_reorg_depth` | `integer!` | | +| `last_healthy_ethereum_block_hash` | `bytea` | | +| `last_healthy_ethereum_block_number` | `numeric` | | +| `debug_fork` | `text` | | +| `synced_at` | `timestamptz` | time when deployment first reach chain head | +| `synced_at_block_number` | `integer` | block number where deployment first reach chain head | The columns `reorg_count`, `current_reorg_depth`, and `max_reorg_depth` are set during indexing. They are used to determine whether a reorg happened while a query was running, and whether that reorg could have affected the query. -### `subgraph_manifest` +### `subgraphs.subgraph_manifest` Details about a deployment that rarely change. Maintained in the shard alongside the deployment's data in `sgdNNN`. | Column | Type | Use | -|-------------------------|------------|------------------------------------------------------| +| ----------------------- | ---------- | ---------------------------------------------------- | | `id` | `integer!` | primary key, same as `deployment_schemas.id` | | `spec_version` | `text!` | | | `description` | `text` | | @@ -106,26 +121,27 @@ shard alongside the deployment's data in `sgdNNN`. | `start_block_hash` | `bytea` | Parent of the smallest start block from the manifest | | `start_block_number` | `int4` | | | `on_sync` | `text` | Additional behavior when deployment becomes synced | +| `history_blocks` | `int4!` | How many blocks of history to keep | -### `subgraph_deployment_assignment` +### `subgraphs.subgraph_deployment_assignment` Tracks which index node is indexing a deployment. Maintained in the primary, but there is a background job that periodically copies the table from the primary to all other shards. | Column | Type | Use | -|---------|-------|---------------------------------------------| +| ------- | ----- | ------------------------------------------- | | id | int4! | primary key, ref to `deployment_schemas.id` | | node_id | text! | name of index node | This table could simply be a column on `deployment_schemas`. -### `dynamic_ethereum_contract_data_source` +### `subgraphs.dynamic_ethereum_contract_data_source` Stores the dynamic data sources for all subgraphs (will be turned into a table that lives in each subgraph's namespace `sgdNNN` soon) -### `subgraph_error` +### `subgraphs.subgraph_error` Stores details about errors that subgraphs encounter during indexing. @@ -140,3 +156,16 @@ correctly across index node restarts. The table `subgraphs.table_stats` stores which tables for a deployment should have the 'account-like' optimization turned on. + +### `subgraphs.subgraph_features` + +Details about features that a deployment uses, Maintained in the primary. + +| Column | Type | Use | +| -------------- | --------- | ----------- | +| `id` | `text!` | primary key | +| `spec_version` | `text!` | | +| `api_version` | `text` | | +| `features` | `text[]!` | | +| `data_sources` | `text[]!` | | +| `handlers` | `text[]!` | | diff --git a/docs/implementation/offchain.md b/docs/implementation/offchain.md new file mode 100644 index 00000000000..268aba5157b --- /dev/null +++ b/docs/implementation/offchain.md @@ -0,0 +1,25 @@ +# Offchain data sources + +### Summary + +Graph Node supports syncing offchain data sources in a subgraph, such as IPFS files. The documentation for subgraph developers can be found in the official docs. This document describes the implementation of offchain data sources and how support for a new kinds offchain data source can be added. + +### Implementation Overview + +The implementation of offchain data sources has multiple reusable components and data structures, seeking to simplify the addition of new kinds of file data sources. The initially supported data source kind is `file/ipfs`, so in particular any new file kind should be able to reuse a lot the existing code. + +The data structures that represent an offchain data source, along with the code that parses it from the manifest or creates it as a dynamic data source, lives in the `graph` crate, in `data_source/offchain.rs`. A new file kind would probably only need a new `enum Source` variant, and the kind would need to be added to `const OFFCHAIN_KINDS`. + +The `OffchainMonitor` is responsible for tracking and fetching the offchain data. It currently lives in `subgraph/context.rs`. When an offchain data source is created from a template, `fn add_source` is called. It is expected that a background task will monitor the source for relevant events, in the case of a file that means the file becoming available and the event is the file content. To process these events, the subgraph runner calls `fn ready_offchain_events` periodically. + +If the data source kind being added relies on polling to check the availability of the monitored object, the generic `PollingMonitor` component can be used. Then the only implementation work is implementing the polling logic itself, as a `tower` service. The `IpfsService` serves as an example of how to do that. + +### Testing + +Automated testing for this functionality can be tricky, and will need to be discussed in each case, but the `file_data_sources` test in the `runner_tests.rs` can serve as a starting point of how to write an integration test using offchain data source. + +### Notes + +- Offchain data sources currently can only exist as dynamic data sources, instantiated from templates, and not as static data sources configured in the manifest. +- Some parts of the existing support for offchain data sources assumes they are 'one shot', meaning only a single trigger is ever handled by each offchain data source. This works well for files, the file is found, handled, and that's it. More complex offchain data sources will require additional planning. +- Entities from offchain data sources do not currently influence the PoI. Causality region ids are not deterministic. diff --git a/docs/implementation/pruning.md b/docs/implementation/pruning.md new file mode 100644 index 00000000000..4faf66f4e31 --- /dev/null +++ b/docs/implementation/pruning.md @@ -0,0 +1,99 @@ +## Pruning deployments + +Subgraphs, by default, store a full version history for entities, allowing +consumers to query the subgraph as of any historical block. Pruning is an +operation that deletes entity versions from a deployment older than a +certain block, so it is no longer possible to query the deployment as of +prior blocks. In GraphQL, those are only queries with a constraint `block { +number: } }` or a similar constraint by block hash where `n` is before +the block to which the deployment is pruned. Queries that are run at a +block height greater than that are not affected by pruning, and there is no +difference between running these queries against an unpruned and a pruned +deployment. + +Because pruning reduces the amount of data in a deployment, it reduces the +amount of storage needed for that deployment, and is beneficial for both +query performance and indexing speed. Especially compared to the default of +keeping all history for a deployment, it can often reduce the amount of +data for a deployment by a very large amount and speed up queries +considerably. See [caveats](#caveats) below for the downsides. + +The block `b` to which a deployment is pruned is controlled by how many +blocks `history_blocks` of history to retain; `b` is calculated internally +using `history_blocks` and the latest block of the deployment when the +prune operation is performed. When pruning finishes, it updates the +`earliest_block` for the deployment. The `earliest_block` can be retrieved +through the `index-node` status API, and `graph-node` will return an error +for any query that tries to time-travel to a point before +`earliest_block`. The value of `history_blocks` must be greater than +`ETHEREUM_REORG_THRESHOLD` to make sure that reverts can never conflict +with pruning. + +Pruning is started by running `graphman prune`. That command will perform +an initial prune of the deployment and set the subgraph's `history_blocks` +setting which is used to periodically check whether the deployment has +accumulated more history than that. Whenever the deployment does contain +more history than that, the deployment is automatically repruned. If +ongoing pruning is not desired, pass the `--once` flag to `graphman +prune`. Ongoing pruning can be turned off by setting `history_blocks` to a +very large value with the `--history` flag. + +Repruning is performed whenever the deployment has more than +`history_blocks * GRAPH_STORE_HISTORY_SLACK_FACTOR` blocks of history. The +environment variable `GRAPH_STORE_HISTORY_SLACK_FACTOR` therefore controls +how often repruning is performed: with +`GRAPH_STORE_HISTORY_SLACK_FACTOR=1.5` and `history_blocks` set to 10,000, +a reprune will happen every 5,000 blocks. After the initial pruning, a +reprune therefore happens every `history_blocks * (1 - +GRAPH_STORE_HISTORY_SLACK_FACTOR)` blocks. This value should be set high +enough so that repruning occurs relatively infrequently to not cause too +much database work. + +Pruning uses two different strategies for how to remove unneeded data: +rebuilding tables and deleting old entity versions. Deleting old entity +versions is straightforward: this strategy deletes rows from the underlying +tables. Rebuilding tables will copy the data that should be kept from the +existing tables into new tables and then replaces the existing tables with +these much smaller tables. Which strategy to use is determined for each +table individually, and governed by the settings for +`GRAPH_STORE_HISTORY_REBUILD_THRESHOLD` and +`GRAPH_STORE_HISTORY_DELETE_THRESHOLD`, both numbers between 0 and 1: if we +estimate that we will remove more than `REBUILD_THRESHOLD` of the table, +the table will be rebuilt. If we estimate that we will remove a fraction +between `REBUILD_THRESHOLD` and `DELETE_THRESHOLD` of the table, unneeded +entity versions will be deleted. If we estimate to remove less than +`DELETE_THRESHOLD`, the table is not changed at all. With both strategies, +operations are broken into batches that should each take +`GRAPH_STORE_BATCH_TARGET_DURATION` seconds to avoid causing very +long-running transactions. + +Pruning, in most cases, runs in parallel with indexing and does not block +it. When the rebuild strategy is used, pruning does block indexing while it +copies non-final entities from the existing table to the new table. + +The initial prune started by `graphman prune` prints a progress report on +the console. For the ongoing prune runs that are periodically performed, +the following information is logged: a message `Start pruning historical +entities` which includes the earliest and latest block, a message `Analyzed +N tables`, and a message `Finished pruning entities` with details about how +much was deleted or copied and how long that took. Pruning analyzes tables, +if that seems necessary, because its estimates of how much of a table is +likely not needed are based on Postgres statistics. + +### Caveats + +Pruning is a user-visible operation and does affect some of the things that +can be done with a deployment: + +* because it removes history, it restricts how far back time-travel queries + can be performed. This will only be an issue for entities that keep + lifetime statistics about some object (e.g., a token) and are used to + produce time series: after pruning, it is only possible to produce a time + series that goes back no more than `history_blocks`. It is very + beneficial though for entities that keep daily or similar statistics + about some object as it removes data that is not needed once the time + period is over, and does not affect how far back time series based on + these objects can be retrieved. +* it restricts how far back a graft can be performed. Because it removes + history, it becomes impossible to graft more than `history_blocks` before + the current deployment head. diff --git a/docs/implementation/schema-generation.md b/docs/implementation/schema-generation.md index c8bba833681..fbd227f9de6 100644 --- a/docs/implementation/schema-generation.md +++ b/docs/implementation/schema-generation.md @@ -5,13 +5,13 @@ table definition in Postgres. Schema generation follows a few simple rules: -* the data for a subgraph is entirely stored in a Postgres namespace whose +- the data for a subgraph is entirely stored in a Postgres namespace whose name is `sgdNNNN`. The mapping between namespace name and deployment id is kept in `deployment_schemas` -* the data for each entity type is stored in a table whose structure follows +- the data for each entity type is stored in a table whose structure follows the declaration of the type in the GraphQL schema -* enums in the GraphQL schema are stored as enum types in Postgres -* interfaces are not stored in the database, only the concrete types that +- enums in the GraphQL schema are stored as enum types in Postgres +- interfaces are not stored in the database, only the concrete types that implement the interface are stored Any table for an entity type has the following structure: @@ -32,20 +32,20 @@ queries](./time-travel.md). The attributes of the GraphQL type correspond directly to columns in the generated table. The types of these columns are -* the `id` column can have type `ID`, `String`, and `Bytes`, where `ID` is +- the `id` column can have type `ID`, `String`, and `Bytes`, where `ID` is an alias for `String` for historical reasons. -* if the attribute has a primitive type, the column has the SQL type that +- if the attribute has a primitive type, the column has the SQL type that most closely mirrors the GraphQL type. `BigDecimal` and `BigInt` are stored as `numeric`, `Bytes` is stored as `bytea`, etc. -* if the attribute references another entity, the column has the type of the +- if the attribute references another entity, the column has the type of the `id` type of the referenced entity type. We do not use foreign key constraints to allow storing an entity that references an entity that will only be created later. Foreign key constraint violations will therefore only be detected when a query is issued, or simply lead to the reference missing from the query result. -* if the attribute has an enum type, we generate a SQL enum type and use +- if the attribute has an enum type, we generate a SQL enum type and use that as the type of the column. -* if the attribute has a list type, like `[String]`, the corresponding +- if the attribute has a list type, like `[String]`, the corresponding column uses an array type. We do not allow nested arrays like `[[String]]` in GraphQL, so arrays will only ever contain entries of a primitive type. @@ -70,6 +70,22 @@ constraint `unique(id)` to such tables, and can avoid expensive GiST indexes in favor of simple BTree indexes since the `block$` column is an integer. +### Timeseries + +Entity types declared with `@entity(timeseries: true)` are represented in +the same way as immutable entities. The only difference is that timeseries +also must have a `timestamp` attribute. + +### Aggregations + +Entity types declared with `@aggregation` are represented by several tables, +one for each `interval` from the `@aggregation` directive. The tables are +named `TYPE_INTERVAL` where `TYPE` is the name of the aggregation, and +`INTERVAL` is the name of the interval; they do not support mutating +entities as aggregations are never updated, only appended to. The tables +have one column for each dimension and aggregate. The type of the columns is +determined in the same way as for those of normal entity types. + ## Indexing We do not know ahead of time which queries will be issued and therefore @@ -79,17 +95,17 @@ are open issues at this time. We generate the following indexes for each table: -* for mutable entity types - * an exclusion index over `(id, block_range)` that ensures that the +- for mutable entity types + - an exclusion index over `(id, block_range)` that ensures that the versions for the same entity `id` have disjoint block ranges - * a BRIN index on `(lower(block_range), COALESCE(upper(block_range), - 2147483647), vid)` that helps speed up some operations, especially + - a BRIN index on `(lower(block_range), COALESCE(upper(block_range), +2147483647), vid)` that helps speed up some operations, especially reversion, in tables that have good data locality, for example, tables where entities are never updated or deleted -* for immutable entity types - * a unique index on `id` - * a BRIN index on `(block$, vid)` -* for each attribute, an index called `attr_N_M_..` where `N` is the number +- for immutable and timeseries entity types + - a unique index on `id` + - a BRIN index on `(block$, vid)` +- for each attribute, an index called `attr_N_M_..` where `N` is the number of the entity type in the GraphQL schema, and `M` is the number of the attribute within that type. For attributes of a primitive type, the index is a BTree index. For attributes that reference other entities, the index diff --git a/docs/implementation/sql-interface.md b/docs/implementation/sql-interface.md new file mode 100644 index 00000000000..6b90fe6da9c --- /dev/null +++ b/docs/implementation/sql-interface.md @@ -0,0 +1,89 @@ +# SQL Queries + +**This interface is extremely experimental. There is no guarantee that this +interface will ever be brought to production use. It's solely here to help +evaluate the utility of such an interface** + +**The interface is only available if the environment variable `GRAPH_ENABLE_SQL_QUERIES` is set to `true`** + +SQL queries can be issued by posting a JSON document to +`/subgraphs/sql`. The server will respond with a JSON response that +contains the records matching the query in JSON form. + +The body of the request must contain the following keys: + +* `deployment`: the hash of the deployment against which the query should + be run +* `query`: the SQL query +* `mode`: either `info` or `data`. When the mode is `info` only some + information of the response is reported, with a mode of `data` the query + result is sent in the response + +The SQL query can use all the tables of the given subgraph. Table and +attribute names for normal `@entity` types are snake-cased from their form +in the GraphQL schema, so that data for `SomeDailyStuff` is stored in a +table `some_daily_stuff`. For `@aggregation` types, the table can be +accessed as `()`, for example, `my_stats('hour')` for +`type MyStats @aggregation(..) { .. }` + +The query can use fairly arbitrary SQL, including aggregations and most +functions built into PostgreSQL. + +## Example + +For a subgraph whose schema defines an entity `Block`, the following query +```json +{ + "query": "select number, hash, parent_hash, timestamp from block order by number desc limit 2", + "deployment": "QmSoMeThInG", + "mode": "data" +} +``` + +might result in this response +```json +{ + "data": [ + { + "hash": "\\x5f91e535ee4d328725b869dd96f4c42059e3f2728dfc452c32e5597b28ce68d6", + "number": 5000, + "parent_hash": "\\x82e95c1ee3a98cd0646225b5ae6afc0b0229367b992df97aeb669c898657a4bb", + "timestamp": "2015-07-30T20:07:44+00:00" + }, + { + "hash": "\\x82e95c1ee3a98cd0646225b5ae6afc0b0229367b992df97aeb669c898657a4bb", + "number": 4999, + "parent_hash": "\\x875c9a0f8215258c3b17fd5af5127541121cca1f594515aae4fbe5a7fbef8389", + "timestamp": "2015-07-30T20:07:36+00:00" + } + ] +} +``` + +## Limitations/Ideas/Disclaimers + +Most of these are fairly easy to address: + +- bind variables/query parameters are not supported, only literal SQL + queries +* queries must finish within `GRAPH_SQL_STATEMENT_TIMEOUT` (unlimited by + default) +* queries are always executed at the subgraph head. It would be easy to add + a way to specify a block at which the query should be executed +* the interface right now pretty much exposes the raw SQL schema for a + subgraph, though system columns like `vid` or `block_range` are made + inaccessible. +* it is not possible to join across subgraphs, though it would be possible + to add that. Implenting that would require some additional plumbing that + hides the effects of sharding. +* JSON as the response format is pretty terrible, and we should change that + to something that isn't so inefficient +* the response contains data that's pretty raw; as the example shows, + binary data uses Postgres' notation for hex strings +* because of how broad the supported SQL is, it is pretty easy to issue + queries that take a very long time. It will therefore not be hard to take + down a `graph-node`, especially when no query timeout is set + +Most importantly: while quite a bit of effort has been put into making this +interface safe, in particular, making sure it's not possible to write +through this interface, there's no guarantee that this works without bugs. diff --git a/docs/log-store.md b/docs/log-store.md new file mode 100644 index 00000000000..8be1cddecd8 --- /dev/null +++ b/docs/log-store.md @@ -0,0 +1,853 @@ +# Log Store Configuration and Usage + +This guide explains how to configure subgraph indexing logs storage in graph-node. + +## Table of Contents + +- [Overview](#overview) +- [How Log Stores Work](#how-log-stores-work) +- [Log Store Types](#log-store-types) + - [File-based Logs](#file-based-logs) + - [Elasticsearch](#elasticsearch) + - [Loki](#loki) + - [Disabled](#disabled) +- [Configuration](#configuration) + - [Environment Variables](#environment-variables) + - [CLI Arguments](#cli-arguments) + - [Configuration Precedence](#configuration-precedence) +- [Querying Logs](#querying-logs) +- [Migrating from Deprecated Configuration](#migrating-from-deprecated-configuration) +- [Choosing the Right Backend](#choosing-the-right-backend) +- [Best Practices](#best-practices) +- [Troubleshooting](#troubleshooting) + +## Overview + +Graph Node supports multiple logs storage backends for subgraph indexing logs. Subgraph indexing logs include: +- **User-generated logs**: Explicit logging from subgraph mapping code (`log.info()`, `log.error()`, etc.) +- **Runtime logs**: Handler execution, event processing, data source activity +- **System logs**: Warnings, errors, and diagnostics from the indexing system + +**Available backends:** +- **File**: JSON Lines files on local filesystem (for local development) +- **Elasticsearch**: Enterprise-grade search and analytics (for production) +- **Loki**: Grafana's lightweight log aggregation system (for production) +- **Disabled**: No log storage (default) + +All backends share the same query interface through GraphQL, making it easy to switch between them. + +**Important Note:** When log storage is disabled (the default), subgraph logs still appear in stdout/stderr as they always have. The "disabled" setting simply means logs are not stored separately in a queryable format. You can still see logs in your terminal or container logs - they just won't be available via the `_logs` GraphQL query. + +## How Log Stores Work + +### Architecture + +``` +┌─────────────────┐ +│ Subgraph Code │ +│ (mappings) │ +└────────┬────────┘ + │ log.info(), log.error(), etc. + ▼ +┌─────────────────┐ +│ Graph Runtime │ +│ (WebAssembly) │ +└────────┬────────┘ + │ Log events + ▼ +┌─────────────────┐ +│ Log Drain │ ◄─── slog-based logging system +└────────┬────────┘ + │ Write + ▼ +┌─────────────────┐ +│ Log Store │ ◄─── Configurable backend +│ (ES/Loki/File) │ +└────────┬────────┘ + │ + ▼ +┌─────────────────┐ +│ GraphQL API │ ◄─── Unified query interface +│ (port 8000) │ +└─────────────────┘ +``` + +### Log Flow + +1. **Log sources** generate logs from: + - User mapping code (explicit `log.info()`, `log.error()`, etc. calls) + - Subgraph runtime (handler execution, event processing, data source triggers) + - System warnings and errors (indexing issues, constraint violations, etc.) +2. **Graph runtime** captures these logs with metadata (timestamp, level, source location) +3. **Log drain** formats logs and writes to configured backend +4. **Log store** persists logs and handles queries +5. **GraphQL API** exposes logs through the `_logs` query + +### Log Entry Structure + +Each log entry contains: +- **`id`**: Unique identifier +- **`subgraphId`**: Deployment hash (QmXxx...) +- **`timestamp`**: ISO 8601 timestamp (e.g., `2024-01-15T10:30:00.123456789Z`) +- **`level`**: CRITICAL, ERROR, WARNING, INFO, or DEBUG +- **`text`**: Log message +- **`arguments`**: Key-value pairs from structured logging +- **`meta`**: Source location (module, line, column) + +## Log Store Types + +### File-based Logs + +**Best for:** Local development, testing + +#### How It Works + +File-based logs store each subgraph's logs in a separate JSON Lines (`.jsonl`) file: + +``` +graph-logs/ +├── QmSubgraph1Hash.jsonl +├── QmSubgraph2Hash.jsonl +└── QmSubgraph3Hash.jsonl +``` + +Each line in the file is a complete JSON object representing one log entry. + +#### Storage Format + +```json +{"id":"QmTest-2024-01-15T10:30:00.123456789Z","subgraphId":"QmTest","timestamp":"2024-01-15T10:30:00.123456789Z","level":"error","text":"Handler execution failed, retries: 3","arguments":[{"key":"retries","value":"3"}],"meta":{"module":"mapping.ts","line":42,"column":10}} +``` + +#### Query Performance + +File-based logs stream through files line-by-line with bounded memory usage. + +**Performance characteristics:** +- Query time: O(n) where n = number of log entries +- Memory usage: O(skip + first) - only matching entries kept in memory +- Suitable for: Development and testing + +#### Configuration + +**Minimum configuration (CLI):** +```bash +graph-node \ + --postgres-url postgresql://graph:pass@localhost/graph-node \ + --ethereum-rpc mainnet:https://... \ + --ipfs 127.0.0.1:5001 \ + --log-store-backend file \ + --log-store-file-dir ./graph-logs +``` + +**Full configuration (environment variables):** +```bash +export GRAPH_LOG_STORE_BACKEND=file +export GRAPH_LOG_STORE_FILE_DIR=/var/log/graph-node +export GRAPH_LOG_STORE_FILE_MAX_SIZE=104857600 # 100MB +export GRAPH_LOG_STORE_FILE_RETENTION_DAYS=30 +``` + +#### Features + +**Advantages:** +- No external dependencies +- Simple setup (just specify a directory) +- Human-readable format (JSON Lines) +- Easy to inspect with standard tools (`jq`, `grep`, etc.) +- Good for debugging during development + +**Limitations:** +- Not suitable for production with high log volume +- No indexing (O(n) query time scales with file size) +- No automatic log rotation or retention management +- Single file per subgraph (no sharding) + +#### When to Use + +Use file-based logs when: +- Developing subgraphs locally +- Testing on a development machine +- Running low-traffic subgraphs (< 1000 total logs/day including system logs) +- You want simple log access without external services + +### Elasticsearch + +**Best for:** Production deployments, high log volume, advanced search + +#### How It Works + +Elasticsearch stores logs in indices with full-text search capabilities, making it ideal for production deployments with high log volume. + +**Architecture:** +``` +graph-node → Elasticsearch HTTP API → Elasticsearch cluster + → Index: subgraph-logs-* + → Query DSL for filtering +``` + +#### Features + +**Advantages:** +- **Indexed searching**: Fast queries even with millions of logs +- **Full-text search**: Powerful text search across log messages +- **Scalability**: Handles billions of log entries +- **High availability**: Supports clustering and replication +- **Kibana integration**: Rich visualization and dashboards for operators +- **Time-based indices**: Efficient retention management + +**Considerations:** +- Requires Elasticsearch cluster (infrastructure overhead) +- Resource-intensive (CPU, memory, disk) + +#### Configuration + +**Minimum configuration (CLI):** +```bash +graph-node \ + --postgres-url postgresql://graph:pass@localhost/graph-node \ + --ethereum-rpc mainnet:https://... \ + --ipfs 127.0.0.1:5001 \ + --log-store-backend elasticsearch \ + --log-store-elasticsearch-url http://localhost:9200 +``` + +**Full configuration with authentication:** +```bash +graph-node \ + --postgres-url postgresql://graph:pass@localhost/graph-node \ + --ethereum-rpc mainnet:https://... \ + --ipfs 127.0.0.1:5001 \ + --log-store-backend elasticsearch \ + --log-store-elasticsearch-url https://es.example.com:9200 \ + --log-store-elasticsearch-user elastic \ + --log-store-elasticsearch-password secret \ + --log-store-elasticsearch-index subgraph-logs +``` + +**Environment variables:** +```bash +export GRAPH_LOG_STORE_BACKEND=elasticsearch +export GRAPH_LOG_STORE_ELASTICSEARCH_URL=http://localhost:9200 +export GRAPH_LOG_STORE_ELASTICSEARCH_USER=elastic +export GRAPH_LOG_STORE_ELASTICSEARCH_PASSWORD=secret +export GRAPH_LOG_STORE_ELASTICSEARCH_INDEX=subgraph-logs +``` + +#### Index Configuration + +Logs are stored in the configured index (default: `subgraph`). The index mapping is automatically created. + +**Recommended index settings for production:** +```json +{ + "settings": { + "number_of_shards": 3, + "number_of_replicas": 1, + "refresh_interval": "5s" + } +} +``` + +#### Query Performance + +**Performance characteristics:** +- Query time: O(log n) with indexing +- Memory usage: Minimal (server-side filtering) +- Suitable for: Millions to billions of log entries + +#### When to Use + +Use Elasticsearch when: +- Running production deployments +- High log volume +- Need advanced search and filtering +- Want to build dashboards with Kibana +- Need high availability and scalability +- Have DevOps resources to manage Elasticsearch or can set up a managed ElasticSearch deployment + +### Loki + +**Best for:** Production deployments, Grafana users, cost-effective at scale + +#### How It Works + +Loki is Grafana's log aggregation system, designed to be cost-effective and easy to operate. Unlike Elasticsearch, Loki only indexes metadata (not full-text), making it more efficient for time-series log data. + +**Architecture:** +``` +graph-node → Loki HTTP API → Loki + → Stores compressed chunks + → Indexes labels only +``` + +#### Features + +**Advantages:** +- **Cost-effective**: Lower storage costs than Elasticsearch +- **Grafana integration**: Native integration with Grafana +- **Horizontal scalability**: Designed for cloud-native deployments +- **Multi-tenancy**: Built-in tenant isolation +- **Efficient compression**: Optimized for log data +- **LogQL**: Powerful query language similar to PromQL +- **Lower resource usage**: Less CPU/memory than Elasticsearch + +**Considerations:** +- No full-text indexing (slower text searches) +- Best used with Grafana (less tooling than Elasticsearch) +- Younger ecosystem than Elasticsearch +- Query performance depends on label cardinality + +#### Configuration + +**Minimum configuration (CLI):** +```bash +graph-node \ + --postgres-url postgresql://graph:pass@localhost/graph-node \ + --ethereum-rpc mainnet:https://... \ + --ipfs 127.0.0.1:5001 \ + --log-store-backend loki \ + --log-store-loki-url http://localhost:3100 +``` + +**With multi-tenancy:** +```bash +graph-node \ + --postgres-url postgresql://graph:pass@localhost/graph-node \ + --ethereum-rpc mainnet:https://... \ + --ipfs 127.0.0.1:5001 \ + --log-store-backend loki \ + --log-store-loki-url http://localhost:3100 \ + --log-store-loki-tenant-id my-graph-node +``` + +**Environment variables:** +```bash +export GRAPH_LOG_STORE_BACKEND=loki +export GRAPH_LOG_STORE_LOKI_URL=http://localhost:3100 +export GRAPH_LOG_STORE_LOKI_TENANT_ID=my-graph-node +``` + +#### Labels + +Loki uses labels for indexing. Graph Node automatically creates labels: +- `subgraph_id`: Deployment hash +- `level`: Log level +- `job`: "graph-node" + +#### Query Performance + +**Performance characteristics:** +- Query time: O(n) for text searches, O(log n) for label queries +- Memory usage: Minimal (server-side processing) +- Suitable for: Millions to billions of log entries +- Best performance with label-based filtering + +#### When to Use + +Use Loki when: +- Already using Grafana for monitoring +- Need cost-effective log storage at scale +- Want simpler operations than Elasticsearch +- Multi-tenancy is required +- Log volume is very high (> 1M logs/day) +- Full-text search is not critical + +### Disabled + +**Best for:** Minimalist deployments, reduced overhead + +#### How It Works + +When log storage is disabled (the default), subgraph logs are **still written to stdout/stderr** along with all other graph-node logs. They are just **not stored separately** in a queryable format. + +**Important:** "Disabled" does NOT mean logs are discarded. It means: +- Logs appear in stdout/stderr (traditional behavior) +- Logs are not stored in a separate queryable backend +- The `_logs` GraphQL query returns empty results + +This is the default behavior - logs continue to work exactly as they did before this feature was added. + +#### Configuration + +**Explicitly disable:** +```bash +export GRAPH_LOG_STORE_BACKEND=disabled +``` + +**Or simply don't configure a backend** (defaults to disabled): +```bash +# No log store configuration = disabled +graph-node \ + --postgres-url postgresql://graph:pass@localhost/graph-node \ + --ethereum-rpc mainnet:https://... \ + --ipfs 127.0.0.1:5001 +``` + +#### Features + +**Advantages:** +- Zero additional overhead +- No external dependencies +- Minimal configuration +- Logs still appear in stdout/stderr for debugging + +**Limitations:** +- Cannot query logs via GraphQL (`_logs` returns empty results) +- No separation of subgraph logs from other graph-node logs in stdout +- Logs mixed with system logs (harder to filter programmatically) +- No structured querying or filtering capabilities + +#### When to Use + +Use disabled log storage when: +- Running minimal test deployments with less dependencies +- Exposing logs to users is not required for your use case +- You'd like subgraph logs sent to external log collection (e.g., container logs) + +## Configuration + +### Environment Variables + +Environment variables are the recommended way to configure log stores, especially in containerized deployments. + +#### Backend Selection + +```bash +GRAPH_LOG_STORE_BACKEND= +``` +Valid values: `disabled`, `elasticsearch`, `loki`, `file` + +#### Elasticsearch + +```bash +GRAPH_LOG_STORE_ELASTICSEARCH_URL=http://localhost:9200 +GRAPH_LOG_STORE_ELASTICSEARCH_USER=elastic # Optional +GRAPH_LOG_STORE_ELASTICSEARCH_PASSWORD=secret # Optional +GRAPH_LOG_STORE_ELASTICSEARCH_INDEX=subgraph # Default: "subgraph" +``` + +#### Loki + +```bash +GRAPH_LOG_STORE_LOKI_URL=http://localhost:3100 +GRAPH_LOG_STORE_LOKI_TENANT_ID=my-tenant # Optional +``` + +#### File + +```bash +GRAPH_LOG_STORE_FILE_DIR=/var/log/graph-node +GRAPH_LOG_STORE_FILE_MAX_SIZE=104857600 # Default: 100MB +GRAPH_LOG_STORE_FILE_RETENTION_DAYS=30 # Default: 30 +``` + +### CLI Arguments + +CLI arguments provide the same functionality as environment variables and the two can be mixed together. + +#### Backend Selection + +```bash +--log-store-backend +``` + +#### Elasticsearch + +```bash +--log-store-elasticsearch-url +--log-store-elasticsearch-user +--log-store-elasticsearch-password +--log-store-elasticsearch-index +``` + +#### Loki + +```bash +--log-store-loki-url +--log-store-loki-tenant-id +``` + +#### File + +```bash +--log-store-file-dir +--log-store-file-max-size +--log-store-file-retention-days +``` + +### Configuration Precedence + +When multiple configuration methods are used: + +1. **CLI arguments** take highest precedence +2. **Environment variables** are used if no CLI args provided +3. **Defaults** are used if neither is set + +## Querying Logs + +All log backends share the same GraphQL query interface. Logs are queried through the subgraph-specific GraphQL endpoint: + +- **Subgraph by deployment**: `http://localhost:8000/subgraphs/id/` +- **Subgraph by name**: `http://localhost:8000/subgraphs/name/` + +The `_logs` query is automatically scoped to the subgraph in the URL, so you don't need to pass a `subgraphId` parameter. + +**Note**: Queries return all log types - both user-generated logs from mapping code and system-generated runtime logs (handler execution, events, warnings, etc.). Use the `search` filter to search for specific messages, or `level` to filter by severity. + +### Basic Query + +Query the `_logs` field at your subgraph's GraphQL endpoint: + +```graphql +query { + _logs( + first: 100 + ) { + id + timestamp + level + text + } +} +``` + +**Example endpoint**: `http://localhost:8000/subgraphs/id/QmYourDeploymentHash` + +### Query with Filters + +```graphql +query { + _logs( + level: ERROR + from: "2024-01-01T00:00:00Z" + to: "2024-01-31T23:59:59Z" + search: "timeout" + first: 50 + skip: 0 + ) { + id + timestamp + level + text + arguments { + key + value + } + meta { + module + line + column + } + } +} +``` + +### Available Filters + +| Filter | Type | Description | +|--------|------|-------------| +| `level` | LogLevel | Filter by level: CRITICAL, ERROR, WARNING, INFO, DEBUG | +| `from` | String | Start timestamp (ISO 8601) | +| `to` | String | End timestamp (ISO 8601) | +| `search` | String | Case-insensitive substring search in log messages | +| `first` | Int | Number of results to return (default: 100, max: 1000) | +| `skip` | Int | Number of results to skip for pagination (max: 10000) | + +### Response Fields + +| Field | Type | Description | +|-------|------|-------------| +| `id` | String | Unique log entry ID | +| `timestamp` | String | ISO 8601 timestamp with nanosecond precision | +| `level` | LogLevel | Log level (CRITICAL, ERROR, WARNING, INFO, DEBUG) | +| `text` | String | Complete log message with arguments | +| `arguments` | [(String, String)] | Structured key-value pairs | +| `meta.module` | String | Source file name | +| `meta.line` | Int | Line number | +| `meta.column` | Int | Column number | + +### Query Examples + +#### Recent Errors + +```graphql +query RecentErrors { + _logs( + level: ERROR + first: 20 + ) { + timestamp + text + meta { + module + line + } + } +} +``` + +#### Search for Specific Text + +```graphql +query SearchTimeout { + _logs( + search: "timeout" + first: 50 + ) { + timestamp + level + text + } +} +``` + +#### Handler Execution Logs + +```graphql +query HandlerLogs { + _logs( + search: "handler" + first: 50 + ) { + timestamp + level + text + } +} +``` + +#### Time Range Query + +```graphql +query LogsInRange { + _logs( + from: "2024-01-15T00:00:00Z" + to: "2024-01-15T23:59:59Z" + first: 1000 + ) { + timestamp + level + text + } +} +``` + +#### Pagination + +```graphql +# First page +query Page1 { + _logs( + first: 100 + skip: 0 + ) { + id + text + } +} + +# Second page +query Page2 { + _logs( + first: 100 + skip: 100 + ) { + id + text + } +} +``` + +### Querying the logs store using cURL + +```bash +curl -X POST http://localhost:8000/subgraphs/id/ \ + -H "Content-Type: application/json" \ + -d '{ + "query": "{ _logs(level: ERROR, first: 10) { timestamp level text } }" + }' +``` + +### Performance Considerations + +**File-based:** _for development only_ +- Streams through files line-by-line (bounded memory usage) +- Memory usage limited to O(skip + first) entries +- Query time is O(n) where n = total log entries in file + +**Elasticsearch:** +- Indexed queries are fast regardless of size +- Text searches are optimized with full-text indexing +- Can handle billions of log entries +- Best for production with high query volume + +**Loki:** +- Label-based queries are fast (indexed) +- Text searches scan compressed chunks (slower than Elasticsearch) +- Good performance with proper label filtering +- Best for production with Grafana integration + +## Choosing the Right Backend + +### Decision Matrix + +| Scenario | Recommended Backend | Reason | +|----------|-------------------|-----------------------------------------------------------------------------------| +| Local development | **File** | Simple, no dependencies, easy to inspect | +| Testing/staging | **File** or **Elasticsearch** | File for simplicity, ES if testing production config | +| Production | **Elasticsearch** or **Loki** | Both handle scale well | +| Using Grafana | **Loki** | Native integration | +| Cost-sensitive at scale | **Loki** | Lower storage costs | +| Want rich ecosystem | **Elasticsearch** | More tools and plugins | +| Minimal deployment | **Disabled** | No overhead | + +### Resource Requirements + +#### File-based +- **Disk**: Minimal (log files only) +- **Memory**: Depends on file size during queries +- **CPU**: Minimal +- **Network**: None +- **External services**: None + +#### Elasticsearch +- **Disk**: High (indices + replicas) +- **Memory**: 4-8GB minimum for small deployments +- **CPU**: Medium to high +- **Network**: HTTP API calls +- **External services**: Elasticsearch cluster + +#### Loki +- **Disk**: Medium (compressed chunks) +- **Memory**: 2-4GB minimum +- **CPU**: Low to medium +- **Network**: HTTP API calls +- **External services**: Loki server + +## Best Practices + +### General + +1. **Start with file-based for development** - Simplest setup, easy debugging +2. **Use Elasticsearch or Loki for production** - Better performance and features +3. **Monitor log volume** - Set up alerts if log volume grows unexpectedly (includes both user logs and system-generated runtime logs) +4. **Set retention policies** - Don't keep logs forever (disk space and cost) +5. **Use structured logging** - Pass key-value pairs to log functions for better filtering + +### File-based Logs + +1. **Monitor file size** - While queries use bounded memory, larger files take longer to scan (O(n) query time) +2. **Archive old logs** - Manually archive/delete old files or implement external rotation +3. **Monitor disk usage** - Files can grow quickly with verbose logging +4. **Use JSON tools** - `jq` is excellent for inspecting .jsonl files locally + +**Example local inspection:** +```bash +# Count logs by level +cat graph-logs/QmExample.jsonl | jq -r '.level' | sort | uniq -c + +# Find errors in last 1000 lines +tail -n 1000 graph-logs/QmExample.jsonl | jq 'select(.level == "error")' + +# Search for specific text +cat graph-logs/QmExample.jsonl | jq 'select(.text | contains("timeout"))' +``` + +### Elasticsearch + +1. **Use index patterns** - Time-based indices for easier management +2. **Configure retention** - Use Index Lifecycle Management (ILM) +3. **Monitor cluster health** - Set up Elasticsearch monitoring +4. **Tune for your workload** - Adjust shards/replicas based on log volume +5. **Use Kibana** - Visualize and explore logs effectively + +**Example Elasticsearch retention policy:** +```json +{ + "policy": "graph-logs-policy", + "phases": { + "hot": { "min_age": "0ms", "actions": {} }, + "warm": { "min_age": "7d", "actions": {} }, + "delete": { "min_age": "30d", "actions": { "delete": {} } } + } +} +``` + +### Loki + +1. **Use proper labels** - Don't over-index, keep label cardinality low +2. **Configure retention** - Set retention period in Loki config +3. **Use Grafana** - Native integration provides best experience +4. **Compress efficiently** - Loki's compression works best with batch writes +5. **Multi-tenancy** - Use tenant IDs if running multiple environments + +**Example Grafana query:** +```logql +{subgraph_id="QmExample", level="error"} |= "timeout" +``` + +## Troubleshooting + +### File-based Logs + +**Problem: Log file doesn't exist** +- Check `GRAPH_LOG_STORE_FILE_DIR` is set correctly +- Verify directory is writable by graph-node + +**Problem: Queries are slow** +- Subgraph logs file may be very large +- Consider archiving old logs or implementing retention +- For high-volume production use, switch to Elasticsearch or Loki + +**Problem: Disk filling up** +- Implement log rotation +- Reduce log verbosity in subgraph code +- Set up monitoring for disk usage + +### Elasticsearch + +**Problem: Cannot connect to Elasticsearch** +- Verify `GRAPH_LOG_STORE_ELASTICSEARCH_URL` is correct +- Check Elasticsearch is running: `curl http://localhost:9200` +- Verify authentication credentials if using security features +- Check network connectivity and firewall rules + +**Problem: No logs appearing in Elasticsearch** +- Check Elasticsearch cluster health +- Verify index exists: `curl http://localhost:9200/_cat/indices` +- Check graph-node logs for write errors +- Verify Elasticsearch has disk space + +**Problem: Queries are slow** +- Check Elasticsearch cluster health and resources +- Verify indices are not over-sharded +- Consider adding replicas for query performance +- Review query patterns and add appropriate indices + +### Loki + +**Problem: Cannot connect to Loki** +- Verify `GRAPH_LOG_STORE_LOKI_URL` is correct +- Check Loki is running: `curl http://localhost:3100/ready` +- Verify tenant ID if using multi-tenancy +- Check network connectivity + +**Problem: No logs appearing in Loki** +- Check Loki service health +- Verify Loki has disk space for chunks +- Check graph-node logs for write errors +- Verify Loki retention settings aren't deleting logs immediately + +**Problem: Queries return no results in Grafana** +- Check label selectors match what graph-node is sending +- Verify time range includes when logs were written +- Check Loki retention period +- Verify tenant ID matches if using multi-tenancy + +## Further Reading + +- [Environment Variables Reference](environment-variables.md) +- [Graph Node Configuration](config.md) +- [Elasticsearch Documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/index.html) +- [Grafana Loki Documentation](https://grafana.com/docs/loki/latest/) diff --git a/docs/metrics.md b/docs/metrics.md index 7a545b1469a..61c223f8256 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -27,7 +27,7 @@ Track the **last reverted block** for a subgraph deployment - `deployment_sync_secs` total **time spent syncing** - `deployment_transact_block_operations_duration` -Measures **duration of commiting all the entity operations** in a block and **updating the subgraph pointer** +Measures **duration of committing all the entity operations** in a block and **updating the subgraph pointer** - `deployment_trigger_processing_duration` Measures **duration of trigger processing** for a subgraph deployment - `eth_rpc_errors` @@ -57,6 +57,9 @@ the **maximum size of a query result** (in CacheWeight) the **size of the result of successful GraphQL queries** (in CacheWeight) - `query_semaphore_wait_ms` Moving **average of time spent on waiting for postgres query semaphore** +- `query_blocks_behind` +A histogram for how many blocks behind the subgraph head queries are being made at. +This helps inform pruning decisions. - `query_kill_rate` The rate at which the load manager kills queries - `registered_metrics` @@ -66,4 +69,4 @@ The **number of Postgres connections** currently **checked out** - `store_connection_error_count` The **number of Postgres connections errors** - `store_connection_wait_time_ms` -**Average connection wait time** +**Average connection wait time** \ No newline at end of file diff --git a/docs/plans/gnd-cli-expansion.md b/docs/plans/gnd-cli-expansion.md new file mode 100644 index 00000000000..b1bef8bc108 --- /dev/null +++ b/docs/plans/gnd-cli-expansion.md @@ -0,0 +1,418 @@ +# Plan: Extend gnd with graph-cli functionality + +## Overview + +Extend the existing `gnd` (Graph Node Dev) CLI to be a **drop-in replacement** for the TypeScript-based `graph-cli`. The goal is identical CLI behavior, flags, output format, and byte-for-byte identical AssemblyScript code generation. + +**Spec document**: `docs/specs/gnd-cli-expansion.md` + +**Scope**: + +- Ethereum protocol only (other protocols are future work) +- All graph-cli commands except `local` and `node` subcommand +- Match latest graph-cli version (0.98.x) + +**Current state of gnd** (`~/code/graph-node/gnd/`): + +- ~700 lines of Rust across 3 files +- Single command with flags (no subcommands) +- Already uses clap, imports graph crates +- Runs graph-node in dev mode with file watching + +## Status Summary (Updated 2026-01-19) + +**Overall Progress**: ~99% complete (implementation + docs + automated tests done, only publish/test commands need manual verification) + +| Phase | Status | Notes | +|-------|--------|-------| +| Phase 1: CLI Restructure | ✅ Complete | | +| Phase 2: Infrastructure | ✅ Complete | | +| Phase 3: Simple Commands | ✅ Complete | | +| Phase 4: Code Generation | ✅ Complete | 11 verification fixtures passing | +| Phase 5: Migrations | ✅ Complete | | +| Phase 6: Build Command | ✅ Complete | Tested via `just test-gnd-commands` | +| Phase 7: Deploy Command | ✅ Complete | Tested via `just test-gnd-cli` | +| Phase 8: Init Command | ✅ Complete | Tested via `just test-gnd-commands` | +| Phase 9: Add Command | ✅ Complete | | +| Phase 10: Publish Command | 🟡 Needs Testing | Manual end-to-end verification | +| Phase 11: Test Command | 🟡 Needs Testing | Manual Matchstick test | +| Phase 12: Testing & Polish | ✅ Complete | Documentation done, test porting analyzed | +| Phase 13: CLI Integration Tests | ✅ Complete | Test infrastructure created, ready for manual verification | + +**Total LOC**: ~15,000 lines of Rust (158 unit tests, 11 verification tests passing) + +**Documentation**: +- `gnd/README.md` - CLI command reference with examples +- `gnd/docs/MIGRATING_FROM_GRAPH_CLI.md` - Migration guide from graph-cli + +## Git Workflow + +**Branch**: All work should be committed to the `gnd-cli` branch. + +**Commit discipline**: + +- Commit work in small, reviewable chunks +- Each commit should be self-contained and pass all checks +- Prefer many small commits over few large ones +- Each commit message should clearly describe what it does + +**Before each commit**: + +```bash +just format +just lint +just test-unit # only run the tests in gnd/ +``` + +- MANDATORY: Work must be committed, a task is only done when work is committed +- MANDATORY: Make sure to follow the commit discipline above + +## Commands + +| Command | Description | Complexity | +| ------------- | ----------------------------------- | ----------------------- | +| `gnd dev` | Existing gnd functionality | Done (restructure only) | +| `gnd codegen` | Generate AssemblyScript types | High | +| `gnd build` | Compile to WASM | Medium | +| `gnd deploy` | Deploy to Graph Node | Medium | +| `gnd init` | Scaffold new subgraph | High | +| `gnd add` | Add datasource to existing subgraph | Medium | +| `gnd create` | Register subgraph name | Low | +| `gnd remove` | Unregister subgraph name | Low | +| `gnd auth` | Set deploy key | Low | +| `gnd publish` | Publish to decentralized network | Medium | +| `gnd test` | Run Matchstick tests | Low (shell out) | +| `gnd clean` | Remove build artifacts | Low | + +**Not implemented** (documented differences): + +- `gnd local` - Use existing test infrastructure +- `gnd node` - Use graphman for node management +- `--uncrashable` flag - Float Capital third-party feature + +## Reusable from graph-node + +| Feature | Location | Status | +| ------------------- | ------------------------------------- | ---------------------------- | +| Manifest parsing | `graph/src/data/subgraph/mod.rs` | Ready | +| Manifest validation | `graph/src/data/subgraph/mod.rs` | Ready (may need refactoring) | +| GraphQL schema | `graph/src/schema/input/` | Ready | +| ABI parsing | `ethabi` + chain/ethereum wrappers | Ready | +| IPFS client | `graph/src/ipfs/` | Ready | +| Link resolution | `graph/src/components/link_resolver/` | Ready | +| File watching | `gnd/src/watcher.rs` | Ready | +| CLI framework | `clap` (already in gnd) | Ready | + +## Module Structure + +``` +gnd/src/ +├── main.rs # Entry point, clap setup +├── lib.rs +├── commands/ +│ ├── mod.rs +│ ├── dev.rs # Existing gnd functionality +│ ├── codegen.rs +│ ├── build.rs +│ ├── deploy.rs +│ ├── init.rs +│ ├── add.rs +│ ├── create.rs +│ ├── remove.rs +│ ├── auth.rs +│ ├── publish.rs +│ ├── test.rs +│ └── clean.rs +├── codegen/ +│ ├── mod.rs +│ ├── schema.rs # Entity class generation +│ ├── abi.rs # ABI binding generation +│ └── template.rs # Template binding generation +├── scaffold/ +│ ├── mod.rs +│ ├── manifest.rs # Generate subgraph.yaml +│ ├── schema.rs # Generate schema.graphql +│ └── mapping.rs # Generate mapping.ts +├── migrations/ +│ ├── mod.rs +│ └── ... # One module per migration version +├── compiler/ +│ ├── mod.rs +│ └── asc.rs # Shell out to asc +├── services/ +│ ├── mod.rs +│ ├── etherscan.rs # Etherscan API client +│ ├── sourcify.rs # Sourcify API client +│ ├── registry.rs # Network registry client +│ └── graph_node.rs # Graph Node JSON-RPC client +├── config/ +│ ├── mod.rs +│ ├── auth.rs # ~/.graphprotocol/ management +│ └── networks.rs # networks.json handling +├── output/ +│ ├── mod.rs +│ └── spinner.rs # Progress/spinner output (match TS CLI exactly) +└── watcher.rs # Existing file watcher +``` + +## Dependencies to Add + +| Crate | Purpose | +| ----------- | ------------------------------------------- | +| `inquire` | Interactive prompts (init) | +| `minijinja` | Template rendering (scaffold) | +| `indicatif` | Progress bars and spinners | +| `reqwest` | HTTP client (Etherscan, Sourcify, registry) | + +## TODO List + +### Phase 1: CLI Restructure + +- [x] Create `gnd-cli` branch +- [x] Restructure main.rs with clap subcommands +- [x] Move existing gnd logic to `commands/dev.rs` +- [x] Add empty command stubs for all commands +- [x] Verify `gnd dev` works exactly as before +- [x] Add `--version` output showing both gnd and graph-cli versions + +### Phase 2: Infrastructure + +- [x] Implement `output/spinner.rs` matching TS CLI output format exactly +- [x] Implement `config/auth.rs` for `~/.graph-cli.json` management (in commands/auth.rs) +- [x] Implement `config/networks.rs` for networks.json parsing +- [x] Add reqwest dependency and basic HTTP client setup (in services/graph_node.rs) +- [x] Implement `services/registry.rs` for @pinax/graph-networks-registry (implemented as `NetworksRegistry` in services/contract.rs) + +### Phase 3: Simple Commands + +- [x] Implement `gnd clean` command +- [x] Implement `gnd auth` command +- [x] Implement `gnd create` command +- [x] Implement `gnd remove` command +- [x] Add tests for simple commands + +### Phase 4: Code Generation (High Priority, High Risk) + +- [x] Study TS CLI codegen output in detail +- [x] Implement `codegen/schema.rs` - entity classes from GraphQL +- [x] Implement `codegen/typescript.rs` - AST builders for TypeScript/AssemblyScript +- [x] Implement `codegen/types.rs` - type conversion utilities +- [x] Implement `codegen/abi.rs` - ABI bindings +- [x] Implement `codegen/template.rs` - template bindings +- [x] Integrate prettier formatting (shell out) +- [x] Implement `commands/codegen.rs` with all flags +- [x] Implement `--watch` mode using existing file watcher +- [x] Create snapshot tests comparing output to TS CLI (11 fixtures from graph-cli validation tests) +- [x] Verify byte-for-byte identical output - tests pass with documented known differences: + - Int8 import always included (gnd simplicity) + - Trailing commas in multi-line constructs (gnd style) + - 2D array accessors use correct `toStringMatrix()` (gnd fixes graph-cli bug) + +### Phase 5: Migrations + +- [x] Study TS CLI migrations in `/packages/cli/src/migrations/` +- [x] Implement migration framework in `migrations/mod.rs` +- [x] Implement each migration version (0.0.1 → current) +- [x] Add `--skip-migrations` flag support +- [x] Test migrations with old manifest versions + +### Phase 6: Build Command + +- [x] Implement `compiler/asc.rs` - shell out to asc +- [x] Implement `commands/build.rs` with all flags (~1084 lines, full build pipeline) +- [x] Handle network file resolution +- [x] Implement `--watch` mode +- [x] Implement optional IPFS upload with deduplication +- [x] Test build output matches TS CLI - Covered by `just test-gnd-commands` (test_build_after_codegen) + +### Phase 7: Deploy Command + +- [x] Implement `services/graph_node.rs` - JSON-RPC client (~258 lines, create/remove/deploy) +- [x] Implement `commands/deploy.rs` with all flags (~239 lines) +- [x] Support all deploy targets (local, studio, hosted service) - defaults to Subgraph Studio +- [x] Handle access token / deploy key authentication +- [x] Test deployment to local Graph Node - Covered by `just test-gnd-cli` (block-handlers, value-roundtrip, int8) + +### Phase 8: Init Command + +- [x] Add `inquire` dependency for interactive prompts +- [x] Implement `services/contract.rs` - ABI fetching (Etherscan, Blockscout, Sourcify) (~730 lines) +- [x] Implement networks registry loading +- [x] Implement `scaffold/manifest.rs` (~271 lines) +- [x] Implement `scaffold/schema.rs` (~206 lines) +- [x] Implement `scaffold/mapping.rs` (~205 lines) +- [x] Implement `commands/init.rs` with all flags (~1060 lines including --from-subgraph) +- [x] Implement --from-example mode +- [x] Implement --from-contract mode (uses ContractService for ABI fetching) +- [x] Add interactive prompts when required options are missing +- [x] Implement --from-subgraph mode (fetches manifest from IPFS, extracts immutable entities) +- [x] Test scaffold output matches TS CLI - Covered by `just test-gnd-commands` (test_init_from_example, test_init_from_contract_with_abi) + +### Phase 9: Add Command + +- [x] Implement `commands/add.rs` (~773 lines, fully functional) +- [x] Reuse scaffold components for actual implementation +- [x] ABI validation, entity generation, mapping creation, manifest updates +- [x] Test adding datasource to existing subgraph (unit tests for helper functions) +- [x] Comprehensive error handling + +### Phase 10: Publish Command + +- [x] Implement `commands/publish.rs` (~230 lines, fully functional) +- [x] Build subgraph and upload to IPFS (reuses build command) +- [x] Open browser to webapp URL with query params (same approach as graph-cli) +- [x] Support --ipfs-hash flag to skip build +- [x] Support --subgraph-id, --api-key for updating existing subgraphs +- [x] Support --protocol-network (arbitrum-one, arbitrum-sepolia) +- [x] Interactive prompt before opening browser +- [ ] Test publish workflow (NEEDS HUMAN: manual end-to-end verification) + +### Phase 11: Test Command + +- [x] Implement Matchstick binary download/detection +- [x] Implement `commands/test.rs` (~254 lines, shell out to Matchstick) +- [x] Handle Docker mode (Dockerfile generation on demand) +- [x] Recompilation flag support +- [ ] Test with actual Matchstick tests (NEEDS HUMAN: install matchstick + run end-to-end) + +### Phase 12: Testing & Polish + +- [x] Port all tests from TS CLI test suite not already covered by gnd tests + - Location: `/home/lutter/code/subgraphs/graph-cli/packages/cli/tests/` + - Review existing coverage and identify gaps + - Analysis: gnd has 11/12 success fixtures from graph-cli validation tests (missing `near-is-valid` which is out of scope for Ethereum-only support). Error/validation fixtures test manifest parsing which is handled by graph-node's validation layer. +- [x] Add snapshot tests for code generation scenarios (11 fixtures from graph-cli validation tests) +- [x] Add tests for overloaded events/functions in ABI codegen (disambiguation) +- [x] Add tests for simple array fields in schema codegen +- [x] Add tests for nested array types (`[[String]]` etc.) - schema codegen refactored to track list depth +- [x] Add tests for tuple/struct types in ABI codegen (functions, events, nested, arrays) +- [x] Add tests for array types in ABI codegen (array params in events, 2D matrices) +- [x] Codegen verification test framework (gnd/tests/codegen_verification.rs) +- [x] Add comprehensive tests for prompt module (network completer, source type) +- [x] Add comprehensive tests for init command (interactive mode detection) +- [ ] Ensure all edge cases are covered (ongoing) +- [x] Documentation: + - [x] CLI usage docs (README with command reference, examples, common workflows) + - [x] Migration guide from graph-cli to gnd (differences, compatibility notes) +- [x] Shell completions (bash, elvish, fish, powershell, zsh via clap_complete) + +### Phase 13: CLI Integration Tests + +Verify gnd works as a drop-in replacement for graph-cli by running integration tests with `GRAPH_CLI=../target/debug/gnd`. + +**Key Insight**: The existing integration test infrastructure uses `CONFIG.graph_cli` which can be set via the `GRAPH_CLI` environment variable. By setting this to the gnd binary, existing integration tests will use gnd for the deployment flow (codegen, create, deploy). + +#### Integration Deployment Tests (`tests/tests/gnd_cli_tests.rs`) + +- [x] Create test file that sets `GRAPH_CLI` env var to gnd binary +- [x] Verify gnd binary exists (fail fast with clear message) +- [x] Run subset of integration tests: `block-handlers`, `value-roundtrip`, `int8` +- [x] Reuse `Contract::deploy_all()`, `CONFIG.reset_database()`, `CONFIG.spawn_graph_node()` +- [x] Use `Subgraph::deploy()` which calls: `gnd codegen`, `gnd create`, `gnd deploy` +- [x] Verify subgraphs sync and queries return expected data + +**Commands Tested via Subgraph::deploy()**: +- `gnd codegen` - via `Subgraph::deploy()` +- `gnd create` - via `Subgraph::deploy()` +- `gnd deploy` - via `Subgraph::deploy()` + +#### Standalone Command Tests (`gnd/tests/cli_commands.rs`) + +Tests for commands that don't require running graph-node: + +**`gnd init` Tests**: +- [x] `test_init_from_example` - Run `gnd init --from-example ethereum-gravatar`, verify scaffold created +- [x] `test_init_from_contract_with_abi` - Run with `--from-contract` + `--abi`, verify manifest/schema/mapping created +- [ ] `test_init_from_subgraph` - Run with `--from-subgraph`, verify immutable entities extracted (requires IPFS) - **Skipped**: Requires IPFS with subgraph data + +**`gnd add` Tests**: +- [x] `test_add_datasource` - Create base subgraph with init, then add contract, verify manifest updated + +**`gnd build` Tests**: +- [x] `test_build_after_codegen` - Copy fixture, run codegen + build, verify `build/` directory created with WASM + +**`gnd codegen` Tests**: +- [x] `test_codegen_generates_types` - Run codegen on scaffolded subgraph, verify `generated/schema.ts` exists + +**`gnd clean` Tests**: +- [x] `test_clean_removes_artifacts` - Verify clean removes `generated/` and `build/` directories + +**ABIs Available**: `tests/contracts/abis/*.json` + +#### Justfile Targets + +- [x] Add `test-gnd-cli` - Run gnd CLI integration tests (deployment with gnd as CLI) +- [x] Add `test-gnd-commands` - Run gnd standalone command tests (init, add, build) + +#### Service Requirements + +Tests use the same services as `just test-integration`: + +| Service | Port | Start Command | +|---------|------|---------------| +| PostgreSQL | 3011 | `nix run .#integration` | +| IPFS | 3001 | (included above) | +| Anvil | 3021 | (included above) | + +#### Verification + +1. `just build` - Build gnd binary +2. `just test-gnd-cli` - Run integration deployment tests with gnd +3. `just test-gnd-commands` - Run standalone command tests + +Expected output: All tests pass, showing gnd works as drop-in replacement. + +## Key Decisions + +| Decision | Choice | Rationale | +| ---------------- | ---------------------------- | ------------------------------------------- | +| Binary name | `gnd` | Existing name, avoid confusion with `graph` | +| Protocol support | Ethereum only | Simplify initial scope | +| Compatibility | Drop-in replacement | Same flags, output, exit codes | +| Code generation | Byte-for-byte identical | Snapshot testable, no surprises | +| Formatting | Shell out to prettier | Guarantees identical output | +| Compilation | Shell out to asc | Same as TS CLI | +| Testing | Shell out to Matchstick | Same as TS CLI | +| Debug logging | RUST_LOG | Rust standard, not DEBUG env var | +| Validation | Use graph-node's | Refactor if needed, single source of truth | +| Network registry | Fetch at runtime | Registry contents change over time | +| Error messages | Same info, format may differ | Convey same information | +| TS CLI bugs | Fix them | Don't replicate bugs | + +## Risk Assessment + +| Risk | Severity | Mitigation | +| --------------------------- | -------- | ------------------------------------ | +| Code generation fidelity | High | Snapshot tests against TS CLI output | +| Migration correctness | High | Test with real old manifests | +| Breaking existing gnd users | Medium | Keep `gnd dev` behavior identical | +| `asc` version compat | Medium | Version detection, clear errors | +| Network API changes | Low | Error handling, clear messages | + +## TS CLI References + +Key files to study in `/home/lutter/code/subgraphs/graph-cli/packages/cli/src/`: + +| Feature | TS CLI Location | +| ---------------- | ----------------------------------- | +| Commands | `commands/*.ts` | +| Type generator | `type-generator.ts` | +| Schema codegen | `codegen/schema.ts` | +| ABI codegen | `protocols/ethereum/codegen/abi.ts` | +| Template codegen | `codegen/template.ts` | +| Compiler | `compiler/index.ts` | +| Migrations | `migrations/` | +| Scaffold | `scaffold/` | +| Spinner/output | `command-helpers/spinner.ts` | +| Auth | `command-helpers/auth.ts` | +| Network config | `command-helpers/network.ts` | +| Tests | `../tests/cli/` | + +## Effort Estimate + +**~3-4 person-months** for full implementation because: + +- Code generation must be byte-for-byte identical (requires careful study) +- All migrations must be implemented +- Full test coverage needed +- But we reuse: manifest parsing, validation, IPFS client, file watching, CLI framework diff --git a/docs/plans/runner-refactor.md b/docs/plans/runner-refactor.md new file mode 100644 index 00000000000..c5ce612f8db --- /dev/null +++ b/docs/plans/runner-refactor.md @@ -0,0 +1,281 @@ +# Runner Refactor Implementation Plan + +This document outlines the implementation plan for the runner refactor described in [the spec](../specs/runner-refactor.md). + +## Overview + +The refactor transforms `core/src/subgraph/runner.rs` from a complex nested-loop structure into a cleaner state machine with explicit pipeline stages. + +## Git Workflow + +**Branch**: All work should be committed to the `runner-refactor` branch. + +**Commit discipline**: + +- Commit work in small, reviewable chunks +- Each commit should be self-contained and pass all checks +- Prefer many small commits over few large ones +- Each commit message should clearly describe what it does +- Each step in the implementation phases should correspond to one or more commits + +**Before each commit**: + +```bash +just format +just lint +just test-unit +just test-runner +``` + +- MANDATORY: Work must be committed, a task is only done when work is committed +- MANDATORY: Make sure to follow the commit discipline above +- IMPORTANT: The runner tests produce output in `tests/runner-tests.log`. Use that to investigate failures. + +## Implementation Phases + +### Phase 1: Extract TriggerRunner Component + +**Goal:** Eliminate duplicated trigger processing code (lines 616-656 vs 754-790). + +**Files to modify:** + +- `core/src/subgraph/runner.rs` - Extract logic +- Create `core/src/subgraph/runner/trigger_runner.rs` + +**Steps:** + +1. Create `TriggerRunner` struct with execute method +2. Replace first trigger loop (lines 616-656) with `TriggerRunner::execute()` +3. Replace second trigger loop (lines 754-790) with same call +4. Verify tests pass + +**Verification:** + +- `just test-unit` passes +- `just test-runner` passes +- No behavioral changes + +### Phase 2: Define RunnerState Enum + +**Goal:** Introduce explicit state machine types without changing control flow yet. + +**Files to modify:** + +- Create `core/src/subgraph/runner/state.rs` +- `core/src/subgraph/runner.rs` - Add state field + +**Steps:** + +1. Define `RunnerState` enum with all variants +2. Define `RestartReason` and `StopReason` enums +3. Add `state: RunnerState` field to `SubgraphRunner` +4. Initialize state in constructor +5. Verify tests pass (no behavioral changes yet) + +**Verification:** + +- Code compiles +- Tests pass unchanged + +### Phase 3: Refactor run_inner to State Machine + +**Goal:** Replace nested loops with explicit state transitions. + +**Files to modify:** + +- `core/src/subgraph/runner.rs` - Rewrite `run_inner` + +**Steps:** + +1. Extract `initialize()` method for pre-loop setup +2. Extract `await_block()` method for stream event handling +3. Extract `restart()` method for restart logic +4. Extract `finalize()` method for cleanup +5. Rewrite `run_inner` as state machine loop +6. Remove nested loop structure +7. Verify tests pass + +**Verification:** + +- `just test-unit` passes +- `just test-runner` passes +- Same behavior, cleaner structure + +### Phase 4: Define Pipeline Stages + +**Goal:** Break `process_block` into explicit stages. + +**Files to modify:** + +- Create `core/src/subgraph/runner/pipeline.rs` +- `core/src/subgraph/runner.rs` - Refactor `process_block` + +**Steps:** + +1. Extract `match_triggers()` stage method +2. Extract `execute_triggers()` stage method (uses `TriggerRunner`) +3. Extract `process_dynamic_data_sources()` stage method +4. Extract `process_offchain_triggers()` stage method +5. Extract `persist_block_state()` stage method +6. Rewrite `process_block` to call stages in sequence +7. Verify tests pass + +**Verification:** + +- `just test-unit` passes +- `just test-runner` passes +- Same behavior, cleaner structure + +### Phase 5: Consolidate Error Handling + +**Goal:** Unify scattered error handling into explicit classification. + +**Files to modify:** + +- `graph/src/components/subgraph/error.rs` (or wherever `ProcessingError` lives) +- `core/src/subgraph/runner.rs` - Use new error methods + +**Steps:** + +1. Add `ProcessingErrorKind` enum with Deterministic/NonDeterministic/PossibleReorg variants +2. Add `kind()` method to `ProcessingError` +3. Add helper methods: `should_stop_processing()`, `should_restart()`, `is_retryable()` +4. Replace scattered error checks in `process_block` with unified logic +5. Replace scattered error checks in dynamic DS handling +6. Replace scattered error checks in `handle_offchain_triggers` +7. Document error handling invariants in code comments +8. Verify tests pass + +**Verification:** + +- `just test-unit` passes +- `just test-runner` passes +- Error behavior unchanged (same semantics, cleaner code) + +### Phase 6: Add BlockState Checkpoints + +**Goal:** Enable rollback capability with minimal overhead. + +**Files to modify:** + +- `graph/src/prelude.rs` or wherever `BlockState` is defined +- `core/src/subgraph/runner.rs` - Use checkpoints + +**Steps:** + +1. Add `checkpoint()` method to `BlockState` +2. Add `BlockStateCheckpoint` struct +3. Add `restore()` method to `BlockState` +4. Use checkpoint before dynamic DS processing +5. Verify tests pass + +**Verification:** + +- `just test-unit` passes +- No performance regression (checkpoints are lightweight) + +### Phase 7: Module Organization + +**Goal:** Organize code into proper module structure. + +**Files to create/modify:** + +- `core/src/subgraph/runner/mod.rs` +- Move/organize existing extracted modules + +**Steps:** + +1. Create `runner/` directory +2. Move `state.rs`, `pipeline.rs`, `trigger_runner.rs` into it +3. Update `runner.rs` to re-export from module +4. Update imports in dependent files +5. Verify tests pass + +**Verification:** + +- `just test-unit` passes +- `just test-runner` passes +- `just lint` shows no warnings + +## Completion Criteria + +Each phase is complete when: + +1. `just format` - Code is formatted +2. `just lint` - Zero warnings +3. `just check --release` - Builds in release mode +4. `just test-unit` - Unit tests pass +5. `just test-runner` - Runner tests pass + +## Progress Checklist + +### Phase 1: Extract TriggerRunner Component + +- [x] Create `TriggerRunner` struct with execute method +- [x] Replace first trigger loop (lines 616-656) +- [x] Replace second trigger loop (lines 754-790) +- [x] Verify tests pass + +### Phase 2: Define RunnerState Enum + +- [x] Define `RunnerState` enum with all variants +- [x] Define `RestartReason` and `StopReason` enums +- [x] Add `state: RunnerState` field to `SubgraphRunner` +- [x] Initialize state in constructor +- [x] Verify tests pass + +### Phase 3: Refactor run_inner to State Machine + +- [x] Extract `initialize()` method +- [x] Extract `await_block()` method +- [x] Extract `restart()` method +- [x] Extract `finalize()` method +- [x] Rewrite `run_inner` as state machine loop +- [x] Remove nested loop structure +- [x] Verify tests pass + +### Phase 4: Define Pipeline Stages + +- [x] Extract `match_triggers()` stage method +- [x] Extract `execute_triggers()` stage method +- [x] Extract `process_dynamic_data_sources()` stage method +- [x] Extract `process_offchain_triggers()` stage method +- [x] Extract `persist_block_state()` stage method +- [x] Rewrite `process_block` to call stages in sequence +- [x] Verify tests pass + +### Phase 5: Consolidate Error Handling + +- [x] Add `ProcessingErrorKind` enum +- [x] Add `kind()` method to `ProcessingError` +- [x] Add helper methods (`should_stop_processing()`, `should_restart()`, `is_retryable()`) +- [x] Replace scattered error checks in `process_block` +- [x] Replace scattered error checks in dynamic DS handling (preserved existing behavior per spec) +- [x] Replace scattered error checks in `handle_offchain_triggers` (preserved existing behavior per spec) +- [x] Document error handling invariants +- [x] Verify tests pass + +### Phase 6: Add BlockState Checkpoints + +- [x] Add `BlockStateCheckpoint` struct +- [x] Add `checkpoint()` method to `BlockState` +- [x] Add `restore()` method to `BlockState` +- [x] Use checkpoint before dynamic DS processing +- [x] Verify tests pass + +### Phase 7: Module Organization + +- [x] Create `runner_components/` directory (named to avoid conflict with `runner.rs`) +- [x] Move `state.rs`, `trigger_runner.rs` into it (pipeline.rs was not created as stages are methods) +- [x] Create `runner_components/mod.rs` with re-exports +- [x] Update imports in `runner.rs` to use the new module path +- [x] Verify tests pass +- [x] `just lint` shows zero warnings + +## Notes + +- Each phase should be a separate, reviewable PR +- Phases 1-4 can potentially be combined if changes are small +- Phase 3 (FSM refactor of run_inner) is the most invasive and should be reviewed carefully +- Phase 5 (error handling) can be done earlier if it helps simplify other phases +- Preserve all existing behavior - this is a refactor, not a feature change diff --git a/docs/sharding.md b/docs/sharding.md new file mode 100644 index 00000000000..de2015be22a --- /dev/null +++ b/docs/sharding.md @@ -0,0 +1,158 @@ +# Sharding + +When a `graph-node` installation grows beyond what a single Postgres +instance can handle, it is possible to scale the system horizontally by +adding more Postgres instances. This is called _sharding_ and each Postgres +instance is called a _shard_. The resulting `graph-node` system uses all +these Postgres instances together, essentially forming a distributed +database. Sharding relies heavily on the fact that in almost all cases the +traffic for a single subgraph can be handled by a single Postgres instance, +and load can be distributed by storing different subgraphs in different +shards. + +In a sharded setup, one shard is special, and is called the _primary_. The +primary is used to store system-wide metadata such as the mapping of +subgraph names to IPFS hashes, a directory of all subgraphs and the shards +in which each is stored, or the list of configured chains. In general, +metadata that rarely changes is stored in the primary whereas metadata that +changes frequently such as the subgraph head pointer is stored in the +shards. The details of which metadata tables are stored where can be found +in [this document](./implementation/metadata.md). + +## Setting up + +Sharding requires that `graph-node` uses a [configuration file](./config.md) +rather than the older mechanism of configuring `graph-node` entirely with +environment variables. It is configured by adding additional +`[store.]` entries to `graph-node.toml` as described +[here](./config.md#configuring-multiple-databases) + +In a sharded setup, shards communicate with each other using the +[`postgres_fdw`](https://www.postgresql.org/docs/current/postgres-fdw.html) +extension. `graph-node` sets up the required foreign servers and foreign +tables to achieve this. It uses the connection information from the +configuration file for that which requires that the `connection` string for +each shard is in the form `postgres://USER:PASSWORD@HOST[:PORT]/DB` since +`graph-node` needs to parse the connection string to extract these +components. + +Before setting up sharding, it is important to make sure that the shards can +talk to each other. That requires in particular that firewall rules allow +traffic from each shard to each other shard, and that authentication +configuration like `pg_hba.conf` allows connections from all the other +shards using the target shard's credentials. + +When a new shard is added to the configuration file, `graph-node` will +initialize the database schema of that shard during startup. Once the schema +has been initialized, it is possible to manually check inter-shard +connectivity by running `select count(*) from primary_public.chains;` and +`select count(*) from shard__subgraphs.subgraph` --- the result of +these queries doesn't matter, it only matters that they succeed. + +With multiple shards, `graph-node` will periodically copy some metadata from +the primary to all the other shards. The metadata that gets copied is the +metadata that is needed to respond to queries as each query needs the +primary to find the shard that stores the subgraph's data. The copies of the +metadata are used when the primary is down to ensure that queries can still +be answered. + +## Best practices + +Usually, a `graph-node` installation starts out with a single shard. When a +new shard is added, the original shard, which is now called the _primary_, +can still be used in the same way it was used before, and existing subgraphs +and block caches can remain in the primary. + +Data can be added to new shards by setting up [deployment +rules](./config.md#controlling-deployment) that send certain subgraphs to +the new shard. It is also possible to store the block cache for new chains +in a new shard by setting the `shard` attribute of the [chain +definition](./config.md#configuring-ethereum-providers) + +With shards, there are many possibilities how data can be split between +them. One possible setup is: + +- a small primary that mostly stores metadata +- multiple shards for low-traffic subgraphs with a large number of subgraphs + per shard +- one or a small number of shards for high-traffic subgraphs with a small + number of subgraphs per shard +- one or more dedicated shards that store only block caches + +## Copying between shards + +Besides deployment rules for new subgraphs, it is also possible to copy and +move subgraphs between shards. The command `graphman copy create` starts the +process of copying a subgraph from one shard to another. It is possible to +have a copy of the same deployment, identified by an IPFS hash, in multiple +shards, but only one copy can exist in each shard. If a deployment has +multiple copies, exactly one of them is marked as `active` and is the one +that is used to respond to queries. The copies are indexed independently +from each other, according to how they are assigned to index nodes. + +By default, `graphman copy create` will copy the data of the source subgraph +up to the point where the copy was initiated and then start indexing the +subgraph independently from its source. When the `--activate` flag is passed +to `graphman copy create`, the copy process will mark the copy as `active` +once copying has finished and the copy has caught up to the chain head. When +the `--replace` flag is passed, the copy process will also mark the source +of the copy as unused, so that the unused deployment reaper built into +`graph-node` will eventually delete it. In the default configuration, the +source will be deleted about 8 hours after the copy has synced to the chain +head. + +When a subgraph has multiple copies, copies that are not `active` can be +made eligible for deletion by simply unassigning them. The unused deployment +reaper will eventually delete them. + +Copying a deployment can, depending on the size of the deployment, take a +long time. The command `graphman copy stats sgdDEST` can be used to check on +the progress of the copy. Copying also periodically logs progress messages. +After the data has been copied, the copy process has to perform a few +operations that can take a very long time with not much output. In +particular, it has to count all the entities in a subgraph to update the +`entity_count` of the copy. + +During copying, `graph-node` creates a namespace in the destination shard +that has the same `sgdNNN` identifier as the deployment in the source shard +and maps all tables from the source into the destination shard. That +namespace in the destination will be automatically deleted when the copy +finishes. + +The command `graphman copy list` can be used to list all currently active or +pending copy operations. The number of active copy operations is restricted +to 5 for each source shard/destination shard pair to limit the amount of +load that copying can put on the shards. + +## Namespaces + +Sharding creates a few namespaces ('schemas') within Postgres which are used +to import data from one shard into another. These namespaces are: + +- `primary_public`: maps some important tables from the primary into each shard +- `shard__subgraphs`: maps some important tables from each shard into + every other shard + +The code that sets up these mappings is in `ForeignServer::map_primary` and +`ForeignServer::map_metadata` +[here](https://github.com/graphprotocol/graph-node/blob/master/store/postgres/src/connection_pool.rs) + +The mappings can be rebuilt by running `graphman database remap`. + +The split of metadata between the primary and the shards currently poses +some issues for dashboarding data that requires information from both the +primary and a shard. That will be improved in a future release. + +## Removing a shard + +When a shard is no longer needed, it can be removed from the configuration. +This requires that nothing references that shard anymore. In particular that +means that there is no deployment that is still stored in that shard, and +that no chain is stored in it. If these two conditions are met, removing a +shard is as simple as deleting its declaration from the configuration file. + +Removing a shard in this way will leave the foreign tables in +`shard__subgraphs`, the user mapping and foreign server definition in +all the other shards behind. Those will not hamper the operation of +`graph-node` but can be removed by running the corresponding `DROP` commands +via `psql`. diff --git a/docs/specs/dump-restore.md b/docs/specs/dump-restore.md new file mode 100644 index 00000000000..6d7e117b78d --- /dev/null +++ b/docs/specs/dump-restore.md @@ -0,0 +1,269 @@ +# Spec: Parquet Dump/Restore for Subgraph Data + +## Problem + +Subgraph entity data lives exclusively in PostgreSQL. There's no way to export a subgraph's data for backup, migration between environments, or sharing. We need a file-based dump/restore mechanism. + +## Goals + +1. **Dump** a subgraph's entity data to parquet files +2. **Restore** a subgraph from parquet files +3. **Incremental append** -- add newly arrived data to an existing dump without rewriting it +4. Long-term: make dumping an **ongoing process** (not just a one-off CLI operation) + +## Non-goals (for now) + +- S3/GCS output +- Schema evolution / migration between different schema versions + +## Data Model Recap + +Each subgraph deployment has: +- A PostgreSQL schema (e.g., `sgd123`) +- One table per entity type, with columns: + - `vid` (bigserial) -- row version ID, primary key + - `block_range` (int4range) for mutable entities OR `block$` (int) for immutable + - `causality_region` (int, optional) -- for offchain data sources + - Data columns matching GraphQL fields (text, int, bigint, numeric, bytea, bool, timestamptz, arrays, enums) +- `data_sources$` table -- dynamic data sources created at runtime (defined in `dynds/private.rs`, separate from Layout): + - `vid` (int, identity PK), `block_range` (int4range), `causality_region` (int) + - `manifest_idx` (int), `parent` (int, self-ref FK), `id` (bytea) + - `param` (bytea, nullable), `context` (jsonb, nullable), `done_at` (int, nullable) + - Note: `parent` and `id` exist in the DDL but are currently unused by `insert()`, `load()`, and `copy_to()`. The dump should include them for completeness (they may contain data in older deployments). +- `poi2$` table -- Proof of Indexing data (a **mutable** entity table in the Layout, i.e. it uses `block_range` not `block$`; has `digest` (bytea), `id` (text), optionally `block_time$` (int8); `has_causality_region: false`). Conditionally created when `catalog.use_poi` is true. Excluded from some Layout operations like `find_changes()`. +- Metadata in `subgraphs.subgraph_manifest`, `subgraphs.deployment`, and `subgraphs.head` tables + +## Dump Format + +### Directory layout + +``` +/ + metadata.json -- deployment metadata + per-table state + schema.graphql -- raw GraphQL schema text + subgraph.yaml -- raw subgraph manifest YAML (optional) + / + chunk_000000.parquet -- rows ordered by vid + chunk_000001.parquet -- incremental append + ... + data_sources$/ + chunk_000000.parquet -- dynamic data sources +``` + +One parquet file per entity type (each has a different columnar schema). The `data_sources$` table is dumped alongside entity tables. The `poi2$` table, when present, is a regular entity table in the `Layout` (conditionally created when `catalog.use_poi` is true) and appears as any other entity type directory. Incremental dumps produce new chunk files rather than rewriting existing ones. + +The GraphQL schema and subgraph manifest YAML are stored as separate files rather than embedded in `metadata.json`. This matches the existing dump code in `dump.rs` and keeps the files human-readable and diffable. + +### Parquet schema per entity type + +System columns (always present): +- `vid` -> Int64 +- Immutable entities: `block$` -> Int32 +- Mutable entities: `block_range_start` -> Int32, `block_range_end` -> Int32 (nullable; null = unbounded/current) +- `causality_region` -> Int32 (only if table has it) + +Data columns mapped from `ColumnType` (all 10 variants defined in `relational.rs:1342`): + +| ColumnType | Arrow DataType | Notes | +|--------------------|---------------------------|-------------------------------| +| Boolean | Boolean | | +| Int | Int32 | | +| Int8 | Int64 | | +| Bytes | Binary | Raw bytes | +| BigInt | Utf8 | Arbitrary precision as string | +| BigDecimal | Utf8 | Arbitrary precision as string | +| Timestamp | TimestampMicrosecond(None) | Matches Value::Timestamp | +| String | Utf8 | | +| Enum(EnumType) | Utf8 | String value of enum variant | +| TSVector(FulltextConfig) | **Skip** | Generated; rebuild on restore | + +**List/array columns:** `List(T)` is not a `ColumnType` variant. Whether a column is an array is determined by `Column.is_list()` (delegates to the GraphQL `field_type`). A `[String]` field has `column_type: ColumnType::String` with a list-typed `field_type`. For Arrow mapping, check `column.is_list()` and wrap the base Arrow type in `List`. In `OidValue`, arrays have separate variants (`StringArray`, `BytesArray`, `BoolArray`, `Ints`, `Int8Array`, `BigDecimalArray`, `TimestampArray`). + +Nullability follows the GraphQL schema (non-null fields -> non-nullable Arrow columns). + +### metadata.json + +Contains everything needed to reconstruct the deployment's table structure, plus diagnostic information (health, indexes) captured at dump time. The GraphQL schema and manifest YAML are stored in separate files (`schema.graphql`, `subgraph.yaml`), not embedded here. + +The struct backing this file is `Metadata` (evolved from the existing `Control` struct in `dump.rs`). + +```json +{ + "version": 1, + "deployment": "Qm...", + "network": "mainnet", + + "manifest": { + "spec_version": "1.0.0", + "description": "Optional subgraph description", + "repository": "https://github.com/...", + "features": ["..."], + "entities_with_causality_region": ["EntityType1"], + "history_blocks": 2147483647 + }, + + "earliest_block_number": 12345, + "start_block": { "number": 12345, "hash": "0xabc..." }, + "head_block": { "number": 99999, "hash": "0xdef..." }, + "entity_count": 150000, + + "graft_base": null, + "graft_block": null, + "debug_fork": null, + + "health": { + "failed": false, + "health": "healthy", + "fatal_error": null, + "non_fatal_errors": [] + }, + + "indexes": { + "token": [ + "CREATE INDEX CONCURRENTLY IF NOT EXISTS attr_0_0_id ON sgd.token USING btree (id)" + ] + }, + + "tables": { + "Token": { + "immutable": true, + "has_causality_region": false, + "chunks": [ + { "file": "Token/chunk_000000.parquet", "min_vid": 0, "max_vid": 50000, "row_count": 50000 } + ], + "max_vid": 50000 + }, + "data_sources$": { + "immutable": false, + "has_causality_region": true, + "chunks": [ + { "file": "data_sources$/chunk_000000.parquet", "min_vid": 0, "max_vid": 100, "row_count": 100 } + ], + "max_vid": 100 + } + } +} +``` + +**Field sources:** + +| Field | Source | Code path | +|-------|--------|-----------| +| `manifest.*` | `subgraphs.subgraph_manifest` | `SubgraphManifestEntity` via `deployment_entity()` in `detail.rs` | +| `start_block` | `subgraphs.subgraph_manifest` | `start_block_number`, `start_block_hash` columns; available via `StoredSubgraphManifest` in `detail.rs:542-543`, assembled into `SubgraphDeploymentEntity.start_block` | +| `earliest_block_number` | `subgraphs.deployment` | `SubgraphDeploymentEntity.earliest_block_number` | +| `graft_base`, `graft_block` | `subgraphs.deployment` | `SubgraphDeploymentEntity.graft_base`, `.graft_block` | +| `debug_fork` | `subgraphs.deployment` | `SubgraphDeploymentEntity.debug_fork` | +| `head_block` | `subgraphs.head` | `SubgraphDeploymentEntity.latest_block` | +| `entity_count` | `subgraphs.head` | `DeploymentDetail.entity_count` (i64 in DB, usize in Rust) | +| `health.*` | `subgraphs.deployment` + `subgraph_error` | `SubgraphDeploymentEntity.{failed, health, fatal_error, non_fatal_errors}` | +| `indexes` | `pg_indexes` catalog | `IndexList::load()` → `CreateIndex::to_sql()` (existing code in `dump.rs:163-179`) | +| `network` | `deployment_schemas` | `Site.network` | +| `tables.*` | `Layout.tables` | `Table.{immutable, has_causality_region}` | + +**Notes:** +- `use_bytea_prefix` is not stored in the dump. It is hardcoded to `true` in `create_deployment` (deployment.rs:1302) and will always be set to `true` on restore. +- `health` and `indexes` are point-in-time diagnostic snapshots. They are not used during restore (a restored deployment starts healthy; indexes are auto-created by `Layout::create_relational_schema()`). They are included for inspection and debugging. +- `indexes` are serialized as SQL strings using `CreateIndex::with_nsp("sgd")` + `to_sql(true, true)`, producing `CREATE INDEX CONCURRENTLY IF NOT EXISTS` statements with a normalized `sgd` namespace. +- The `manifest` fields mirror the existing `Manifest` struct in `dump.rs` (derived from `SubgraphManifestEntity`). The `schema` and `raw_yaml` fields of `SubgraphManifestEntity` are written to separate files instead. +- The `poi2$` table, when present, is a regular mutable entity table in `Layout.tables` and appears in the `tables` map like any other entity. It does not need special handling. + +The raw GraphQL schema (in `schema.graphql`) is sufficient to reconstruct the full relational layout via `InputSchema::parse(spec_version, schema, deployment_hash)` → `Layout::new()`. The `InputSchema::parse()` call requires `manifest.spec_version` for version-specific parsing logic. + +## Dump Process + +**Existing code:** There is already a metadata-only dump in `store/postgres/src/relational/dump.rs` (`Layout::dump()`) that writes `control.json`, `schema.graphql`, and `subgraph.yaml`. It is called via `DeploymentStore::dump()` (deployment_store.rs:901) which loads `Layout` + `IndexList` and passes both to `Layout::dump()`. The connection is `AsyncPgConnection` via `pool.get_permitted()`. The new parquet dump extends this to include entity data. + +The existing `Control` struct is renamed to `Metadata` and extended with the fields described above. The existing `Manifest`, `BlockPtr`, `Health`, and `Error` structs in `dump.rs` are reused and extended. + +1. Resolve the deployment (by name, hash, or sgdN) +2. Read deployment metadata from `subgraph_manifest` + `deployment` + `head` tables (via `deployment_entity()` in `detail.rs`) +3. Write `schema.graphql` and `subgraph.yaml` (existing behavior) +4. For each entity type table in `Layout.tables` (sorted by name for determinism; includes `poi2$` when present): + a. Query rows in vid order, batched (adaptive sizing like `VidBatcher`) + b. Convert PG rows directly to Arrow `RecordBatch` (no JSON intermediate) + c. Write batches to parquet file + d. Record chunk info (file path, min_vid, max_vid, row_count) +5. Dump `data_sources$` table (fixed schema, same batch approach; include all DDL columns: `vid`, `block_range`, `causality_region`, `manifest_idx`, `parent`, `id`, `param`, `context`, `done_at`). Note: `parent` and `id` are in the DDL (`private.rs:68-69`) but not in the `DataSourcesTable` struct — dumping them requires raw SQL or extending the struct. +6. Write `metadata.json` atomically (write to tmp file, rename) + +### Incremental append + +- Read existing `metadata.json` to get `max_vid` per entity type +- Query rows with `vid > max_vid` +- Write as new chunk files (`chunk_000001.parquet`, etc.) +- Update metadata atomically + +## Restore Process + +1. Read `metadata.json` and `schema.graphql` +2. Parse schema via `InputSchema::parse(manifest.spec_version, schema_text, deployment_hash)` +3. Create a `Site` entry in `deployment_schemas` (needed for the deployment to be discoverable) +4. Create deployment via `create_deployment(conn, site, DeploymentCreate { .. })` -- this populates three tables: + - `subgraphs.head` (block pointers, entity count -- initially null/0) + - `subgraphs.deployment` (deployment hash, earliest_block, graft info, health) + - `subgraphs.subgraph_manifest` (schema from file, features/spec_version/etc. from metadata) +5. Create tables via `Layout::create_relational_schema()` -- this generates DDL from the parsed schema and creates all entity tables with default indexes +6. Restore `data_sources$` table via DDL from `DataSourcesTable::new().as_ddl()` + batch-insert +7. For each entity type (including `poi2$` if present), read all parquet chunks in order, batch-insert into PG +8. Reset vid sequences to `max_vid + 1` for all entity tables and data_sources$ +9. Update `subgraphs.head` with `head_block.number`, `head_block.hash`, and `entity_count` from dump metadata + +## PG Read Strategy: OidValue-based Dynamic Columns + +Use the existing `dsl::Table::select_cols()` + `DynamicRow` pattern (see `store/postgres/src/relational/dsl.rs` and `store/postgres/src/relational/value.rs`). This already solves dynamic-schema typed extraction through the connection pool: + +1. `select_cols()` builds a typed SELECT using `DynamicSelectClause` for any set of columns +2. Results are `DynamicRow` where `OidValue` dispatches on PG OID at runtime +3. `OidValue` captures all needed types: String, Bytes, Bool, Int, Int8, BigDecimal, Timestamp, plus array variants +4. Convert `OidValue` -> Arrow `ArrayBuilder` (analogous to existing `OidValue` -> `Entity` in `FromOidRow`) + +No JSON, no separate connection. The existing `DeploymentStore::dump()` already uses `AsyncPgConnection` via `pool.get_permitted()`. + +**Block range handling:** Add `OidValue::Int4Range(Bound, Bound)` variant (OID 3904). Diesel already has `FromSql, Pg>` for `(Bound, Bound)` which parses the binary format. ~15 lines of code in `value.rs` + fix the `BLOCK_RANGE_COL` placeholder in `dsl.rs:46-49` (currently `ColumnType::Bytes`, with comment "we can't deserialize in4range"). This resolves the existing TODO at dsl.rs line 294. + +**Key existing code:** +- `dsl::Table::select_cols()` (`store/postgres/src/relational/dsl.rs:305`) +- `OidValue` enum and `FromSql` impl (`store/postgres/src/relational/value.rs:33`) +- `FromOidRow` trait for result deserialization (`value.rs:206`) +- `selected_columns()` for building column list with system columns (`dsl.rs:246`) + +### Vid continuity on restore + +Preserve original vid values. Needed for incremental consistency and simpler to implement. Reset vid sequence to `max_vid + 1` after restore. + +## Where Code Lives + +- `store/postgres/src/relational/dump.rs` -- **Existing** metadata-only dump. Contains `Manifest`, `BlockPtr`, `Error`, `Health`, `Control` structs. `Control` will be renamed to `Metadata` and extended. The existing helper structs (`Manifest`, `BlockPtr`, `Health`, `Error`) are reused. Called via `DeploymentStore::dump()` → `Layout::dump()`. +- `store/postgres/src/parquet/` -- New module for parquet read/write/schema mapping +- `node/src/manager/commands/dump.rs` -- **Existing** CLI command skeleton (resolves deployment, calls `SubgraphStore::dump()`) +- `node/src/manager/commands/restore.rs` -- New CLI command for restore +- Expose via `command_support` in `store/postgres/src/lib.rs` + +## Dependencies to Add + +- `parquet = "=57.3.0"` (same version as existing `arrow`) to workspace and `store/postgres` +- `arrow` workspace dep to `store/postgres` + +## Existing Code to Reuse + +- `Layout`, `Table`, `Column`, `ColumnType` (`store/postgres/src/relational.rs`) -- schema introspection; `poi2$` is in Layout as a mutable entity table (conditionally, when `catalog.use_poi` is true) +- `Column.is_list()` (`relational.rs:1574`) -- determines if a column is an array type (delegates to GraphQL `field_type.is_list()`) +- `DataSourcesTable` (`store/postgres/src/dynds/private.rs`) -- `data_sources$` DDL via `as_ddl()` method; note that `parent` and `id` columns are in the DDL but not in the struct's typed fields +- `VidBatcher` (`store/postgres/src/vid_batcher.rs`) -- adaptive batch iteration using PG histogram statistics +- `copy.rs` pattern -- progress reporting, batch operation lifecycle; already handles both entity tables and `data_sources$` copying +- `InputSchema::parse(spec_version, raw, id)` (`graph/src/schema/input/mod.rs:965`) -- schema reconstruction from text +- `DeploymentSearch` (`node/src/manager/deployment.rs`) -- CLI deployment resolution (supports name, Qm hash, sgdN namespace) +- `create_deployment` (`store/postgres/src/deployment.rs:1224`) -- populates `head`, `deployment`, and `subgraph_manifest` tables +- `DeploymentCreate` + `SubgraphManifestEntity` (`graph/src/data/subgraph/schema.rs:103`) -- structs needed by `create_deployment` +- `deployment_entity()` (`store/postgres/src/detail.rs`) -- reads deployment metadata into `SubgraphDeploymentEntity`; note that `start_block_*` is in the DB table (`StoredSubgraphManifest`) but only partially exposed through `SubgraphDeploymentEntity` +- `IndexList::load()` + `CreateIndex::to_sql()` (`store/postgres/src/relational/index.rs`) -- loads and serializes indexes +- Existing `dump.rs` (`store/postgres/src/relational/dump.rs`) -- metadata serialization types: `Manifest`, `BlockPtr`, `Health`, `Error`, `Control` (to be renamed `Metadata`) + +## Implementation Order + +1. Schema mapping + metadata types (foundation, unit-testable) +2. Parquet writer (dump from PG) + graphman `dump` command +3. Incremental append support +4. Parquet reader (restore to PG) + graphman `restore` command +5. Ongoing dump integration (run as part of graph-node, not just CLI) diff --git a/docs/specs/gnd-cli-expansion.md b/docs/specs/gnd-cli-expansion.md new file mode 100644 index 00000000000..7d0cf0d37fe --- /dev/null +++ b/docs/specs/gnd-cli-expansion.md @@ -0,0 +1,891 @@ +# Spec: gnd CLI Expansion + +This spec describes the expansion of `gnd` (Graph Node Dev) to include functionality currently provided by the TypeScript-based `graph-cli`. The goal is a drop-in replacement for subgraph development workflows, implemented in Rust and integrated with graph-node's existing infrastructure. + +## Goals + +- **Drop-in replacement**: Same commands, flags, output format, and exit codes as `graph-cli` +- **Identical code generation**: AssemblyScript output must be byte-for-byte identical after formatting +- **Ethereum only**: Initial scope limited to Ethereum protocol +- **Reuse graph-node internals**: Leverage existing manifest parsing, validation, IPFS client, etc. + +## Non-Goals + +- Multi-protocol support (NEAR, Cosmos, Arweave, Substreams) - future work +- Rollout/migration plan from TS CLI +- Performance optimization beyond reasonable behavior + +## Commands + +### Command Matrix + +| Command | TS CLI | gnd | Notes | +| --------- | ------ | --- | ------------------------------------------ | +| `codegen` | Yes | Yes | Generate AssemblyScript types | +| `build` | Yes | Yes | Compile to WASM | +| `deploy` | Yes | Yes | Deploy to Graph Node | +| `init` | Yes | Yes | Scaffold new subgraph | +| `add` | Yes | Yes | Add datasource to existing subgraph | +| `remove` | Yes | Yes | Unregister subgraph name | +| `create` | Yes | Yes | Register subgraph name | +| `auth` | Yes | Yes | Set deploy key | +| `publish` | Yes | Yes | Publish to decentralized network | +| `test` | Yes | Yes | Run Matchstick tests | +| `clean` | Yes | Yes | Remove build artifacts | +| `dev` | No | Yes | Run graph-node in dev mode (existing gnd) | +| `local` | Yes | No | Skipped - use existing test infrastructure | +| `node` | Yes | No | Skipped - use graphman for node management | + +### Known Differences from TS CLI + +**Commands:** + +1. **`local` command**: Not implemented. Users should use existing integration test infrastructure. +2. **`node` subcommand**: Not implemented. Use `graphman` for node management operations. +3. **`--uncrashable` flag on codegen**: Not implemented. Float Capital's uncrashable helper generation is a niche third-party feature. +4. **Debug output**: Uses `RUST_LOG` environment variable instead of `DEBUG=graph-cli:*`. + +**Code generation** (intentional, documented in verification tests): 5. **Int8 import**: gnd always imports `Int8` for simplicity, even when not used. 6. **Trailing commas**: gnd uses trailing commas in multi-line constructs. 7. **2D array accessors**: gnd uses correct `toStringMatrix()` while graph-cli has a bug using `toStringArray()` for 2D GraphQL array types. 8. **Tuple/struct component names**: gnd correctly extracts component names from raw ABI JSON, while graph-cli's ethabi parser loses nested component names for some tuple types. 9. **Subgraph data source file naming**: gnd uses the data source name (`subgraph-SourceName.ts`) instead of IPFS hash (`subgraph-QmHash.ts`) for stable import paths that don't break when source subgraphs are redeployed. + +## CLI Interface + +### Binary and Invocation + +``` +gnd [options] [arguments] +``` + +The binary name is `gnd`. All graph-cli commands become gnd subcommands. + +### Version Output + +``` +$ gnd --version +gnd 0.1.0 (graph-cli compatible: 0.98.1) +``` + +Shows both gnd version and the graph-cli version it emulates. + +### Flag Compatibility + +All flags must match the TS CLI exactly: + +- Same long names (`--output-dir`) +- Same short names (`-o`) +- Same defaults +- Same validation behavior + +Reference: Each command section below lists flags with references to TS CLI source. + +### Output Format + +Output must match TS CLI format exactly, including: + +- Spinner/progress indicators +- Success checkmarks (`✔`) +- Step descriptions +- File paths displayed +- Error formatting (information must match; exact wording may differ) + +Reference: `/packages/cli/src/command-helpers/spinner.ts` + +### Exit Codes + +Exit codes must match TS CLI behavior: + +- `0`: Success +- `1`: Error (validation, compilation, deployment failure, etc.) + +### Configuration Files + +Use same paths and formats as TS CLI: + +| File | Path | Purpose | +| -------------- | ------------------------------ | ----------------------------- | +| Auth tokens | `~/.graphprotocol/` | Deploy keys and access tokens | +| Network config | `networks.json` (project root) | Network-specific addresses | + +Reference: `/packages/cli/src/command-helpers/auth.ts` + +## Command Specifications + +### `gnd codegen` + +Generates AssemblyScript types from subgraph manifest. + +**Usage:** + +``` +gnd codegen [subgraph-manifest] +``` + +**Arguments:** + +- `subgraph-manifest`: Path to manifest file (default: `subgraph.yaml`) + +**Flags:** +| Flag | Short | Default | Description | +|------|-------|---------|-------------| +| `--output-dir` | `-o` | `generated/` | Output directory for generated types | +| `--skip-migrations` | | `false` | Skip subgraph migrations | +| `--watch` | `-w` | `false` | Regenerate on file changes | +| `--ipfs` | `-i` | `https://api.thegraph.com/ipfs/` | IPFS node URL | +| `--help` | `-h` | | Show help | + +**Behavior:** + +1. Load and validate manifest +2. Apply migrations (unless `--skip-migrations`) +3. Assert minimum API version (0.0.5) and graph-ts version (0.25.0) +4. Generate entity classes from GraphQL schema +5. Generate ABI bindings for each contract +6. Generate template datasource bindings +7. Format output with prettier +8. Write to output directory + +**Output Structure:** + +``` +generated/ +├── schema.ts # Entity classes +├── / +│ └── .ts # ABI bindings +├── templates/ +│ └── / +│ └── .ts # Template ABI bindings +└── subgraph-.ts # Entity types for subgraph data sources +``` + +**TS CLI References:** + +- Command: `/packages/cli/src/commands/codegen.ts` +- Type generator: `/packages/cli/src/type-generator.ts` +- Schema codegen: `/packages/cli/src/codegen/schema.ts` +- ABI codegen: `/packages/cli/src/protocols/ethereum/codegen/abi.ts` + +### `gnd build` + +Compiles subgraph to WASM. + +**Usage:** + +``` +gnd build [subgraph-manifest] +``` + +**Arguments:** + +- `subgraph-manifest`: Path to manifest file (default: `subgraph.yaml`) + +**Flags:** +| Flag | Short | Default | Description | +|------|-------|---------|-------------| +| `--output-dir` | `-o` | `build/` | Output directory | +| `--output-format` | `-t` | `wasm` | Output format: `wasm` or `wast` | +| `--skip-migrations` | | `false` | Skip subgraph migrations | +| `--watch` | `-w` | `false` | Rebuild on file changes | +| `--ipfs` | `-i` | | IPFS node URL (uploads if provided) | +| `--network` | | | Network to use from networks.json | +| `--network-file` | | `networks.json` | Path to networks config | +| `--help` | `-h` | | Show help | + +**Behavior:** + +1. Run codegen (unless types already exist) +2. Apply migrations (unless `--skip-migrations`) +3. Resolve network-specific values from networks.json +4. Shell out to `asc` (AssemblyScript compiler) for each mapping +5. Copy ABIs and schema to build directory +6. Copy template ABIs to build/templates// directories +7. Generate build manifest +8. Optionally upload to IPFS + +**Build Output Structure:** + +``` +build/ +├── schema.graphql +├── subgraph.yaml +├── / +│ ├── .wasm +│ └── .json # ABI +└── templates/ + └── / + ├── .wasm + └── .json # ABI +``` + +**Note:** When multiple data sources or templates share the same mapping.ts file, gnd compiles it once and copies the resulting WASM to all required output locations. + +**TS CLI References:** + +- Command: `/packages/cli/src/commands/build.ts` +- Compiler: `/packages/cli/src/compiler/index.ts` + +### `gnd deploy` + +Deploys subgraph to a Graph Node. + +**Usage:** + +``` +gnd deploy [subgraph-name] [subgraph-manifest] +``` + +**Arguments:** + +- `subgraph-name`: Name to deploy as (e.g., `user/subgraph`) +- `subgraph-manifest`: Path to manifest file (default: `subgraph.yaml`) + +**Flags:** +| Flag | Short | Default | Description | +|------|-------|---------|-------------| +| `--product` | | | Product: `subgraph-studio` or `hosted-service` | +| `--studio` | | `false` | Shorthand for `--product subgraph-studio` | +| `--node` | `-g` | | Graph Node URL | +| `--ipfs` | `-i` | | IPFS node URL | +| `--access-token` | | | Access token for authentication | +| `--deploy-key` | | | Deploy key (alias for access-token) | +| `--version-label` | `-l` | | Version label | +| `--headers` | | | Additional HTTP headers (JSON) | +| `--debug-fork` | | | Fork subgraph for debugging | +| `--skip-migrations` | | `false` | Skip subgraph migrations | +| `--network` | | | Network from networks.json | +| `--network-file` | | `networks.json` | Path to networks config | +| `--output-dir` | `-o` | `build/` | Build output directory | +| `--help` | `-h` | | Show help | + +**Deploy Targets:** + +- Local Graph Node (via `--node` and `--ipfs`) +- Subgraph Studio (`--product subgraph-studio` or `--studio`) +- Hosted Service (`--product hosted-service`) +- Decentralized network (via `publish` command) + +**Behavior:** + +1. Build subgraph (runs build command) +2. Upload build artifacts to IPFS +3. Send deployment request to Graph Node via JSON-RPC + +**TS CLI References:** + +- Command: `/packages/cli/src/commands/deploy.ts` + +### `gnd init` + +Scaffolds a new subgraph project. + +**Usage:** + +``` +gnd init [directory] +``` + +**Arguments:** + +- `directory`: Directory to create subgraph in + +**Flags:** +| Flag | Short | Default | Description | +|------|-------|---------|-------------| +| `--protocol` | | | Protocol: `ethereum` | +| `--product` | | | Product for deployment | +| `--studio` | | `false` | Initialize for Subgraph Studio | +| `--from-contract` | | | Contract address to generate from | +| `--from-example` | | | Example subgraph to clone | +| `--contract-name` | | | Name for the contract | +| `--index-events` | | `false` | Index all contract events | +| `--start-block` | | | Start block for indexing | +| `--network` | | `mainnet` | Network name | +| `--abi` | | | Path to ABI file | +| `--spkg` | | | Path to Substreams package | +| `--allow-simple-name` | | `false` | Allow simple subgraph names | +| `--help` | `-h` | | Show help | + +**Example Subgraphs:** + +The `--from-example ` flag clones from `https://github.com/graphprotocol/graph-tooling/tree/main/examples`. +The example name is **required** - no default is provided. Available examples include: +`ethereum-gravatar`, `aggregations`, `ethereum-basic-event-handlers`, `substreams-powered-subgraph`, etc. +Legacy name `ethereum/gravatar` is automatically converted to `ethereum-gravatar`. + +**Behavior:** + +1. Prompt for missing information (protocol, network, contract, etc.) +2. Fetch ABI from Etherscan/Sourcify if `--from-contract` and no `--abi` +3. Generate scaffold: + - `subgraph.yaml` manifest + - `schema.graphql` with entities for events + - `src/mapping.ts` with event handlers + - `package.json` with dependencies + - `tsconfig.json` + - ABIs directory +4. Optionally initialize git repository +5. Install dependencies + +**External APIs:** + +- Etherscan API: Fetch verified contract ABIs +- Sourcify API: Fetch verified contract ABIs (fallback) +- Network registry: `@pinax/graph-networks-registry` for chain configuration + +**TS CLI References:** + +- Command: `/packages/cli/src/commands/init.ts` +- Scaffold: `/packages/cli/src/scaffold/index.ts` +- Schema generation: `/packages/cli/src/scaffold/schema.ts` +- Mapping generation: `/packages/cli/src/scaffold/mapping.ts` +- Etherscan client: `/packages/cli/src/command-helpers/contracts.ts` + +### `gnd add` + +Adds a new datasource to an existing subgraph. + +**Usage:** + +``` +gnd add
[subgraph-manifest] +``` + +**Arguments:** + +- `address`: Contract address +- `subgraph-manifest`: Path to manifest file (default: `subgraph.yaml`) + +**Flags:** +| Flag | Short | Default | Description | +|------|-------|---------|-------------| +| `--abi` | | | Path to ABI file | +| `--contract-name` | | | Name for the contract | +| `--merge-entities` | | `false` | Merge with existing entities | +| `--network-file` | | `networks.json` | Path to networks config | +| `--start-block` | | | Start block | +| `--help` | `-h` | | Show help | + +**TS CLI References:** + +- Command: `/packages/cli/src/commands/add.ts` + +### `gnd create` + +Registers a subgraph name with a Graph Node. + +**Usage:** + +``` +gnd create +``` + +**Arguments:** + +- `subgraph-name`: Name to register + +**Flags:** +| Flag | Short | Default | Description | +|------|-------|---------|-------------| +| `--node` | `-g` | | Graph Node URL | +| `--access-token` | | | Access token | +| `--help` | `-h` | | Show help | + +**TS CLI References:** + +- Command: `/packages/cli/src/commands/create.ts` + +### `gnd remove` + +Unregisters a subgraph name from a Graph Node. + +**Usage:** + +``` +gnd remove +``` + +**Arguments:** + +- `subgraph-name`: Name to unregister + +**Flags:** +| Flag | Short | Default | Description | +|------|-------|---------|-------------| +| `--node` | `-g` | | Graph Node URL | +| `--access-token` | | | Access token | +| `--help` | `-h` | | Show help | + +**TS CLI References:** + +- Command: `/packages/cli/src/commands/remove.ts` + +### `gnd auth` + +Sets the deploy key for a Graph Node. + +**Usage:** + +``` +gnd auth +``` + +**Arguments:** + +- `deploy-key`: Deploy key to store + +**Flags:** +| Flag | Short | Default | Description | +|------|-------|---------|-------------| +| `--product` | | | Product: `subgraph-studio` or `hosted-service` | +| `--studio` | | `false` | Shorthand for subgraph-studio | +| `--help` | `-h` | | Show help | + +**Behavior:** +Stores deploy key in `~/.graphprotocol/` for later use by deploy/publish commands. + +**TS CLI References:** + +- Command: `/packages/cli/src/commands/auth.ts` +- Auth helpers: `/packages/cli/src/command-helpers/auth.ts` + +### `gnd publish` + +Publishes a subgraph to The Graph's decentralized network. + +**Usage:** + +``` +gnd publish [subgraph-manifest] +``` + +**Arguments:** + +- `subgraph-manifest`: Path to manifest file (default: `subgraph.yaml`) + +**Flags:** +| Flag | Short | Default | Description | +|------|-------|---------|-------------| +| `--subgraph-id` | | | Subgraph ID to publish to | +| `--ipfs` | `-i` | | IPFS node URL | +| `--protocol-network` | | | Protocol network (e.g., `arbitrum-one`) | +| `--help` | `-h` | | Show help | + +**TS CLI References:** + +- Command: `/packages/cli/src/commands/publish.ts` + +### `gnd test` + +Runs Matchstick tests for the subgraph. + +**Usage:** + +``` +gnd test [datasource] +``` + +**Arguments:** + +- `datasource`: Specific datasource to test (optional) + +**Flags:** +| Flag | Short | Default | Description | +|------|-------|---------|-------------| +| `--coverage` | `-c` | `false` | Run with coverage | +| `--docker` | `-d` | `false` | Run in Docker container | +| `--force` | `-f` | `false` | Force recompilation | +| `--logs` | `-l` | `false` | Show logs | +| `--recompile` | `-r` | `false` | Recompile before testing | +| `--version` | `-v` | | Matchstick version | +| `--help` | `-h` | | Show help | + +**Behavior:** + +1. Download Matchstick binary (if not present) +2. Shell out to Matchstick with appropriate flags +3. Report test results + +**TS CLI References:** + +- Command: `/packages/cli/src/commands/test.ts` + +### `gnd clean` + +Removes build artifacts and generated files. + +**Usage:** + +``` +gnd clean +``` + +**Flags:** +| Flag | Short | Default | Description | +|------|-------|---------|-------------| +| `--codegen-dir` | | `generated/` | Codegen output directory | +| `--build-dir` | | `build/` | Build output directory | +| `--help` | `-h` | | Show help | + +**Behavior:** +Removes `generated/` and `build/` directories (or custom paths if specified). + +**TS CLI References:** + +- Command: `/packages/cli/src/commands/clean.ts` + +### `gnd dev` + +Runs graph-node in development mode with file watching. + +This is the existing `gnd` functionality, preserved as a subcommand. The implementation can be adjusted to fit the new subcommand structure. + +**Usage:** + +``` +gnd dev [options] +``` + +**Flags:** +Preserve existing gnd flags, adjusted as needed for subcommand structure. + +## Code Generation + +Code generation is the most complex component and must produce byte-for-byte identical output to the TS CLI (after prettier formatting). + +### Generated File Types + +#### 1. Entity Classes (`schema.ts`) + +Generated from GraphQL schema. Each entity type becomes an AssemblyScript class with: + +- Constructor +- Static `load(id)` method +- Static `loadInBlock(id)` method +- `save()` method +- Getters/setters for each field +- Proper type mappings (GraphQL → AssemblyScript) + +**TS CLI Reference:** `/packages/cli/src/codegen/schema.ts` + +#### 2. ABI Bindings (`.ts`) + +Generated from contract ABI. Includes: + +- Event classes with typed parameters +- Function call result classes +- Contract class with typed call methods +- Proper Ethereum type mappings + +**Tuple/Struct Handling:** +When an event or function has tuple parameters with named components (e.g., a struct), gnd generates proper struct classes with named getters: + +```typescript +class AssetTransfer__ParamsAssetStruct extends ethereum.Tuple { + get addr(): Address { + return this[0].toAddress(); + } + get amount(): BigInt { + return this[1].toBigInt(); + } + get active(): boolean { + return this[2].toBoolean(); + } +} +``` + +gnd parses the raw ABI JSON to extract component names, which ethabi loses during parsing. + +**TS CLI Reference:** `/packages/cli/src/protocols/ethereum/codegen/abi.ts` + +#### 3. Template Bindings + +Generated for template datasources with the same structure as ABI bindings. + +**TS CLI Reference:** `/packages/cli/src/codegen/template.ts` + +#### 4. Subgraph Data Source Bindings (`subgraph-.ts`) + +Generated for `kind: subgraph` data sources. When a manifest contains a subgraph data source: + +```yaml +dataSources: + - kind: subgraph + name: SourceSubgraph + source: + address: "QmRWTEejPDDwALaquFGm6X2GBbbh5osYDXwCRRkoZ6KQhb" +``` + +gnd will: + +1. Validate that all subgraph data source names are unique (error if duplicates found) +2. Fetch the referenced subgraph's manifest from IPFS +3. Extract and fetch the schema from the manifest +4. Generate entity types (without store methods) to `generated/subgraph-{DataSourceName}.ts` + +**Note:** gnd uses the data source name (e.g., `SourceSubgraph`) for the generated filename rather than the IPFS hash. This produces stable import paths like `import { Entity } from '../generated/subgraph-SourceSubgraph'` that don't break when the source subgraph is redeployed with a new hash. This differs from graph-cli which uses the IPFS hash. + +The generated types include entity classes with getters for all fields, but no `save()`, `load()`, or `loadInBlock()` methods since these are read-only types from the source subgraph. + +**TS CLI Reference:** `/packages/cli/src/type-generator.ts` lines 72-136 + +### Type Mappings + +#### GraphQL → AssemblyScript + +| GraphQL | AssemblyScript | +| ---------------- | -------------- | +| `ID` | `string` | +| `String` | `string` | +| `Int` | `i32` | +| `BigInt` | `BigInt` | +| `BigDecimal` | `BigDecimal` | +| `Bytes` | `Bytes` | +| `Boolean` | `boolean` | +| `[T]` | `Array` | +| Entity reference | `string` (ID) | + +**TS CLI Reference:** `/packages/cli/src/codegen/schema.ts` (look for type mapping functions) + +#### Ethereum ABI → AssemblyScript + +| Solidity | AssemblyScript | +| --------- | --------------- | +| `address` | `Address` | +| `bool` | `boolean` | +| `bytes` | `Bytes` | +| `bytesN` | `Bytes` | +| `intN` | `BigInt` | +| `uintN` | `BigInt` | +| `string` | `string` | +| `T[]` | `Array` | +| tuple | Generated class | + +**TS CLI Reference:** `/packages/cli/src/protocols/ethereum/codegen/abi.ts` + +### API Version Handling + +Different `apiVersion` values in the manifest affect code generation. gnd must support all versions that the TS CLI supports. + +**TS CLI Reference:** `/packages/cli/src/codegen/` (version-specific logic throughout) + +### Formatting + +All generated code must be formatted with prettier before writing: + +- Shell out to `prettier` with same configuration as TS CLI +- Parser: `typescript` + +## Migrations + +Migrations update older manifest formats to newer versions. gnd must implement all migrations that TS CLI supports. + +**Migration Chain:** + +``` +0.0.1 → 0.0.2 → 0.0.3 → 0.0.4 → 0.0.5 → ... → current +``` + +**TS CLI Reference:** `/packages/cli/src/migrations/` + +Each migration is a transformation function that: + +1. Checks manifest version +2. Applies necessary changes +3. Updates version number + +## External Dependencies + +### Runtime Dependencies (shell out) + +| Tool | Purpose | Required | +| ------------ | ----------------------- | ------------- | +| `asc` | AssemblyScript compiler | For `build` | +| `prettier` | Code formatting | For `codegen` | +| `matchstick` | Test runner | For `test` | + +### Network APIs + +| API | Purpose | +| -------------------------------- | --------------------------------------- | +| Etherscan | Fetch verified contract ABIs | +| Sourcify | Fetch verified contract ABIs (fallback) | +| `@pinax/graph-networks-registry` | Network configuration (chain IDs, etc.) | + +The network registry should be fetched at runtime to get current network configurations. + +### graph-node Reuse + +| Component | graph-node Location | Purpose | +| ------------------- | ------------------------------------- | --------------------------- | +| Manifest parsing | `graph/src/data/subgraph/` | Load subgraph.yaml | +| Manifest validation | `graph/src/data/subgraph/` | Validate manifest structure | +| GraphQL schema | `graph/src/schema/input/` | Parse schema.graphql | +| IPFS client | `graph/src/ipfs/` | Upload to IPFS | +| Link resolver | `graph/src/components/link_resolver/` | Resolve file references | +| File watcher | `gnd/src/watcher.rs` | Watch mode | + +Refactor graph-node components as needed to make them reusable. + +## Module Structure + +``` +gnd/src/ +├── main.rs # Entry point, clap setup +├── lib.rs +├── formatter.rs # Prettier integration for code formatting +├── prompt.rs # Interactive prompts (network selection, etc.) +├── watcher.rs # File watcher for --watch modes +├── commands/ +│ ├── mod.rs +│ ├── add.rs +│ ├── auth.rs +│ ├── build.rs +│ ├── clean.rs +│ ├── codegen.rs +│ ├── create.rs +│ ├── deploy.rs +│ ├── dev.rs # Existing gnd functionality +│ ├── init.rs +│ ├── publish.rs +│ ├── remove.rs +│ └── test.rs +├── codegen/ +│ ├── mod.rs +│ ├── abi.rs # ABI binding generation +│ ├── schema.rs # Entity class generation +│ ├── template.rs # Template binding generation +│ ├── types.rs # Type conversion utilities +│ └── typescript.rs # AST builders for TypeScript/AssemblyScript +├── scaffold/ +│ ├── mod.rs +│ ├── manifest.rs # Generate subgraph.yaml +│ ├── mapping.rs # Generate mapping.ts +│ └── schema.rs # Generate schema.graphql +├── migrations/ +│ ├── mod.rs +│ ├── api_version.rs # API version migrations +│ ├── spec_version.rs # Spec version migrations +│ └── versions.rs # Version definitions +├── compiler/ +│ ├── mod.rs +│ └── asc.rs # Shell out to asc +├── services/ +│ ├── mod.rs +│ ├── contract.rs # ABI fetching (Etherscan, Blockscout, Sourcify, registry) +│ ├── graph_node.rs # Graph Node JSON-RPC client +│ └── ipfs.rs # IPFS client for uploads +├── config/ +│ ├── mod.rs +│ └── networks.rs # networks.json handling +└── output/ + ├── mod.rs + └── spinner.rs # Progress/spinner output +``` + +## Testing + +### Current Status + +- **166 unit tests** passing (`cargo test -p gnd --lib`) +- **12 codegen verification tests** passing (`cargo test -p gnd --test codegen_verification`) +- **7 CLI command tests** passing (`cargo test -p gnd --test cli_commands`) +- Test fixtures from graph-cli validation tests in `gnd/tests/fixtures/codegen_verification/` + +### Test Strategy + +1. **Unit tests**: Test individual functions (type mappings, migrations, etc.) +2. **Snapshot tests**: Compare generated output against TS CLI output +3. **Integration tests**: End-to-end command execution + +### Test Corpus + +Use the same test corpus as graph-cli. Tests must cover at least everything the TS CLI tests cover. + +**TS CLI Test Location:** `/packages/cli/tests/` + +### Codegen Verification Tests + +Located in `gnd/tests/codegen_verification.rs`, these tests verify compatibility with graph-cli: + +1. Copy fixture to temp directory (excluding `generated/`) +2. Run `gnd codegen` on the fixture +3. Compare output against expected `generated/` directory +4. Allow known differences (Int8 import, trailing commas, 2D array fix) + +Fixtures can be regenerated with `./tests/fixtures/regenerate.sh`. + +### Edge Cases + +When edge case bugs are discovered in the TS CLI, gnd should fix them rather than replicate them. Document any behavioral differences that result from bug fixes. + +### CLI Integration Tests + +Located in `tests/tests/gnd_cli_tests.rs`, these tests verify gnd works as a drop-in replacement for graph-cli by running the integration test suite with `GRAPH_CLI` environment variable pointing to the gnd binary. + +**How it works:** + +- The integration test infrastructure uses `CONFIG.graph_cli` for deployment commands +- Setting `GRAPH_CLI=../target/debug/gnd` makes tests use gnd instead of graph-cli +- `Subgraph::deploy()` calls `gnd codegen`, `gnd create`, `gnd deploy` + +**Commands tested:** + +- `gnd codegen` - Generate AssemblyScript types +- `gnd create` - Register subgraph name with Graph Node +- `gnd deploy` - Deploy subgraph to Graph Node + +**Running:** + +```bash +just test-gnd-cli +``` + +### Standalone Command Tests + +Located in `gnd/tests/cli_commands.rs`, these tests verify commands that don't require a running Graph Node: + +**Commands tested:** + +- `gnd init` - Scaffold generation (--from-example, --from-contract, --from-subgraph) +- `gnd add` - Add datasource to existing subgraph +- `gnd build` - WASM compilation + +**Running:** + +```bash +just test-gnd-commands +``` + +## Dependencies to Add + +### Cargo.toml additions + +```toml +[dependencies] +# CLI framework (already present) +clap = { version = "...", features = ["derive"] } + +# Interactive prompts (for init) +inquire = "..." + +# HTTP client (for Etherscan, Sourcify, registry) +reqwest = { version = "...", features = ["json"] } + +# Progress/spinner output +indicatif = "..." + +# Template rendering (for scaffold) +minijinja = "..." + +# JSON handling +serde_json = "..." +``` + +## Open Questions + +None at this time. All major decisions have been made. + +## References + +- TS CLI repository: https://github.com/graphprotocol/graph-tooling +- TS CLI source: `/packages/cli/src/` +- Local checkout: `/home/lutter/code/subgraphs/graph-cli` +- Original gnd expansion plan: `/LOCAL/plans/gnd-cli-expansion.md` diff --git a/docs/specs/runner-refactor.md b/docs/specs/runner-refactor.md new file mode 100644 index 00000000000..ee36ebe3342 --- /dev/null +++ b/docs/specs/runner-refactor.md @@ -0,0 +1,380 @@ +# Subgraph Runner Simplification Spec + +## Problem Statement + +`core/src/subgraph/runner.rs` is complex and hard to modify. Key issues: + +1. **Duplicated trigger processing** (lines 616-656 vs 754-790): Nearly identical loops +2. **Control flow confusion**: Nested loops in `run_inner` with 6 exit paths +3. **State management**: Mixed patterns (mutable fields, `std::mem::take`, drains) +4. **`process_block` monolith**: ~260 lines handling triggers, dynamic DS, offchain, persistence + +## Design Decisions + +| Aspect | Decision | +|--------|----------| +| Control flow | Enum-based FSM for full runner lifecycle | +| Trigger processing | New `TriggerRunner` component | +| Block processing | Pipeline with explicitly defined stages | +| State management | Mutable accumulator with checkpoints | +| Breaking changes | Moderate (internal APIs can change) | + +## Target Architecture + +### 1. Runner State Machine + +Replace nested loops in `run_inner` with an explicit enum FSM covering the full lifecycle: + +```rust +enum RunnerState { + /// Initial state, ready to start block stream + Initializing, + + /// Block stream active, waiting for next event + AwaitingBlock { + block_stream: Cancelable>>, + }, + + /// Processing a block through the pipeline + ProcessingBlock { + block: BlockWithTriggers, + cursor: FirehoseCursor, + }, + + /// Handling a revert event + Reverting { + to_ptr: BlockPtr, + cursor: FirehoseCursor, + }, + + /// Restarting block stream (new filters, store restart, etc.) + Restarting { + reason: RestartReason, + }, + + /// Terminal state + Stopped { + reason: StopReason, + }, +} + +enum RestartReason { + DynamicDataSourceCreated, + DataSourceExpired, + StoreError, + PossibleReorg, +} + +enum StopReason { + MaxEndBlockReached, + Canceled, + Unassigned, +} +``` + +The main loop becomes: + +```rust +async fn run(mut self) -> Result<(), SubgraphRunnerError> { + loop { + self.state = match self.state { + RunnerState::Initializing => self.initialize().await?, + RunnerState::AwaitingBlock { stream } => self.await_block(stream).await?, + RunnerState::ProcessingBlock { block, cursor } => { + self.process_block(block, cursor).await? + } + RunnerState::Reverting { to_ptr, cursor } => { + self.handle_revert(to_ptr, cursor).await? + } + RunnerState::Restarting { reason } => self.restart(reason).await?, + RunnerState::Stopped { reason } => return self.finalize(reason).await, + }; + } +} +``` + +### 2. Block Processing Pipeline + +Replace the `process_block` monolith with explicit stages: + +```rust +/// Pipeline stages for block processing +mod pipeline { + pub struct TriggerMatchStage; + pub struct TriggerExecuteStage; + pub struct DynamicDataSourceStage; + pub struct OffchainTriggerStage; + pub struct PersistStage; +} + +/// Result of block processing pipeline +pub struct BlockProcessingResult { + pub action: Action, + pub block_state: BlockState, +} + +impl SubgraphRunner { + async fn process_block( + &mut self, + block: BlockWithTriggers, + cursor: FirehoseCursor, + ) -> Result { + let block = Arc::new(block.block); + let triggers = block.trigger_data; + + // Stage 1: Match triggers to hosts and decode + let runnables = self.match_triggers(&block, triggers).await?; + + // Stage 2: Execute triggers (unified for initial + dynamic DS) + let mut block_state = self.execute_triggers(&block, runnables).await?; + + // Checkpoint before dynamic DS processing + let checkpoint = block_state.checkpoint(); + + // Stage 3: Process dynamic data sources (loop until none created) + block_state = self.process_dynamic_data_sources(&block, &cursor, block_state).await?; + + // Stage 4: Handle offchain triggers + let offchain_result = self.process_offchain_triggers(&block, &mut block_state).await?; + + // Stage 5: Persist to store + self.persist_block_state(block_state, offchain_result).await?; + + // Determine next state + Ok(self.determine_next_state()) + } +} +``` + +### 3. Error Handling Strategy + +Consolidate scattered error handling into explicit classification: + +```rust +/// Unified error classification for trigger processing +pub enum ProcessingErrorKind { + /// Stop processing, persist PoI only + Deterministic(anyhow::Error), + /// Retry with backoff, attempt to unfail + NonDeterministic(anyhow::Error), + /// Restart block stream cleanly (don't persist) + PossibleReorg(anyhow::Error), +} + +impl ProcessingError { + /// Classify error once, use classification throughout + pub fn kind(&self) -> ProcessingErrorKind { ... } + + /// Whether this error should stop processing the current block + pub fn should_stop_processing(&self) -> bool { + matches!(self.kind(), ProcessingErrorKind::Deterministic(_)) + } + + /// Whether this error requires a clean restart + pub fn should_restart(&self) -> bool { + matches!(self.kind(), ProcessingErrorKind::PossibleReorg(_)) + } + + /// Whether this error is retryable with backoff + pub fn is_retryable(&self) -> bool { + matches!(self.kind(), ProcessingErrorKind::NonDeterministic(_)) + } +} +``` + +**Key Invariant (must be preserved):** +``` +Deterministic → Stop processing block, persist PoI only +NonDeterministic → Retry with backoff +PossibleReorg → Restart cleanly (don't persist) +``` + +Currently this logic is scattered across: +- `process_block` early return for PossibleReorg (line 664-677) +- Dynamic data sources error mapping (line 792-802) +- `transact_block_state` (line 405-430) +- `handle_offchain_triggers` (line 1180-1190) + +Consolidating into helper methods eliminates these scattered special cases. + +### 4. TriggerRunner Component + +Extract trigger execution into a dedicated component: + +```rust +/// Handles matching, decoding, and executing triggers +pub struct TriggerRunner<'a, C: Blockchain, T: RuntimeHostBuilder> { + decoder: &'a Decoder, + processor: &'a dyn TriggerProcessor, + logger: &'a Logger, + metrics: &'a SubgraphMetrics, + debug_fork: &'a Option>, + instrument: bool, +} + +impl<'a, C, T> TriggerRunner<'a, C, T> +where + C: Blockchain, + T: RuntimeHostBuilder, +{ + /// Execute triggers against hosts, accumulating state + pub async fn execute( + &self, + block: &Arc, + runnables: Vec>, + mut block_state: BlockState, + proof_of_indexing: &SharedProofOfIndexing, + causality_region: &PoICausalityRegion, + ) -> Result { + for runnable in runnables { + block_state = self.processor + .process_trigger( + self.logger, + runnable.hosted_triggers, + block, + block_state, + proof_of_indexing, + causality_region, + self.debug_fork, + self.metrics, + self.instrument, + ) + .await + .map_err(|e| e.add_trigger_context(&runnable.trigger))?; + } + Ok(block_state) + } +} +``` + +This eliminates the duplicated loops (lines 616-656 and 754-790). + +### 5. State Management with Checkpoints + +**Explicit Input/Output Types for Pipeline Stages:** + +```rust +/// Input to trigger processing - makes dependencies explicit +struct TriggerProcessingContext<'a> { + block: &'a Arc, + proof_of_indexing: &'a SharedProofOfIndexing, + causality_region: &'a PoICausalityRegion, +} + +/// Output from trigger processing - makes results explicit +struct TriggerProcessingResult { + block_state: BlockState, + restart_needed: bool, +} +``` + +**Add checkpoint capability to `BlockState` for rollback scenarios:** + +```rust +impl BlockState { + /// Create a lightweight checkpoint for rollback + pub fn checkpoint(&self) -> BlockStateCheckpoint { + BlockStateCheckpoint { + created_data_sources_count: self.created_data_sources.len(), + persisted_data_sources_count: self.persisted_data_sources.len(), + // Note: entity_cache changes cannot be easily checkpointed + // Rollback clears the cache (acceptable per current behavior) + } + } + + /// Restore state to checkpoint (partial rollback) + pub fn restore(&mut self, checkpoint: BlockStateCheckpoint) { + self.created_data_sources.truncate(checkpoint.created_data_sources_count); + self.persisted_data_sources.truncate(checkpoint.persisted_data_sources_count); + // Entity cache is cleared on rollback (matches current behavior) + } +} +``` + +### 6. File Structure + +``` +core/src/subgraph/ +├── runner.rs # Main SubgraphRunner with FSM +├── runner/ +│ ├── mod.rs +│ ├── state.rs # RunnerState enum and transitions +│ ├── pipeline.rs # Pipeline stage definitions +│ └── trigger_runner.rs # TriggerRunner component +├── context.rs # IndexingContext (unchanged) +├── inputs.rs # IndexingInputs (unchanged) +└── state.rs # IndexingState (unchanged) +``` + +## Deferred Concerns + +### Fishy Block Refetch (Preserve Behavior) + +The TODO at lines 721-729 notes unclear behavior around block refetching in the dynamic DS loop. The restructure preserves this behavior without attempting to fix it. Investigate separately. + +## Key Interfaces + +### RunnerState Transitions + +``` +Initializing ──────────────────────────────────┐ + │ │ + v │ +AwaitingBlock ◄─────────────────────────────────┤ + │ │ + ├── ProcessBlock event ──► ProcessingBlock │ + │ │ │ + │ ├── success ┼──► AwaitingBlock + │ │ │ + │ └── restart ┼──► Restarting + │ │ + ├── Revert event ──────────► Reverting ────┤ + │ │ + ├── Error ─────────────────► Restarting ───┤ + │ │ + └── Cancel/MaxBlock ───────► Stopped │ + │ +Restarting ─────────────────────────────────────┘ +``` + +### Pipeline Data Flow + +``` +BlockWithTriggers + │ + v +┌──────────────────┐ +│ TriggerMatchStage│ ─► Vec +└──────────────────┘ + │ + v +┌────────────────────┐ +│ TriggerExecuteStage│ ─► BlockState (mutated) +└────────────────────┘ + │ + v (loop while has_created_data_sources) +┌─────────────────────────┐ +│ DynamicDataSourceStage │ ─► BlockState (mutated), new hosts added +└─────────────────────────┘ + │ + v +┌─────────────────────┐ +│ OffchainTriggerStage│ ─► offchain_mods, processed_ds +└─────────────────────┘ + │ + v +┌─────────────┐ +│ PersistStage│ ─► Store transaction +└─────────────┘ +``` + +## Verification + +After implementation, verify: + +1. **Unit tests pass**: `just test-unit` +2. **Runner tests pass**: `just test-runner` +3. **Lint clean**: `just lint` (zero warnings) +4. **Build succeeds**: `just check --release` + +For behavioral verification, the existing runner tests should catch regressions. No new integration tests required for a refactor that preserves behavior. diff --git a/docs/subgraph-manifest.md b/docs/subgraph-manifest.md index 14b47b059dc..caad7943e84 100644 --- a/docs/subgraph-manifest.md +++ b/docs/subgraph-manifest.md @@ -34,7 +34,7 @@ Any data format that has a well-defined 1:1 mapping with the [IPLD Canonical For | --- | --- | --- | | **kind** | *String | The type of data source. Possible values: *ethereum/contract*.| | **name** | *String* | The name of the source data. Will be used to generate APIs in the mapping and also for self-documentation purposes. | -| **network** | *String* | For blockchains, this describes which network the subgraph targets. For Ethereum, this can be any of "mainnet", "rinkeby", "kovan", "ropsten", "goerli", "poa-core", "poa-sokol", "xdai", "matic", "mumbai", "fantom", "bsc" or "clover". Developers could look for an up to date list in the graph-cli [*code*](https://github.com/graphprotocol/graph-cli/blob/main/packages/cli/src/protocols/index.js#L70-L107).| +| **network** | *String* | For blockchains, this describes which network the subgraph targets. For Ethereum, this can be any of "mainnet", "rinkeby", "kovan", "ropsten", "goerli", "poa-core", "poa-sokol", "xdai", "matic", "mumbai", "fantom", "bsc" or "clover". Developers could look for an up to date list in the graph-cli [*code*](https://github.com/graphprotocol/graph-tooling/blob/main/packages/cli/src/protocols/index.ts#L76-L117).| | **source** | [*EthereumContractSource*](#151-ethereumcontractsource) | The source data on a blockchain such as Ethereum. | | **mapping** | [*Mapping*](#152-mapping) | The transformation logic applied to the data prior to being indexed. | @@ -74,6 +74,7 @@ The `mapping` field may be one of the following supported mapping manifests: | **event** | *String* | An identifier for an event that will be handled in the mapping script. For Ethereum contracts, this must be the full event signature to distinguish from events that may share the same name. No alias types can be used. For example, uint will not work, uint256 must be used.| | **handler** | *String* | The name of an exported function in the mapping script that should handle the specified event. | | **topic0** | optional *String* | A `0x` prefixed hex string. If provided, events whose topic0 is equal to this value will be processed by the given handler. When topic0 is provided, _only_ the topic0 value will be matched, and not the hash of the event signature. This is useful for processing anonymous events in Solidity, which can have their topic0 set to anything. By default, topic0 is equal to the hash of the event signature. | +| **calls** | optional [*CallDecl*](#153-declaring-calls) | A list of predeclared `eth_calls` that will be made before running the handler | #### 1.5.2.3 CallHandler @@ -95,6 +96,40 @@ The `mapping` field may be one of the following supported mapping manifests: | --- | --- | --- | | **kind** | *String* | The selected block handler filter. Only option for now: `call`: This will only run the handler if the block contains at least one call to the data source contract. | +### 1.5.3 Declaring calls + +_Available from spec version 1.2.0. Struct field access available from spec version 1.4.0_ + +Declared calls are performed in parallel before the handler is run and can +greatly speed up syncing. Mappings access the call results simply by using +`ethereum.call` from the mappings. The **calls** are a map of key value pairs: + +| Field | Type | Description | +| --- | --- | --- | +| **label** | *String* | A label for the call for error messages etc. | +| **call** | *String* | See below | + +Each call is of the form `[
].()`: + +| Field | Type | Description | +| --- | --- | --- | +| **ABI** | *String* | The name of an ABI from the `abis` section | +| **address** | *Expr* | The address of a contract that follows the `ABI` | +| **function** | *String* | The name of a view function in the contract | +| **args** | *[Expr]* | The arguments to pass to the function | + +#### Expression Types + +The `Expr` can be one of the following: + +| Expression | Description | +| --- | --- | +| **event.address** | The address of the contract that emitted the event | +| **event.params.<name>** | A simple parameter from the event | +| **event.params.<name>.<index>** | A field from a struct parameter by numeric index | +| **event.params.<name>.<fieldName>** | A field from a struct parameter by field name (spec version 1.4.0+) | + + ## 1.6 Path A path has one field `path`, which either refers to a path of a file on the local dev machine or an [IPLD link](https://github.com/ipld/specs/). diff --git a/entitlements.plist b/entitlements.plist new file mode 100644 index 00000000000..d9ce520f2e1 --- /dev/null +++ b/entitlements.plist @@ -0,0 +1,12 @@ + + + + + com.apple.security.cs.allow-jit + + com.apple.security.cs.allow-unsigned-executable-memory + + com.apple.security.cs.disable-executable-page-protection + + + \ No newline at end of file diff --git a/flake.lock b/flake.lock new file mode 100644 index 00000000000..7d69cf5ac8c --- /dev/null +++ b/flake.lock @@ -0,0 +1,182 @@ +{ + "nodes": { + "fenix": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ], + "rust-analyzer-src": "rust-analyzer-src" + }, + "locked": { + "lastModified": 1755585599, + "narHash": "sha256-tl/0cnsqB/Yt7DbaGMel2RLa7QG5elA8lkaOXli6VdY=", + "owner": "nix-community", + "repo": "fenix", + "rev": "6ed03ef4c8ec36d193c18e06b9ecddde78fb7e42", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "fenix", + "type": "github" + } + }, + "flake-parts": { + "inputs": { + "nixpkgs-lib": "nixpkgs-lib" + }, + "locked": { + "lastModified": 1754487366, + "narHash": "sha256-pHYj8gUBapuUzKV/kN/tR3Zvqc7o6gdFB9XKXIp1SQ8=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "af66ad14b28a127c5c0f3bbb298218fc63528a18", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, + "flake-utils": { + "locked": { + "lastModified": 1644229661, + "narHash": "sha256-1YdnJAsNy69bpcjuoKdOYQX0YxZBiCYZo4Twxerqv7k=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "3cecb5b042f7f209c56ffd8371b2711a290ec797", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "foundry": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs" + }, + "locked": { + "lastModified": 1760001025, + "narHash": "sha256-j/jPNlW2W4zwV2rSqgw5ts1VoTXVe5n3CNXFvtQhK3Q=", + "owner": "shazow", + "repo": "foundry.nix", + "rev": "b7adb89167832516589c899addcd25ca2a78dcfe", + "type": "github" + }, + "original": { + "owner": "shazow", + "repo": "foundry.nix", + "rev": "b7adb89167832516589c899addcd25ca2a78dcfe", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1666753130, + "narHash": "sha256-Wff1dGPFSneXJLI2c0kkdWTgxnQ416KE6X4KnFkgPYQ=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "f540aeda6f677354f1e7144ab04352f61aaa0118", + "type": "github" + }, + "original": { + "id": "nixpkgs", + "type": "indirect" + } + }, + "nixpkgs-lib": { + "locked": { + "lastModified": 1753579242, + "narHash": "sha256-zvaMGVn14/Zz8hnp4VWT9xVnhc8vuL3TStRqwk22biA=", + "owner": "nix-community", + "repo": "nixpkgs.lib", + "rev": "0f36c44e01a6129be94e3ade315a5883f0228a6e", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "nixpkgs.lib", + "type": "github" + } + }, + "nixpkgs_2": { + "locked": { + "lastModified": 1756128520, + "narHash": "sha256-R94HxJBi+RK1iCm8Y4Q9pdrHZl0GZoDPIaYwjxRNPh4=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "c53baa6685261e5253a1c355a1b322f82674a824", + "type": "github" + }, + "original": { + "owner": "nixos", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "process-compose-flake": { + "locked": { + "lastModified": 1749418557, + "narHash": "sha256-wJHHckWz4Gvj8HXtM5WVJzSKXAEPvskQANVoRiu2w1w=", + "owner": "Platonic-Systems", + "repo": "process-compose-flake", + "rev": "91dcc48a6298e47e2441ec76df711f4e38eab94e", + "type": "github" + }, + "original": { + "owner": "Platonic-Systems", + "repo": "process-compose-flake", + "type": "github" + } + }, + "root": { + "inputs": { + "fenix": "fenix", + "flake-parts": "flake-parts", + "foundry": "foundry", + "nixpkgs": "nixpkgs_2", + "process-compose-flake": "process-compose-flake", + "services-flake": "services-flake" + } + }, + "rust-analyzer-src": { + "flake": false, + "locked": { + "lastModified": 1755504847, + "narHash": "sha256-VX0B9hwhJypCGqncVVLC+SmeMVd/GAYbJZ0MiiUn2Pk=", + "owner": "rust-lang", + "repo": "rust-analyzer", + "rev": "a905e3b21b144d77e1b304e49f3264f6f8d4db75", + "type": "github" + }, + "original": { + "owner": "rust-lang", + "ref": "nightly", + "repo": "rust-analyzer", + "type": "github" + } + }, + "services-flake": { + "locked": { + "lastModified": 1755996515, + "narHash": "sha256-1RQQIDhshp1g4PP5teqibcFLfk/ckTDOJRckecAHiU0=", + "owner": "juspay", + "repo": "services-flake", + "rev": "e316d6b994fd153f0c35d54bd07d60e53f0ad9a9", + "type": "github" + }, + "original": { + "owner": "juspay", + "repo": "services-flake", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 00000000000..933a35ef04c --- /dev/null +++ b/flake.nix @@ -0,0 +1,209 @@ +{ + inputs = { + nixpkgs.url = "github:nixos/nixpkgs/nixpkgs-unstable"; + foundry.url = "github:shazow/foundry.nix/b7adb89167832516589c899addcd25ca2a78dcfe"; + fenix = { + url = "github:nix-community/fenix"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + process-compose-flake.url = "github:Platonic-Systems/process-compose-flake"; + services-flake.url = "github:juspay/services-flake"; + flake-parts.url = "github:hercules-ci/flake-parts"; + }; + + outputs = inputs @ { + flake-parts, + process-compose-flake, + services-flake, + nixpkgs, + fenix, + foundry, + ... + }: + flake-parts.lib.mkFlake {inherit inputs;} { + imports = [process-compose-flake.flakeModule]; + systems = [ + "x86_64-linux" # 64-bit Intel/AMD Linux + "aarch64-linux" # 64-bit ARM Linux + "x86_64-darwin" # 64-bit Intel macOS + "aarch64-darwin" # 64-bit ARM macOS + ]; + + perSystem = { + config, + self', + inputs', + pkgs, + system, + ... + }: let + overlays = [ + fenix.overlays.default + foundry.overlay + ]; + + pkgs = import nixpkgs { + inherit overlays system; + }; + + toolchain = with fenix.packages.${system}; + combine [ + (fromToolchainFile { + file = ./rust-toolchain.toml; + sha256 = "sha256-+9FmLhAOezBZCOziO0Qct1NOrfpjNsXxc/8I0c7BdKE="; + }) + stable.rust-src # This is needed for rust-analyzer to find stdlib symbols. Should use the same channel as the toolchain. + ]; + in { + formatter = pkgs.alejandra; + devShells.default = pkgs.mkShell { + packages = with pkgs; [ + toolchain + foundry-bin + solc + protobuf + uv + cmake + corepack + nodejs + postgresql + just + cargo-nextest + ]; + }; + + process-compose = let + inherit (services-flake.lib) multiService; + ipfs = multiService ./nix/ipfs.nix; + anvil = multiService ./nix/anvil.nix; + + # Helper function to create postgres configuration with graph-specific defaults + mkPostgresConfig = { + name, + port, + user, + password, + database, + dataDir, + }: { + enable = true; + inherit port dataDir; + initialScript = { + before = '' + CREATE ROLE postgres WITH LOGIN; + CREATE USER \"${user}\" WITH PASSWORD '${password}' SUPERUSER; + ''; + }; + initialDatabases = [ + { + inherit name; + schemas = [ + (pkgs.writeText "init-${name}.sql" '' + CREATE EXTENSION IF NOT EXISTS pg_trgm; + CREATE EXTENSION IF NOT EXISTS btree_gist; + CREATE EXTENSION IF NOT EXISTS postgres_fdw; + CREATE EXTENSION IF NOT EXISTS pg_stat_statements; + GRANT USAGE ON FOREIGN DATA WRAPPER postgres_fdw TO "${user}"; + ALTER DATABASE "${database}" OWNER TO "${user}"; + '') + ]; + } + ]; + settings = { + shared_preload_libraries = "pg_stat_statements"; + log_statement = "all"; + default_text_search_config = "pg_catalog.english"; + max_connections = 500; + }; + }; + in { + # Unit tests configuration + unit = { + imports = [ + services-flake.processComposeModules.default + ipfs + anvil + ]; + + cli = { + environment.PC_DISABLE_TUI = true; + options = { + port = 8881; + }; + }; + + services.postgres."postgres-unit" = mkPostgresConfig { + name = "graph-test"; + port = 5432; + dataDir = "./.data/unit/postgres"; + user = "graph"; + password = "graph"; + database = "graph-test"; + }; + + # Set PGUSER so psql connects as the OS user (matching initdb's superuser) + settings.processes."postgres-unit-init".environment = { + PGUSER = builtins.getEnv "USER"; + }; + + services.ipfs."ipfs-unit" = { + enable = true; + dataDir = "./.data/unit/ipfs"; + port = 5001; + gateway = 8080; + }; + }; + + # Integration tests configuration + integration = { + imports = [ + services-flake.processComposeModules.default + ipfs + anvil + ]; + + cli = { + environment.PC_DISABLE_TUI = true; + options = { + port = 8882; + }; + }; + + services.postgres."postgres-integration" = mkPostgresConfig { + name = "graph-node"; + port = 3011; + dataDir = "./.data/integration/postgres"; + user = "graph-node"; + password = "let-me-in"; + database = "graph-node"; + }; + + # Set PGUSER so psql connects as the OS user (matching initdb's superuser) + settings.processes."postgres-integration-init".environment = { + PGUSER = builtins.getEnv "USER"; + }; + + services.ipfs."ipfs-integration" = { + enable = true; + dataDir = "./.data/integration/ipfs"; + port = 3001; + gateway = 3002; + }; + + services.anvil."anvil-integration" = { + enable = true; + package = pkgs.foundry-bin; + port = 3021; + timestamp = 1743944919; + gasLimit = 100000000000; + baseFee = 1; + blockTime = null; + state = "./.data/integration/anvil/state.json"; + stateInterval = 30; + preserveHistoricalStates = true; + }; + }; + }; + }; + }; +} diff --git a/gnd/Cargo.toml b/gnd/Cargo.toml new file mode 100644 index 00000000000..754949a4722 --- /dev/null +++ b/gnd/Cargo.toml @@ -0,0 +1,92 @@ +[package] +name = "gnd" +version.workspace = true +edition.workspace = true + +[[bin]] +name = "gnd" +path = "src/main.rs" + +[[test]] +name = "cli_commands" +path = "tests/cli_commands.rs" + +[[test]] +name = "codegen_verification" +path = "tests/codegen_verification.rs" + +[[test]] +name = "gnd_test" +path = "tests/gnd_test.rs" + +[dependencies] +# Core graph dependencies +graph = { path = "../graph" } +graph-chain-ethereum = { path = "../chain/ethereum" } +graph-core = { path = "../core" } +graph-node = { path = "../node" } +graph-graphql = { path = "../graphql" } +graph-store-postgres = { path = "../store/postgres" } + +# Test command dependencies +hex = "0.4" +async-trait = { workspace = true } +tower = { workspace = true } + +# Direct dependencies from current dev.rs +anyhow = { workspace = true } +clap = { workspace = true } +clap_complete = { workspace = true } +env_logger = { workspace = true } +git-testament = { workspace = true } +lazy_static = { workspace = true } +serde = { workspace = true } +tokio = { workspace = true } +tokio-util = { workspace = true } + +# File watching +notify = "8.2.0" +globset = "0.4.18" + +# Config and auth +url = { workspace = true } +serde_json = { workspace = true } +pq-sys = { version = "0.7.5", features = ["bundled"] } + +# HTTP client for Graph Node API +reqwest = { workspace = true } +thiserror = { workspace = true } + +# Console output +similar = "3" +indicatif = { workspace = true } +console = { workspace = true } + +# Code generation +graphql-tools = { workspace = true } +regex = { workspace = true } +serde_yaml = { workspace = true } +Inflector = { workspace = true } + +# Migrations +semver = { workspace = true } + +# Build command +sha1 = "0.11" +wasmparser = { workspace = true } + +# Interactive prompts +inquire = "0.9" + +# Temp directories for init +tempfile = "3" + +# Browser opening for publish command +open = "5" + +[target.'cfg(unix)'.dependencies] +pgtemp = { git = "https://github.com/graphprotocol/pgtemp", branch = "initdb-args" } + +[dev-dependencies] +tempfile = "3" +walkdir = "2" diff --git a/gnd/README.md b/gnd/README.md new file mode 100644 index 00000000000..375e1ef0e01 --- /dev/null +++ b/gnd/README.md @@ -0,0 +1,524 @@ +# gnd - Graph Node Dev CLI + +A drop-in replacement for `graph-cli` written in Rust. `gnd` provides all the subgraph development commands with identical output, flags, and behavior. + +## Installation + +Build from source: + +```bash +cargo build -p gnd --release +``` + +The binary will be at `target/release/gnd`. + +## Quick Start + +```bash +# Create a new subgraph from a contract +gnd init --from-contract 0x1234... --network mainnet my-subgraph + +# Generate AssemblyScript types +gnd codegen + +# Build the subgraph +gnd build + +# Deploy to a local Graph Node +gnd create --node http://localhost:8020 my-name/my-subgraph +gnd deploy --node http://localhost:8020 --ipfs http://localhost:5001 -l v0.0.1 my-name/my-subgraph + +# Or deploy to Subgraph Studio +gnd auth YOUR_DEPLOY_KEY +gnd deploy -l v0.0.1 my-name/my-subgraph +``` + +## Commands + +### `gnd init` + +Create a new subgraph with basic scaffolding. + +```bash +gnd init [SUBGRAPH_NAME] [DIRECTORY] +``` + +**Flags:** + +| Flag | Short | Description | +|------|-------|-------------| +| `--protocol` | | Protocol: `ethereum`, `near`, `cosmos`, `arweave`, `substreams` | +| `--from-contract` | | Create from an existing contract address | +| `--from-example` | | Create from an example subgraph template | +| `--from-subgraph` | | Create from an existing deployed subgraph | +| `--contract-name` | | Name for the contract (with `--from-contract`) | +| `--index-events` | | Index all contract events as entities | +| `--network` | | Network the contract is deployed to | +| `--start-block` | | Block number to start indexing from | +| `--abi` | | Path to the contract ABI file | +| `--node` | `-g` | Graph Node URL | +| `--ipfs` | `-i` | IPFS node URL | +| `--skip-install` | | Skip installing npm dependencies | +| `--skip-git` | | Skip initializing a Git repository | + +**Examples:** + +```bash +# Interactive mode (prompts for all options) +gnd init + +# From contract with ABI fetched from Etherscan +gnd init --from-contract 0x1234... --network mainnet my-subgraph + +# From contract with local ABI +gnd init --from-contract 0x1234... --abi ./MyContract.json my-subgraph + +# From an existing deployed subgraph +gnd init --from-subgraph QmHash... my-subgraph +``` + +### `gnd codegen` + +Generate AssemblyScript types from the subgraph manifest. + +```bash +gnd codegen [MANIFEST] +``` + +**Arguments:** +- `MANIFEST`: Path to subgraph manifest (default: `subgraph.yaml`) + +**Flags:** + +| Flag | Short | Description | +|------|-------|-------------| +| `--output-dir` | `-o` | Output directory (default: `generated/`) | +| `--skip-migrations` | | Skip manifest migrations | +| `--watch` | `-w` | Regenerate on file changes | +| `--ipfs` | `-i` | IPFS node URL | + +**Examples:** + +```bash +# Generate types +gnd codegen + +# Generate to custom directory +gnd codegen -o src/generated/ + +# Watch mode +gnd codegen --watch +``` + +### `gnd build` + +Compile the subgraph to WASM. + +```bash +gnd build [MANIFEST] +``` + +**Arguments:** +- `MANIFEST`: Path to subgraph manifest (default: `subgraph.yaml`) + +**Flags:** + +| Flag | Short | Description | +|------|-------|-------------| +| `--output-dir` | `-o` | Output directory (default: `build/`) | +| `--output-format` | `-t` | Output format: `wasm` or `wast` (default: `wasm`) | +| `--skip-migrations` | | Skip manifest migrations | +| `--watch` | `-w` | Rebuild on file changes | +| `--ipfs` | `-i` | IPFS node URL (uploads if provided) | +| `--network` | | Network from networks.json | +| `--network-file` | | Path to networks config (default: `networks.json`) | + +**Examples:** + +```bash +# Build subgraph +gnd build + +# Build and upload to IPFS +gnd build --ipfs http://localhost:5001 + +# Build for specific network +gnd build --network mainnet +``` + +### `gnd deploy` + +Deploy a subgraph to a Graph Node. + +```bash +gnd deploy [MANIFEST] +``` + +**Arguments:** +- `SUBGRAPH_NAME`: Name to deploy as (e.g., `user/subgraph`) +- `MANIFEST`: Path to subgraph manifest (default: `subgraph.yaml`) + +**Flags:** + +| Flag | Short | Description | +|------|-------|-------------| +| `--node` | `-g` | Graph Node URL (defaults to Subgraph Studio) | +| `--ipfs` | `-i` | IPFS node URL | +| `--deploy-key` | | Deploy key for authentication | +| `--version-label` | `-l` | Version label for the deployment (required in non-interactive mode) | +| `--ipfs-hash` | | IPFS hash of already-uploaded manifest | +| `--output-dir` | `-o` | Build output directory (default: `build/`) | +| `--skip-migrations` | | Skip manifest migrations | +| `--network` | | Network from networks.json | +| `--network-file` | | Path to networks config | +| `--debug-fork` | | Fork subgraph ID for debugging | + +If `--version-label` is omitted in an interactive terminal, `gnd deploy` prompts for it. +In non-interactive environments (CI/scripts), you must pass `--version-label`. + +**Examples:** + +```bash +# Deploy to Subgraph Studio (uses saved auth key) +gnd deploy -l v1.0.0 my-name/my-subgraph + +# Deploy to local Graph Node +gnd deploy --node http://localhost:8020 --ipfs http://localhost:5001 -l v1.0.0 my-name/my-subgraph + +# Deploy without --version-label in an interactive terminal (prompts) +gnd deploy my-name/my-subgraph +``` + +### `gnd publish` + +Publish a subgraph to The Graph's decentralized network. + +```bash +gnd publish [MANIFEST] +``` + +**Arguments:** +- `MANIFEST`: Path to subgraph manifest (default: `subgraph.yaml`) + +**Flags:** + +| Flag | Short | Description | +|------|-------|-------------| +| `--ipfs` | `-i` | IPFS node URL | +| `--ipfs-hash` | | Skip build, use existing IPFS hash | +| `--subgraph-id` | | Subgraph ID for updating existing subgraphs | +| `--protocol-network` | | Network: `arbitrum-one` or `arbitrum-sepolia` | +| `--api-key` | | API key (required when updating existing) | +| `--output-dir` | `-o` | Build output directory | +| `--skip-migrations` | | Skip manifest migrations | +| `--network` | | Network from networks.json | +| `--network-file` | | Path to networks config | + +**Examples:** + +```bash +# Publish new subgraph +gnd publish + +# Update existing subgraph +gnd publish --subgraph-id Qm... --api-key YOUR_KEY +``` + +### `gnd add` + +Add a new data source to an existing subgraph. + +```bash +gnd add
[MANIFEST] +``` + +**Arguments:** +- `ADDRESS`: Contract address to add +- `MANIFEST`: Path to subgraph manifest (default: `subgraph.yaml`) + +**Flags:** + +| Flag | Short | Description | +|------|-------|-------------| +| `--abi` | | Path to the contract ABI | +| `--contract-name` | | Name for the new data source | +| `--merge-entities` | | Merge with existing entities of same name | +| `--network` | | Network the contract is deployed to | +| `--start-block` | | Block number to start indexing from | + +**Examples:** + +```bash +# Add contract with ABI from Etherscan +gnd add 0x1234... --network mainnet + +# Add contract with local ABI +gnd add 0x1234... --abi ./NewContract.json --contract-name MyContract +``` + +### `gnd create` + +Register a subgraph name with a Graph Node. + +```bash +gnd create --node +``` + +**Flags:** + +| Flag | Short | Description | +|------|-------|-------------| +| `--node` | `-g` | Graph Node URL (required) | +| `--access-token` | | Access token for authentication | + +### `gnd remove` + +Unregister a subgraph name from a Graph Node. + +```bash +gnd remove --node +``` + +**Flags:** + +| Flag | Short | Description | +|------|-------|-------------| +| `--node` | `-g` | Graph Node URL (required) | +| `--access-token` | | Access token for authentication | + +### `gnd auth` + +Store a deploy key for authentication. + +```bash +gnd auth +``` + +**Flags:** + +| Flag | Short | Description | +|------|-------|-------------| +| `--node` | `-g` | Graph Node URL (default: Subgraph Studio) | + +Keys are stored in `~/.graph-cli.json`. + +### `gnd test` + +Run subgraph tests. + +```bash +gnd test [TEST_FILES...] +``` + +**Arguments:** +- `PATHS`: Test JSON files or directories to scan. Defaults to `tests/` when nothing is specified. + +**Flags:** + +| Flag | Short | Description | +|------|-------|-------------| +| `--manifest` | `-m` | Path to subgraph manifest (default: `subgraph.yaml`) | +| `--skip-build` | `-s` | Skip building the subgraph before testing | +| `--postgres-url` | | PostgreSQL connection URL (env: `POSTGRES_URL`) | +| `--matchstick` | | Use legacy Matchstick runner (**deprecated** — migrate to JSON-based tests) | +| `--docker` | `-d` | Run Matchstick in Docker (requires `--matchstick`) | +| `--coverage` | `-c` | Run with coverage reporting (requires `--matchstick`) | +| `--recompile` | `-r` | Force recompilation (requires `--matchstick`) | +| `--force` | `-f` | Force redownload of Matchstick binary (requires `--matchstick`) | + +**Examples:** + +```bash +# Run all tests in tests/ directory (default) +gnd test + +# Run specific test files +gnd test transfer.json approval.json +gnd test tests/transfer.json + +# Scan a custom directory +gnd test my-tests/ + +# Use a different manifest +gnd test -m subgraph.staging.yaml tests/transfer.json + +# Skip automatic build +gnd test -s +``` + +### `gnd clean` + +Remove build artifacts and generated files. + +```bash +gnd clean +``` + +**Flags:** + +| Flag | Short | Description | +|------|-------|-------------| +| `--codegen-dir` | | Codegen directory (default: `generated/`) | +| `--build-dir` | | Build directory (default: `build/`) | + +### `gnd dev` + +Run graph-node in development mode with file watching. + +```bash +gnd dev [OPTIONS] +``` + +**Flags:** + +| Flag | Short | Description | +|------|-------|-------------| +| `--watch` | | Watch build directory for changes | +| `--manifests` | | Subgraph manifest locations | +| `--sources` | | Source manifest locations for aliases | +| `--database-dir` | | Database directory (default: `./build`) | +| `--postgres-url` | | PostgreSQL connection URL | +| `--ethereum-rpc` | | Ethereum RPC URL | +| `--ipfs` | | IPFS node URL | + +### `gnd indexer` + +Manage indexer operations via [`indexer-cli`](https://github.com/graphprotocol/indexer/tree/main/packages/indexer-cli). + +Requires `graph-indexer` to be installed and on `$PATH`: + +```bash +npm install -g @graphprotocol/indexer-cli +``` + +```bash +gnd indexer [args...] +``` + +**Help:** + +There are two ways to get help: + +| Command | What it shows | +|---------|---------------| +| `gnd indexer --help` | gnd's own help for the indexer subcommand (works without `graph-indexer` installed) | +| `gnd indexer help` | Full `graph-indexer` help with all available commands (requires `graph-indexer`) | + +**Examples:** + +```bash +# Check indexer status +gnd indexer status --network arbitrum-one + +# Manage indexing rules +gnd indexer rules get all --network mainnet +gnd indexer rules set decisionBasis always --network mainnet + +# Manage allocations +gnd indexer allocations get --network arbitrum-one + +# Manage cost models +gnd indexer cost get + +# View available indexer commands +gnd indexer help + +# Check graph-indexer version +gnd indexer version +``` + +### `gnd completions` + +Generate shell completions. + +```bash +gnd completions +``` + +**Arguments:** +- `SHELL`: One of `bash`, `elvish`, `fish`, `powershell`, `zsh` + +**Examples:** + +```bash +# Bash +gnd completions bash > ~/.bash_completion.d/gnd + +# Zsh +gnd completions zsh > ~/.zfunc/_gnd + +# Fish +gnd completions fish > ~/.config/fish/completions/gnd.fish +``` + +## Configuration Files + +### `~/.graph-cli.json` + +Stores deploy keys for different Graph Node URLs. Created by `gnd auth`. + +### `networks.json` + +Network-specific configuration for contract addresses and start blocks: + +```json +{ + "mainnet": { + "MyContract": { + "address": "0x1234...", + "startBlock": 12345678 + } + }, + "sepolia": { + "MyContract": { + "address": "0x5678...", + "startBlock": 1000000 + } + } +} +``` + +Use with `--network mainnet` on build/deploy commands. + +## Differences from graph-cli + +`gnd` is designed as a drop-in replacement for `graph-cli`. Some intentional differences: + +### Commands Not Implemented + +- **`local`**: Use graph-node's integration test infrastructure instead +- **`node`**: Use `graphman` for node management operations + +### Code Generation Differences + +These are documented and tested: + +1. **Int8 import**: Always imported for simplicity, even when not used +2. **Trailing commas**: Used in multi-line constructs +3. **2D array accessors**: Uses correct `toStringMatrix()` (fixes a bug in graph-cli) + +### Other Differences + +- **Debug logging**: Uses `RUST_LOG` environment variable instead of `DEBUG=graph-cli:*` +- **`--uncrashable` flag**: Not implemented (Float Capital third-party feature) + +## Environment Variables + +| Variable | Description | +|----------|-------------| +| `RUST_LOG` | Debug logging level (e.g., `RUST_LOG=gnd=debug`) | +| `POSTGRES_URL` | PostgreSQL URL for `gnd dev` | +| `ETHEREUM_RPC` | Ethereum RPC URL for `gnd dev` | + +## Version + +```bash +gnd --version +# gnd 0.1.0 (graph-cli compatible: 0.98.1) +``` + +Shows both the gnd version and the graph-cli version it emulates. + +## License + +Apache-2.0 OR MIT diff --git a/gnd/docs/gnd-test.md b/gnd/docs/gnd-test.md new file mode 100644 index 00000000000..6fcc118fc29 --- /dev/null +++ b/gnd/docs/gnd-test.md @@ -0,0 +1,939 @@ +# gnd test + +Mock-based subgraph test runner that feeds JSON-defined blocks through real graph-node infrastructure (store, WASM runtime, trigger processing) with only the blockchain layer mocked. + +## Quick Start + +```bash +# Run all tests in tests/ directory +gnd test + +# Run a specific test file +gnd test tests/transfer.json + +# Skip automatic build (if subgraph already built) +gnd test --skip-build + +# Use legacy Matchstick runner +gnd test --matchstick +``` + +## Test File Format + +Tests are JSON files that define: +- Mock blockchain blocks with events +- Mock Ethereum RPC responses (`eth_call`, `eth_getBalance`, `eth_getCode`) +- GraphQL assertions to validate entity state + +Place test files in a `tests/` directory with `.json` or `.test.json` extension. + +### Basic Example + +```json +{ + "name": "Transfer creates entity", + "blocks": [ + { + "number": 1, + "timestamp": 1672531200, + "events": [ + { + "address": "0x1234...", + "event": "Transfer(address indexed from, address indexed to, uint256 value)", + "params": { + "from": "0xaaaa...", + "to": "0xbbbb...", + "value": "1000" + } + } + ], + "ethCalls": [ + { + "address": "0x1234...", + "function": "balanceOf(address)(uint256)", + "params": ["0xaaaa..."], + "returns": ["1000000000000000000"] + } + ] + } + ], + "assertions": [ + { + "query": "{ transfer(id: \"1\") { from to value } }", + "expected": { + "transfer": { + "from": "0xaaaa...", + "to": "0xbbbb...", + "value": "1000" + } + } + } + ] +} +``` + +## Block Fields + +| Field | Required | Default | Description | +|-------|----------|---------|-------------| +| `number` | No | Auto-increments from lowest defined `startBlock` in the manifest file, or from `0` if no `startBlock` are defined | Block number | +| `hash` | No | `keccak256(block_number)` | Block hash | +| `timestamp` | No | `block_number` | Unix timestamp | +| `baseFeePerGas` | No | None (pre-EIP-1559) | Base fee in wei | +| `events` | No | Empty array | Log events in this block | +| `ethCalls` | No | Empty array | Mock `eth_call` responses | +| `getBalanceCalls` | No | Empty array | Mock `eth_getBalance` responses for `ethereum.getBalance()` | +| `hasCodeCalls` | No | Empty array | Mock `eth_getCode` responses for `ethereum.hasCode()` | + +### Empty Blocks + +Empty blocks (no events) still trigger block handlers: + +```json +{ + "name": "Test block handlers", + "blocks": [ + { + "number": 1, + "events": [...] + }, + {} // Block 2 with no events - block handlers still fire + ] +} +``` + +## Event Fields + +| Field | Required | Default | Description | +|-------|----------|---------|-------------| +| `address` | Yes | — | Contract address (lowercase hex with 0x prefix) | +| `event` | Yes | — | Full event signature with `indexed` keywords | +| `params` | No | Empty object | Event parameter values | +| `txHash` | No | `keccak256(block_number \|\| log_index)` | Transaction hash | + +### Event Signature Format + +**Important:** Include `indexed` keywords in the signature: + +```json +{ + "event": "Transfer(address indexed from, address indexed to, uint256 value)" +} +``` + +Not: +```json +{ + "event": "Transfer(address,address,uint256)" // ❌ Missing indexed keywords +} +``` + +### Parameter Types + +Event parameters are automatically ABI-encoded based on the signature. Supported formats: + +```json +{ + "params": { + "from": "0xaaaa...", // address + "to": "0xbbbb...", // address + "value": "1000", // uint256 (string or number) + "amount": 1000, // uint256 (number) + "enabled": true, // bool + "data": "0x1234...", // bytes + "name": "Token" // string + } +} +``` + +## Transaction Receipts + +Mock receipts are constructed for every log trigger and attached only to handlers that declare `receipt: true` in the manifest, mirroring production behaviour. Handlers without `receipt: true` receive a null receipt — the same as on a real node. + +**Limitation:** Only `receipt.logs` reflects your test data. All other receipt fields (`from`, `to`, `gas_used`, `status`, etc.) are hardcoded stubs and do not correspond to real transaction data. If your handler reads those fields, the values will be fixed defaults regardless of what you put in the test JSON. + +### How receipts are built + +Every event gets a mock receipt attached automatically. The key rule is **`txHash` grouping**: + +- Events sharing the same `txHash` share **one receipt** — `event.receipt!.logs` contains all of their logs in declaration order. +- Events without an explicit `txHash` each get a unique auto-generated hash (`keccak256(block_number || log_index)`), so each gets its own single-log receipt. + +### Example: Two events sharing a receipt + +```json +{ + "events": [ + { + "address": "0x1234...", + "event": "Transfer(address indexed from, address indexed to, uint256 value)", + "params": { "from": "0xaaaa...", "to": "0xbbbb...", "value": "100" }, + "txHash": "0xdeadbeef0000000000000000000000000000000000000000000000000000000" + }, + { + "address": "0x1234...", + "event": "Transfer(address indexed from, address indexed to, uint256 value)", + "params": { "from": "0xbbbb...", "to": "0xcccc...", "value": "50" }, + "txHash": "0xdeadbeef0000000000000000000000000000000000000000000000000000000" + } + ] +} +``` + +Both handlers receive a receipt where `receipt.logs` has two entries, in declaration order. + +### Mock receipt defaults + +| Field | Value | +|-------|-------| +| `status` | success | +| `cumulative_gas_used` | `21000` | +| `gas_used` | `21000` | +| transaction type | `2` (EIP-1559) | +| `from` | `0x000...000` | +| `to` | `null` | +| `effective_gas_price` | `0` | + +Handlers without `receipt: true` in the manifest are unaffected — they never access `event.receipt`. + +## Block Handlers + +Block handlers are **automatically triggered** for every block. You don't need to specify block triggers in the JSON. + +### How Block Handlers Work + +The test runner auto-injects both `Start` and `End` block triggers for each block, ensuring all block handler filters work correctly: + +- **`once` filter** → Fires once at `startBlock` (via `Start` trigger) +- **No filter** → Fires on every block (via `End` trigger) +- **`polling` filter** → Fires every N blocks based on formula: `(block_number - startBlock) % every == 0` + +### Example: Basic Block Handlers + +```json +{ + "name": "Block handlers test", + "blocks": [ + {}, // Block 0 - both 'once' and regular block handlers fire + {} // Block 1 - only regular block handlers fire + ], + "assertions": [ + { + "query": "{ blocks { number } }", + "expected": { + "blocks": [ + {"number": "0"}, + {"number": "1"} + ] + } + }, + { + "query": "{ blockOnces { msg } }", + "expected": { + "blockOnces": [ + {"msg": "This fires only once at block 0"} + ] + } + } + ] +} +``` + +### Polling Block Handlers + +Polling handlers fire at regular intervals specified by the `every` parameter. The handler fires when: + +``` +(block_number - startBlock) % every == 0 +``` + +**Manifest example:** +```yaml +blockHandlers: + - handler: handleEveryThreeBlocks + filter: + kind: polling + every: 3 +``` + +**Test example (startBlock: 0):** +```json +{ + "name": "Polling handler test", + "blocks": [ + {}, // Block 0 - handler fires (0 % 3 == 0) + {}, // Block 1 - handler doesn't fire + {}, // Block 2 - handler doesn't fire + {}, // Block 3 - handler fires (3 % 3 == 0) + {}, // Block 4 - handler doesn't fire + {}, // Block 5 - handler doesn't fire + {} // Block 6 - handler fires (6 % 3 == 0) + ], + "assertions": [ + { + "query": "{ pollingBlocks(orderBy: number) { number } }", + "expected": { + "pollingBlocks": [ + {"number": "0"}, + {"number": "3"}, + {"number": "6"} + ] + } + } + ] +} +``` + +**With non-zero startBlock:** + +When your data source has `startBlock > 0`, the polling interval is calculated from that starting point. + +**Manifest:** +```yaml +dataSources: + - name: Token + source: + startBlock: 100 + mapping: + blockHandlers: + - handler: handlePolling + filter: + kind: polling + every: 5 +``` + +**Test:** +```json +{ + "name": "Polling from block 100", + "blocks": [ + {"number": 100}, // Fires: (100-100) % 5 == 0 + {"number": 101}, // Doesn't fire + {"number": 102}, // Doesn't fire + {"number": 103}, // Doesn't fire + {"number": 104}, // Doesn't fire + {"number": 105}, // Fires: (105-100) % 5 == 0 + {"number": 106}, // Doesn't fire + {"number": 107}, // Doesn't fire + {"number": 108}, // Doesn't fire + {"number": 109}, // Doesn't fire + {"number": 110} // Fires: (110-100) % 5 == 0 + ], + "assertions": [ + { + "query": "{ pollingBlocks(orderBy: number) { number } }", + "expected": { + "pollingBlocks": [ + {"number": "100"}, + {"number": "105"}, + {"number": "110"} + ] + } + } + ] +} +``` + +**Note:** The test runner automatically handles `startBlock > 0`, so blocks default to numbering from the manifest's `startBlock`. + +## eth_call Mocking + +Mock contract calls made from mapping handlers using `contract.call()`: + +```json +{ + "ethCalls": [ + { + "address": "0x1234...", + "function": "balanceOf(address)(uint256)", + "params": ["0xaaaa..."], + "returns": ["1000000000000000000"] + } + ] +} +``` + +### ethCall Fields + +| Field | Required | Description | +|-------|----------|-------------| +| `address` | Yes | Contract address | +| `function` | Yes | Full signature: `"functionName(inputTypes)(returnTypes)"` | +| `params` | Yes | Array of input parameters (as strings) | +| `returns` | Yes | Array of return values (as strings, ignored if `reverts: true`) | +| `reverts` | No | Default `false`. If `true`, the mock transport returns an RPC error | + +### Function Signature Format + +Use full signatures with input and return types: + +```json +{ + "function": "symbol()(string)", // No inputs, returns string + "function": "balanceOf(address)(uint256)", // One input, returns uint256 + "function": "decimals()(uint8)" // No inputs, returns uint8 +} +``` + +### Mocking Reverts + +```json +{ + "address": "0x1234...", + "function": "transfer(address,uint256)(bool)", + "params": ["0xaaaa...", "1000"], + "returns": [], + "reverts": true +} +``` + +### Real-World Example + +From the ERC20 test: + +```json +{ + "ethCalls": [ + { + "address": "0x731a10897d267e19b34503ad902d0a29173ba4b1", + "function": "symbol()(string)", + "params": [], + "returns": ["GRT"] + }, + { + "address": "0x731a10897d267e19b34503ad902d0a29173ba4b1", + "function": "name()(string)", + "params": [], + "returns": ["TheGraph"] + }, + { + "address": "0x731a10897d267e19b34503ad902d0a29173ba4b1", + "function": "balanceOf(address)(uint256)", + "params": ["0xaaaa000000000000000000000000000000000000"], + "returns": ["3000000000000000000"] + } + ] +} +``` + +## ethereum.getBalance() Mocking + +Mock balance lookups made from mapping handlers using `ethereum.getBalance()`: + +```json +{ + "getBalanceCalls": [ + { + "address": "0xaaaa000000000000000000000000000000000000", + "value": "1000000000000000000" + } + ] +} +``` + +### getBalanceCalls Fields + +| Field | Required | Description | +|-------|----------|-------------| +| `address` | Yes | Account address (checksummed or lowercase hex) | +| `value` | Yes | Balance in Wei as a decimal string | + +## ethereum.hasCode() Mocking + +Mock code existence checks made from mapping handlers using `ethereum.hasCode()`: + +```json +{ + "hasCodeCalls": [ + { + "address": "0x1234000000000000000000000000000000000000", + "hasCode": true + } + ] +} +``` + +### hasCodeCalls Fields + +| Field | Required | Description | +|-------|----------|-------------| +| `address` | Yes | Contract address (checksummed or lowercase hex) | +| `hasCode` | Yes | Whether the address has deployed bytecode | + +## File Data Sources + +Mock IPFS and Arweave file contents for file data source handlers. Files are defined at the top level of the test JSON (not inside blocks). + +### IPFS Files + +```json +{ + "name": "File data source test", + "files": [ + { + "cid": "QmExample...", + "content": "{\"name\": \"Token\", \"description\": \"A token\"}" + }, + { + "cid": "QmAnother...", + "file": "fixtures/metadata.json" + } + ], + "blocks": [...], + "assertions": [...] +} +``` + +#### files Fields + +| Field | Required | Description | +|-------|----------|-------------| +| `cid` | Yes | IPFS CID (`Qm...` or `bafy...`). The mock ignores hash/content relationship | +| `content` | One of `content`/`file` | Inline UTF-8 content | +| `file` | One of `content`/`file` | File path, resolved relative to the test JSON | + +### Arweave Files + +```json +{ + "name": "Arweave data source test", + "arweaveFiles": [ + { + "txId": "abc123", + "content": "{\"name\": \"Token\"}" + }, + { + "txId": "def456/metadata.json", + "file": "fixtures/arweave-data.json" + } + ], + "blocks": [...], + "assertions": [...] +} +``` + +#### arweaveFiles Fields + +| Field | Required | Description | +|-------|----------|-------------| +| `txId` | Yes | Arweave transaction ID or bundle path (e.g. `"txid/filename.json"`) | +| `content` | One of `content`/`file` | Inline UTF-8 content | +| `file` | One of `content`/`file` | File path, resolved relative to the test JSON | + +## Assertions + +GraphQL queries to validate the indexed entity state after processing all blocks. + +### Assertion Fields + +| Field | Required | Description | +|-------|----------|-------------| +| `query` | Yes | GraphQL query string | +| `expected` | Yes | Expected JSON response | + +### Comparison Behavior + +| Aspect | Behavior | +|--------|----------| +| Objects | Key-compared, order-insensitive | +| Arrays | **Order-insensitive** (set comparison) | +| String vs Number | Coerced — `"123"` matches `123` | +| Nulls/Booleans | Strict equality | +| Hex strings | **Must be lowercase** — graph-node returns all hex values in lowercase | + +**Important:** Arrays are compared as sets (order doesn't matter). + +**Important:** graph-node returns all hex-encoded values (addresses, transaction hashes, byte arrays) in **lowercase**. Expected values in assertions must match exactly — mixed-case hex will not match: + +```json +{ + "expected": { + "transfer": { + "from": "0xaaaa000000000000000000000000000000000000", // ✅ lowercase + "to": "0xBBBB000000000000000000000000000000000000" // ❌ will not match + } + } +} +``` + +Note: hex values in event inputs (`params`, `address`) are normalized automatically and can be mixed case. If you need ordered results, use `orderBy` in your GraphQL query: + +```json +{ + "query": "{ transfers(orderBy: timestamp, orderDirection: asc) { id from to value } }", + "expected": { ... } +} +``` + +### Multiple Assertions + +You can have multiple assertions per test. They run sequentially after all blocks are processed: + +```json +{ + "assertions": [ + { + "query": "{ tokens { id name symbol } }", + "expected": { ... } + }, + { + "query": "{ accounts { id balance } }", + "expected": { ... } + } + ] +} +``` + +### Nested Entity Queries + +Test relationships and nested entities: + +```json +{ + "query": "{ accounts { id balances { token { symbol } amount } } }", + "expected": { + "accounts": [ + { + "id": "0xbbbb...", + "balances": [ + { + "token": { "symbol": "GRT" }, + "amount": "5000000000000000000" + } + ] + } + ] + } +} +``` + +## startBlock Handling + +The test runner automatically reads `startBlock` from your subgraph manifest and handles it correctly — **no real blockchain connection needed**. + +### How It Works + +1. Extracts the **minimum `startBlock`** across all data sources in your manifest +2. If min > 0, creates a `start_block_override` to bypass graph-node's on-chain block validation +3. Test blocks without explicit `"number"` auto-increment starting from that minimum `startBlock` + +### Default Block Numbering + +The starting block number depends on your manifest: + +| Manifest Configuration | Test Block Numbers | +|----------------------|-------------------| +| `startBlock: 0` (or unset) | 0, 1, 2, ... | +| `startBlock: 100` | 100, 101, 102, ... | +| Multiple data sources: `startBlock: 50` and `startBlock: 200` | 50, 51, 52, ... (uses minimum) | + +### Example: Single Data Source + +**Manifest:** +```yaml +dataSources: + - name: Token + source: + startBlock: 1000 +``` + +**Test:** +```json +{ + "blocks": [ + {}, // Block 1000 (auto-numbered) + {} // Block 1001 (auto-numbered) + ] +} +``` + +### Example: Explicit Block Numbers + +Override auto-numbering by specifying `"number"`: + +```json +{ + "blocks": [ + { + "number": 5000, + "events": [...] + }, + { + "number": 5001, + "events": [...] + } + ] +} +``` + +### Multi-Data Source Testing + +When your subgraph has multiple data sources with different `startBlock` values, you may need to use explicit block numbers. + +**Scenario:** DataSource A at `startBlock: 50` (Transfer events), DataSource B at `startBlock: 200` (Approval events). You want to test only DataSource B. + +**Manifest:** +```yaml +dataSources: + - name: TokenTransfers + source: + startBlock: 50 + mapping: + eventHandlers: + - event: Transfer(...) + handler: handleTransfer + - name: TokenApprovals + source: + startBlock: 200 + mapping: + eventHandlers: + - event: Approval(...) + handler: handleApproval +``` + +**Test:** +```json +{ + "name": "Test Approval handler", + "blocks": [ + { + "number": 200, // Explicit number >= DataSource B's startBlock + "events": [ + { + "address": "0x5678...", + "event": "Approval(address indexed owner, address indexed spender, uint256 value)", + "params": { + "owner": "0xaaaa...", + "spender": "0xbbbb...", + "value": "500" + } + } + ] + }, + { + "number": 201, + "events": [...] + } + ] +} +``` + +**Why explicit numbers are needed:** +- Default numbering starts at the **minimum** `startBlock` across all data sources (50 in this case) +- Blocks 50-199 are below DataSource B's `startBlock: 200`, so its handlers won't fire +- Use explicit `"number": 200` to ensure the block is in DataSource B's active range + +**Note:** DataSource A is still "active" from block 50 onward, but it simply sees no matching Transfer events in blocks 200-201, so no handlers fire for it. This is normal behavior — graph-node doesn't error on inactive handlers. + +## Test Organization + +### Directory Structure + +``` +my-subgraph/ +├── subgraph.yaml +├── schema.graphql +├── src/ +│ └── mapping.ts +└── tests/ + ├── transfer.json + ├── approval.json + └── edge-cases.test.json +``` + +### Naming Conventions + +- Use `.json` or `.test.json` extension +- Descriptive names: `transfer.json`, `mint-burn.json`, `edge-cases.json` +- The test runner discovers all `*.json` and `*.test.json` files in the test directory + +## Known Limitations + +| Feature | Status | +|---------|--------| +| Log events | ✅ Supported | +| Block handlers (all filters) | ✅ Supported | +| `eth_call` mocking | ✅ Supported | +| `ethereum.getBalance()` mocking | ✅ Supported | +| `ethereum.hasCode()` mocking | ✅ Supported | +| Dynamic/template data sources | ✅ Supported | +| Transaction receipts (`receipt: true`) | ⚠️ Partial — `receipt.logs` is populated and grouped by `txHash`; other fields (gas, from, to, etc.) are hardcoded stubs (see [Transaction Receipts](#transaction-receipts)) | +| File data sources (IPFS + Arweave) | ✅ Supported | +| Call triggers (traces) | ❌ Not implemented | +| `--json` CI output | ❌ Not implemented | +| Parallel test execution | ❌ Not implemented | +| Test name filtering (`--filter`) | ❌ Not implemented | + +## Tips & Best Practices + +### Use Lowercase Hex in Assertions + +graph-node returns **all** hex-encoded values in lowercase — addresses, transaction hashes, and any `Bytes`/`ID` fields. Expected values in assertions must use lowercase hex: + +```json +{ + "expected": { + "transfer": { + "from": "0xaaaa000000000000000000000000000000000000" // ✅ lowercase + } + } +} +``` + +Not: +```json +{ + "expected": { + "transfer": { + "from": "0xAAAA000000000000000000000000000000000000" // ❌ will not match + } + } +} +``` + +Event inputs (`address`, `params`) are normalized automatically and can be mixed case. + +### Test One Thing at a Time + +Write focused tests that validate a single behavior: + +```json +// ✅ Good - tests one scenario +{ + "name": "Transfer event creates TransferEvent entity", + "blocks": [...], + "assertions": [...] +} +``` + +```json +// ❌ Avoid - tests too many things +{ + "name": "Test everything", + "blocks": [/* 50 blocks */], + "assertions": [/* 20 assertions */] +} +``` + +### Order GraphQL Results + +If your assertion needs specific ordering, use `orderBy`: + +```json +{ + "query": "{ transfers(first: 10, orderBy: timestamp, orderDirection: asc) { id } }", + "expected": { ... } +} +``` + +### Test Block Handlers with Empty Blocks + +Use empty blocks to test that block handlers fire even without events: + +```json +{ + "blocks": [ + {}, // Empty block - block handlers still fire + {} + ] +} +``` + +### Split Complex Tests + +Instead of one large test with many blocks, split into multiple focused test files: + +``` +tests/ +├── transfer-basic.json # Basic transfer functionality +├── transfer-zero-value.json # Edge case: zero value +└── transfer-same-account.json # Edge case: self-transfer +``` + +## Architecture + +The test runner reuses real graph-node infrastructure: + +``` +test.json + ↓ +Parse & ABI encode events + ↓ +Mock block stream (StaticStreamBuilder) + ↓ +Real graph-node indexer + ├── WASM runtime + ├── Trigger processing + └── Entity storage (pgtemp database) + ↓ +GraphQL queries → Assertions +``` + +**Key design principles:** + +- **Isolated database per test:** Each test gets a pgtemp database dropped on completion (default), or a shared persistent database with post-test cleanup (`--postgres-url`) +- **Mock transport layer:** A mock Alloy transport serves `eth_call`, `eth_getBalance`, and `eth_getCode` from test JSON data. All three flow through the real production code path — only the transport returns mock responses. Unmocked RPC calls fail immediately with a descriptive error. +- **No IPFS for manifest:** Uses `FileLinkResolver` to load manifest/WASM from build directory + +## Troubleshooting + +### Test Fails: "Entity not found" + +**Cause:** Handler didn't create the expected entity. + +**Fix:** +1. Check event signature matches ABI (include `indexed` keywords) +2. Verify contract address matches manifest +3. Check block number is >= data source's `startBlock` +4. Add debug logging to your mapping handler + +### Test Timeout + +**Cause:** Indexer took longer than 60 seconds (default timeout). + +**Fix:** +1. Reduce number of blocks in test +2. Simplify mapping logic +3. Check for infinite loops in handler code + +### Unmocked RPC Call + +**Cause:** A mapping handler calls `ethereum.call`, `ethereum.getBalance`, or `ethereum.hasCode` for a call that has no matching mock entry. + +**Symptom:** Test fails immediately with a descriptive error like: +``` +gnd test: unmocked eth_call to 0x1234... at block hash 0xabcd... +Add a matching 'ethCalls' entry to this block in your test JSON. +``` + +**Fix:** +1. Add the missing mock to the appropriate field in your test block (`ethCalls`, `getBalanceCalls`, or `hasCodeCalls`) +2. If the call is not supposed to happen, check the mapping logic — a code path may be executing unexpectedly + +### Block Handler Not Firing + +**Cause:** Block handlers auto-fire, but might be outside data source's active range. + +**Fix:** +1. Check data source's `startBlock` in manifest +2. Use explicit `"number"` in test blocks to ensure they're >= `startBlock` +3. Verify handler is defined in manifest's `blockHandlers` section + +## Legacy Matchstick Mode + +Fall back to the external Matchstick test runner for backward compatibility: + +```bash +gnd test --matchstick +``` + +This is useful if: +- You have existing Matchstick tests +- You need features not yet supported by the mock-based runner +- You're migrating gradually from Matchstick to the new test format + +## See Also + +- [Subgraph Manifest Documentation](https://thegraph.com/docs/en/developing/creating-a-subgraph/) +- [AssemblyScript Mapping API](https://thegraph.com/docs/en/developing/assemblyscript-api/) +- [GraphQL Schema](https://thegraph.com/docs/en/developing/creating-a-subgraph/#the-graph-ql-schema) diff --git a/gnd/docs/migrating-from-graph-cli.md b/gnd/docs/migrating-from-graph-cli.md new file mode 100644 index 00000000000..6203a6f2f21 --- /dev/null +++ b/gnd/docs/migrating-from-graph-cli.md @@ -0,0 +1,303 @@ +# Migrating from graph-cli to gnd + +`gnd` is designed as a drop-in replacement for `graph-cli`. This guide covers the differences and how to migrate your workflow. + +## TL;DR + +For most users, migration is straightforward: + +```bash +# Instead of: +graph codegen +graph build +graph deploy --studio my-subgraph + +# Use: +gnd codegen +gnd build +gnd deploy my-subgraph # defaults to Studio; prompts for version label in interactive terminals +``` + +## Command Mapping + +| graph-cli | gnd | Notes | +|-----------|-----|-------| +| `graph init` | `gnd init` | Identical flags | +| `graph codegen` | `gnd codegen` | Identical flags | +| `graph build` | `gnd build` | Identical flags | +| `graph deploy` | `gnd deploy` | Defaults to Studio if `--node` not provided; pass `--version-label` in non-interactive mode | +| `graph create` | `gnd create` | Identical flags | +| `graph remove` | `gnd remove` | Identical flags | +| `graph auth` | `gnd auth` | Identical flags | +| `graph add` | `gnd add` | Identical flags | +| `graph test` | `gnd test` | Identical flags | +| `graph clean` | `gnd clean` | Identical flags (new in graph-cli 0.80+) | +| `graph publish` | `gnd publish` | Identical flags | +| `graph indexer` | `gnd indexer` | Delegates to `graph-indexer` — requires `indexer-cli` installed | +| `graph local` | N/A | Not implemented - use graph-node's test infrastructure | +| `graph node` | N/A | Not implemented - use `graphman` | + +## What's the Same + +### Flag Compatibility + +All flags use the same names, short forms, and defaults: + +```bash +# Both work identically +graph codegen -o generated/ --skip-migrations +gnd codegen -o generated/ --skip-migrations + +graph build -t wasm --network mainnet +gnd build -t wasm --network mainnet + +graph deploy -l v1.0.0 --ipfs http://localhost:5001 my-subgraph +gnd deploy -l v1.0.0 --ipfs http://localhost:5001 my-subgraph +``` + +### Output Format + +Same success checkmarks, step descriptions, and progress indicators: + +``` +✔ Generate types for data source: Gravity (...) +✔ Write types to generated/Gravity/Gravity.ts (...) +``` + +### Exit Codes + +- `0` for success +- `1` for any error +- `gnd indexer` passes through the exit code from `graph-indexer` + +### Configuration Files + +`gnd` uses the same configuration files as `graph-cli`: + +- `~/.graph-cli.json` - Deploy keys (from `gnd auth`) +- `networks.json` - Network-specific addresses + +Your existing auth keys work with both tools. + +### Generated Code + +Code generation produces identical AssemblyScript output (after formatting). Your existing subgraph code will work without changes. + +## What's Different + +### Indexer Commands + +`gnd indexer` provides access to indexer management commands via [`indexer-cli`](https://github.com/graphprotocol/indexer/tree/main/packages/indexer-cli). Requires `graph-indexer` on `$PATH`: + +```bash +npm install -g @graphprotocol/indexer-cli + +gnd indexer status --network arbitrum-one +gnd indexer rules get all --network mainnet +gnd indexer allocations get --network arbitrum-one +``` + +Note: `gnd indexer --help` shows gnd's own help and works without `graph-indexer` installed. Use `gnd indexer help` to see the full list of `graph-indexer` commands. + +### Commands Not Available + +#### `graph local` + +Not implemented in `gnd`. Use graph-node's built-in integration test infrastructure or Docker compose setups instead. + +#### `graph node` + +Not implemented. Use `graphman` for node management operations: + +```bash +# Instead of graph node commands, use graphman: +graphman info subgraph-name +graphman reassign subgraph-name shard +``` + +### Debug Logging + +```bash +# graph-cli uses: +DEBUG=graph-cli:* graph codegen + +# gnd uses: +RUST_LOG=gnd=debug gnd codegen +``` + +### The `--uncrashable` Flag + +The `--uncrashable` flag on `codegen` (Float Capital's uncrashable helper generation) is not implemented. This is a third-party feature that most subgraphs don't use. + +### Minor Code Generation Differences + +These differences are intentional and documented: + +1. **Int8 import**: `gnd` always imports `Int8` in generated code for simplicity, even when not used. This doesn't affect functionality. + +2. **Trailing commas**: `gnd` uses trailing commas in multi-line constructs: + ```typescript + // gnd output + new Foo( + param1, + param2, // trailing comma + ) + + // graph-cli output + new Foo( + param1, + param2 + ) + ``` + +3. **2D array accessors**: `gnd` uses the correct `toStringMatrix()` method for 2D GraphQL arrays, fixing a bug in graph-cli that used `toStringArray()`. + +## Migration Steps + +### Step 1: Install gnd + +Build from source in the graph-node repository: + +```bash +cargo build -p gnd --release +# Binary at target/release/gnd +``` + +### Step 2: Verify Your Auth + +Your existing `~/.graph-cli.json` works with both tools. Verify with: + +```bash +# Check stored keys +cat ~/.graph-cli.json + +# Or just run a deploy dry-run to see if auth is working +gnd deploy --help +``` + +### Step 3: Test Locally + +Run your normal workflow with gnd: + +```bash +gnd codegen +gnd build +``` + +Compare the output to graph-cli if you want to verify: + +```bash +# graph-cli +graph codegen -o generated-graph/ +graph build -o build-graph/ + +# gnd +gnd codegen -o generated-gnd/ +gnd build -o build-gnd/ + +# Compare (should be identical after formatting) +diff -r generated-graph generated-gnd +diff -r build-graph build-gnd +``` + +### Step 4: Update CI/CD + +Replace `graph` with `gnd` in your CI configuration: + +```yaml +# Before +- run: graph codegen +- run: graph build +- run: graph deploy --studio -l ${{ github.sha }} ${{ secrets.SUBGRAPH_NAME }} + +# After +- run: gnd codegen +- run: gnd build +- run: gnd deploy -l ${{ github.sha }} ${{ secrets.SUBGRAPH_NAME }} +``` + +### Step 5: Update package.json Scripts (Optional) + +If you have npm scripts calling graph-cli: + +```json +{ + "scripts": { + "codegen": "gnd codegen", + "build": "gnd build", + "deploy": "gnd deploy -l $VERSION_LABEL" + } +} +``` + +Set `VERSION_LABEL` in your environment (for example, from a git tag or commit SHA). + +## Troubleshooting + +### "Command not found: gnd" + +Make sure the gnd binary is in your PATH: + +```bash +export PATH="$PATH:/path/to/graph-node/target/release" +``` + +### "prettier: command not found" + +`gnd codegen` shells out to `prettier` for formatting. Install it: + +```bash +npm install -g prettier +# or +pnpm add -g prettier +``` + +### "asc: command not found" + +`gnd build` shells out to the AssemblyScript compiler. Install it: + +```bash +npm install -g assemblyscript +# or have it in your subgraph's node_modules +npm install --save-dev assemblyscript +``` + +### Different Output After codegen + +If you notice differences in generated code: + +1. Check if it's one of the documented differences (Int8 import, trailing commas) +2. Run prettier on both outputs to normalize formatting +3. Report any unexpected differences as a bug + +### Authentication Issues + +Keys stored by graph-cli work with gnd and vice versa. If you have issues: + +```bash +# Re-authenticate +gnd auth YOUR_DEPLOY_KEY + +# Verify the key was saved +cat ~/.graph-cli.json +``` + +## Why Migrate? + +### Benefits of gnd + +- **Native performance**: Rust implementation is faster for large subgraphs +- **Integrated with graph-node**: Same codebase, consistent behavior +- **Better error messages**: Leverages graph-node's validation +- **`gnd dev` command**: Run graph-node in development mode + +### When to Keep Using graph-cli + +- If you need multi-protocol support (NEAR, Cosmos, Arweave, Substreams) - gnd currently focuses on Ethereum +- If you use the `--uncrashable` flag +- If you need the `graph local` or `graph node` commands + +## Getting Help + +- File issues at: https://github.com/graphprotocol/graph-node/issues +- Check gnd version: `gnd --version` diff --git a/gnd/npm/README.md b/gnd/npm/README.md new file mode 100644 index 00000000000..acca621b4dc --- /dev/null +++ b/gnd/npm/README.md @@ -0,0 +1,35 @@ +# The Graph CLI + +This package installs the CLI for developing subgraphs for [The Graph +Network](https://thegraph.com/subgraphs/) It supports all aspects of +[developing, testing, and deploying subgraphs](https://thegraph.com/docs/en/subgraphs/quick-start/) locally and +on the network. + +This package is a complete replacement for the older +[graph-cli](https://www.npmjs.com/package/@graphprotocol/graph-cli) which +will over time be migrated to be a simple wrapper for `gnd`. Older +documentation will reference `graph-cli` in its instructions; running +`alias graph=gnd` in the shell make it possible to follow these +instructions verbatim with `gnd`. + +Besides the tools to develop subgraphs, `gnd` also contains a version of +[graph-node](https://github.com/graphprotocol/graph-node) tailored to +running subgraphs locally via `gnd dev`. + + +## Getting started + +Run `npm install -g @graphprotocol/gnd` to install `gnd`. + +After installation, you can create and run an example subgraph locally with +```bash +gnd init --from-example ethereum-gravatar 'My new subgraph' new-subgraph +cd new-subgraph +gnd codegen +gnd build +gnd dev --ethereum-rpc 'mainnet:' +``` + +If you have an existing contract for which you want to write a subgraph, +have a look at `gnd help init` and the `--from-contract` option. You can +also simply run `gnd init` and follow the prompts to set up your subgraph. diff --git a/gnd/npm/bin/gnd.js b/gnd/npm/bin/gnd.js new file mode 100644 index 00000000000..82a7ae5bf09 --- /dev/null +++ b/gnd/npm/bin/gnd.js @@ -0,0 +1,33 @@ +#!/usr/bin/env node + +const { execFileSync } = require("child_process"); +const path = require("path"); + +const PLATFORMS = { + "darwin-arm64": "@graphprotocol/gnd-darwin-arm64", + "darwin-x64": "@graphprotocol/gnd-darwin-x64", + "linux-arm64": "@graphprotocol/gnd-linux-arm64", + "linux-x64": "@graphprotocol/gnd-linux-x64", + "win32-x64": "@graphprotocol/gnd-win32-x64", +}; + +const key = `${process.platform}-${process.arch}`; +const pkg = PLATFORMS[key]; +if (!pkg) { + console.error(`Unsupported platform: ${key}`); + process.exit(1); +} + +const bin = process.platform === "win32" ? "gnd.exe" : "gnd"; +const binPath = path.join( + require.resolve(`${pkg}/package.json`), + "..", + "bin", + bin +); + +try { + execFileSync(binPath, process.argv.slice(2), { stdio: "inherit" }); +} catch (e) { + process.exit(e.status ?? 1); +} diff --git a/gnd/src/abi.rs b/gnd/src/abi.rs new file mode 100644 index 00000000000..0a5fa695cf4 --- /dev/null +++ b/gnd/src/abi.rs @@ -0,0 +1,101 @@ +//! ABI normalization utilities. +//! +//! Handles extraction of bare ABI arrays from various artifact formats +//! (raw arrays, Hardhat/Foundry, Truffle). + +use anyhow::{Context, Result, anyhow}; + +/// Normalize ABI JSON to extract the actual ABI array from various artifact formats. +/// +/// Supports: +/// - Raw ABI array: `[{...}]` +/// - Foundry/Hardhat format: `{"abi": [...], ...}` +/// - Truffle format: `{"compilerOutput": {"abi": [...], ...}, ...}` +pub fn normalize_abi_json(abi_str: &str) -> Result { + let value: serde_json::Value = + serde_json::from_str(abi_str).context("Failed to parse ABI JSON")?; + + // Case 1: Already an array - return as-is + if value.is_array() { + return Ok(value); + } + + // Case 2: Object with "abi" field (Foundry/Hardhat format) + if let Some(abi) = value.get("abi") + && abi.is_array() + { + return Ok(abi.clone()); + } + + // Case 3: Object with "compilerOutput.abi" field (Truffle format) + if let Some(compiler_output) = value.get("compilerOutput") + && let Some(abi) = compiler_output.get("abi") + && abi.is_array() + { + return Ok(abi.clone()); + } + + Err(anyhow!( + "Invalid ABI format: expected an array or an object with 'abi' field" + )) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_normalize_abi_json_raw_array() { + let raw_abi = r#"[{"type": "event", "name": "Transfer"}]"#; + let result = normalize_abi_json(raw_abi).unwrap(); + assert!(result.is_array()); + assert_eq!(result.as_array().unwrap().len(), 1); + } + + #[test] + fn test_normalize_abi_json_hardhat_format() { + let hardhat_abi = r#"{ + "_format": "hh-sol-artifact-1", + "contractName": "MyContract", + "abi": [{"type": "event", "name": "Transfer"}], + "bytecode": "0x..." + }"#; + let result = normalize_abi_json(hardhat_abi).unwrap(); + assert!(result.is_array()); + assert_eq!(result.as_array().unwrap().len(), 1); + assert_eq!( + result.as_array().unwrap()[0].get("name").unwrap(), + "Transfer" + ); + } + + #[test] + fn test_normalize_abi_json_truffle_format() { + let truffle_abi = r#"{ + "contractName": "MyContract", + "compilerOutput": { + "abi": [{"type": "event", "name": "Transfer"}] + } + }"#; + let result = normalize_abi_json(truffle_abi).unwrap(); + assert!(result.is_array()); + assert_eq!(result.as_array().unwrap().len(), 1); + assert_eq!( + result.as_array().unwrap()[0].get("name").unwrap(), + "Transfer" + ); + } + + #[test] + fn test_normalize_abi_json_invalid_format() { + let invalid_abi = r#"{"contractName": "MyContract"}"#; + let result = normalize_abi_json(invalid_abi); + assert!(result.is_err()); + assert!( + result + .unwrap_err() + .to_string() + .contains("Invalid ABI format") + ); + } +} diff --git a/gnd/src/codegen/abi.rs b/gnd/src/codegen/abi.rs new file mode 100644 index 00000000000..9b881fdcbe9 --- /dev/null +++ b/gnd/src/codegen/abi.rs @@ -0,0 +1,1955 @@ +//! ABI code generation for Ethereum contracts. +//! +//! Generates AssemblyScript bindings from contract ABIs: +//! - Event classes with typed parameters +//! - Call classes for function calls with inputs/outputs +//! - Contract class with typed call methods + +use std::collections::HashMap; + +use graph::abi::{ + DynSolType, Event, EventParam, Function, FunctionExt, JsonAbi, Param, StateMutability, +}; +use regex::Regex; + +use super::typescript::{self as ts, Class, ClassMember, Method, ModuleImports, Param as TsParam}; +use crate::shared::{capitalize, handle_reserved_word}; + +/// Resolve a `Param`'s type to `DynSolType`. +fn resolve_param_type(param: &Param) -> DynSolType { + param + .selector_type() + .parse::() + .expect("valid ABI type") +} + +/// Resolve an `EventParam`'s type to `DynSolType`. +fn resolve_event_param_type(param: &EventParam) -> DynSolType { + param + .selector_type() + .parse::() + .expect("valid ABI type") +} + +const GRAPH_TS_MODULE: &str = "@graphprotocol/graph-ts"; + +/// ABI code generator. +pub struct AbiCodeGenerator { + contract: JsonAbi, + name: String, +} + +impl AbiCodeGenerator { + /// Create a new ABI code generator. + pub fn new(contract: JsonAbi, name: impl Into) -> Self { + let mut name = name.into(); + // Sanitize name to be a valid class name + let re = Regex::new(r#"[!@#$%^&*()+\-=\[\]{};':\"|,.<>/?]+"#).unwrap(); + name = re.replace_all(&name, "_").to_string(); + Self { contract, name } + } + + /// Generate module imports for the ABI file. + pub fn generate_module_imports(&self) -> Vec { + vec![ModuleImports::new( + vec![ + "ethereum".to_string(), + "JSONValue".to_string(), + "TypedMap".to_string(), + "Entity".to_string(), + "Bytes".to_string(), + "Address".to_string(), + "BigInt".to_string(), + ], + GRAPH_TS_MODULE, + )] + } + + /// Generate all types from the ABI. + pub fn generate_types(&self) -> Vec { + let mut classes = Vec::new(); + classes.extend(self.generate_event_types()); + classes.extend(self.generate_smart_contract_class()); + classes.extend(self.generate_call_types()); + classes + } + + /// Generate event type classes. + fn generate_event_types(&self) -> Vec { + let mut classes = Vec::new(); + let events = self.disambiguate_events(); + + for (event, alias) in events { + let event_class_name = alias.clone(); + let mut tuple_classes = Vec::new(); + + // Generate params class + let params_class_name = [&event_class_name, "__Params"].concat(); + let mut params_class = ts::klass(¶ms_class_name).exported(); + params_class.add_member(ClassMember::new("_event", &event_class_name)); + params_class.add_method(Method::new( + "constructor", + vec![TsParam::new("event", ts::NamedType::new(&event_class_name))], + None, + "this._event = event", + )); + + // Generate getters for event params + let inputs = self.disambiguate_event_params(&event.inputs, "param"); + for (index, (param, param_name)) in inputs.iter().enumerate() { + let param_object = self.generate_event_param( + param, + param_name, + index, + &event_class_name, + &mut tuple_classes, + ); + params_class.add_method(param_object); + } + + // Generate event class + let mut event_class = ts::klass(&event_class_name) + .exported() + .extends("ethereum.Event"); + event_class.add_method(Method::new( + "get params", + vec![], + Some(ts::NamedType::new(¶ms_class_name).into()), + format!("return new {}(this)", params_class_name), + )); + + classes.push(event_class); + classes.push(params_class); + classes.extend(tuple_classes); + } + + classes + } + + /// Generate the smart contract class with call methods. + fn generate_smart_contract_class(&self) -> Vec { + let mut classes = Vec::new(); + + let mut contract_class = ts::klass(&self.name) + .exported() + .extends("ethereum.SmartContract"); + + // Add static bind method + let contract_name = &self.name; + contract_class.add_static_method(ts::StaticMethod::new( + "bind", + vec![TsParam::new("address", ts::NamedType::new("Address"))], + ts::NamedType::new(&self.name), + format!("return new {}('{}', address)", contract_name, contract_name), + )); + + // Get callable functions and sort alphabetically for deterministic output + let mut functions = self.get_callable_functions(); + functions.sort_by(|a, b| a.name.cmp(&b.name)); + let disambiguated = self.disambiguate_functions(&functions); + + for (func, alias) in disambiguated { + let (method, try_method, result_classes) = self.generate_function_methods(func, &alias); + contract_class.add_method(method); + contract_class.add_method(try_method); + classes.extend(result_classes); + } + + classes.push(contract_class); + classes + } + + /// Generate call type classes. + fn generate_call_types(&self) -> Vec { + let mut classes = Vec::new(); + let mut functions = self.get_call_functions(); + functions.sort_by(|a, b| a.name.cmp(&b.name)); + let disambiguated = self.disambiguate_call_functions(&functions); + + for (func, alias) in disambiguated { + let cap_alias = capitalize(&alias); + let call_class_name = format!("{}Call", cap_alias); + let mut tuple_classes = Vec::new(); + + // Generate inputs class + let inputs_class_name = [&call_class_name, "__Inputs"].concat(); + let mut inputs_class = ts::klass(&inputs_class_name).exported(); + inputs_class.add_member(ClassMember::new("_call", &call_class_name)); + inputs_class.add_method(Method::new( + "constructor", + vec![TsParam::new("call", ts::NamedType::new(&call_class_name))], + None, + "this._call = call", + )); + + let inputs = self.disambiguate_params(&func.inputs, "value"); + for (index, (param, param_name)) in inputs.iter().enumerate() { + let getter = self.generate_input_output_getter( + param, + param_name, + index, + &call_class_name, + "call", + "inputValues", + &mut tuple_classes, + ); + inputs_class.add_method(getter); + } + + // Generate outputs class + let outputs_class_name = [&call_class_name, "__Outputs"].concat(); + let mut outputs_class = ts::klass(&outputs_class_name).exported(); + outputs_class.add_member(ClassMember::new("_call", &call_class_name)); + outputs_class.add_method(Method::new( + "constructor", + vec![TsParam::new("call", ts::NamedType::new(&call_class_name))], + None, + "this._call = call", + )); + + let outputs = self.disambiguate_params(&func.outputs, "value"); + for (index, (param, param_name)) in outputs.iter().enumerate() { + let getter = self.generate_input_output_getter( + param, + param_name, + index, + &call_class_name, + "call", + "outputValues", + &mut tuple_classes, + ); + outputs_class.add_method(getter); + } + + // Generate call class + let mut call_class = ts::klass(&call_class_name) + .exported() + .extends("ethereum.Call"); + call_class.add_method(Method::new( + "get inputs", + vec![], + Some(ts::NamedType::new(&inputs_class_name).into()), + format!("return new {}(this)", inputs_class_name), + )); + call_class.add_method(Method::new( + "get outputs", + vec![], + Some(ts::NamedType::new(&outputs_class_name).into()), + format!("return new {}(this)", outputs_class_name), + )); + + classes.push(call_class); + classes.push(inputs_class); + classes.push(outputs_class); + classes.extend(tuple_classes); + } + + classes + } + + /// Generate a getter method for an event parameter. + fn generate_event_param( + &self, + param: &EventParam, + name: &str, + index: usize, + event_class_name: &str, + tuple_classes: &mut Vec, + ) -> Method { + let param_type = resolve_event_param_type(param); + + // Handle indexed params - strings, bytes and arrays are hashed to bytes32 + let value_type = if param.indexed { + indexed_input_type(¶m_type) + } else { + param_type.clone() + }; + + if contains_tuple_type(&value_type) { + self.generate_tuple_getter_for_event_param( + param, + ¶m_type, + name, + index, + event_class_name, + "event", + "parameters", + tuple_classes, + ) + } else { + let asc_type = asc_type_for_ethereum(&value_type); + let access = format!("this._event.parameters[{}].value", index); + let conversion = ethereum_to_asc(&access, &value_type, None); + Method::new( + format!("get {}", name), + vec![], + Some(ts::TypeExpr::Raw(asc_type)), + format!("return {}", conversion), + ) + } + } + + /// Generate a getter for call inputs/outputs. + #[allow(clippy::too_many_arguments)] + fn generate_input_output_getter( + &self, + param: &Param, + name: &str, + index: usize, + parent_class: &str, + parent_type: &str, + parent_field: &str, + tuple_classes: &mut Vec, + ) -> Method { + let param_type = resolve_param_type(param); + if contains_tuple_type(¶m_type) { + self.generate_tuple_getter( + param, + ¶m_type, + name, + index, + parent_class, + parent_type, + parent_field, + tuple_classes, + ) + } else { + let asc_type = asc_type_for_ethereum(¶m_type); + let access = format!("this._{}.{}[{}].value", parent_type, parent_field, index); + let conversion = ethereum_to_asc(&access, ¶m_type, None); + Method::new( + format!("get {}", name), + vec![], + Some(ts::TypeExpr::Raw(asc_type)), + format!("return {}", conversion), + ) + } + } + + /// Generate a tuple getter and its associated classes (for `Param`). + #[allow(clippy::too_many_arguments)] + fn generate_tuple_getter( + &self, + param: &Param, + param_type: &DynSolType, + name: &str, + index: usize, + parent_class: &str, + parent_type: &str, + parent_field: &str, + tuple_classes: &mut Vec, + ) -> Method { + let cap_name = capitalize(name); + let tuple_identifier = format!("{}{}", parent_class, cap_name); + let tuple_class_name = if parent_field == "outputValues" { + format!("{}OutputStruct", tuple_identifier) + } else { + format!("{}Struct", tuple_identifier) + }; + + let is_tuple = matches!(param_type, DynSolType::Tuple(_)); + let access_code = if parent_type == "tuple" { + format!("this[{}]", index) + } else { + format!("this._{}.{}[{}].value", parent_type, parent_field, index) + }; + + let return_value = ethereum_to_asc(&access_code, param_type, Some(&tuple_class_name)); + + let return_type = if is_tuple_matrix_type(param_type) { + format!("Array>", tuple_class_name) + } else if is_tuple_array_type(param_type) { + format!("Array<{}>", tuple_class_name) + } else { + tuple_class_name.clone() + }; + + let body = if is_tuple { + format!("return changetype<{}>({})", tuple_class_name, return_value) + } else { + format!("return {}", return_value) + }; + + // Generate tuple class from Param's components + let components = get_tuple_param_components(param); + if !components.is_empty() { + let mut tuple_class = ts::klass(&tuple_class_name) + .exported() + .extends("ethereum.Tuple"); + + let component_params = disambiguate_tuple_components(components); + for (idx, (component, component_name)) in component_params.iter().enumerate() { + let component_getter = self.generate_tuple_component_getter( + component, + component_name, + idx, + &tuple_identifier, + tuple_classes, + ); + tuple_class.add_method(component_getter); + } + + tuple_classes.push(tuple_class); + } + + Method::new( + format!("get {}", name), + vec![], + Some(ts::TypeExpr::Raw(return_type)), + body, + ) + } + + /// Generate a tuple getter and its associated classes (for `EventParam`). + #[allow(clippy::too_many_arguments)] + fn generate_tuple_getter_for_event_param( + &self, + param: &EventParam, + param_type: &DynSolType, + name: &str, + index: usize, + parent_class: &str, + parent_type: &str, + parent_field: &str, + tuple_classes: &mut Vec, + ) -> Method { + let cap_name = capitalize(name); + let tuple_identifier = format!("{}{}", parent_class, cap_name); + let tuple_class_name = if parent_field == "outputValues" { + format!("{}OutputStruct", tuple_identifier) + } else { + format!("{}Struct", tuple_identifier) + }; + + let is_tuple = matches!(param_type, DynSolType::Tuple(_)); + let access_code = if parent_type == "tuple" { + format!("this[{}]", index) + } else { + format!("this._{}.{}[{}].value", parent_type, parent_field, index) + }; + + let return_value = ethereum_to_asc(&access_code, param_type, Some(&tuple_class_name)); + + let return_type = if is_tuple_matrix_type(param_type) { + format!("Array>", tuple_class_name) + } else if is_tuple_array_type(param_type) { + format!("Array<{}>", tuple_class_name) + } else { + tuple_class_name.clone() + }; + + let body = if is_tuple { + format!("return changetype<{}>({})", tuple_class_name, return_value) + } else { + format!("return {}", return_value) + }; + + // Generate tuple class from EventParam's components + // EventParam.components is Vec, same as Param.components + let components = ¶m.components; + if !components.is_empty() { + let mut tuple_class = ts::klass(&tuple_class_name) + .exported() + .extends("ethereum.Tuple"); + + let component_params = disambiguate_tuple_components(components); + for (idx, (component, component_name)) in component_params.iter().enumerate() { + let component_getter = self.generate_tuple_component_getter( + component, + component_name, + idx, + &tuple_identifier, + tuple_classes, + ); + tuple_class.add_method(component_getter); + } + + tuple_classes.push(tuple_class); + } + + Method::new( + format!("get {}", name), + vec![], + Some(ts::TypeExpr::Raw(return_type)), + body, + ) + } + + /// Generate a getter for a tuple component. + fn generate_tuple_component_getter( + &self, + param: &Param, + name: &str, + index: usize, + parent_class: &str, + tuple_classes: &mut Vec, + ) -> Method { + let param_type = resolve_param_type(param); + if contains_tuple_type(¶m_type) { + self.generate_tuple_getter( + param, + ¶m_type, + name, + index, + parent_class, + "tuple", + "", + tuple_classes, + ) + } else { + let asc_type = asc_type_for_ethereum(¶m_type); + let access = format!("this[{}]", index); + let conversion = ethereum_to_asc(&access, ¶m_type, None); + Method::new( + format!("get {}", name), + vec![], + Some(ts::TypeExpr::Raw(asc_type)), + format!("return {}", conversion), + ) + } + } + + /// Generate methods for a callable function. + fn generate_function_methods( + &self, + func: &Function, + alias: &str, + ) -> (Method, Method, Vec) { + let mut result_classes = Vec::new(); + let fn_signature = func.signature_compat(); + let contract_name = &self.name; + let tuple_result_parent_type = [contract_name, "__", alias, "Result"].concat(); + let tuple_input_parent_type = [contract_name, "__", alias, "Input"].concat(); + + // Disambiguate outputs + let outputs = self.disambiguate_params(&func.outputs, "value"); + + // Determine return type + let (return_type, simple_return_type) = if outputs.len() > 1 { + // Multiple outputs - create a result struct + let result_class = self.generate_result_class( + &outputs, + &tuple_result_parent_type, + &mut result_classes, + ); + result_classes.push(result_class.clone()); + (result_class.name.clone(), false) + } else if !outputs.is_empty() { + let (param, _) = &outputs[0]; + let param_type = resolve_param_type(param); + if contains_tuple_type(¶m_type) { + let tuple_name = self.generate_tuple_return_type( + param, + ¶m_type, + 0, + &tuple_result_parent_type, + &mut result_classes, + ); + (tuple_name, true) + } else { + (asc_type_for_ethereum(¶m_type), true) + } + } else { + ("void".to_string(), true) + }; + + // Disambiguate inputs + let inputs = self.disambiguate_params(&func.inputs, "param"); + + // Generate tuple types for inputs + for (index, (param, _)) in inputs.iter().enumerate() { + let param_type = resolve_param_type(param); + if contains_tuple_type(¶m_type) { + self.generate_tuple_class_for_input( + param, + index, + &tuple_input_parent_type, + &mut result_classes, + ); + } + } + + // Build params + let params: Vec = inputs + .iter() + .enumerate() + .map(|(index, (param, name))| { + let p_type = resolve_param_type(param); + let param_type_str = + get_param_type_for_input(&p_type, index, &tuple_input_parent_type); + TsParam::new(name.clone(), ts::TypeExpr::Raw(param_type_str)) + }) + .collect(); + + // Build call arguments + let call_args: Vec = inputs + .iter() + .map(|(param, name)| { + let p_type = resolve_param_type(param); + ethereum_from_asc(name, &p_type) + }) + .collect(); + + let func_name = &func.name; + let call_args_str = call_args.join(", "); + let super_inputs = format!("'{}', '{}', [{}]", func_name, fn_signature, call_args_str); + + // Generate method body + let method_body = self.generate_call_body( + &outputs, + &return_type, + simple_return_type, + &super_inputs, + &tuple_result_parent_type, + false, + ); + + let try_method_body = self.generate_call_body( + &outputs, + &return_type, + simple_return_type, + &super_inputs, + &tuple_result_parent_type, + true, + ); + + let method = Method::new( + alias.to_string(), + params.clone(), + Some(ts::TypeExpr::Raw(return_type.clone())), + method_body, + ); + + let try_method = Method::new( + format!("try_{}", alias), + params, + Some(ts::TypeExpr::Raw(format!( + "ethereum.CallResult<{}>", + return_type + ))), + try_method_body, + ); + + (method, try_method, result_classes) + } + + /// Generate call method body. + fn generate_call_body( + &self, + outputs: &[(&Param, String)], + return_type: &str, + simple_return_type: bool, + super_inputs: &str, + tuple_result_parent_type: &str, + is_try: bool, + ) -> String { + let nl = "\n"; + let (call_stmt, result_var) = if is_try { + let mut lines = Vec::new(); + lines.push(format!("let result = super.tryCall({})", super_inputs)); + lines.push(" if (result.reverted) {".to_string()); + lines.push(" return new ethereum.CallResult()".to_string()); + lines.push(" }".to_string()); + lines.push(" let value = result.value".to_string()); + (lines.join(nl), "value") + } else { + ( + format!("let result = super.call({})", super_inputs), + "result", + ) + }; + + let return_val = if simple_return_type { + if outputs.is_empty() { + String::new() + } else { + let (param, _) = &outputs[0]; + let p_type = resolve_param_type(param); + let tuple_name = if is_tuple_array_type(&p_type) { + Some(tuple_type_name(0, tuple_result_parent_type)) + } else { + None + }; + let val = ethereum_to_asc( + &format!("{}[0]", result_var), + &p_type, + tuple_name.as_deref(), + ); + if matches!(p_type, DynSolType::Tuple(_)) { + format!("changetype<{}>({})", return_type, val) + } else { + val + } + } + } else { + let conversions: Vec = outputs + .iter() + .enumerate() + .map(|(index, (param, _))| { + let p_type = resolve_param_type(param); + let tuple_name = if is_tuple_array_type(&p_type) { + Some(tuple_type_name(index, tuple_result_parent_type)) + } else { + None + }; + let val = ethereum_to_asc( + &format!("{}[{}]", result_var, index), + &p_type, + tuple_name.as_deref(), + ); + if matches!(p_type, DynSolType::Tuple(_)) { + let tn = tuple_type_name(index, tuple_result_parent_type); + format!("changetype<{}>({})", tn, val) + } else { + val + } + }) + .collect(); + let conv_str = conversions.join(", "); + format!("new {}({})", return_type, conv_str) + }; + + if is_try { + [ + &call_stmt, + nl, + " return ethereum.CallResult.fromValue(", + &return_val, + ")", + ] + .concat() + } else if outputs.is_empty() { + call_stmt + } else { + [&call_stmt, nl, nl, " return (", &return_val, ")"].concat() + } + } + + /// Generate a result class for multiple outputs. + fn generate_result_class( + &self, + outputs: &[(&Param, String)], + tuple_result_parent_type: &str, + result_classes: &mut Vec, + ) -> Class { + let class_name = tuple_result_parent_type.to_string(); + let mut klass = ts::klass(&class_name).exported(); + + // Add constructor + let constructor_params: Vec = outputs + .iter() + .enumerate() + .map(|(index, (param, _))| { + let p_type = resolve_param_type(param); + let param_type_str = + get_param_type_for_input(&p_type, index, tuple_result_parent_type); + TsParam::new(format!("value{}", index), ts::TypeExpr::Raw(param_type_str)) + }) + .collect(); + + let nl = "\n"; + let constructor_body: Vec = outputs + .iter() + .enumerate() + .map(|(index, _)| format!("this.value{} = value{}", index, index)) + .collect(); + + klass.add_method(Method::new( + "constructor", + constructor_params, + None, + constructor_body.join(&format!("{} ", nl)), + )); + + // Add toMap method + let map_entries: Vec = outputs + .iter() + .enumerate() + .map(|(index, (param, _))| { + let p_type = resolve_param_type(param); + let this_val = format!("this.value{}", index); + let from_asc = ethereum_from_asc(&this_val, &p_type); + format!("map.set('value{}', {})", index, from_asc) + }) + .collect(); + + let map_body = [ + "let map = new TypedMap()", + nl, + " ", + &map_entries.join(&format!("{} ", nl)), + nl, + " return map", + ] + .concat(); + + klass.add_method(Method::new( + "toMap", + vec![], + Some(ts::TypeExpr::Raw( + "TypedMap".to_string(), + )), + map_body, + )); + + // Add members + for (index, (param, _)) in outputs.iter().enumerate() { + let p_type = resolve_param_type(param); + let param_type_str = get_param_type_for_input(&p_type, index, tuple_result_parent_type); + klass.add_member(ClassMember::new(format!("value{}", index), param_type_str)); + } + + // Add getters for outputs + for (index, (param, _)) in outputs.iter().enumerate() { + let getter_name = if param.name.trim().is_empty() { + format!("getValue{}", index) + } else { + let cap = capitalize(¶m.name); + format!("get{}", cap) + }; + let p_type = resolve_param_type(param); + let param_type_str = get_param_type_for_input(&p_type, index, tuple_result_parent_type); + klass.add_method(Method::new( + getter_name, + vec![], + Some(ts::TypeExpr::Raw(param_type_str)), + format!("return this.value{}", index), + )); + } + + // Generate tuple classes for outputs + for (index, (param, _)) in outputs.iter().enumerate() { + let p_type = resolve_param_type(param); + if contains_tuple_type(&p_type) { + self.generate_tuple_class_for_input( + param, + index, + tuple_result_parent_type, + result_classes, + ); + } + } + + klass + } + + /// Generate tuple return type name and classes. + fn generate_tuple_return_type( + &self, + param: &Param, + param_type: &DynSolType, + index: usize, + parent_type: &str, + result_classes: &mut Vec, + ) -> String { + self.generate_tuple_class_for_input(param, index, parent_type, result_classes); + let tn = tuple_type_name(index, parent_type); + if is_tuple_array_type(param_type) { + format!("Array<{}>", tn) + } else if is_tuple_matrix_type(param_type) { + format!("Array>", tn) + } else { + tn + } + } + + /// Generate tuple class for an input/output. + fn generate_tuple_class_for_input( + &self, + param: &Param, + index: usize, + parent_type: &str, + result_classes: &mut Vec, + ) { + let tuple_class_name = tuple_type_name(index, parent_type); + let mut tuple_class = ts::klass(&tuple_class_name) + .exported() + .extends("ethereum.Tuple"); + + let components = get_tuple_param_components(param); + if !components.is_empty() { + let component_params = disambiguate_tuple_components(components); + for (idx, (component, component_name)) in component_params.iter().enumerate() { + let component_type = resolve_param_type(component); + let getter = if contains_tuple_type(&component_type) { + // Recursively generate tuple classes + let cap = capitalize(&format!("{}", index)); + let nested_parent = format!("{}Value{}", parent_type, cap); + self.generate_tuple_class_for_input( + component, + idx, + &nested_parent, + result_classes, + ); + let nested_tuple_name = tuple_type_name(idx, &nested_parent); + let access = format!("this[{}]", idx); + let conversion = + ethereum_to_asc(&access, &component_type, Some(&nested_tuple_name)); + let return_type = if is_tuple_array_type(&component_type) { + format!("Array<{}>", nested_tuple_name) + } else { + nested_tuple_name.clone() + }; + let body = if matches!(component_type, DynSolType::Tuple(_)) { + format!("return changetype<{}>({})", nested_tuple_name, conversion) + } else { + format!("return {}", conversion) + }; + Method::new( + format!("get {}", component_name), + vec![], + Some(ts::TypeExpr::Raw(return_type)), + body, + ) + } else { + let asc_type = asc_type_for_ethereum(&component_type); + let access = format!("this[{}]", idx); + let conversion = ethereum_to_asc(&access, &component_type, None); + Method::new( + format!("get {}", component_name), + vec![], + Some(ts::TypeExpr::Raw(asc_type)), + format!("return {}", conversion), + ) + }; + tuple_class.add_method(getter); + } + } + + result_classes.push(tuple_class); + } + + /// Get callable functions (view, pure, nonpayable, constant with outputs). + fn get_callable_functions(&self) -> Vec<&Function> { + self.contract + .functions() + .filter(|f| { + !f.outputs.is_empty() + && matches!( + f.state_mutability, + StateMutability::View | StateMutability::Pure | StateMutability::NonPayable + ) + }) + .collect() + } + + /// Get functions that can be used as calls (non-view, non-pure functions). + fn get_call_functions(&self) -> Vec<&Function> { + self.contract + .functions() + .filter(|f| { + matches!( + f.state_mutability, + StateMutability::NonPayable | StateMutability::Payable + ) + }) + .collect() + } + + /// Disambiguate events with duplicate names. + fn disambiguate_events(&self) -> Vec<(&Event, String)> { + let mut result = Vec::new(); + let mut collision_counter: HashMap = HashMap::new(); + + for event in self.contract.events() { + let name = handle_reserved_word(&event.name); + let counter = collision_counter.entry(name.clone()).or_insert(0); + let alias = if *counter == 0 { + name.clone() + } else { + format!("{}{}", name, counter) + }; + *counter += 1; + result.push((event, alias)); + } + + result + } + + /// Disambiguate functions. + fn disambiguate_functions<'a>( + &self, + functions: &[&'a Function], + ) -> Vec<(&'a Function, String)> { + let mut result = Vec::new(); + let mut collision_counter: HashMap = HashMap::new(); + + for func in functions { + let name = handle_reserved_word(&func.name); + let counter = collision_counter.entry(name.clone()).or_insert(0); + let alias = if *counter == 0 { + name.clone() + } else { + format!("{}{}", name, counter) + }; + *counter += 1; + result.push((*func, alias)); + } + + result + } + + /// Disambiguate call functions. + fn disambiguate_call_functions<'a>( + &self, + functions: &[&'a Function], + ) -> Vec<(&'a Function, String)> { + let mut result = Vec::new(); + let mut collision_counter: HashMap = HashMap::new(); + + for func in functions { + let name = if func.name.is_empty() { + "default".to_string() + } else { + handle_reserved_word(&func.name) + }; + let counter = collision_counter.entry(name.clone()).or_insert(0); + let alias = if *counter == 0 { + name.clone() + } else { + format!("{}{}", name, counter) + }; + *counter += 1; + result.push((*func, alias)); + } + + result + } + + /// Disambiguate event params. + fn disambiguate_event_params<'a>( + &self, + params: &'a [EventParam], + default_prefix: &str, + ) -> Vec<(&'a EventParam, String)> { + let mut result = Vec::new(); + let mut collision_counter: HashMap = HashMap::new(); + + for (index, param) in params.iter().enumerate() { + let name = if param.name.is_empty() { + format!("{}{}", default_prefix, index) + } else { + handle_reserved_word(¶m.name) + }; + let counter = collision_counter.entry(name.clone()).or_insert(0); + let disambiguated = if *counter == 0 { + name.clone() + } else { + format!("{}{}", name, counter) + }; + *counter += 1; + result.push((param, disambiguated)); + } + + result + } + + /// Disambiguate function params. + fn disambiguate_params<'a>( + &self, + params: &'a [Param], + default_prefix: &str, + ) -> Vec<(&'a Param, String)> { + let mut result = Vec::new(); + let mut collision_counter: HashMap = HashMap::new(); + + for (index, param) in params.iter().enumerate() { + let name = if param.name.is_empty() { + format!("{}{}", default_prefix, index) + } else { + handle_reserved_word(¶m.name) + }; + let counter = collision_counter.entry(name.clone()).or_insert(0); + let disambiguated = if *counter == 0 { + name.clone() + } else { + format!("{}{}", name, counter) + }; + *counter += 1; + result.push((param, disambiguated)); + } + + result + } +} + +/// Get tuple type name for a param. +fn tuple_type_name(index: usize, parent_type: &str) -> String { + format!("{}Value{}Struct", parent_type, index) +} + +/// Get the param type string for an input, handling tuples. +fn get_param_type_for_input(param_type: &DynSolType, index: usize, parent_type: &str) -> String { + if matches!(param_type, DynSolType::Tuple(_)) { + tuple_type_name(index, parent_type) + } else if is_tuple_matrix_type(param_type) { + let tn = tuple_type_name(index, parent_type); + format!("Array>", tn) + } else if is_tuple_array_type(param_type) { + let tn = tuple_type_name(index, parent_type); + format!("Array<{}>", tn) + } else { + asc_type_for_ethereum(param_type) + } +} + +/// Get the tuple components from a `Param`, following through arrays. +/// Returns the `components` from the innermost tuple `Param`. +fn get_tuple_param_components(param: &Param) -> &[Param] { + // If this param has components directly (tuple type), return them + if !param.components.is_empty() { + return ¶m.components; + } + // For array-of-tuple types, the components are on the param itself + // (alloy stores them on the outer param for tuple[] types) + &[] +} + +/// Disambiguate tuple components, reading names directly from `Param.name`. +fn disambiguate_tuple_components(components: &[Param]) -> Vec<(&Param, String)> { + components + .iter() + .enumerate() + .map(|(index, component)| { + let name = if component.name.is_empty() { + format!("value{}", index) + } else { + component.name.clone() + }; + (component, name) + }) + .collect() +} + +/// Get AssemblyScript type for an Ethereum type. +fn asc_type_for_ethereum(param_type: &DynSolType) -> String { + match param_type { + DynSolType::Address => "Address".to_string(), + DynSolType::Bool => "boolean".to_string(), + DynSolType::Bytes => "Bytes".to_string(), + DynSolType::FixedBytes(_) => "Bytes".to_string(), + DynSolType::Int(bits) => { + if *bits <= 32 { + "i32".to_string() + } else { + "BigInt".to_string() + } + } + DynSolType::Uint(bits) => { + if *bits <= 24 { + "i32".to_string() + } else { + "BigInt".to_string() + } + } + DynSolType::String => "string".to_string(), + DynSolType::Array(inner) => { + let inner_type = asc_type_for_ethereum(inner); + format!("Array<{}>", inner_type) + } + DynSolType::FixedArray(inner, _) => { + let inner_type = asc_type_for_ethereum(inner); + format!("Array<{}>", inner_type) + } + DynSolType::Tuple(_) => "ethereum.Tuple".to_string(), + _ => "ethereum.Tuple".to_string(), // Function and other future variants + } +} + +/// Convert ethereum value to AssemblyScript. +fn ethereum_to_asc(code: &str, param_type: &DynSolType, tuple_type: Option<&str>) -> String { + match param_type { + DynSolType::Address => format!("{}.toAddress()", code), + DynSolType::Bool => format!("{}.toBoolean()", code), + DynSolType::Bytes | DynSolType::FixedBytes(_) => format!("{}.toBytes()", code), + DynSolType::Int(bits) => { + if *bits <= 32 { + format!("{}.toI32()", code) + } else { + format!("{}.toBigInt()", code) + } + } + DynSolType::Uint(bits) => { + if *bits <= 24 { + format!("{}.toI32()", code) + } else { + format!("{}.toBigInt()", code) + } + } + DynSolType::String => format!("{}.toString()", code), + DynSolType::Array(inner) | DynSolType::FixedArray(inner, _) => match inner.as_ref() { + DynSolType::Address => format!("{}.toAddressArray()", code), + DynSolType::Bool => format!("{}.toBooleanArray()", code), + DynSolType::Bytes | DynSolType::FixedBytes(_) => { + format!("{}.toBytesArray()", code) + } + DynSolType::Int(bits) => { + if *bits <= 32 { + format!("{}.toI32Array()", code) + } else { + format!("{}.toBigIntArray()", code) + } + } + DynSolType::Uint(bits) => { + if *bits <= 24 { + format!("{}.toI32Array()", code) + } else { + format!("{}.toBigIntArray()", code) + } + } + DynSolType::String => format!("{}.toStringArray()", code), + DynSolType::Tuple(_) => { + if let Some(tuple_name) = tuple_type { + format!("{}.toTupleArray<{}>()", code, tuple_name) + } else { + format!("{}.toTupleArray()", code) + } + } + DynSolType::Array(inner2) | DynSolType::FixedArray(inner2, _) => { + ethereum_to_asc_matrix(code, inner2.as_ref(), tuple_type) + } + _ => format!("{}.toString()", code), // fallback for Function etc. + }, + DynSolType::Tuple(_) => format!("{}.toTuple()", code), + _ => format!("{}.toTuple()", code), // fallback for Function etc. + } +} + +/// Convert matrix type to AssemblyScript. +fn ethereum_to_asc_matrix(code: &str, inner_type: &DynSolType, tuple_type: Option<&str>) -> String { + match inner_type { + DynSolType::Address => format!("{}.toAddressMatrix()", code), + DynSolType::Bool => format!("{}.toBooleanMatrix()", code), + DynSolType::Bytes | DynSolType::FixedBytes(_) => format!("{}.toBytesMatrix()", code), + DynSolType::Int(bits) => { + if *bits <= 32 { + format!("{}.toI32Matrix()", code) + } else { + format!("{}.toBigIntMatrix()", code) + } + } + DynSolType::Uint(bits) => { + if *bits <= 24 { + format!("{}.toI32Matrix()", code) + } else { + format!("{}.toBigIntMatrix()", code) + } + } + DynSolType::String => format!("{}.toStringMatrix()", code), + DynSolType::Tuple(_) => { + if let Some(tuple_name) = tuple_type { + format!("{}.toTupleMatrix<{}>()", code, tuple_name) + } else { + format!("{}.toTupleMatrix()", code) + } + } + _ => format!("{}.toStringMatrix()", code), // fallback + } +} + +/// Convert AssemblyScript value to ethereum value. +fn ethereum_from_asc(code: &str, param_type: &DynSolType) -> String { + match param_type { + DynSolType::Address => format!("ethereum.Value.fromAddress({})", code), + DynSolType::Bool => format!("ethereum.Value.fromBoolean({})", code), + DynSolType::Bytes => format!("ethereum.Value.fromBytes({})", code), + DynSolType::FixedBytes(_) => format!("ethereum.Value.fromFixedBytes({})", code), + DynSolType::Int(bits) => { + if *bits <= 32 { + format!("ethereum.Value.fromI32({})", code) + } else { + format!("ethereum.Value.fromSignedBigInt({})", code) + } + } + DynSolType::Uint(bits) => { + if *bits <= 24 { + format!( + "ethereum.Value.fromUnsignedBigInt(BigInt.fromI32({}))", + code + ) + } else { + format!("ethereum.Value.fromUnsignedBigInt({})", code) + } + } + DynSolType::String => format!("ethereum.Value.fromString({})", code), + DynSolType::Array(inner) | DynSolType::FixedArray(inner, _) => { + ethereum_from_asc_array(code, inner.as_ref()) + } + DynSolType::Tuple(_) => format!("ethereum.Value.fromTuple({})", code), + _ => format!("ethereum.Value.fromTuple({})", code), // fallback + } +} + +/// Convert array to ethereum value. +fn ethereum_from_asc_array(code: &str, inner_type: &DynSolType) -> String { + match inner_type { + DynSolType::Address => format!("ethereum.Value.fromAddressArray({})", code), + DynSolType::Bool => format!("ethereum.Value.fromBooleanArray({})", code), + DynSolType::Bytes => format!("ethereum.Value.fromBytesArray({})", code), + DynSolType::FixedBytes(_) => format!("ethereum.Value.fromFixedBytesArray({})", code), + DynSolType::Int(bits) => { + if *bits <= 32 { + format!("ethereum.Value.fromI32Array({})", code) + } else { + format!("ethereum.Value.fromSignedBigIntArray({})", code) + } + } + DynSolType::Uint(bits) => { + if *bits <= 24 { + format!("ethereum.Value.fromI32Array({})", code) + } else { + format!("ethereum.Value.fromUnsignedBigIntArray({})", code) + } + } + DynSolType::String => format!("ethereum.Value.fromStringArray({})", code), + DynSolType::Tuple(_) => format!("ethereum.Value.fromTupleArray({})", code), + DynSolType::Array(inner2) | DynSolType::FixedArray(inner2, _) => { + ethereum_from_asc_matrix(code, inner2.as_ref()) + } + _ => format!("ethereum.Value.fromStringArray({})", code), // fallback + } +} + +/// Convert matrix to ethereum value. +fn ethereum_from_asc_matrix(code: &str, inner_type: &DynSolType) -> String { + match inner_type { + DynSolType::Address => format!("ethereum.Value.fromAddressMatrix({})", code), + DynSolType::Bool => format!("ethereum.Value.fromBooleanMatrix({})", code), + DynSolType::Bytes => format!("ethereum.Value.fromBytesMatrix({})", code), + DynSolType::FixedBytes(_) => format!("ethereum.Value.fromFixedBytesMatrix({})", code), + DynSolType::Int(bits) => { + if *bits <= 32 { + format!("ethereum.Value.fromI32Matrix({})", code) + } else { + format!("ethereum.Value.fromSignedBigIntMatrix({})", code) + } + } + DynSolType::Uint(bits) => { + if *bits <= 24 { + format!("ethereum.Value.fromI32Matrix({})", code) + } else { + format!("ethereum.Value.fromUnsignedBigIntMatrix({})", code) + } + } + DynSolType::String => format!("ethereum.Value.fromStringMatrix({})", code), + DynSolType::Tuple(_) => format!("ethereum.Value.fromTupleMatrix({})", code), + _ => format!("ethereum.Value.fromStringMatrix({})", code), // fallback + } +} + +/// Check if param type contains a tuple. +fn contains_tuple_type(param_type: &DynSolType) -> bool { + match param_type { + DynSolType::Tuple(_) => true, + DynSolType::Array(inner) | DynSolType::FixedArray(inner, _) => contains_tuple_type(inner), + _ => false, + } +} + +/// Check if param type is a tuple array. +fn is_tuple_array_type(param_type: &DynSolType) -> bool { + matches!( + param_type, + DynSolType::Array(inner) | DynSolType::FixedArray(inner, _) + if matches!(inner.as_ref(), DynSolType::Tuple(_)) + ) +} + +/// Check if param type is a tuple matrix (2D array). +fn is_tuple_matrix_type(param_type: &DynSolType) -> bool { + match param_type { + DynSolType::Array(inner) | DynSolType::FixedArray(inner, _) => is_tuple_array_type(inner), + _ => false, + } +} + +/// Handle indexed input type conversion. +fn indexed_input_type(param_type: &DynSolType) -> DynSolType { + // Strings, bytes, and arrays are encoded and hashed to bytes32 + match param_type { + DynSolType::String | DynSolType::Bytes | DynSolType::Tuple(_) => DynSolType::FixedBytes(32), + DynSolType::Array(_) | DynSolType::FixedArray(_, _) => DynSolType::FixedBytes(32), + _ => param_type.clone(), + } +} + +#[cfg(test)] +mod tests { + + use super::*; + + fn parse_abi(json: &str) -> JsonAbi { + serde_json::from_str(json).unwrap() + } + + #[test] + fn test_simple_event() { + let abi_json = r#"[ + { + "type": "event", + "name": "Transfer", + "inputs": [ + {"name": "from", "type": "address", "indexed": true}, + {"name": "to", "type": "address", "indexed": true}, + {"name": "value", "type": "uint256", "indexed": false} + ], + "anonymous": false + } + ]"#; + + let contract = parse_abi(abi_json); + let generator = AbiCodeGenerator::new(contract, "Token"); + let types = generator.generate_types(); + + assert!(types.iter().any(|c| c.name == "Transfer")); + assert!(types.iter().any(|c| c.name == "Transfer__Params")); + } + + #[test] + fn test_function_with_outputs() { + let abi_json = r#"[ + { + "type": "function", + "name": "balanceOf", + "inputs": [{"name": "owner", "type": "address"}], + "outputs": [{"name": "", "type": "uint256"}], + "stateMutability": "view" + } + ]"#; + + let contract = parse_abi(abi_json); + let generator = AbiCodeGenerator::new(contract, "Token"); + let types = generator.generate_types(); + + assert!(types.iter().any(|c| c.name == "Token")); + let token_class = types.iter().find(|c| c.name == "Token").unwrap(); + + assert!(token_class.methods.iter().any(|m| m.name == "balanceOf")); + assert!( + token_class + .methods + .iter() + .any(|m| m.name == "try_balanceOf") + ); + } + + #[test] + fn test_asc_type_for_ethereum() { + assert_eq!(asc_type_for_ethereum(&DynSolType::Address), "Address"); + assert_eq!(asc_type_for_ethereum(&DynSolType::Bool), "boolean"); + assert_eq!(asc_type_for_ethereum(&DynSolType::Uint(256)), "BigInt"); + assert_eq!(asc_type_for_ethereum(&DynSolType::Uint(8)), "i32"); + assert_eq!(asc_type_for_ethereum(&DynSolType::Int(32)), "i32"); + assert_eq!(asc_type_for_ethereum(&DynSolType::String), "string"); + assert_eq!(asc_type_for_ethereum(&DynSolType::Bytes), "Bytes"); + } + + #[test] + fn test_name_sanitization() { + let generator = AbiCodeGenerator::new(JsonAbi::default(), "Test!Contract@Name"); + assert_eq!(generator.name, "Test_Contract_Name"); + } + + #[test] + fn test_indexed_input_type() { + assert_eq!( + indexed_input_type(&DynSolType::String), + DynSolType::FixedBytes(32) + ); + assert_eq!( + indexed_input_type(&DynSolType::Bytes), + DynSolType::FixedBytes(32) + ); + assert_eq!( + indexed_input_type(&DynSolType::Array(Box::new(DynSolType::Uint(256)))), + DynSolType::FixedBytes(32) + ); + assert_eq!( + indexed_input_type(&DynSolType::Address), + DynSolType::Address + ); + assert_eq!( + indexed_input_type(&DynSolType::Uint(256)), + DynSolType::Uint(256) + ); + } + + /// Test that overloaded events (same name, different inputs) are disambiguated. + /// The TS CLI generates unique names like Transfer, Transfer1, Transfer2. + #[test] + fn test_overloaded_events() { + let abi_json = r#"[ + { + "type": "event", + "name": "Transfer", + "inputs": [], + "anonymous": false + }, + { + "type": "event", + "name": "Transfer", + "inputs": [{"name": "to", "type": "address", "indexed": false}], + "anonymous": false + }, + { + "type": "event", + "name": "Transfer", + "inputs": [ + {"name": "from", "type": "address", "indexed": false}, + {"name": "to", "type": "address", "indexed": false} + ], + "anonymous": false + } + ]"#; + + let contract = parse_abi(abi_json); + let generator = AbiCodeGenerator::new(contract, "Token"); + let types = generator.generate_types(); + + // Get all event class names + let event_names: Vec<&str> = types + .iter() + .filter(|c| c.extends == Some("ethereum.Event".to_string())) + .map(|c| c.name.as_str()) + .collect(); + + // Verify we have 3 distinct Transfer events with disambiguation + assert_eq!( + event_names.len(), + 3, + "Should have 3 Transfer event variants" + ); + + // Check that Transfer (with no suffix) exists + assert!( + event_names.contains(&"Transfer"), + "Should have base Transfer event" + ); + + // Check that numbered variants exist (Transfer1, Transfer2) + assert!( + event_names.contains(&"Transfer1"), + "Should have Transfer1 event" + ); + assert!( + event_names.contains(&"Transfer2"), + "Should have Transfer2 event" + ); + } + + /// Test that overloaded functions are disambiguated. + #[test] + fn test_overloaded_functions() { + let abi_json = r#"[ + { + "type": "function", + "name": "getSomething", + "inputs": [], + "outputs": [{"name": "result", "type": "bytes32"}], + "stateMutability": "view" + }, + { + "type": "function", + "name": "getSomething", + "inputs": [{"name": "owner", "type": "address"}], + "outputs": [{"name": "result", "type": "bytes32"}], + "stateMutability": "view" + } + ]"#; + + let contract = parse_abi(abi_json); + let generator = AbiCodeGenerator::new(contract, "Token"); + let types = generator.generate_types(); + + // Find the Token contract class + let token_class = types.iter().find(|c| c.name == "Token").unwrap(); + + // Get all method names (excluding try_ variants and bind) + let method_names: Vec<&str> = token_class + .methods + .iter() + .map(|m| m.name.as_str()) + .filter(|n| !n.starts_with("try_") && *n != "bind") + .collect(); + + // Verify we have disambiguated getSomething variants + assert!( + method_names.contains(&"getSomething"), + "Should have base getSomething method" + ); + assert!( + method_names.contains(&"getSomething1"), + "Should have getSomething1 method" + ); + } + + /// Test that tuple/struct types in function inputs and outputs are handled correctly. + #[test] + fn test_tuple_types_in_functions() { + let abi_json = r#"[ + { + "type": "function", + "name": "doSomething", + "inputs": [ + { + "name": "data", + "type": "tuple", + "components": [ + {"name": "owner", "type": "address"}, + {"name": "value", "type": "uint256"} + ] + } + ], + "outputs": [ + { + "name": "result", + "type": "tuple", + "components": [ + {"name": "success", "type": "bool"}, + {"name": "newValue", "type": "uint256"} + ] + } + ], + "stateMutability": "nonpayable" + } + ]"#; + + let contract = parse_abi(abi_json); + let generator = AbiCodeGenerator::new(contract, "TestContract"); + let types = generator.generate_types(); + + // Get class names + let class_names: Vec<&str> = types.iter().map(|c| c.name.as_str()).collect(); + + // Should have main contract class + assert!( + class_names.contains(&"TestContract"), + "Should have TestContract class" + ); + + // Should have struct classes for input and output tuples + // Input struct: TestContract__doSomethingInputValue0Struct + // Output struct: TestContract__doSomethingResultValue0Struct + let has_input_struct = class_names + .iter() + .any(|n| n.contains("Input") && n.contains("Struct")); + let has_output_struct = class_names + .iter() + .any(|n| n.contains("Result") && n.contains("Struct")); + + assert!( + has_input_struct, + "Should have input struct class, found: {:?}", + class_names + ); + assert!( + has_output_struct, + "Should have output/result struct class, found: {:?}", + class_names + ); + + // Verify the output struct extends ethereum.Tuple + let output_struct = types + .iter() + .find(|c| c.name.contains("Result") && c.name.contains("Struct")) + .expect("Should find output struct"); + assert_eq!( + output_struct.extends, + Some("ethereum.Tuple".to_string()), + "Output struct should extend ethereum.Tuple" + ); + + // Verify the output struct has getters for its components + // With alloy, component names are preserved from the ABI + let method_names: Vec<&str> = output_struct + .methods + .iter() + .map(|m| m.name.as_str()) + .collect(); + assert!( + method_names.iter().any(|n| n.contains("success")), + "Should have success getter for first component, found methods: {:?}", + method_names + ); + assert!( + method_names.iter().any(|n| n.contains("newValue")), + "Should have newValue getter for second component, found methods: {:?}", + method_names + ); + } + + /// Test that tuple types in events are handled correctly. + #[test] + fn test_tuple_types_in_events() { + let abi_json = r#"[ + { + "type": "event", + "name": "DataUpdated", + "inputs": [ + {"name": "id", "type": "uint256", "indexed": true}, + { + "name": "data", + "type": "tuple", + "indexed": false, + "components": [ + {"name": "timestamp", "type": "uint256"}, + {"name": "value", "type": "bytes32"} + ] + } + ], + "anonymous": false + } + ]"#; + + let contract = parse_abi(abi_json); + let generator = AbiCodeGenerator::new(contract, "TestContract"); + let types = generator.generate_types(); + + // Get class names + let class_names: Vec<&str> = types.iter().map(|c| c.name.as_str()).collect(); + + // Should have event class + assert!( + class_names.contains(&"DataUpdated"), + "Should have DataUpdated event class" + ); + + // Should have params class + assert!( + class_names.contains(&"DataUpdated__Params"), + "Should have DataUpdated__Params class" + ); + + // Should have struct class for the tuple parameter + let has_struct = class_names + .iter() + .any(|n| n.contains("Struct") && n.contains("DataUpdated")); + assert!( + has_struct, + "Should have struct class for tuple parameter, found: {:?}", + class_names + ); + } + + /// Test that nested tuple types (struct with struct field) are handled. + #[test] + fn test_nested_tuple_types() { + let abi_json = r#"[ + { + "type": "function", + "name": "getNestedData", + "inputs": [], + "outputs": [ + { + "name": "result", + "type": "tuple", + "components": [ + {"name": "id", "type": "uint256"}, + { + "name": "inner", + "type": "tuple", + "components": [ + {"name": "x", "type": "uint256"}, + {"name": "y", "type": "uint256"} + ] + } + ] + } + ], + "stateMutability": "view" + } + ]"#; + + let contract = parse_abi(abi_json); + let generator = AbiCodeGenerator::new(contract, "TestContract"); + let types = generator.generate_types(); + + // Get class names + let class_names: Vec<&str> = types.iter().map(|c| c.name.as_str()).collect(); + + // Should have main contract class + assert!( + class_names.contains(&"TestContract"), + "Should have TestContract class" + ); + + // Should have at least two struct classes (outer and inner) + let struct_count = class_names.iter().filter(|n| n.contains("Struct")).count(); + assert!( + struct_count >= 2, + "Should have at least 2 struct classes for nested tuple, found {} in {:?}", + struct_count, + class_names + ); + } + + /// Test that tuple arrays are handled correctly. + #[test] + fn test_tuple_array_types() { + let abi_json = r#"[ + { + "type": "function", + "name": "getAllItems", + "inputs": [], + "outputs": [ + { + "name": "items", + "type": "tuple[]", + "components": [ + {"name": "id", "type": "uint256"}, + {"name": "name", "type": "string"} + ] + } + ], + "stateMutability": "view" + } + ]"#; + + let contract = parse_abi(abi_json); + let generator = AbiCodeGenerator::new(contract, "TestContract"); + let types = generator.generate_types(); + + // Find the contract class + let contract_class = types.iter().find(|c| c.name == "TestContract").unwrap(); + + // Find the getAllItems method + let method = contract_class + .methods + .iter() + .find(|m| m.name == "getAllItems") + .expect("Should have getAllItems method"); + + // The return type should be an Array of the struct type + let return_type = method + .return_type + .as_ref() + .expect("Should have return type"); + let return_type_str = return_type.to_string(); + assert!( + return_type_str.starts_with("Array<"), + "Return type should be Array<...>, got: {}", + return_type_str + ); + } + + /// Test that array types in events are handled correctly. + #[test] + fn test_array_types_in_events() { + let abi_json = r#"[ + { + "type": "event", + "name": "Airdropped", + "inputs": [ + {"name": "sender", "type": "address", "indexed": true}, + {"name": "recipients", "type": "address[]", "indexed": false}, + {"name": "amounts", "type": "uint256[]", "indexed": false} + ], + "anonymous": false + } + ]"#; + + let contract = parse_abi(abi_json); + let generator = AbiCodeGenerator::new(contract, "Token"); + let types = generator.generate_types(); + + // Find the params class + let params_class = types + .iter() + .find(|c| c.name == "Airdropped__Params") + .expect("Should have Airdropped__Params class"); + + // Check that the array getters exist and have correct return types + let method_names: Vec<&str> = params_class + .methods + .iter() + .map(|m| m.name.as_str()) + .collect(); + + assert!( + method_names.iter().any(|n| n.contains("recipients")), + "Should have recipients getter, found: {:?}", + method_names + ); + assert!( + method_names.iter().any(|n| n.contains("amounts")), + "Should have amounts getter, found: {:?}", + method_names + ); + + // Verify the recipients getter returns Array
+ let recipients_getter = params_class + .methods + .iter() + .find(|m| m.name.contains("recipients")) + .expect("Should have recipients getter"); + let return_type_str = recipients_getter + .return_type + .as_ref() + .expect("Should have return type") + .to_string(); + assert!( + return_type_str.contains("Array
"), + "recipients should return Array
, got: {}", + return_type_str + ); + + // Verify the amounts getter returns Array + let amounts_getter = params_class + .methods + .iter() + .find(|m| m.name.contains("amounts")) + .expect("Should have amounts getter"); + let amounts_type_str = amounts_getter + .return_type + .as_ref() + .expect("Should have return type") + .to_string(); + assert!( + amounts_type_str.contains("Array"), + "amounts should return Array, got: {}", + amounts_type_str + ); + } + + /// Test that 2D array types (matrices) are handled correctly in functions. + #[test] + fn test_matrix_types_in_functions() { + let abi_json = r#"[ + { + "type": "function", + "name": "getMatrix", + "inputs": [], + "outputs": [ + {"name": "data", "type": "uint256[][]"} + ], + "stateMutability": "view" + } + ]"#; + + let contract = parse_abi(abi_json); + let generator = AbiCodeGenerator::new(contract, "TestContract"); + let types = generator.generate_types(); + + // Find the contract class + let contract_class = types.iter().find(|c| c.name == "TestContract").unwrap(); + + // Find the getMatrix method + let method = contract_class + .methods + .iter() + .find(|m| m.name == "getMatrix") + .expect("Should have getMatrix method"); + + // The return type should be Array> + let return_type = method + .return_type + .as_ref() + .expect("Should have return type"); + let return_type_str = return_type.to_string(); + assert!( + return_type_str.contains("Array>"), + "Return type should be Array>, got: {}", + return_type_str + ); + } +} diff --git a/gnd/src/codegen/mod.rs b/gnd/src/codegen/mod.rs new file mode 100644 index 00000000000..4c8f5fdbd98 --- /dev/null +++ b/gnd/src/codegen/mod.rs @@ -0,0 +1,20 @@ +//! Code generation for subgraph AssemblyScript types. +//! +//! This module generates AssemblyScript types from: +//! - GraphQL schema (entity classes) +//! - Contract ABIs (event and call bindings) +//! - Data source templates + +mod abi; +mod schema; +mod template; +mod types; +mod typescript; + +pub use abi::AbiCodeGenerator; +pub use schema::SchemaCodeGenerator; +pub use template::{Template, TemplateCodeGenerator, TemplateKind}; +pub use typescript::{ + ArrayType, Class, ClassMember, GENERATED_FILE_NOTE, Method, ModuleImports, NamedType, + NullableType, Param, StaticMethod, +}; diff --git a/gnd/src/codegen/schema.rs b/gnd/src/codegen/schema.rs new file mode 100644 index 00000000000..a8a276f1ca0 --- /dev/null +++ b/gnd/src/codegen/schema.rs @@ -0,0 +1,1289 @@ +//! Schema code generation. +//! +//! Generates AssemblyScript entity classes from GraphQL schemas. + +use anyhow::{Result, anyhow}; +use graphql_tools::parser::schema::{ + Definition, Document, Field, ObjectType, Type, TypeDefinition, +}; + +use super::types::{asc_type_for_value, value_from_asc, value_to_asc}; +use super::typescript::{ + self as ts, ArrayType, Class, Method, ModuleImports, NamedType, NullableType, Param, + StaticMethod, TypeExpr, +}; +use crate::shared::handle_reserved_word; + +/// Type of the ID field. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum IdFieldKind { + String, + Bytes, + Int8, +} + +impl IdFieldKind { + /// Get the AssemblyScript type name for this ID type. + pub fn type_name(&self) -> &'static str { + match self { + IdFieldKind::String => "string", + IdFieldKind::Bytes => "Bytes", + IdFieldKind::Int8 => "i64", + } + } + + /// Get the GraphQL type name for this ID type. + pub fn gql_type_name(&self) -> &'static str { + match self { + IdFieldKind::String => "String", + IdFieldKind::Bytes => "Bytes", + IdFieldKind::Int8 => "Int8", + } + } + + /// Get the code to create a Value from the ID. + pub fn value_from(&self) -> &'static str { + match self { + IdFieldKind::String => "Value.fromString(id)", + IdFieldKind::Bytes => "Value.fromBytes(id)", + IdFieldKind::Int8 => "Value.fromI64(id)", + } + } + + /// Get the ValueKind for this ID type. + pub fn value_kind(&self) -> &'static str { + match self { + IdFieldKind::String => "ValueKind.STRING", + IdFieldKind::Bytes => "ValueKind.BYTES", + IdFieldKind::Int8 => "ValueKind.INT8", + } + } + + /// Get the code to convert a Value to a string representation. + pub fn value_to_string(&self) -> &'static str { + match self { + IdFieldKind::String => "id.toString()", + IdFieldKind::Bytes => "id.toBytes().toHexString()", + IdFieldKind::Int8 => "id.toI64().toString()", + } + } + + /// Get the code to convert the ID to a string. + pub fn id_to_string_code(self) -> &'static str { + match self { + IdFieldKind::String => "id", + IdFieldKind::Bytes => "id.toHexString()", + IdFieldKind::Int8 => "id.toString()", + } + } + + /// Determine the ID field kind from a type name. + pub fn from_type_name(type_name: &str) -> Self { + match type_name { + "Bytes" => IdFieldKind::Bytes, + "Int8" => IdFieldKind::Int8, + _ => IdFieldKind::String, + } + } + + /// Returns true if this ID type supports auto-increment (optional constructor arg). + pub fn supports_auto(&self) -> bool { + matches!(self, IdFieldKind::Int8 | IdFieldKind::Bytes) + } + + /// Get the constructor parameter type (with nullability for auto-increment types). + pub fn constructor_param_type(&self) -> &'static str { + match self { + IdFieldKind::String => "string", + IdFieldKind::Bytes => "Bytes | null", + IdFieldKind::Int8 => "i64", // primitive, can't be nullable + } + } + + /// Get the default value for constructor parameter (for auto-increment types). + pub fn constructor_default(&self) -> Option<&'static str> { + match self { + IdFieldKind::String => None, + IdFieldKind::Bytes => Some("null"), + IdFieldKind::Int8 => Some("i64.MIN_VALUE"), + } + } + + /// Get the condition to check if id was provided (not auto-increment). + pub fn auto_check_condition(&self) -> &'static str { + match self { + IdFieldKind::String => "", // N/A + // For Bytes we end up generating 'if (id) { .. }' Using 'id != + // null' crashes the AssemblyScript compiler 0.19.23 because id + // has type 'Bytes | null' and the compiler doesn't strip the + // null from the type in the body of the if + IdFieldKind::Bytes => "id", + IdFieldKind::Int8 => "id != i64.MIN_VALUE", + } + } +} + +/// Get the base type name from a GraphQL type (stripping NonNull and List wrappers). +fn get_base_type_name(ty: &Type<'_, String>) -> String { + match ty { + Type::NamedType(name) => name.clone(), + Type::NonNullType(inner) => get_base_type_name(inner), + Type::ListType(inner) => get_base_type_name(inner), + } +} + +/// Check if a type is nullable (not wrapped in NonNull). +fn is_nullable(ty: &Type<'_, String>) -> bool { + !matches!(ty, Type::NonNullType(_)) +} + +/// Count the list nesting depth of a type. +/// `String` -> 0, `[String]` -> 1, `[[String]]` -> 2, etc. +fn list_depth(ty: &Type<'_, String>) -> u8 { + match ty { + Type::ListType(inner) => 1 + list_depth(inner), + Type::NonNullType(inner) => list_depth(inner), + Type::NamedType(_) => 0, + } +} + +/// Check if the innermost list members are nullable. +/// For `[String]` returns true, for `[String!]` returns false. +/// For non-list types, returns false. +fn is_list_member_nullable(ty: &Type<'_, String>) -> bool { + match ty { + Type::ListType(inner) => { + // Check the immediate inner type + is_nullable(inner) + } + Type::NonNullType(inner) => is_list_member_nullable(inner), + Type::NamedType(_) => false, + } +} + +/// Check if a field has the @derivedFrom directive. +fn is_derived_field(field: &Field<'_, String>) -> bool { + field.directives.iter().any(|d| d.name == "derivedFrom") +} + +/// Check if an object type has the @entity directive. +fn is_entity_type(obj: &ObjectType<'_, String>) -> bool { + obj.directives.iter().any(|d| d.name == "entity") +} + +/// Collected entity info for code generation. +struct EntityInfo { + name: String, + id_kind: IdFieldKind, + fields: Vec, +} + +/// Collected field info. +struct FieldInfo { + name: String, + is_derived: bool, + base_type: String, + is_nullable: bool, + /// The nesting depth of list wrappers. 0 = scalar, 1 = [T], 2 = [[T]], etc. + list_depth: u8, + /// Whether list members are nullable. Only meaningful when list_depth > 0. + member_nullable: bool, +} + +/// Schema code generator. +pub struct SchemaCodeGenerator { + entities: Vec, + entity_names: std::collections::HashSet, + /// Maps entity name to its ID field kind, for resolving entity reference types. + entity_id_kinds: std::collections::HashMap, +} + +impl SchemaCodeGenerator { + /// Create a new schema code generator from a parsed GraphQL document. + /// + /// Returns an error if the schema contains invalid patterns like non-nullable + /// lists with nullable members (e.g., `[Something]!`). + pub fn new(document: &Document<'_, String>) -> Result { + let mut entities = Vec::new(); + let mut entity_names = std::collections::HashSet::new(); + let mut entity_id_kinds = std::collections::HashMap::new(); + + // First pass: collect entity names and their ID types + for def in &document.definitions { + if let Definition::TypeDefinition(TypeDefinition::Object(obj)) = def + && is_entity_type(obj) + { + entity_names.insert(obj.name.clone()); + let id_field = obj.fields.iter().find(|f| f.name == "id"); + let id_kind = id_field + .map(|f| IdFieldKind::from_type_name(&get_base_type_name(&f.field_type))) + .unwrap_or(IdFieldKind::String); + entity_id_kinds.insert(obj.name.clone(), id_kind); + } + } + + // Second pass: collect entity info + for def in &document.definitions { + if let Definition::TypeDefinition(TypeDefinition::Object(obj)) = def + && is_entity_type(obj) + { + let name = obj.name.clone(); + + // Find ID field + let id_field = obj.fields.iter().find(|f| f.name == "id"); + let id_kind = id_field + .map(|f| IdFieldKind::from_type_name(&get_base_type_name(&f.field_type))) + .unwrap_or(IdFieldKind::String); + + // Collect field info + let fields: Vec<_> = obj + .fields + .iter() + .map(|f| FieldInfo { + name: f.name.clone(), + is_derived: is_derived_field(f), + base_type: get_base_type_name(&f.field_type), + is_nullable: is_nullable(&f.field_type), + list_depth: list_depth(&f.field_type), + member_nullable: is_list_member_nullable(&f.field_type), + }) + .collect(); + + entities.push(EntityInfo { + name, + id_kind, + fields, + }); + } + } + + // Validate: non-nullable lists must have non-nullable members + for entity in &entities { + for field in &entity.fields { + if field.list_depth > 0 && !field.is_nullable && field.member_nullable { + return Err(anyhow!( + "Codegen can't generate code for GraphQL field '{}' of type '[{}]!' since the inner type is nullable.\n\ + Suggestion: add an '!' to the inner type, e.g., '[{}!]!'", + field.name, + field.base_type, + field.base_type + )); + } + } + } + + Ok(Self { + entities, + entity_names, + entity_id_kinds, + }) + } + + /// Generate module imports for the schema file. + pub fn generate_module_imports(&self) -> Vec { + vec![ModuleImports::new( + vec![ + "TypedMap".to_string(), + "Entity".to_string(), + "Value".to_string(), + "ValueKind".to_string(), + "store".to_string(), + "Bytes".to_string(), + "BigInt".to_string(), + "BigDecimal".to_string(), + "Int8".to_string(), + ], + "@graphprotocol/graph-ts", + )] + } + + /// Generate entity classes from the schema. + pub fn generate_types(&self, generate_store_methods: bool) -> Vec { + self.entities + .iter() + .map(|entity| self.generate_entity_type(entity, generate_store_methods)) + .collect() + } + + /// Generate derived loaders for fields with @derivedFrom. + pub fn generate_derived_loaders(&self) -> Vec { + let mut loaders = Vec::new(); + let mut seen_types = std::collections::HashSet::new(); + + for entity in &self.entities { + for field in &entity.fields { + if field.is_derived && !seen_types.contains(&field.base_type) { + // Only generate loaders for entity types, not interfaces + if self.entity_names.contains(&field.base_type) { + seen_types.insert(field.base_type.clone()); + loaders.push(self.generate_derived_loader(&field.base_type)); + } + } + } + } + + loaders + } + + fn generate_entity_type(&self, entity: &EntityInfo, generate_store_methods: bool) -> Class { + let mut klass = ts::klass(&entity.name).exported().extends("Entity"); + + // Generate constructor + klass.add_method(self.generate_constructor(&entity.id_kind)); + + // Generate store methods + if generate_store_methods { + for method in self.generate_store_methods(&entity.name, &entity.id_kind) { + match method { + StoreMethod::Regular(m) => klass.add_method(m), + StoreMethod::Static(m) => klass.add_static_method(m), + } + } + } + + // Generate field getters and setters + for field in &entity.fields { + if let Some(getter) = self.generate_field_getter(&entity.name, field, &entity.id_kind) { + klass.add_method(getter); + } + if let Some(setter) = self.generate_field_setter(field) { + klass.add_method(setter); + } + } + + klass + } + + fn generate_constructor(&self, id_kind: &IdFieldKind) -> Method { + if id_kind.supports_auto() { + // For Int8 and Bytes, make id optional to support auto-increment + let param_type = id_kind.constructor_param_type(); + let default_value = id_kind.constructor_default().unwrap(); + let check_condition = id_kind.auto_check_condition(); + + Method::new( + "constructor", + vec![Param::with_default( + "id", + TypeExpr::Raw(param_type.to_string()), + default_value, + )], + None, + format!( + r#" + super() + if ({}) {{ + this.set('id', {}) + }}"#, + check_condition, + id_kind.value_from() + ), + ) + .with_doc("Leaving out the id argument uses an autoincrementing id.") + } else { + // For String IDs, keep the existing behavior (required parameter) + Method::new( + "constructor", + vec![Param::new("id", NamedType::new(id_kind.type_name()))], + None, + format!( + r#" + super() + this.set('id', {})"#, + id_kind.value_from() + ), + ) + } + } + + fn generate_store_methods(&self, entity_name: &str, id_kind: &IdFieldKind) -> Vec { + // Generate save() method - different for auto-increment vs string IDs + let save_method = if id_kind.supports_auto() { + // For Int8 and Bytes, check if id is null/unset and use "auto" as the key + Method::new( + "save", + vec![], + Some(NamedType::new("void").into()), + format!( + r#" + let id = this.get('id') + if (id == null || id.kind == ValueKind.NULL) {{ + store.set('{}', 'auto', this) + }} else {{ + assert(id.kind == {}, + `Entities of type {} must have an ID of type {} but the id '${{id.displayData()}}' is of type ${{id.displayKind()}}`) + store.set('{}', {}, this) + }}"#, + entity_name, + id_kind.value_kind(), + entity_name, + id_kind.gql_type_name(), + entity_name, + id_kind.value_to_string() + ), + ) + } else { + // For String IDs, keep the existing behavior (require ID) + Method::new( + "save", + vec![], + Some(NamedType::new("void").into()), + format!( + r#" + let id = this.get('id') + assert(id != null, + 'Cannot save {} entity without an ID') + if (id) {{ + assert(id.kind == {}, + `Entities of type {} must have an ID of type {} but the id '${{id.displayData()}}' is of type ${{id.displayKind()}}`) + store.set('{}', {}, this) + }}"#, + entity_name, + id_kind.value_kind(), + entity_name, + id_kind.gql_type_name(), + entity_name, + id_kind.value_to_string() + ), + ) + }; + + vec![ + StoreMethod::Regular(save_method), + // loadInBlock() static method + StoreMethod::Static(StaticMethod::new( + "loadInBlock", + vec![Param::new("id", NamedType::new(id_kind.type_name()))], + NullableType::new(NamedType::new(entity_name)), + format!( + r#" + return changetype<{} | null>(store.get_in_block('{}', {}))"#, + entity_name, + entity_name, + id_kind.id_to_string_code() + ), + )), + // load() static method + StoreMethod::Static(StaticMethod::new( + "load", + vec![Param::new("id", NamedType::new(id_kind.type_name()))], + NullableType::new(NamedType::new(entity_name)), + format!( + r#" + return changetype<{} | null>(store.get('{}', {}))"#, + entity_name, + entity_name, + id_kind.id_to_string_code() + ), + )), + ] + } + + fn generate_field_getter( + &self, + entity_name: &str, + field: &FieldInfo, + id_kind: &IdFieldKind, + ) -> Option { + let safe_name = handle_reserved_word(&field.name); + + // Handle derived fields + if field.is_derived { + return self.generate_derived_field_getter(entity_name, field, &safe_name, id_kind); + } + + let value_type = self.value_type_from_field(field); + let return_type = self.type_from_field(field); + let nullable = field.is_nullable; + + let primitive_default = match &return_type { + TypeExpr::Named(t) => t.get_primitive_default(), + _ => None, + }; + + let get_code = if nullable { + format!( + r#" + let value = this.get('{}') + if (!value || value.kind == ValueKind.NULL) {{ + return null + }} else {{ + return {} + }}"#, + field.name, + value_to_asc("value", &value_type) + ) + } else { + let null_handling = match primitive_default { + Some(default) => format!("return {}", default), + None => "throw new Error('Cannot return null for a required field.')".to_string(), + }; + format!( + r#" + let value = this.get('{}') + if (!value || value.kind == ValueKind.NULL) {{ + {} + }} else {{ + return {} + }}"#, + field.name, + null_handling, + value_to_asc("value", &value_type) + ) + }; + + Some(Method::new( + format!("get {}", safe_name), + vec![], + Some(return_type), + get_code, + )) + } + + fn generate_derived_field_getter( + &self, + entity_name: &str, + field: &FieldInfo, + safe_name: &str, + id_kind: &IdFieldKind, + ) -> Option { + let loader_name = format!("{}Loader", field.base_type); + + let id_conversion = match id_kind { + IdFieldKind::Bytes => "this.get('id')!.toBytes().toHexString()", + _ => "this.get('id')!.toString()", + }; + + Some(Method::new( + format!("get {}", safe_name), + vec![], + Some(NamedType::new(&loader_name).into()), + format!( + r#" + return new {}('{}', {}, '{}')"#, + loader_name, entity_name, id_conversion, field.name + ), + )) + } + + fn generate_field_setter(&self, field: &FieldInfo) -> Option { + // No setters for derived fields + if field.is_derived { + return None; + } + + let safe_name = handle_reserved_word(&field.name); + let value_type = self.value_type_from_field(field); + let param_type = self.type_from_field(field); + let nullable = field.is_nullable; + + let set_code = if nullable { + let inner_type = match ¶m_type { + TypeExpr::Nullable(n) => n.inner.to_string(), + other => other.to_string(), + }; + format!( + r#" + if (!value) {{ + this.unset('{}') + }} else {{ + this.set('{}', {}) + }}"#, + field.name, + field.name, + value_from_asc(&format!("<{}>value", inner_type), &value_type) + ) + } else { + format!( + r#" + this.set('{}', {})"#, + field.name, + value_from_asc("value", &value_type) + ) + }; + + Some(Method::new( + format!("set {}", safe_name), + vec![Param::new("value", param_type)], + None, + set_code, + )) + } + + fn generate_derived_loader(&self, type_name: &str) -> Class { + let loader_name = format!("{}Loader", type_name); + let mut klass = ts::klass(&loader_name).exported().extends("Entity"); + + // Add members + klass.add_member(ts::klass_member("_entity", "string")); + klass.add_member(ts::klass_member("_field", "string")); + klass.add_member(ts::klass_member("_id", "string")); + + // Add constructor + klass.add_method(Method::new( + "constructor", + vec![ + Param::new("entity", NamedType::new("string")), + Param::new("id", NamedType::new("string")), + Param::new("field", NamedType::new("string")), + ], + None, + r#" + super(); + this._entity = entity; + this._id = id; + this._field = field;"# + .to_string(), + )); + + // Add load() method + klass.add_method(Method::new( + "load", + vec![], + Some(TypeExpr::Raw(format!("{}[]", type_name))), + format!( + r#" + let value = store.loadRelated(this._entity, this._id, this._field); + return changetype<{}[]>(value);"#, + type_name + ), + )); + + klass + } + + /// Get the value type string for a field. + /// + /// Returns the GraphQL-style value type string: + /// - Scalars: `String`, `Int`, `BigInt`, etc. + /// - Arrays: `[String]`, `[Int]`, etc. + /// - Nested arrays: `[[String]]`, `[[Int]]`, etc. + /// - Entity references are converted to the referenced entity's ID type + fn value_type_from_field(&self, field: &FieldInfo) -> String { + let base = if let Some(id_kind) = self.entity_id_kinds.get(&field.base_type) { + id_kind.gql_type_name().to_string() + } else { + field.base_type.clone() + }; + + // Wrap with brackets for each level of list nesting + let mut result = base; + for _ in 0..field.list_depth { + result = format!("[{}]", result); + } + result + } + + /// Convert field info to an AssemblyScript TypeExpr. + /// + /// Creates the correct type expression including nested arrays: + /// - Scalars: `string`, `i32`, `BigInt`, etc. + /// - Arrays: `Array`, `Array`, etc. + /// - Nested arrays: `Array>`, etc. + fn type_from_field(&self, field: &FieldInfo) -> TypeExpr { + let type_name = if let Some(id_kind) = self.entity_id_kinds.get(&field.base_type) { + id_kind.type_name() + } else { + asc_type_for_value(&field.base_type) + }; + + let named = NamedType::new(type_name); + + if field.list_depth > 0 { + // Use ArrayType::with_depth to create nested array types + let array_type = ArrayType::with_depth(named, field.list_depth); + if field.is_nullable { + NullableType::new(array_type).into() + } else { + array_type + } + } else if field.is_nullable && !named.is_primitive() { + NullableType::new(named).into() + } else { + named.into() + } + } +} + +enum StoreMethod { + Regular(Method), + Static(StaticMethod), +} + +#[cfg(test)] +mod tests { + use super::*; + + use graphql_tools::parser::parse_schema; + + #[test] + fn test_simple_entity() { + let schema = r#" + type Transfer @entity { + id: ID! + from: Bytes! + to: Bytes! + value: BigInt! + } + "#; + let doc = parse_schema::(schema).unwrap(); + let generator = SchemaCodeGenerator::new(&doc).unwrap(); + + let classes = generator.generate_types(true); + assert_eq!(classes.len(), 1); + + let transfer = &classes[0]; + assert_eq!(transfer.name, "Transfer"); + assert_eq!(transfer.extends, Some("Entity".to_string())); + assert!(transfer.export); + } + + #[test] + fn test_nullable_field() { + let schema = r#" + type Token @entity { + id: ID! + name: String + symbol: String! + } + "#; + let doc = parse_schema::(schema).unwrap(); + let generator = SchemaCodeGenerator::new(&doc).unwrap(); + + let classes = generator.generate_types(true); + assert_eq!(classes.len(), 1); + + // Check that we have methods for nullable and non-nullable fields + let token = &classes[0]; + let method_names: Vec<_> = token.methods.iter().map(|m| m.name.as_str()).collect(); + assert!(method_names.contains(&"get name")); + assert!(method_names.contains(&"set name")); + assert!(method_names.contains(&"get symbol")); + assert!(method_names.contains(&"set symbol")); + } + + #[test] + fn test_id_field_types() { + assert_eq!(IdFieldKind::String.type_name(), "string"); + assert_eq!(IdFieldKind::Bytes.type_name(), "Bytes"); + assert_eq!(IdFieldKind::Int8.type_name(), "i64"); + } + + #[test] + fn test_auto_increment_support() { + // String IDs don't support auto-increment + assert!(!IdFieldKind::String.supports_auto()); + assert!(IdFieldKind::String.constructor_default().is_none()); + + // Int8 IDs support auto-increment + assert!(IdFieldKind::Int8.supports_auto()); + assert_eq!(IdFieldKind::Int8.constructor_param_type(), "i64"); + assert_eq!( + IdFieldKind::Int8.constructor_default(), + Some("i64.MIN_VALUE") + ); + assert_eq!( + IdFieldKind::Int8.auto_check_condition(), + "id != i64.MIN_VALUE" + ); + + // Bytes IDs support auto-increment + assert!(IdFieldKind::Bytes.supports_auto()); + assert_eq!(IdFieldKind::Bytes.constructor_param_type(), "Bytes | null"); + assert_eq!(IdFieldKind::Bytes.constructor_default(), Some("null")); + assert_eq!(IdFieldKind::Bytes.auto_check_condition(), "id"); + } + + #[test] + fn test_int8_id_auto_increment_codegen() { + let schema = r#" + type Counter @entity { + id: Int8! + value: BigInt! + } + "#; + let doc = parse_schema::(schema).unwrap(); + let generator = SchemaCodeGenerator::new(&doc).unwrap(); + + let classes = generator.generate_types(true); + assert_eq!(classes.len(), 1); + + let counter = &classes[0]; + let output = counter.to_string(); + + // Constructor should have optional id with sentinel default + assert!( + output.contains("constructor(id: i64 = i64.MIN_VALUE)"), + "Int8 ID constructor should have default sentinel value, got: {}", + output + ); + + // Constructor should have doc comment + assert!( + output.contains("Leaving out the id argument uses an autoincrementing id"), + "Constructor should have auto-increment doc comment" + ); + + // Constructor body should conditionally set id + assert!( + output.contains("if (id != i64.MIN_VALUE)"), + "Constructor should check for sentinel value" + ); + + // save() method should use "auto" when id is null + assert!( + output.contains("store.set('Counter', 'auto', this)"), + "save() should use 'auto' key for auto-increment, got: {}", + output + ); + } + + #[test] + fn test_bytes_id_auto_increment_codegen() { + let schema = r#" + type Event @entity { + id: Bytes! + data: String! + } + "#; + let doc = parse_schema::(schema).unwrap(); + let generator = SchemaCodeGenerator::new(&doc).unwrap(); + + let classes = generator.generate_types(true); + assert_eq!(classes.len(), 1); + + let event = &classes[0]; + let output = event.to_string(); + + // Constructor should have nullable id with null default + assert!( + output.contains("constructor(id: Bytes | null = null)"), + "Bytes ID constructor should have null default, got: {}", + output + ); + + // Constructor should have doc comment + assert!( + output.contains("Leaving out the id argument uses an autoincrementing id"), + "Constructor should have auto-increment doc comment" + ); + + // Constructor body should conditionally set id + assert!( + output.contains("if (id)"), + "Constructor should check for null" + ); + + // save() method should use "auto" when id is null + assert!( + output.contains("store.set('Event', 'auto', this)"), + "save() should use 'auto' key for auto-increment, got: {}", + output + ); + } + + #[test] + fn test_string_id_no_auto_increment() { + let schema = r#" + type User @entity { + id: ID! + name: String! + } + "#; + let doc = parse_schema::(schema).unwrap(); + let generator = SchemaCodeGenerator::new(&doc).unwrap(); + + let classes = generator.generate_types(true); + assert_eq!(classes.len(), 1); + + let user = &classes[0]; + let output = user.to_string(); + + // Constructor should have required id (no default) + assert!( + output.contains("constructor(id: string)"), + "String ID constructor should have required parameter, got: {}", + output + ); + + // Constructor should NOT have auto-increment doc comment + assert!( + !output.contains("autoincrementing"), + "String ID should not mention auto-increment" + ); + + // save() method should NOT use "auto" + assert!( + !output.contains("'auto'"), + "String ID save() should not use 'auto' key" + ); + + // save() should require ID + assert!( + output.contains("Cannot save User entity without an ID"), + "String ID save() should require ID" + ); + } + + #[test] + fn test_entity_reference() { + let schema = r#" + type User @entity { + id: ID! + name: String! + } + type Post @entity { + id: ID! + author: User! + } + "#; + let doc = parse_schema::(schema).unwrap(); + let generator = SchemaCodeGenerator::new(&doc).unwrap(); + + // The Post.author field should be treated as a string (entity ID reference) + assert!(generator.entity_names.contains("User")); + assert!(generator.entity_names.contains("Post")); + } + + #[test] + fn test_simple_array_field() { + let schema = r#" + type Token @entity { + id: ID! + holders: [String!]! + } + "#; + let doc = parse_schema::(schema).unwrap(); + let generator = SchemaCodeGenerator::new(&doc).unwrap(); + + let classes = generator.generate_types(true); + assert_eq!(classes.len(), 1); + + let token = &classes[0]; + let output = token.to_string(); + + // Verify array field getter/setter are generated + assert!( + output.contains("get holders()"), + "Should have holders getter" + ); + assert!( + output.contains("set holders("), + "Should have holders setter" + ); + + // Check the type is Array + assert!( + output.contains("Array"), + "Array field should use Array type" + ); + } + + #[test] + fn test_nested_array_field() { + let schema = r#" + type Matrix @entity { + id: ID! + stringMatrix: [[String!]!]! + intMatrix: [[Int!]!] + bigIntMatrix: [[BigInt!]!]! + } + "#; + let doc = parse_schema::(schema).unwrap(); + let generator = SchemaCodeGenerator::new(&doc).unwrap(); + + let classes = generator.generate_types(true); + assert_eq!(classes.len(), 1); + + let matrix = &classes[0]; + let output = matrix.to_string(); + + // Verify nested array field getter/setter are generated + assert!( + output.contains("get stringMatrix()"), + "Should have stringMatrix getter" + ); + assert!( + output.contains("set stringMatrix("), + "Should have stringMatrix setter" + ); + + // Check the type is Array> + assert!( + output.contains("Array>"), + "Nested array field should use Array> type, got: {}", + output + ); + + // Check that toStringMatrix() and fromStringMatrix() are used + assert!( + output.contains("toStringMatrix()"), + "Should use toStringMatrix() for nested string arrays" + ); + assert!( + output.contains("fromStringMatrix("), + "Should use Value.fromStringMatrix() for nested string arrays" + ); + + // Check BigInt matrix uses correct methods + assert!( + output.contains("Array>"), + "BigInt matrix should use Array> type" + ); + assert!( + output.contains("toBigIntMatrix()"), + "Should use toBigIntMatrix() for nested BigInt arrays" + ); + assert!( + output.contains("fromBigIntMatrix("), + "Should use Value.fromBigIntMatrix() for nested BigInt arrays" + ); + + // Check nullable nested array has correct type + assert!( + output.contains("Array> | null"), + "Nullable nested array should be Array> | null" + ); + } + + #[test] + fn test_list_depth() { + use graphql_tools::parser::parse_schema; + + // Helper to get list depth from schema field type + fn get_field_list_depth(schema_str: &str) -> u8 { + let doc = parse_schema::(schema_str).unwrap(); + for def in &doc.definitions { + if let Definition::TypeDefinition(TypeDefinition::Object(obj)) = def { + for field in &obj.fields { + if field.name == "field" { + return list_depth(&field.field_type); + } + } + } + } + panic!("Field not found"); + } + + // Scalar + assert_eq!( + get_field_list_depth("type T @entity { id: ID!, field: String! }"), + 0 + ); + + // Simple array + assert_eq!( + get_field_list_depth("type T @entity { id: ID!, field: [String!]! }"), + 1 + ); + + // Nested array (matrix) + assert_eq!( + get_field_list_depth("type T @entity { id: ID!, field: [[String!]!]! }"), + 2 + ); + + // Triple nested array + assert_eq!( + get_field_list_depth("type T @entity { id: ID!, field: [[[String!]!]!]! }"), + 3 + ); + } + + #[test] + fn test_value_type_from_field_nested() { + let schema = r#" + type Test @entity { + id: ID! + scalar: String! + array: [String!]! + matrix: [[String!]!]! + } + "#; + let doc = parse_schema::(schema).unwrap(); + let generator = SchemaCodeGenerator::new(&doc).unwrap(); + + // Find the entity + let entity = &generator.entities[0]; + + // Find each field and check its value type + let scalar_field = entity.fields.iter().find(|f| f.name == "scalar").unwrap(); + let array_field = entity.fields.iter().find(|f| f.name == "array").unwrap(); + let matrix_field = entity.fields.iter().find(|f| f.name == "matrix").unwrap(); + + assert_eq!(generator.value_type_from_field(scalar_field), "String"); + assert_eq!(generator.value_type_from_field(array_field), "[String]"); + assert_eq!(generator.value_type_from_field(matrix_field), "[[String]]"); + } + + #[test] + fn test_bytes_id_entity_reference() { + let schema = r#" + type Token @entity { + id: Bytes! + name: String! + } + type Balance @entity { + id: ID! + token: Token! + amount: BigInt! + } + "#; + let doc = parse_schema::(schema).unwrap(); + let generator = SchemaCodeGenerator::new(&doc).unwrap(); + + let classes = generator.generate_types(true); + let balance = classes.iter().find(|c| c.name == "Balance").unwrap(); + let output = balance.to_string(); + + // Getter should return Bytes and use toBytes() + assert!( + output.contains("value.toBytes()"), + "Bytes-ID entity reference getter should use toBytes(), got: {}", + output + ); + + // Setter should use Value.fromBytes() + assert!( + output.contains("Value.fromBytes("), + "Bytes-ID entity reference setter should use Value.fromBytes(), got: {}", + output + ); + + // Return type should be Bytes, not string + assert!( + output.contains("get token(): Bytes"), + "Bytes-ID entity reference getter should return Bytes, got: {}", + output + ); + } + + #[test] + fn test_int8_id_entity_reference() { + let schema = r#" + type Counter @entity { + id: Int8! + value: BigInt! + } + type Snapshot @entity { + id: ID! + counter: Counter! + } + "#; + let doc = parse_schema::(schema).unwrap(); + let generator = SchemaCodeGenerator::new(&doc).unwrap(); + + let classes = generator.generate_types(true); + let snapshot = classes.iter().find(|c| c.name == "Snapshot").unwrap(); + let output = snapshot.to_string(); + + // Getter should return i64 and use toI64() + assert!( + output.contains("value.toI64()"), + "Int8-ID entity reference getter should use toI64(), got: {}", + output + ); + + // Setter should use Value.fromI64() + assert!( + output.contains("Value.fromI64("), + "Int8-ID entity reference setter should use Value.fromI64(), got: {}", + output + ); + + // Return type should be i64 + assert!( + output.contains("get counter(): i64"), + "Int8-ID entity reference getter should return i64, got: {}", + output + ); + } + + #[test] + fn test_mixed_id_entity_references() { + let schema = r#" + type User @entity { + id: ID! + name: String! + } + type Token @entity { + id: Bytes! + owner: User! + } + "#; + let doc = parse_schema::(schema).unwrap(); + let generator = SchemaCodeGenerator::new(&doc).unwrap(); + + let classes = generator.generate_types(true); + let token = classes.iter().find(|c| c.name == "Token").unwrap(); + let output = token.to_string(); + + // Token.owner references User which has String ID + assert!( + output.contains("get owner(): string"), + "Reference to String-ID entity should use string type, got: {}", + output + ); + assert!( + output.contains("value.toString()"), + "Reference to String-ID entity should use toString(), got: {}", + output + ); + } + + #[test] + fn test_nullable_bytes_id_entity_reference() { + let schema = r#" + type Token @entity { + id: Bytes! + name: String! + } + type Balance @entity { + id: ID! + token: Token + amount: BigInt! + } + "#; + let doc = parse_schema::(schema).unwrap(); + let generator = SchemaCodeGenerator::new(&doc).unwrap(); + + let classes = generator.generate_types(true); + let balance = classes.iter().find(|c| c.name == "Balance").unwrap(); + let output = balance.to_string(); + + // Nullable Bytes reference should be `Bytes | null` + assert!( + output.contains("get token(): Bytes | null"), + "Nullable Bytes-ID reference should return Bytes | null, got: {}", + output + ); + } + + #[test] + fn test_derived_field_with_bytes_id_parent() { + let schema = r#" + type Token @entity { + id: Bytes! + balances: [Balance!]! @derivedFrom(field: "token") + } + type Balance @entity { + id: ID! + token: Token! + amount: BigInt! + } + "#; + let doc = parse_schema::(schema).unwrap(); + let generator = SchemaCodeGenerator::new(&doc).unwrap(); + + let classes = generator.generate_types(true); + let token = classes.iter().find(|c| c.name == "Token").unwrap(); + let output = token.to_string(); + + // Derived field getter on Bytes-ID entity should use toBytes().toHexString() + assert!( + output.contains("this.get('id')!.toBytes().toHexString()"), + "Derived field on Bytes-ID entity should use toBytes().toHexString(), got: {}", + output + ); + } +} diff --git a/gnd/src/codegen/template.rs b/gnd/src/codegen/template.rs new file mode 100644 index 00000000000..72a27f9e733 --- /dev/null +++ b/gnd/src/codegen/template.rs @@ -0,0 +1,286 @@ +//! Data source template code generation. +//! +//! Generates AssemblyScript classes for subgraph templates that allow +//! dynamic data source creation at runtime. + +use super::typescript::{self as ts, Class, ModuleImports, Param, StaticMethod}; + +/// The kind of a data source template. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum TemplateKind { + /// Ethereum contract template + Ethereum, + /// IPFS file template + FileIpfs, + /// Arweave file template + FileArweave, +} + +impl TemplateKind { + /// Parse a template kind from a string (e.g., "ethereum/contract", "file/ipfs"). + pub fn from_str_kind(kind: &str) -> Option { + match kind { + "ethereum/contract" | "ethereum" => Some(TemplateKind::Ethereum), + "file/ipfs" => Some(TemplateKind::FileIpfs), + "file/arweave" => Some(TemplateKind::FileArweave), + _ => None, + } + } +} + +/// A data source template from the subgraph manifest. +pub struct Template { + /// The name of the template. + pub name: String, + /// The kind of template. + pub kind: TemplateKind, +} + +impl Template { + /// Create a new template. + pub fn new(name: impl Into, kind: TemplateKind) -> Self { + Self { + name: name.into(), + kind, + } + } +} + +const GRAPH_TS_MODULE: &str = "@graphprotocol/graph-ts"; + +/// Template code generator. +pub struct TemplateCodeGenerator { + templates: Vec